1// SPDX-License-Identifier: GPL-2.0 2/* 3 * s390 specific pci instructions 4 * 5 * Copyright IBM Corp. 2013 6 */ 7 8#include <linux/export.h> 9#include <linux/errno.h> 10#include <linux/delay.h> 11#include <linux/jump_label.h> 12#include <asm/asm-extable.h> 13#include <asm/facility.h> 14#include <asm/pci_insn.h> 15#include <asm/pci_debug.h> 16#include <asm/pci_io.h> 17#include <asm/processor.h> 18 19#define ZPCI_INSN_BUSY_DELAY 1 /* 1 microsecond */ 20 21struct zpci_err_insn_data { 22 u8 insn; 23 u8 cc; 24 u8 status; 25 union { 26 struct { 27 u64 req; 28 u64 offset; 29 }; 30 struct { 31 u64 addr; 32 u64 len; 33 }; 34 }; 35} __packed; 36 37static inline void zpci_err_insn_req(int lvl, u8 insn, u8 cc, u8 status, 38 u64 req, u64 offset) 39{ 40 struct zpci_err_insn_data data = { 41 .insn = insn, .cc = cc, .status = status, 42 .req = req, .offset = offset}; 43 44 zpci_err_hex_level(lvl, &data, sizeof(data)); 45} 46 47static inline void zpci_err_insn_addr(int lvl, u8 insn, u8 cc, u8 status, 48 u64 addr, u64 len) 49{ 50 struct zpci_err_insn_data data = { 51 .insn = insn, .cc = cc, .status = status, 52 .addr = addr, .len = len}; 53 54 zpci_err_hex_level(lvl, &data, sizeof(data)); 55} 56 57/* Modify PCI Function Controls */ 58static inline u8 __mpcifc(u64 req, struct zpci_fib *fib, u8 *status) 59{ 60 u8 cc; 61 62 asm volatile ( 63 " .insn rxy,0xe300000000d0,%[req],%[fib]\n" 64 " ipm %[cc]\n" 65 " srl %[cc],28\n" 66 : [cc] "=d" (cc), [req] "+d" (req), [fib] "+Q" (*fib) 67 : : "cc"); 68 *status = req >> 24 & 0xff; 69 return cc; 70} 71 72u8 zpci_mod_fc(u64 req, struct zpci_fib *fib, u8 *status) 73{ 74 bool retried = false; 75 u8 cc; 76 77 do { 78 cc = __mpcifc(req, fib, status); 79 if (cc == 2) { 80 msleep(ZPCI_INSN_BUSY_DELAY); 81 if (!retried) { 82 zpci_err_insn_req(1, 'M', cc, *status, req, 0); 83 retried = true; 84 } 85 } 86 } while (cc == 2); 87 88 if (cc) 89 zpci_err_insn_req(0, 'M', cc, *status, req, 0); 90 else if (retried) 91 zpci_err_insn_req(1, 'M', cc, *status, req, 0); 92 93 return cc; 94} 95EXPORT_SYMBOL_GPL(zpci_mod_fc); 96 97/* Refresh PCI Translations */ 98static inline u8 __rpcit(u64 fn, u64 addr, u64 range, u8 *status) 99{ 100 union register_pair addr_range = {.even = addr, .odd = range}; 101 u8 cc; 102 103 asm volatile ( 104 " .insn rre,0xb9d30000,%[fn],%[addr_range]\n" 105 " ipm %[cc]\n" 106 " srl %[cc],28\n" 107 : [cc] "=d" (cc), [fn] "+d" (fn) 108 : [addr_range] "d" (addr_range.pair) 109 : "cc"); 110 *status = fn >> 24 & 0xff; 111 return cc; 112} 113 114int zpci_refresh_trans(u64 fn, u64 addr, u64 range) 115{ 116 bool retried = false; 117 u8 cc, status; 118 119 do { 120 cc = __rpcit(fn, addr, range, &status); 121 if (cc == 2) { 122 udelay(ZPCI_INSN_BUSY_DELAY); 123 if (!retried) { 124 zpci_err_insn_addr(1, 'R', cc, status, addr, range); 125 retried = true; 126 } 127 } 128 } while (cc == 2); 129 130 if (cc) 131 zpci_err_insn_addr(0, 'R', cc, status, addr, range); 132 else if (retried) 133 zpci_err_insn_addr(1, 'R', cc, status, addr, range); 134 135 if (cc == 1 && (status == 4 || status == 16)) 136 return -ENOMEM; 137 138 return (cc) ? -EIO : 0; 139} 140 141/* Set Interruption Controls */ 142int zpci_set_irq_ctrl(u16 ctl, u8 isc, union zpci_sic_iib *iib) 143{ 144 if (!test_facility(72)) 145 return -EIO; 146 147 asm volatile( 148 ".insn rsy,0xeb00000000d1,%[ctl],%[isc],%[iib]\n" 149 : : [ctl] "d" (ctl), [isc] "d" (isc << 27), [iib] "Q" (*iib)); 150 151 return 0; 152} 153EXPORT_SYMBOL_GPL(zpci_set_irq_ctrl); 154 155/* PCI Load */ 156static inline int ____pcilg(u64 *data, u64 req, u64 offset, u8 *status) 157{ 158 union register_pair req_off = {.even = req, .odd = offset}; 159 int cc = -ENXIO; 160 u64 __data; 161 162 asm volatile ( 163 " .insn rre,0xb9d20000,%[data],%[req_off]\n" 164 "0: ipm %[cc]\n" 165 " srl %[cc],28\n" 166 "1:\n" 167 EX_TABLE(0b, 1b) 168 : [cc] "+d" (cc), [data] "=d" (__data), 169 [req_off] "+&d" (req_off.pair) :: "cc"); 170 *status = req_off.even >> 24 & 0xff; 171 *data = __data; 172 return cc; 173} 174 175static inline int __pcilg(u64 *data, u64 req, u64 offset, u8 *status) 176{ 177 u64 __data; 178 int cc; 179 180 cc = ____pcilg(&__data, req, offset, status); 181 if (!cc) 182 *data = __data; 183 184 return cc; 185} 186 187int __zpci_load(u64 *data, u64 req, u64 offset) 188{ 189 bool retried = false; 190 u8 status; 191 int cc; 192 193 do { 194 cc = __pcilg(data, req, offset, &status); 195 if (cc == 2) { 196 udelay(ZPCI_INSN_BUSY_DELAY); 197 if (!retried) { 198 zpci_err_insn_req(1, 'l', cc, status, req, offset); 199 retried = true; 200 } 201 } 202 } while (cc == 2); 203 204 if (cc) 205 zpci_err_insn_req(0, 'l', cc, status, req, offset); 206 else if (retried) 207 zpci_err_insn_req(1, 'l', cc, status, req, offset); 208 209 return (cc > 0) ? -EIO : cc; 210} 211EXPORT_SYMBOL_GPL(__zpci_load); 212 213static inline int zpci_load_fh(u64 *data, const volatile void __iomem *addr, 214 unsigned long len) 215{ 216 struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)]; 217 u64 req = ZPCI_CREATE_REQ(READ_ONCE(entry->fh), entry->bar, len); 218 219 return __zpci_load(data, req, ZPCI_OFFSET(addr)); 220} 221 222static inline int __pcilg_mio(u64 *data, u64 ioaddr, u64 len, u8 *status) 223{ 224 union register_pair ioaddr_len = {.even = ioaddr, .odd = len}; 225 int cc = -ENXIO; 226 u64 __data; 227 228 asm volatile ( 229 " .insn rre,0xb9d60000,%[data],%[ioaddr_len]\n" 230 "0: ipm %[cc]\n" 231 " srl %[cc],28\n" 232 "1:\n" 233 EX_TABLE(0b, 1b) 234 : [cc] "+d" (cc), [data] "=d" (__data), 235 [ioaddr_len] "+&d" (ioaddr_len.pair) :: "cc"); 236 *status = ioaddr_len.odd >> 24 & 0xff; 237 *data = __data; 238 return cc; 239} 240 241int zpci_load(u64 *data, const volatile void __iomem *addr, unsigned long len) 242{ 243 u8 status; 244 int cc; 245 246 if (!static_branch_unlikely(&have_mio)) 247 return zpci_load_fh(data, addr, len); 248 249 cc = __pcilg_mio(data, (__force u64) addr, len, &status); 250 if (cc) 251 zpci_err_insn_addr(0, 'L', cc, status, (__force u64) addr, len); 252 253 return (cc > 0) ? -EIO : cc; 254} 255EXPORT_SYMBOL_GPL(zpci_load); 256 257/* PCI Store */ 258static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status) 259{ 260 union register_pair req_off = {.even = req, .odd = offset}; 261 int cc = -ENXIO; 262 263 asm volatile ( 264 " .insn rre,0xb9d00000,%[data],%[req_off]\n" 265 "0: ipm %[cc]\n" 266 " srl %[cc],28\n" 267 "1:\n" 268 EX_TABLE(0b, 1b) 269 : [cc] "+d" (cc), [req_off] "+&d" (req_off.pair) 270 : [data] "d" (data) 271 : "cc"); 272 *status = req_off.even >> 24 & 0xff; 273 return cc; 274} 275 276int __zpci_store(u64 data, u64 req, u64 offset) 277{ 278 bool retried = false; 279 u8 status; 280 int cc; 281 282 do { 283 cc = __pcistg(data, req, offset, &status); 284 if (cc == 2) { 285 udelay(ZPCI_INSN_BUSY_DELAY); 286 if (!retried) { 287 zpci_err_insn_req(1, 's', cc, status, req, offset); 288 retried = true; 289 } 290 } 291 } while (cc == 2); 292 293 if (cc) 294 zpci_err_insn_req(0, 's', cc, status, req, offset); 295 else if (retried) 296 zpci_err_insn_req(1, 's', cc, status, req, offset); 297 298 return (cc > 0) ? -EIO : cc; 299} 300EXPORT_SYMBOL_GPL(__zpci_store); 301 302static inline int zpci_store_fh(const volatile void __iomem *addr, u64 data, 303 unsigned long len) 304{ 305 struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)]; 306 u64 req = ZPCI_CREATE_REQ(READ_ONCE(entry->fh), entry->bar, len); 307 308 return __zpci_store(data, req, ZPCI_OFFSET(addr)); 309} 310 311static inline int __pcistg_mio(u64 data, u64 ioaddr, u64 len, u8 *status) 312{ 313 union register_pair ioaddr_len = {.even = ioaddr, .odd = len}; 314 int cc = -ENXIO; 315 316 asm volatile ( 317 " .insn rre,0xb9d40000,%[data],%[ioaddr_len]\n" 318 "0: ipm %[cc]\n" 319 " srl %[cc],28\n" 320 "1:\n" 321 EX_TABLE(0b, 1b) 322 : [cc] "+d" (cc), [ioaddr_len] "+&d" (ioaddr_len.pair) 323 : [data] "d" (data) 324 : "cc", "memory"); 325 *status = ioaddr_len.odd >> 24 & 0xff; 326 return cc; 327} 328 329int zpci_store(const volatile void __iomem *addr, u64 data, unsigned long len) 330{ 331 u8 status; 332 int cc; 333 334 if (!static_branch_unlikely(&have_mio)) 335 return zpci_store_fh(addr, data, len); 336 337 cc = __pcistg_mio(data, (__force u64) addr, len, &status); 338 if (cc) 339 zpci_err_insn_addr(0, 'S', cc, status, (__force u64) addr, len); 340 341 return (cc > 0) ? -EIO : cc; 342} 343EXPORT_SYMBOL_GPL(zpci_store); 344 345/* PCI Store Block */ 346static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status) 347{ 348 int cc = -ENXIO; 349 350 asm volatile ( 351 " .insn rsy,0xeb00000000d0,%[req],%[offset],%[data]\n" 352 "0: ipm %[cc]\n" 353 " srl %[cc],28\n" 354 "1:\n" 355 EX_TABLE(0b, 1b) 356 : [cc] "+d" (cc), [req] "+d" (req) 357 : [offset] "d" (offset), [data] "Q" (*data) 358 : "cc"); 359 *status = req >> 24 & 0xff; 360 return cc; 361} 362 363int __zpci_store_block(const u64 *data, u64 req, u64 offset) 364{ 365 bool retried = false; 366 u8 status; 367 int cc; 368 369 do { 370 cc = __pcistb(data, req, offset, &status); 371 if (cc == 2) { 372 udelay(ZPCI_INSN_BUSY_DELAY); 373 if (!retried) { 374 zpci_err_insn_req(0, 'b', cc, status, req, offset); 375 retried = true; 376 } 377 } 378 } while (cc == 2); 379 380 if (cc) 381 zpci_err_insn_req(0, 'b', cc, status, req, offset); 382 else if (retried) 383 zpci_err_insn_req(1, 'b', cc, status, req, offset); 384 385 return (cc > 0) ? -EIO : cc; 386} 387EXPORT_SYMBOL_GPL(__zpci_store_block); 388 389static inline int zpci_write_block_fh(volatile void __iomem *dst, 390 const void *src, unsigned long len) 391{ 392 struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(dst)]; 393 u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, len); 394 u64 offset = ZPCI_OFFSET(dst); 395 396 return __zpci_store_block(src, req, offset); 397} 398 399static inline int __pcistb_mio(const u64 *data, u64 ioaddr, u64 len, u8 *status) 400{ 401 int cc = -ENXIO; 402 403 asm volatile ( 404 " .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[data]\n" 405 "0: ipm %[cc]\n" 406 " srl %[cc],28\n" 407 "1:\n" 408 EX_TABLE(0b, 1b) 409 : [cc] "+d" (cc), [len] "+d" (len) 410 : [ioaddr] "d" (ioaddr), [data] "Q" (*data) 411 : "cc"); 412 *status = len >> 24 & 0xff; 413 return cc; 414} 415 416int zpci_write_block(volatile void __iomem *dst, 417 const void *src, unsigned long len) 418{ 419 u8 status; 420 int cc; 421 422 if (!static_branch_unlikely(&have_mio)) 423 return zpci_write_block_fh(dst, src, len); 424 425 cc = __pcistb_mio(src, (__force u64) dst, len, &status); 426 if (cc) 427 zpci_err_insn_addr(0, 'B', cc, status, (__force u64) dst, len); 428 429 return (cc > 0) ? -EIO : cc; 430} 431EXPORT_SYMBOL_GPL(zpci_write_block); 432 433static inline void __pciwb_mio(void) 434{ 435 asm volatile (".insn rre,0xb9d50000,0,0\n"); 436} 437 438void zpci_barrier(void) 439{ 440 if (static_branch_likely(&have_mio)) 441 __pciwb_mio(); 442} 443EXPORT_SYMBOL_GPL(zpci_barrier); 444