1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * IBM Accelerator Family 'GenWQE' 4 * 5 * (C) Copyright IBM Corp. 2013 6 * 7 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com> 8 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com> 9 * Author: Michael Jung <mijung@gmx.net> 10 * Author: Michael Ruettger <michael@ibmra.de> 11 */ 12 13/* 14 * Device Driver Control Block (DDCB) queue support. Definition of 15 * interrupt handlers for queue support as well as triggering the 16 * health monitor code in case of problems. The current hardware uses 17 * an MSI interrupt which is shared between error handling and 18 * functional code. 19 */ 20 21#include <linux/types.h> 22#include <linux/sched.h> 23#include <linux/wait.h> 24#include <linux/pci.h> 25#include <linux/string.h> 26#include <linux/dma-mapping.h> 27#include <linux/delay.h> 28#include <linux/module.h> 29#include <linux/interrupt.h> 30#include <linux/crc-itu-t.h> 31 32#include "card_base.h" 33#include "card_ddcb.h" 34 35/* 36 * N: next DDCB, this is where the next DDCB will be put. 37 * A: active DDCB, this is where the code will look for the next completion. 38 * x: DDCB is enqueued, we are waiting for its completion. 39 40 * Situation (1): Empty queue 41 * +---+---+---+---+---+---+---+---+ 42 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 43 * | | | | | | | | | 44 * +---+---+---+---+---+---+---+---+ 45 * A/N 46 * enqueued_ddcbs = A - N = 2 - 2 = 0 47 * 48 * Situation (2): Wrapped, N > A 49 * +---+---+---+---+---+---+---+---+ 50 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 51 * | | | x | x | | | | | 52 * +---+---+---+---+---+---+---+---+ 53 * A N 54 * enqueued_ddcbs = N - A = 4 - 2 = 2 55 * 56 * Situation (3): Queue wrapped, A > N 57 * +---+---+---+---+---+---+---+---+ 58 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 59 * | x | x | | | x | x | x | x | 60 * +---+---+---+---+---+---+---+---+ 61 * N A 62 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 2) = 6 63 * 64 * Situation (4a): Queue full N > A 65 * +---+---+---+---+---+---+---+---+ 66 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 67 * | x | x | x | x | x | x | x | | 68 * +---+---+---+---+---+---+---+---+ 69 * A N 70 * 71 * enqueued_ddcbs = N - A = 7 - 0 = 7 72 * 73 * Situation (4a): Queue full A > N 74 * +---+---+---+---+---+---+---+---+ 75 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 76 * | x | x | x | | x | x | x | x | 77 * +---+---+---+---+---+---+---+---+ 78 * N A 79 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7 80 */ 81 82static int queue_empty(struct ddcb_queue *queue) 83{ 84 return queue->ddcb_next == queue->ddcb_act; 85} 86 87static int queue_enqueued_ddcbs(struct ddcb_queue *queue) 88{ 89 if (queue->ddcb_next >= queue->ddcb_act) 90 return queue->ddcb_next - queue->ddcb_act; 91 92 return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next); 93} 94 95static int queue_free_ddcbs(struct ddcb_queue *queue) 96{ 97 int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1; 98 99 if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */ 100 return 0; 101 } 102 return free_ddcbs; 103} 104 105/* 106 * Use of the PRIV field in the DDCB for queue debugging: 107 * 108 * (1) Trying to get rid of a DDCB which saw a timeout: 109 * pddcb->priv[6] = 0xcc; # cleared 110 * 111 * (2) Append a DDCB via NEXT bit: 112 * pddcb->priv[7] = 0xaa; # appended 113 * 114 * (3) DDCB needed tapping: 115 * pddcb->priv[7] = 0xbb; # tapped 116 * 117 * (4) DDCB marked as correctly finished: 118 * pddcb->priv[6] = 0xff; # finished 119 */ 120 121static inline void ddcb_mark_tapped(struct ddcb *pddcb) 122{ 123 pddcb->priv[7] = 0xbb; /* tapped */ 124} 125 126static inline void ddcb_mark_appended(struct ddcb *pddcb) 127{ 128 pddcb->priv[7] = 0xaa; /* appended */ 129} 130 131static inline void ddcb_mark_cleared(struct ddcb *pddcb) 132{ 133 pddcb->priv[6] = 0xcc; /* cleared */ 134} 135 136static inline void ddcb_mark_finished(struct ddcb *pddcb) 137{ 138 pddcb->priv[6] = 0xff; /* finished */ 139} 140 141static inline void ddcb_mark_unused(struct ddcb *pddcb) 142{ 143 pddcb->priv_64 = cpu_to_be64(0); /* not tapped */ 144} 145 146/** 147 * genwqe_crc16() - Generate 16-bit crc as required for DDCBs 148 * @buff: pointer to data buffer 149 * @len: length of data for calculation 150 * @init: initial crc (0xffff at start) 151 * 152 * Polynomial = x^16 + x^12 + x^5 + 1 (0x1021) 153 * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff 154 * should result in a crc16 of 0x89c3 155 * 156 * Return: crc16 checksum in big endian format ! 157 */ 158static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init) 159{ 160 return crc_itu_t(init, buff, len); 161} 162 163static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue) 164{ 165 int i; 166 struct ddcb *pddcb; 167 unsigned long flags; 168 struct pci_dev *pci_dev = cd->pci_dev; 169 170 spin_lock_irqsave(&cd->print_lock, flags); 171 172 dev_info(&pci_dev->dev, 173 "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n", 174 cd->card_idx, queue->ddcb_act, queue->ddcb_next); 175 176 pddcb = queue->ddcb_vaddr; 177 for (i = 0; i < queue->ddcb_max; i++) { 178 dev_err(&pci_dev->dev, 179 " %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n", 180 i == queue->ddcb_act ? '>' : ' ', 181 i, 182 be16_to_cpu(pddcb->retc_16), 183 be16_to_cpu(pddcb->seqnum_16), 184 pddcb->hsi, 185 pddcb->shi, 186 be64_to_cpu(pddcb->priv_64), 187 pddcb->cmd); 188 pddcb++; 189 } 190 spin_unlock_irqrestore(&cd->print_lock, flags); 191} 192 193struct genwqe_ddcb_cmd *ddcb_requ_alloc(void) 194{ 195 struct ddcb_requ *req; 196 197 req = kzalloc(sizeof(*req), GFP_KERNEL); 198 if (!req) 199 return NULL; 200 201 return &req->cmd; 202} 203 204void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd) 205{ 206 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd); 207 208 kfree(req); 209} 210 211static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req) 212{ 213 return req->req_state; 214} 215 216static inline void ddcb_requ_set_state(struct ddcb_requ *req, 217 enum genwqe_requ_state new_state) 218{ 219 req->req_state = new_state; 220} 221 222static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req) 223{ 224 return req->cmd.ddata_addr != 0x0; 225} 226 227/** 228 * ddcb_requ_finished() - Returns the hardware state of the associated DDCB 229 * @cd: pointer to genwqe device descriptor 230 * @req: DDCB work request 231 * 232 * Status of ddcb_requ mirrors this hardware state, but is copied in 233 * the ddcb_requ on interrupt/polling function. The lowlevel code 234 * should check the hardware state directly, the higher level code 235 * should check the copy. 236 * 237 * This function will also return true if the state of the queue is 238 * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the 239 * shutdown case. 240 */ 241static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req) 242{ 243 return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) || 244 (cd->card_state != GENWQE_CARD_USED); 245} 246 247#define RET_DDCB_APPENDED 1 248#define RET_DDCB_TAPPED 2 249/** 250 * enqueue_ddcb() - Enqueue a DDCB 251 * @cd: pointer to genwqe device descriptor 252 * @queue: queue this operation should be done on 253 * @pddcb: pointer to ddcb structure 254 * @ddcb_no: pointer to ddcb number being tapped 255 * 256 * Start execution of DDCB by tapping or append to queue via NEXT 257 * bit. This is done by an atomic 'compare and swap' instruction and 258 * checking SHI and HSI of the previous DDCB. 259 * 260 * This function must only be called with ddcb_lock held. 261 * 262 * Return: 1 if new DDCB is appended to previous 263 * 2 if DDCB queue is tapped via register/simulation 264 */ 265static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue, 266 struct ddcb *pddcb, int ddcb_no) 267{ 268 unsigned int try; 269 int prev_no; 270 struct ddcb *prev_ddcb; 271 __be32 old, new, icrc_hsi_shi; 272 u64 num; 273 274 /* 275 * For performance checks a Dispatch Timestamp can be put into 276 * DDCB It is supposed to use the SLU's free running counter, 277 * but this requires PCIe cycles. 278 */ 279 ddcb_mark_unused(pddcb); 280 281 /* check previous DDCB if already fetched */ 282 prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1; 283 prev_ddcb = &queue->ddcb_vaddr[prev_no]; 284 285 /* 286 * It might have happened that the HSI.FETCHED bit is 287 * set. Retry in this case. Therefore I expect maximum 2 times 288 * trying. 289 */ 290 ddcb_mark_appended(pddcb); 291 for (try = 0; try < 2; try++) { 292 old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */ 293 294 /* try to append via NEXT bit if prev DDCB is not completed */ 295 if ((old & DDCB_COMPLETED_BE32) != 0x00000000) 296 break; 297 298 new = (old | DDCB_NEXT_BE32); 299 300 wmb(); /* need to ensure write ordering */ 301 icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new); 302 303 if (icrc_hsi_shi == old) 304 return RET_DDCB_APPENDED; /* appended to queue */ 305 } 306 307 /* Queue must be re-started by updating QUEUE_OFFSET */ 308 ddcb_mark_tapped(pddcb); 309 num = (u64)ddcb_no << 8; 310 311 wmb(); /* need to ensure write ordering */ 312 __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */ 313 314 return RET_DDCB_TAPPED; 315} 316 317/** 318 * copy_ddcb_results() - Copy output state from real DDCB to request 319 * @req: pointer to requested DDCB parameters 320 * @ddcb_no: pointer to ddcb number being tapped 321 * 322 * Copy DDCB ASV to request struct. There is no endian 323 * conversion made, since data structure in ASV is still 324 * unknown here. 325 * 326 * This is needed by: 327 * - genwqe_purge_ddcb() 328 * - genwqe_check_ddcb_queue() 329 */ 330static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no) 331{ 332 struct ddcb_queue *queue = req->queue; 333 struct ddcb *pddcb = &queue->ddcb_vaddr[req->num]; 334 335 memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH); 336 337 /* copy status flags of the variant part */ 338 req->cmd.vcrc = be16_to_cpu(pddcb->vcrc_16); 339 req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64); 340 req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64); 341 342 req->cmd.attn = be16_to_cpu(pddcb->attn_16); 343 req->cmd.progress = be32_to_cpu(pddcb->progress_32); 344 req->cmd.retc = be16_to_cpu(pddcb->retc_16); 345 346 if (ddcb_requ_collect_debug_data(req)) { 347 int prev_no = (ddcb_no == 0) ? 348 queue->ddcb_max - 1 : ddcb_no - 1; 349 struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no]; 350 351 memcpy(&req->debug_data.ddcb_finished, pddcb, 352 sizeof(req->debug_data.ddcb_finished)); 353 memcpy(&req->debug_data.ddcb_prev, prev_pddcb, 354 sizeof(req->debug_data.ddcb_prev)); 355 } 356} 357 358/** 359 * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work requests. 360 * @cd: pointer to genwqe device descriptor 361 * @queue: queue to be checked 362 * 363 * Return: Number of DDCBs which were finished 364 */ 365static int genwqe_check_ddcb_queue(struct genwqe_dev *cd, 366 struct ddcb_queue *queue) 367{ 368 unsigned long flags; 369 int ddcbs_finished = 0; 370 struct pci_dev *pci_dev = cd->pci_dev; 371 372 spin_lock_irqsave(&queue->ddcb_lock, flags); 373 374 /* FIXME avoid soft locking CPU */ 375 while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) { 376 377 struct ddcb *pddcb; 378 struct ddcb_requ *req; 379 u16 vcrc, vcrc_16, retc_16; 380 381 pddcb = &queue->ddcb_vaddr[queue->ddcb_act]; 382 383 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 384 0x00000000) 385 goto go_home; /* not completed, continue waiting */ 386 387 wmb(); /* Add sync to decouple prev. read operations */ 388 389 /* Note: DDCB could be purged */ 390 req = queue->ddcb_req[queue->ddcb_act]; 391 if (req == NULL) { 392 /* this occurs if DDCB is purged, not an error */ 393 /* Move active DDCB further; Nothing to do anymore. */ 394 goto pick_next_one; 395 } 396 397 /* 398 * HSI=0x44 (fetched and completed), but RETC is 399 * 0x101, or even worse 0x000. 400 * 401 * In case of seeing the queue in inconsistent state 402 * we read the errcnts and the queue status to provide 403 * a trigger for our PCIe analyzer stop capturing. 404 */ 405 retc_16 = be16_to_cpu(pddcb->retc_16); 406 if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) { 407 u64 errcnts, status; 408 u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr; 409 410 errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS); 411 status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS); 412 413 dev_err(&pci_dev->dev, 414 "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n", 415 __func__, be16_to_cpu(pddcb->seqnum_16), 416 pddcb->hsi, retc_16, errcnts, status, 417 queue->ddcb_daddr + ddcb_offs); 418 } 419 420 copy_ddcb_results(req, queue->ddcb_act); 421 queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */ 422 423 dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num); 424 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 425 426 ddcb_mark_finished(pddcb); 427 428 /* calculate CRC_16 to see if VCRC is correct */ 429 vcrc = genwqe_crc16(pddcb->asv, 430 VCRC_LENGTH(req->cmd.asv_length), 431 0xffff); 432 vcrc_16 = be16_to_cpu(pddcb->vcrc_16); 433 if (vcrc != vcrc_16) { 434 printk_ratelimited(KERN_ERR 435 "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n", 436 GENWQE_DEVNAME, dev_name(&pci_dev->dev), 437 pddcb->pre, VCRC_LENGTH(req->cmd.asv_length), 438 vcrc, vcrc_16); 439 } 440 441 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED); 442 queue->ddcbs_completed++; 443 queue->ddcbs_in_flight--; 444 445 /* wake up process waiting for this DDCB, and 446 processes on the busy queue */ 447 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]); 448 wake_up_interruptible(&queue->busy_waitq); 449 450pick_next_one: 451 queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max; 452 ddcbs_finished++; 453 } 454 455 go_home: 456 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 457 return ddcbs_finished; 458} 459 460/** 461 * __genwqe_wait_ddcb(): Waits until DDCB is completed 462 * @cd: pointer to genwqe device descriptor 463 * @req: pointer to requsted DDCB parameters 464 * 465 * The Service Layer will update the RETC in DDCB when processing is 466 * pending or done. 467 * 468 * Return: > 0 remaining jiffies, DDCB completed 469 * -ETIMEDOUT when timeout 470 * -ERESTARTSYS when ^C 471 * -EINVAL when unknown error condition 472 * 473 * When an error is returned the called needs to ensure that 474 * purge_ddcb() is being called to get the &req removed from the 475 * queue. 476 */ 477int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) 478{ 479 int rc; 480 unsigned int ddcb_no; 481 struct ddcb_queue *queue; 482 struct pci_dev *pci_dev = cd->pci_dev; 483 484 if (req == NULL) 485 return -EINVAL; 486 487 queue = req->queue; 488 if (queue == NULL) 489 return -EINVAL; 490 491 ddcb_no = req->num; 492 if (ddcb_no >= queue->ddcb_max) 493 return -EINVAL; 494 495 rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no], 496 ddcb_requ_finished(cd, req), 497 GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ); 498 499 /* 500 * We need to distinguish 3 cases here: 501 * 1. rc == 0 timeout occurred 502 * 2. rc == -ERESTARTSYS signal received 503 * 3. rc > 0 remaining jiffies condition is true 504 */ 505 if (rc == 0) { 506 struct ddcb_queue *queue = req->queue; 507 struct ddcb *pddcb; 508 509 /* 510 * Timeout may be caused by long task switching time. 511 * When timeout happens, check if the request has 512 * meanwhile completed. 513 */ 514 genwqe_check_ddcb_queue(cd, req->queue); 515 if (ddcb_requ_finished(cd, req)) 516 return rc; 517 518 dev_err(&pci_dev->dev, 519 "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n", 520 __func__, req->num, rc, ddcb_requ_get_state(req), 521 req); 522 dev_err(&pci_dev->dev, 523 "[%s] IO_QUEUE_STATUS=0x%016llx\n", __func__, 524 __genwqe_readq(cd, queue->IO_QUEUE_STATUS)); 525 526 pddcb = &queue->ddcb_vaddr[req->num]; 527 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 528 529 print_ddcb_info(cd, req->queue); 530 return -ETIMEDOUT; 531 532 } else if (rc == -ERESTARTSYS) { 533 return rc; 534 /* 535 * EINTR: Stops the application 536 * ERESTARTSYS: Restartable systemcall; called again 537 */ 538 539 } else if (rc < 0) { 540 dev_err(&pci_dev->dev, 541 "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n", 542 __func__, req->num, rc, ddcb_requ_get_state(req)); 543 return -EINVAL; 544 } 545 546 /* Severe error occured. Driver is forced to stop operation */ 547 if (cd->card_state != GENWQE_CARD_USED) { 548 dev_err(&pci_dev->dev, 549 "[%s] err: DDCB#%d forced to stop (rc=%d)\n", 550 __func__, req->num, rc); 551 return -EIO; 552 } 553 return rc; 554} 555 556/** 557 * get_next_ddcb() - Get next available DDCB 558 * @cd: pointer to genwqe device descriptor 559 * @queue: DDCB queue 560 * @num: internal DDCB number 561 * 562 * DDCB's content is completely cleared but presets for PRE and 563 * SEQNUM. This function must only be called when ddcb_lock is held. 564 * 565 * Return: NULL if no empty DDCB available otherwise ptr to next DDCB. 566 */ 567static struct ddcb *get_next_ddcb(struct genwqe_dev *cd, 568 struct ddcb_queue *queue, 569 int *num) 570{ 571 u64 *pu64; 572 struct ddcb *pddcb; 573 574 if (queue_free_ddcbs(queue) == 0) /* queue is full */ 575 return NULL; 576 577 /* find new ddcb */ 578 pddcb = &queue->ddcb_vaddr[queue->ddcb_next]; 579 580 /* if it is not completed, we are not allowed to use it */ 581 /* barrier(); */ 582 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000) 583 return NULL; 584 585 *num = queue->ddcb_next; /* internal DDCB number */ 586 queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max; 587 588 /* clear important DDCB fields */ 589 pu64 = (u64 *)pddcb; 590 pu64[0] = 0ULL; /* offs 0x00 (ICRC,HSI,SHI,...) */ 591 pu64[1] = 0ULL; /* offs 0x01 (ACFUNC,CMD...) */ 592 593 /* destroy previous results in ASV */ 594 pu64[0x80/8] = 0ULL; /* offs 0x80 (ASV + 0) */ 595 pu64[0x88/8] = 0ULL; /* offs 0x88 (ASV + 0x08) */ 596 pu64[0x90/8] = 0ULL; /* offs 0x90 (ASV + 0x10) */ 597 pu64[0x98/8] = 0ULL; /* offs 0x98 (ASV + 0x18) */ 598 pu64[0xd0/8] = 0ULL; /* offs 0xd0 (RETC,ATTN...) */ 599 600 pddcb->pre = DDCB_PRESET_PRE; /* 128 */ 601 pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++); 602 return pddcb; 603} 604 605/** 606 * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue 607 * @cd: genwqe device descriptor 608 * @req: DDCB request 609 * 610 * This will fail when the request was already FETCHED. In this case 611 * we need to wait until it is finished. Else the DDCB can be 612 * reused. This function also ensures that the request data structure 613 * is removed from ddcb_req[]. 614 * 615 * Do not forget to call this function when genwqe_wait_ddcb() fails, 616 * such that the request gets really removed from ddcb_req[]. 617 * 618 * Return: 0 success 619 */ 620int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) 621{ 622 struct ddcb *pddcb = NULL; 623 unsigned int t; 624 unsigned long flags; 625 struct ddcb_queue *queue = req->queue; 626 struct pci_dev *pci_dev = cd->pci_dev; 627 u64 queue_status; 628 __be32 icrc_hsi_shi = 0x0000; 629 __be32 old, new; 630 631 /* unsigned long flags; */ 632 if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) { 633 dev_err(&pci_dev->dev, 634 "[%s] err: software timeout is not set!\n", __func__); 635 return -EFAULT; 636 } 637 638 pddcb = &queue->ddcb_vaddr[req->num]; 639 640 for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) { 641 642 spin_lock_irqsave(&queue->ddcb_lock, flags); 643 644 /* Check if req was meanwhile finished */ 645 if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) 646 goto go_home; 647 648 /* try to set PURGE bit if FETCHED/COMPLETED are not set */ 649 old = pddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */ 650 if ((old & DDCB_FETCHED_BE32) == 0x00000000) { 651 652 new = (old | DDCB_PURGE_BE32); 653 icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32, 654 old, new); 655 if (icrc_hsi_shi == old) 656 goto finish_ddcb; 657 } 658 659 /* normal finish with HSI bit */ 660 barrier(); 661 icrc_hsi_shi = pddcb->icrc_hsi_shi_32; 662 if (icrc_hsi_shi & DDCB_COMPLETED_BE32) 663 goto finish_ddcb; 664 665 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 666 667 /* 668 * Here the check_ddcb() function will most likely 669 * discover this DDCB to be finished some point in 670 * time. It will mark the req finished and free it up 671 * in the list. 672 */ 673 674 copy_ddcb_results(req, req->num); /* for the failing case */ 675 msleep(100); /* sleep for 1/10 second and try again */ 676 continue; 677 678finish_ddcb: 679 copy_ddcb_results(req, req->num); 680 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED); 681 queue->ddcbs_in_flight--; 682 queue->ddcb_req[req->num] = NULL; /* delete from array */ 683 ddcb_mark_cleared(pddcb); 684 685 /* Move active DDCB further; Nothing to do here anymore. */ 686 687 /* 688 * We need to ensure that there is at least one free 689 * DDCB in the queue. To do that, we must update 690 * ddcb_act only if the COMPLETED bit is set for the 691 * DDCB we are working on else we treat that DDCB even 692 * if we PURGED it as occupied (hardware is supposed 693 * to set the COMPLETED bit yet!). 694 */ 695 icrc_hsi_shi = pddcb->icrc_hsi_shi_32; 696 if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) && 697 (queue->ddcb_act == req->num)) { 698 queue->ddcb_act = ((queue->ddcb_act + 1) % 699 queue->ddcb_max); 700 } 701go_home: 702 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 703 return 0; 704 } 705 706 /* 707 * If the card is dead and the queue is forced to stop, we 708 * might see this in the queue status register. 709 */ 710 queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS); 711 712 dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num); 713 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 714 715 dev_err(&pci_dev->dev, 716 "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n", 717 __func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT, 718 queue_status); 719 720 print_ddcb_info(cd, req->queue); 721 722 return -EFAULT; 723} 724 725int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d) 726{ 727 int len; 728 struct pci_dev *pci_dev = cd->pci_dev; 729 730 if (d == NULL) { 731 dev_err(&pci_dev->dev, 732 "[%s] err: invalid memory for debug data!\n", 733 __func__); 734 return -EFAULT; 735 } 736 737 len = sizeof(d->driver_version); 738 snprintf(d->driver_version, len, "%s", DRV_VERSION); 739 d->slu_unitcfg = cd->slu_unitcfg; 740 d->app_unitcfg = cd->app_unitcfg; 741 return 0; 742} 743 744/** 745 * __genwqe_enqueue_ddcb() - Enqueue a DDCB 746 * @cd: pointer to genwqe device descriptor 747 * @req: pointer to DDCB execution request 748 * @f_flags: file mode: blocking, non-blocking 749 * 750 * Return: 0 if enqueuing succeeded 751 * -EIO if card is unusable/PCIe problems 752 * -EBUSY if enqueuing failed 753 */ 754int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req, 755 unsigned int f_flags) 756{ 757 struct ddcb *pddcb; 758 unsigned long flags; 759 struct ddcb_queue *queue; 760 struct pci_dev *pci_dev = cd->pci_dev; 761 u16 icrc; 762 763 retry: 764 if (cd->card_state != GENWQE_CARD_USED) { 765 printk_ratelimited(KERN_ERR 766 "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n", 767 GENWQE_DEVNAME, dev_name(&pci_dev->dev), 768 __func__, req->num); 769 return -EIO; 770 } 771 772 queue = req->queue = &cd->queue; 773 774 /* FIXME circumvention to improve performance when no irq is 775 * there. 776 */ 777 if (GENWQE_POLLING_ENABLED) 778 genwqe_check_ddcb_queue(cd, queue); 779 780 /* 781 * It must be ensured to process all DDCBs in successive 782 * order. Use a lock here in order to prevent nested DDCB 783 * enqueuing. 784 */ 785 spin_lock_irqsave(&queue->ddcb_lock, flags); 786 787 pddcb = get_next_ddcb(cd, queue, &req->num); /* get ptr and num */ 788 if (pddcb == NULL) { 789 int rc; 790 791 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 792 793 if (f_flags & O_NONBLOCK) { 794 queue->return_on_busy++; 795 return -EBUSY; 796 } 797 798 queue->wait_on_busy++; 799 rc = wait_event_interruptible(queue->busy_waitq, 800 queue_free_ddcbs(queue) != 0); 801 dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n", 802 __func__, rc); 803 if (rc == -ERESTARTSYS) 804 return rc; /* interrupted by a signal */ 805 806 goto retry; 807 } 808 809 if (queue->ddcb_req[req->num] != NULL) { 810 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 811 812 dev_err(&pci_dev->dev, 813 "[%s] picked DDCB %d with req=%p still in use!!\n", 814 __func__, req->num, req); 815 return -EFAULT; 816 } 817 ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED); 818 queue->ddcb_req[req->num] = req; 819 820 pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts); 821 pddcb->cmd = req->cmd.cmd; 822 pddcb->acfunc = req->cmd.acfunc; /* functional unit */ 823 824 /* 825 * We know that we can get retc 0x104 with CRC error, do not 826 * stop the queue in those cases for this command. XDIR = 1 827 * does not work for old SLU versions. 828 * 829 * Last bitstream with the old XDIR behavior had SLU_ID 830 * 0x34199. 831 */ 832 if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull) 833 pddcb->xdir = 0x1; 834 else 835 pddcb->xdir = 0x0; 836 837 838 pddcb->psp = (((req->cmd.asiv_length / 8) << 4) | 839 ((req->cmd.asv_length / 8))); 840 pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts); 841 842 /* 843 * If copying the whole DDCB_ASIV_LENGTH is impacting 844 * performance we need to change it to 845 * req->cmd.asiv_length. But simulation benefits from some 846 * non-architectured bits behind the architectured content. 847 * 848 * How much data is copied depends on the availability of the 849 * ATS field, which was introduced late. If the ATS field is 850 * supported ASIV is 8 bytes shorter than it used to be. Since 851 * the ATS field is copied too, the code should do exactly 852 * what it did before, but I wanted to make copying of the ATS 853 * field very explicit. 854 */ 855 if (genwqe_get_slu_id(cd) <= 0x2) { 856 memcpy(&pddcb->__asiv[0], /* destination */ 857 &req->cmd.__asiv[0], /* source */ 858 DDCB_ASIV_LENGTH); /* req->cmd.asiv_length */ 859 } else { 860 pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats); 861 memcpy(&pddcb->n.asiv[0], /* destination */ 862 &req->cmd.asiv[0], /* source */ 863 DDCB_ASIV_LENGTH_ATS); /* req->cmd.asiv_length */ 864 } 865 866 pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */ 867 868 /* 869 * Calculate CRC_16 for corresponding range PSP(7:4). Include 870 * empty 4 bytes prior to the data. 871 */ 872 icrc = genwqe_crc16((const u8 *)pddcb, 873 ICRC_LENGTH(req->cmd.asiv_length), 0xffff); 874 pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16); 875 876 /* enable DDCB completion irq */ 877 if (!GENWQE_POLLING_ENABLED) 878 pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32; 879 880 dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num); 881 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 882 883 if (ddcb_requ_collect_debug_data(req)) { 884 /* use the kernel copy of debug data. copying back to 885 user buffer happens later */ 886 887 genwqe_init_debug_data(cd, &req->debug_data); 888 memcpy(&req->debug_data.ddcb_before, pddcb, 889 sizeof(req->debug_data.ddcb_before)); 890 } 891 892 enqueue_ddcb(cd, queue, pddcb, req->num); 893 queue->ddcbs_in_flight++; 894 895 if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight) 896 queue->ddcbs_max_in_flight = queue->ddcbs_in_flight; 897 898 ddcb_requ_set_state(req, GENWQE_REQU_TAPPED); 899 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 900 wake_up_interruptible(&cd->queue_waitq); 901 902 return 0; 903} 904 905/** 906 * __genwqe_execute_raw_ddcb() - Setup and execute DDCB 907 * @cd: pointer to genwqe device descriptor 908 * @cmd: user provided DDCB command 909 * @f_flags: file mode: blocking, non-blocking 910 */ 911int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd, 912 struct genwqe_ddcb_cmd *cmd, 913 unsigned int f_flags) 914{ 915 int rc = 0; 916 struct pci_dev *pci_dev = cd->pci_dev; 917 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd); 918 919 if (cmd->asiv_length > DDCB_ASIV_LENGTH) { 920 dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n", 921 __func__, cmd->asiv_length); 922 return -EINVAL; 923 } 924 if (cmd->asv_length > DDCB_ASV_LENGTH) { 925 dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n", 926 __func__, cmd->asiv_length); 927 return -EINVAL; 928 } 929 rc = __genwqe_enqueue_ddcb(cd, req, f_flags); 930 if (rc != 0) 931 return rc; 932 933 rc = __genwqe_wait_ddcb(cd, req); 934 if (rc < 0) /* error or signal interrupt */ 935 goto err_exit; 936 937 if (ddcb_requ_collect_debug_data(req)) { 938 if (copy_to_user((struct genwqe_debug_data __user *) 939 (unsigned long)cmd->ddata_addr, 940 &req->debug_data, 941 sizeof(struct genwqe_debug_data))) 942 return -EFAULT; 943 } 944 945 /* 946 * Higher values than 0x102 indicate completion with faults, 947 * lower values than 0x102 indicate processing faults. Note 948 * that DDCB might have been purged. E.g. Cntl+C. 949 */ 950 if (cmd->retc != DDCB_RETC_COMPLETE) { 951 /* This might happen e.g. flash read, and needs to be 952 handled by the upper layer code. */ 953 rc = -EBADMSG; /* not processed/error retc */ 954 } 955 956 return rc; 957 958 err_exit: 959 __genwqe_purge_ddcb(cd, req); 960 961 if (ddcb_requ_collect_debug_data(req)) { 962 if (copy_to_user((struct genwqe_debug_data __user *) 963 (unsigned long)cmd->ddata_addr, 964 &req->debug_data, 965 sizeof(struct genwqe_debug_data))) 966 return -EFAULT; 967 } 968 return rc; 969} 970 971/** 972 * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished 973 * @cd: pointer to genwqe device descriptor 974 * 975 * We use this as condition for our wait-queue code. 976 */ 977static int genwqe_next_ddcb_ready(struct genwqe_dev *cd) 978{ 979 unsigned long flags; 980 struct ddcb *pddcb; 981 struct ddcb_queue *queue = &cd->queue; 982 983 spin_lock_irqsave(&queue->ddcb_lock, flags); 984 985 if (queue_empty(queue)) { /* empty queue */ 986 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 987 return 0; 988 } 989 990 pddcb = &queue->ddcb_vaddr[queue->ddcb_act]; 991 if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */ 992 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 993 return 1; 994 } 995 996 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 997 return 0; 998} 999 1000/** 1001 * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight 1002 * @cd: pointer to genwqe device descriptor 1003 * 1004 * Keep track on the number of DDCBs which ware currently in the 1005 * queue. This is needed for statistics as well as condition if we want 1006 * to wait or better do polling in case of no interrupts available. 1007 */ 1008int genwqe_ddcbs_in_flight(struct genwqe_dev *cd) 1009{ 1010 unsigned long flags; 1011 int ddcbs_in_flight = 0; 1012 struct ddcb_queue *queue = &cd->queue; 1013 1014 spin_lock_irqsave(&queue->ddcb_lock, flags); 1015 ddcbs_in_flight += queue->ddcbs_in_flight; 1016 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 1017 1018 return ddcbs_in_flight; 1019} 1020 1021static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) 1022{ 1023 int rc, i; 1024 struct ddcb *pddcb; 1025 u64 val64; 1026 unsigned int queue_size; 1027 struct pci_dev *pci_dev = cd->pci_dev; 1028 1029 if (GENWQE_DDCB_MAX < 2) 1030 return -EINVAL; 1031 1032 queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE); 1033 1034 queue->ddcbs_in_flight = 0; /* statistics */ 1035 queue->ddcbs_max_in_flight = 0; 1036 queue->ddcbs_completed = 0; 1037 queue->return_on_busy = 0; 1038 queue->wait_on_busy = 0; 1039 1040 queue->ddcb_seq = 0x100; /* start sequence number */ 1041 queue->ddcb_max = GENWQE_DDCB_MAX; 1042 queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size, 1043 &queue->ddcb_daddr); 1044 if (queue->ddcb_vaddr == NULL) { 1045 dev_err(&pci_dev->dev, 1046 "[%s] **err: could not allocate DDCB **\n", __func__); 1047 return -ENOMEM; 1048 } 1049 queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *), 1050 GFP_KERNEL); 1051 if (!queue->ddcb_req) { 1052 rc = -ENOMEM; 1053 goto free_ddcbs; 1054 } 1055 1056 queue->ddcb_waitqs = kcalloc(queue->ddcb_max, 1057 sizeof(wait_queue_head_t), 1058 GFP_KERNEL); 1059 if (!queue->ddcb_waitqs) { 1060 rc = -ENOMEM; 1061 goto free_requs; 1062 } 1063 1064 for (i = 0; i < queue->ddcb_max; i++) { 1065 pddcb = &queue->ddcb_vaddr[i]; /* DDCBs */ 1066 pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32; 1067 pddcb->retc_16 = cpu_to_be16(0xfff); 1068 1069 queue->ddcb_req[i] = NULL; /* requests */ 1070 init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */ 1071 } 1072 1073 queue->ddcb_act = 0; 1074 queue->ddcb_next = 0; /* queue is empty */ 1075 1076 spin_lock_init(&queue->ddcb_lock); 1077 init_waitqueue_head(&queue->busy_waitq); 1078 1079 val64 = ((u64)(queue->ddcb_max - 1) << 8); /* lastptr */ 1080 __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG, 0x07); /* iCRC/vCRC */ 1081 __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr); 1082 __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq); 1083 __genwqe_writeq(cd, queue->IO_QUEUE_WRAP, val64); 1084 return 0; 1085 1086 free_requs: 1087 kfree(queue->ddcb_req); 1088 queue->ddcb_req = NULL; 1089 free_ddcbs: 1090 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr, 1091 queue->ddcb_daddr); 1092 queue->ddcb_vaddr = NULL; 1093 queue->ddcb_daddr = 0ull; 1094 return rc; 1095 1096} 1097 1098static int ddcb_queue_initialized(struct ddcb_queue *queue) 1099{ 1100 return queue->ddcb_vaddr != NULL; 1101} 1102 1103static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) 1104{ 1105 unsigned int queue_size; 1106 1107 queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE); 1108 1109 kfree(queue->ddcb_req); 1110 queue->ddcb_req = NULL; 1111 1112 if (queue->ddcb_vaddr) { 1113 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr, 1114 queue->ddcb_daddr); 1115 queue->ddcb_vaddr = NULL; 1116 queue->ddcb_daddr = 0ull; 1117 } 1118} 1119 1120static irqreturn_t genwqe_pf_isr(int irq, void *dev_id) 1121{ 1122 u64 gfir; 1123 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id; 1124 struct pci_dev *pci_dev = cd->pci_dev; 1125 1126 /* 1127 * In case of fatal FIR error the queue is stopped, such that 1128 * we can safely check it without risking anything. 1129 */ 1130 cd->irqs_processed++; 1131 wake_up_interruptible(&cd->queue_waitq); 1132 1133 /* 1134 * Checking for errors before kicking the queue might be 1135 * safer, but slower for the good-case ... See above. 1136 */ 1137 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR); 1138 if (((gfir & GFIR_ERR_TRIGGER) != 0x0) && 1139 !pci_channel_offline(pci_dev)) { 1140 1141 if (cd->use_platform_recovery) { 1142 /* 1143 * Since we use raw accessors, EEH errors won't be 1144 * detected by the platform until we do a non-raw 1145 * MMIO or config space read 1146 */ 1147 readq(cd->mmio + IO_SLC_CFGREG_GFIR); 1148 1149 /* Don't do anything if the PCI channel is frozen */ 1150 if (pci_channel_offline(pci_dev)) 1151 goto exit; 1152 } 1153 1154 wake_up_interruptible(&cd->health_waitq); 1155 1156 /* 1157 * By default GFIRs causes recovery actions. This 1158 * count is just for debug when recovery is masked. 1159 */ 1160 dev_err_ratelimited(&pci_dev->dev, 1161 "[%s] GFIR=%016llx\n", 1162 __func__, gfir); 1163 } 1164 1165 exit: 1166 return IRQ_HANDLED; 1167} 1168 1169static irqreturn_t genwqe_vf_isr(int irq, void *dev_id) 1170{ 1171 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id; 1172 1173 cd->irqs_processed++; 1174 wake_up_interruptible(&cd->queue_waitq); 1175 1176 return IRQ_HANDLED; 1177} 1178 1179/** 1180 * genwqe_card_thread() - Work thread for the DDCB queue 1181 * @data: pointer to genwqe device descriptor 1182 * 1183 * The idea is to check if there are DDCBs in processing. If there are 1184 * some finished DDCBs, we process them and wakeup the 1185 * requestors. Otherwise we give other processes time using 1186 * cond_resched(). 1187 */ 1188static int genwqe_card_thread(void *data) 1189{ 1190 int should_stop = 0; 1191 struct genwqe_dev *cd = (struct genwqe_dev *)data; 1192 1193 while (!kthread_should_stop()) { 1194 1195 genwqe_check_ddcb_queue(cd, &cd->queue); 1196 1197 if (GENWQE_POLLING_ENABLED) { 1198 wait_event_interruptible_timeout( 1199 cd->queue_waitq, 1200 genwqe_ddcbs_in_flight(cd) || 1201 (should_stop = kthread_should_stop()), 1); 1202 } else { 1203 wait_event_interruptible_timeout( 1204 cd->queue_waitq, 1205 genwqe_next_ddcb_ready(cd) || 1206 (should_stop = kthread_should_stop()), HZ); 1207 } 1208 if (should_stop) 1209 break; 1210 1211 /* 1212 * Avoid soft lockups on heavy loads; we do not want 1213 * to disable our interrupts. 1214 */ 1215 cond_resched(); 1216 } 1217 return 0; 1218} 1219 1220/** 1221 * genwqe_setup_service_layer() - Setup DDCB queue 1222 * @cd: pointer to genwqe device descriptor 1223 * 1224 * Allocate DDCBs. Configure Service Layer Controller (SLC). 1225 * 1226 * Return: 0 success 1227 */ 1228int genwqe_setup_service_layer(struct genwqe_dev *cd) 1229{ 1230 int rc; 1231 struct ddcb_queue *queue; 1232 struct pci_dev *pci_dev = cd->pci_dev; 1233 1234 if (genwqe_is_privileged(cd)) { 1235 rc = genwqe_card_reset(cd); 1236 if (rc < 0) { 1237 dev_err(&pci_dev->dev, 1238 "[%s] err: reset failed.\n", __func__); 1239 return rc; 1240 } 1241 genwqe_read_softreset(cd); 1242 } 1243 1244 queue = &cd->queue; 1245 queue->IO_QUEUE_CONFIG = IO_SLC_QUEUE_CONFIG; 1246 queue->IO_QUEUE_STATUS = IO_SLC_QUEUE_STATUS; 1247 queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT; 1248 queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN; 1249 queue->IO_QUEUE_OFFSET = IO_SLC_QUEUE_OFFSET; 1250 queue->IO_QUEUE_WRAP = IO_SLC_QUEUE_WRAP; 1251 queue->IO_QUEUE_WTIME = IO_SLC_QUEUE_WTIME; 1252 queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS; 1253 queue->IO_QUEUE_LRW = IO_SLC_QUEUE_LRW; 1254 1255 rc = setup_ddcb_queue(cd, queue); 1256 if (rc != 0) { 1257 rc = -ENODEV; 1258 goto err_out; 1259 } 1260 1261 init_waitqueue_head(&cd->queue_waitq); 1262 cd->card_thread = kthread_run(genwqe_card_thread, cd, 1263 GENWQE_DEVNAME "%d_thread", 1264 cd->card_idx); 1265 if (IS_ERR(cd->card_thread)) { 1266 rc = PTR_ERR(cd->card_thread); 1267 cd->card_thread = NULL; 1268 goto stop_free_queue; 1269 } 1270 1271 rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS); 1272 if (rc) 1273 goto stop_kthread; 1274 1275 /* 1276 * We must have all wait-queues initialized when we enable the 1277 * interrupts. Otherwise we might crash if we get an early 1278 * irq. 1279 */ 1280 init_waitqueue_head(&cd->health_waitq); 1281 1282 if (genwqe_is_privileged(cd)) { 1283 rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED, 1284 GENWQE_DEVNAME, cd); 1285 } else { 1286 rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED, 1287 GENWQE_DEVNAME, cd); 1288 } 1289 if (rc < 0) { 1290 dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq); 1291 goto stop_irq_cap; 1292 } 1293 1294 cd->card_state = GENWQE_CARD_USED; 1295 return 0; 1296 1297 stop_irq_cap: 1298 genwqe_reset_interrupt_capability(cd); 1299 stop_kthread: 1300 kthread_stop(cd->card_thread); 1301 cd->card_thread = NULL; 1302 stop_free_queue: 1303 free_ddcb_queue(cd, queue); 1304 err_out: 1305 return rc; 1306} 1307 1308/** 1309 * queue_wake_up_all() - Handles fatal error case 1310 * @cd: pointer to genwqe device descriptor 1311 * 1312 * The PCI device got unusable and we have to stop all pending 1313 * requests as fast as we can. The code after this must purge the 1314 * DDCBs in question and ensure that all mappings are freed. 1315 */ 1316static int queue_wake_up_all(struct genwqe_dev *cd) 1317{ 1318 unsigned int i; 1319 unsigned long flags; 1320 struct ddcb_queue *queue = &cd->queue; 1321 1322 spin_lock_irqsave(&queue->ddcb_lock, flags); 1323 1324 for (i = 0; i < queue->ddcb_max; i++) 1325 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]); 1326 1327 wake_up_interruptible(&queue->busy_waitq); 1328 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 1329 1330 return 0; 1331} 1332 1333/** 1334 * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces 1335 * @cd: pointer to genwqe device descriptor 1336 * 1337 * Relies on the pre-condition that there are no users of the card 1338 * device anymore e.g. with open file-descriptors. 1339 * 1340 * This function must be robust enough to be called twice. 1341 */ 1342int genwqe_finish_queue(struct genwqe_dev *cd) 1343{ 1344 int i, rc = 0, in_flight; 1345 int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT; 1346 struct pci_dev *pci_dev = cd->pci_dev; 1347 struct ddcb_queue *queue = &cd->queue; 1348 1349 if (!ddcb_queue_initialized(queue)) 1350 return 0; 1351 1352 /* Do not wipe out the error state. */ 1353 if (cd->card_state == GENWQE_CARD_USED) 1354 cd->card_state = GENWQE_CARD_UNUSED; 1355 1356 /* Wake up all requests in the DDCB queue such that they 1357 should be removed nicely. */ 1358 queue_wake_up_all(cd); 1359 1360 /* We must wait to get rid of the DDCBs in flight */ 1361 for (i = 0; i < waitmax; i++) { 1362 in_flight = genwqe_ddcbs_in_flight(cd); 1363 1364 if (in_flight == 0) 1365 break; 1366 1367 dev_dbg(&pci_dev->dev, 1368 " DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n", 1369 i, waitmax, in_flight); 1370 1371 /* 1372 * Severe severe error situation: The card itself has 1373 * 16 DDCB queues, each queue has e.g. 32 entries, 1374 * each DDBC has a hardware timeout of currently 250 1375 * msec but the PFs have a hardware timeout of 8 sec 1376 * ... so I take something large. 1377 */ 1378 msleep(1000); 1379 } 1380 if (i == waitmax) { 1381 dev_err(&pci_dev->dev, " [%s] err: queue is not empty!!\n", 1382 __func__); 1383 rc = -EIO; 1384 } 1385 return rc; 1386} 1387 1388/** 1389 * genwqe_release_service_layer() - Shutdown DDCB queue 1390 * @cd: genwqe device descriptor 1391 * 1392 * This function must be robust enough to be called twice. 1393 */ 1394int genwqe_release_service_layer(struct genwqe_dev *cd) 1395{ 1396 struct pci_dev *pci_dev = cd->pci_dev; 1397 1398 if (!ddcb_queue_initialized(&cd->queue)) 1399 return 1; 1400 1401 free_irq(pci_dev->irq, cd); 1402 genwqe_reset_interrupt_capability(cd); 1403 1404 if (cd->card_thread != NULL) { 1405 kthread_stop(cd->card_thread); 1406 cd->card_thread = NULL; 1407 } 1408 1409 free_ddcb_queue(cd, &cd->queue); 1410 return 0; 1411} 1412