nvme_qpair.c revision 248768
1/*- 2 * Copyright (C) 2012 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/sys/dev/nvme/nvme_qpair.c 248768 2013-03-26 22:06:05Z jimharris $"); 29 30#include <sys/param.h> 31#include <sys/bus.h> 32 33#include <dev/pci/pcivar.h> 34 35#include "nvme_private.h" 36 37static void _nvme_qpair_submit_request(struct nvme_qpair *qpair, 38 struct nvme_request *req); 39 40static boolean_t 41nvme_completion_is_retry(const struct nvme_completion *cpl) 42{ 43 /* 44 * TODO: spec is not clear how commands that are aborted due 45 * to TLER will be marked. So for now, it seems 46 * NAMESPACE_NOT_READY is the only case where we should 47 * look at the DNR bit. 48 */ 49 switch (cpl->status.sct) { 50 case NVME_SCT_GENERIC: 51 switch (cpl->status.sc) { 52 case NVME_SC_ABORTED_BY_REQUEST: 53 case NVME_SC_NAMESPACE_NOT_READY: 54 if (cpl->status.dnr) 55 return (0); 56 else 57 return (1); 58 case NVME_SC_INVALID_OPCODE: 59 case NVME_SC_INVALID_FIELD: 60 case NVME_SC_COMMAND_ID_CONFLICT: 61 case NVME_SC_DATA_TRANSFER_ERROR: 62 case NVME_SC_ABORTED_POWER_LOSS: 63 case NVME_SC_INTERNAL_DEVICE_ERROR: 64 case NVME_SC_ABORTED_SQ_DELETION: 65 case NVME_SC_ABORTED_FAILED_FUSED: 66 case NVME_SC_ABORTED_MISSING_FUSED: 67 case NVME_SC_INVALID_NAMESPACE_OR_FORMAT: 68 case NVME_SC_COMMAND_SEQUENCE_ERROR: 69 case NVME_SC_LBA_OUT_OF_RANGE: 70 case NVME_SC_CAPACITY_EXCEEDED: 71 default: 72 return (0); 73 } 74 case NVME_SCT_COMMAND_SPECIFIC: 75 case NVME_SCT_MEDIA_ERROR: 76 case NVME_SCT_VENDOR_SPECIFIC: 77 default: 78 return (0); 79 } 80} 81 82static void 83nvme_qpair_construct_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr, 84 uint16_t cid) 85{ 86 87 bus_dmamap_create(qpair->dma_tag, 0, &tr->payload_dma_map); 88 bus_dmamap_create(qpair->dma_tag, 0, &tr->prp_dma_map); 89 90 bus_dmamap_load(qpair->dma_tag, tr->prp_dma_map, tr->prp, 91 sizeof(tr->prp), nvme_single_map, &tr->prp_bus_addr, 0); 92 93 callout_init(&tr->timer, 1); 94 tr->cid = cid; 95 tr->qpair = qpair; 96} 97 98static void 99nvme_qpair_complete_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr, 100 struct nvme_completion *cpl, boolean_t print_on_error) 101{ 102 struct nvme_request *req; 103 boolean_t retry, error; 104 105 req = tr->req; 106 error = nvme_completion_is_error(cpl); 107 retry = error && nvme_completion_is_retry(cpl) && 108 req->retries < nvme_retry_count; 109 110 if (error && print_on_error) { 111 nvme_dump_completion(cpl); 112 nvme_dump_command(&req->cmd); 113 } 114 115 qpair->act_tr[cpl->cid] = NULL; 116 117 KASSERT(cpl->cid == req->cmd.cid, ("cpl cid does not match cmd cid\n")); 118 119 if (req->cb_fn && !retry) 120 req->cb_fn(req->cb_arg, cpl); 121 122 mtx_lock(&qpair->lock); 123 callout_stop(&tr->timer); 124 125 if (retry) { 126 req->retries++; 127 nvme_qpair_submit_tracker(qpair, tr); 128 } else { 129 if (req->payload_size > 0 || req->uio != NULL) 130 bus_dmamap_unload(qpair->dma_tag, 131 tr->payload_dma_map); 132 133 nvme_free_request(req); 134 tr->req = NULL; 135 136 TAILQ_REMOVE(&qpair->outstanding_tr, tr, tailq); 137 TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq); 138 139 /* 140 * If the controller is in the middle of resetting, don't 141 * try to submit queued requests here - let the reset logic 142 * handle that instead. 143 */ 144 if (!STAILQ_EMPTY(&qpair->queued_req) && 145 !qpair->ctrlr->is_resetting) { 146 req = STAILQ_FIRST(&qpair->queued_req); 147 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); 148 _nvme_qpair_submit_request(qpair, req); 149 } 150 } 151 152 mtx_unlock(&qpair->lock); 153} 154 155static void 156nvme_qpair_manual_complete_tracker(struct nvme_qpair *qpair, 157 struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr, 158 boolean_t print_on_error) 159{ 160 struct nvme_completion cpl; 161 162 memset(&cpl, 0, sizeof(cpl)); 163 cpl.sqid = qpair->id; 164 cpl.cid = tr->cid; 165 cpl.status.sct = sct; 166 cpl.status.sc = sc; 167 cpl.status.dnr = dnr; 168 nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error); 169} 170 171void 172nvme_qpair_manual_complete_request(struct nvme_qpair *qpair, 173 struct nvme_request *req, uint32_t sct, uint32_t sc, 174 boolean_t print_on_error) 175{ 176 struct nvme_completion cpl; 177 boolean_t error; 178 179 memset(&cpl, 0, sizeof(cpl)); 180 cpl.sqid = qpair->id; 181 cpl.status.sct = sct; 182 cpl.status.sc = sc; 183 184 error = nvme_completion_is_error(&cpl); 185 186 if (error && print_on_error) { 187 nvme_dump_completion(&cpl); 188 nvme_dump_command(&req->cmd); 189 } 190 191 if (req->cb_fn) 192 req->cb_fn(req->cb_arg, &cpl); 193 194 nvme_free_request(req); 195} 196 197void 198nvme_qpair_process_completions(struct nvme_qpair *qpair) 199{ 200 struct nvme_tracker *tr; 201 struct nvme_completion *cpl; 202 203 qpair->num_intr_handler_calls++; 204 205 if (!qpair->is_enabled) 206 /* 207 * qpair is not enabled, likely because a controller reset is 208 * is in progress. Ignore the interrupt - any I/O that was 209 * associated with this interrupt will get retried when the 210 * reset is complete. 211 */ 212 return; 213 214 while (1) { 215 cpl = &qpair->cpl[qpair->cq_head]; 216 217 if (cpl->status.p != qpair->phase) 218 break; 219 220 tr = qpair->act_tr[cpl->cid]; 221 222 if (tr != NULL) { 223 nvme_qpair_complete_tracker(qpair, tr, cpl, TRUE); 224 qpair->sq_head = cpl->sqhd; 225 } else { 226 printf("cpl does not map to outstanding cmd\n"); 227 nvme_dump_completion(cpl); 228 KASSERT(0, ("received completion for unknown cmd\n")); 229 } 230 231 if (++qpair->cq_head == qpair->num_entries) { 232 qpair->cq_head = 0; 233 qpair->phase = !qpair->phase; 234 } 235 236 nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl, 237 qpair->cq_head); 238 } 239} 240 241static void 242nvme_qpair_msix_handler(void *arg) 243{ 244 struct nvme_qpair *qpair = arg; 245 246 nvme_qpair_process_completions(qpair); 247} 248 249void 250nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id, 251 uint16_t vector, uint32_t num_entries, uint32_t num_trackers, 252 uint32_t max_xfer_size, struct nvme_controller *ctrlr) 253{ 254 struct nvme_tracker *tr; 255 uint32_t i; 256 257 qpair->id = id; 258 qpair->vector = vector; 259 qpair->num_entries = num_entries; 260#ifdef CHATHAM2 261 /* 262 * Chatham prototype board starts having issues at higher queue 263 * depths. So use a conservative estimate here of no more than 64 264 * outstanding I/O per queue at any one point. 265 */ 266 if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) 267 num_trackers = min(num_trackers, 64); 268#endif 269 qpair->num_trackers = num_trackers; 270 qpair->max_xfer_size = max_xfer_size; 271 qpair->ctrlr = ctrlr; 272 273 if (ctrlr->msix_enabled) { 274 275 /* 276 * MSI-X vector resource IDs start at 1, so we add one to 277 * the queue's vector to get the corresponding rid to use. 278 */ 279 qpair->rid = vector + 1; 280 281 qpair->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ, 282 &qpair->rid, RF_ACTIVE); 283 284 bus_setup_intr(ctrlr->dev, qpair->res, 285 INTR_TYPE_MISC | INTR_MPSAFE, NULL, 286 nvme_qpair_msix_handler, qpair, &qpair->tag); 287 } 288 289 mtx_init(&qpair->lock, "nvme qpair lock", NULL, MTX_DEF); 290 291 bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), 292 sizeof(uint64_t), PAGE_SIZE, BUS_SPACE_MAXADDR, 293 BUS_SPACE_MAXADDR, NULL, NULL, qpair->max_xfer_size, 294 (qpair->max_xfer_size/PAGE_SIZE)+1, PAGE_SIZE, 0, 295 NULL, NULL, &qpair->dma_tag); 296 297 qpair->num_cmds = 0; 298 qpair->num_intr_handler_calls = 0; 299 300 /* TODO: error checking on contigmalloc, bus_dmamap_load calls */ 301 qpair->cmd = contigmalloc(qpair->num_entries * 302 sizeof(struct nvme_command), M_NVME, M_ZERO | M_NOWAIT, 303 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); 304 qpair->cpl = contigmalloc(qpair->num_entries * 305 sizeof(struct nvme_completion), M_NVME, M_ZERO | M_NOWAIT, 306 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); 307 308 bus_dmamap_create(qpair->dma_tag, 0, &qpair->cmd_dma_map); 309 bus_dmamap_create(qpair->dma_tag, 0, &qpair->cpl_dma_map); 310 311 bus_dmamap_load(qpair->dma_tag, qpair->cmd_dma_map, 312 qpair->cmd, qpair->num_entries * sizeof(struct nvme_command), 313 nvme_single_map, &qpair->cmd_bus_addr, 0); 314 bus_dmamap_load(qpair->dma_tag, qpair->cpl_dma_map, 315 qpair->cpl, qpair->num_entries * sizeof(struct nvme_completion), 316 nvme_single_map, &qpair->cpl_bus_addr, 0); 317 318 qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl); 319 qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl); 320 321 TAILQ_INIT(&qpair->free_tr); 322 TAILQ_INIT(&qpair->outstanding_tr); 323 STAILQ_INIT(&qpair->queued_req); 324 325 for (i = 0; i < qpair->num_trackers; i++) { 326 tr = malloc(sizeof(*tr), M_NVME, M_ZERO | M_NOWAIT); 327 328 if (tr == NULL) { 329 printf("warning: nvme tracker malloc failed\n"); 330 break; 331 } 332 333 nvme_qpair_construct_tracker(qpair, tr, i); 334 TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq); 335 } 336 337 qpair->act_tr = malloc(sizeof(struct nvme_tracker *) * qpair->num_entries, 338 M_NVME, M_ZERO | M_NOWAIT); 339} 340 341static void 342nvme_qpair_destroy(struct nvme_qpair *qpair) 343{ 344 struct nvme_tracker *tr; 345 346 if (qpair->tag) 347 bus_teardown_intr(qpair->ctrlr->dev, qpair->res, qpair->tag); 348 349 if (qpair->res) 350 bus_release_resource(qpair->ctrlr->dev, SYS_RES_IRQ, 351 rman_get_rid(qpair->res), qpair->res); 352 353 if (qpair->cmd) { 354 bus_dmamap_unload(qpair->dma_tag, qpair->cmd_dma_map); 355 bus_dmamap_destroy(qpair->dma_tag, qpair->cmd_dma_map); 356 contigfree(qpair->cmd, 357 qpair->num_entries * sizeof(struct nvme_command), M_NVME); 358 } 359 360 if (qpair->cpl) { 361 bus_dmamap_unload(qpair->dma_tag, qpair->cpl_dma_map); 362 bus_dmamap_destroy(qpair->dma_tag, qpair->cpl_dma_map); 363 contigfree(qpair->cpl, 364 qpair->num_entries * sizeof(struct nvme_completion), 365 M_NVME); 366 } 367 368 if (qpair->dma_tag) 369 bus_dma_tag_destroy(qpair->dma_tag); 370 371 if (qpair->act_tr) 372 free(qpair->act_tr, M_NVME); 373 374 while (!TAILQ_EMPTY(&qpair->free_tr)) { 375 tr = TAILQ_FIRST(&qpair->free_tr); 376 TAILQ_REMOVE(&qpair->free_tr, tr, tailq); 377 bus_dmamap_destroy(qpair->dma_tag, tr->payload_dma_map); 378 bus_dmamap_destroy(qpair->dma_tag, tr->prp_dma_map); 379 free(tr, M_NVME); 380 } 381} 382 383static void 384nvme_admin_qpair_abort_aers(struct nvme_qpair *qpair) 385{ 386 struct nvme_tracker *tr; 387 388 tr = TAILQ_FIRST(&qpair->outstanding_tr); 389 while (tr != NULL) { 390 if (tr->req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST) { 391 nvme_qpair_manual_complete_tracker(qpair, tr, 392 NVME_SCT_GENERIC, NVME_SC_ABORTED_SQ_DELETION, 0, 393 FALSE); 394 tr = TAILQ_FIRST(&qpair->outstanding_tr); 395 } else { 396 tr = TAILQ_NEXT(tr, tailq); 397 } 398 } 399} 400 401void 402nvme_admin_qpair_destroy(struct nvme_qpair *qpair) 403{ 404 405 nvme_admin_qpair_abort_aers(qpair); 406 nvme_qpair_destroy(qpair); 407} 408 409void 410nvme_io_qpair_destroy(struct nvme_qpair *qpair) 411{ 412 413 nvme_qpair_destroy(qpair); 414} 415 416static void 417nvme_abort_complete(void *arg, const struct nvme_completion *status) 418{ 419 struct nvme_tracker *tr = arg; 420 421 /* 422 * If cdw0 == 1, the controller was not able to abort the command 423 * we requested. We still need to check the active tracker array, 424 * to cover race where I/O timed out at same time controller was 425 * completing the I/O. 426 */ 427 if (status->cdw0 == 1 && tr->qpair->act_tr[tr->cid] != NULL) { 428 /* 429 * An I/O has timed out, and the controller was unable to 430 * abort it for some reason. Construct a fake completion 431 * status, and then complete the I/O's tracker manually. 432 */ 433 printf("abort command failed, aborting command manually\n"); 434 nvme_qpair_manual_complete_tracker(tr->qpair, tr, 435 NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST, 0, TRUE); 436 } 437} 438 439static void 440nvme_timeout(void *arg) 441{ 442 struct nvme_tracker *tr = arg; 443 struct nvme_qpair *qpair = tr->qpair; 444 struct nvme_controller *ctrlr = qpair->ctrlr; 445 union csts_register csts; 446 447 /* Read csts to get value of cfs - controller fatal status. */ 448 csts.raw = nvme_mmio_read_4(ctrlr, csts); 449 450 if (ctrlr->enable_aborts && csts.bits.cfs == 0) { 451 /* 452 * If aborts are enabled, only use them if the controller is 453 * not reporting fatal status. 454 */ 455 nvme_ctrlr_cmd_abort(ctrlr, tr->cid, qpair->id, 456 nvme_abort_complete, tr); 457 } else 458 nvme_ctrlr_reset(ctrlr); 459} 460 461void 462nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr) 463{ 464 struct nvme_request *req; 465 struct nvme_controller *ctrlr; 466 467 mtx_assert(&qpair->lock, MA_OWNED); 468 469 req = tr->req; 470 req->cmd.cid = tr->cid; 471 qpair->act_tr[tr->cid] = tr; 472 ctrlr = qpair->ctrlr; 473 474 if (req->timeout) 475#if __FreeBSD_version >= 800030 476 callout_reset_curcpu(&tr->timer, ctrlr->timeout_period * hz, 477 nvme_timeout, tr); 478#else 479 callout_reset(&tr->timer, ctrlr->timeout_period * hz, 480 nvme_timeout, tr); 481#endif 482 483 /* Copy the command from the tracker to the submission queue. */ 484 memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd)); 485 486 if (++qpair->sq_tail == qpair->num_entries) 487 qpair->sq_tail = 0; 488 489 wmb(); 490 nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl, 491 qpair->sq_tail); 492 493 qpair->num_cmds++; 494} 495 496static void 497_nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) 498{ 499 struct nvme_tracker *tr; 500 int err; 501 502 mtx_assert(&qpair->lock, MA_OWNED); 503 504 tr = TAILQ_FIRST(&qpair->free_tr); 505 req->qpair = qpair; 506 507 if (tr == NULL || !qpair->is_enabled) { 508 /* 509 * No tracker is available, or the qpair is disabled due to 510 * an in-progress controller-level reset or controller 511 * failure. 512 */ 513 514 if (qpair->ctrlr->is_failed) { 515 /* 516 * The controller has failed. Post the request to a 517 * task where it will be aborted, so that we do not 518 * invoke the request's callback in the context 519 * of the submission. 520 */ 521 nvme_ctrlr_post_failed_request(qpair->ctrlr, req); 522 } else { 523 /* 524 * Put the request on the qpair's request queue to be 525 * processed when a tracker frees up via a command 526 * completion or when the controller reset is 527 * completed. 528 */ 529 STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq); 530 } 531 return; 532 } 533 534 TAILQ_REMOVE(&qpair->free_tr, tr, tailq); 535 TAILQ_INSERT_TAIL(&qpair->outstanding_tr, tr, tailq); 536 tr->req = req; 537 538 if (req->uio == NULL) { 539 if (req->payload_size > 0) { 540 err = bus_dmamap_load(tr->qpair->dma_tag, 541 tr->payload_dma_map, req->payload, 542 req->payload_size, 543 nvme_payload_map, tr, 0); 544 if (err != 0) 545 panic("bus_dmamap_load returned non-zero!\n"); 546 } else 547 nvme_qpair_submit_tracker(tr->qpair, tr); 548 } else { 549 err = bus_dmamap_load_uio(tr->qpair->dma_tag, 550 tr->payload_dma_map, req->uio, 551 nvme_payload_map_uio, tr, 0); 552 if (err != 0) 553 panic("bus_dmamap_load returned non-zero!\n"); 554 } 555} 556 557void 558nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) 559{ 560 561 mtx_lock(&qpair->lock); 562 _nvme_qpair_submit_request(qpair, req); 563 mtx_unlock(&qpair->lock); 564} 565 566static void 567nvme_qpair_enable(struct nvme_qpair *qpair) 568{ 569 570 qpair->is_enabled = TRUE; 571} 572 573void 574nvme_qpair_reset(struct nvme_qpair *qpair) 575{ 576 577 qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0; 578 579 /* 580 * First time through the completion queue, HW will set phase 581 * bit on completions to 1. So set this to 1 here, indicating 582 * we're looking for a 1 to know which entries have completed. 583 * we'll toggle the bit each time when the completion queue 584 * rolls over. 585 */ 586 qpair->phase = 1; 587 588 memset(qpair->cmd, 0, 589 qpair->num_entries * sizeof(struct nvme_command)); 590 memset(qpair->cpl, 0, 591 qpair->num_entries * sizeof(struct nvme_completion)); 592} 593 594void 595nvme_admin_qpair_enable(struct nvme_qpair *qpair) 596{ 597 struct nvme_tracker *tr; 598 struct nvme_tracker *tr_temp; 599 600 /* 601 * Manually abort each outstanding admin command. Do not retry 602 * admin commands found here, since they will be left over from 603 * a controller reset and its likely the context in which the 604 * command was issued no longer applies. 605 */ 606 TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) { 607 device_printf(qpair->ctrlr->dev, 608 "aborting outstanding admin command\n"); 609 nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 610 NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, TRUE); 611 } 612 613 nvme_qpair_enable(qpair); 614} 615 616void 617nvme_io_qpair_enable(struct nvme_qpair *qpair) 618{ 619 STAILQ_HEAD(, nvme_request) temp; 620 struct nvme_tracker *tr; 621 struct nvme_tracker *tr_temp; 622 struct nvme_request *req; 623 624 /* 625 * Manually abort each outstanding I/O. This normally results in a 626 * retry, unless the retry count on the associated request has 627 * reached its limit. 628 */ 629 TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) { 630 device_printf(qpair->ctrlr->dev, 631 "aborting outstanding i/o\n"); 632 nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 633 NVME_SC_ABORTED_BY_REQUEST, 0, TRUE); 634 } 635 636 mtx_lock(&qpair->lock); 637 638 nvme_qpair_enable(qpair); 639 640 STAILQ_INIT(&temp); 641 STAILQ_SWAP(&qpair->queued_req, &temp, nvme_request); 642 643 while (!STAILQ_EMPTY(&temp)) { 644 req = STAILQ_FIRST(&temp); 645 STAILQ_REMOVE_HEAD(&temp, stailq); 646 device_printf(qpair->ctrlr->dev, 647 "resubmitting queued i/o\n"); 648 nvme_dump_command(&req->cmd); 649 _nvme_qpair_submit_request(qpair, req); 650 } 651 652 mtx_unlock(&qpair->lock); 653} 654 655static void 656nvme_qpair_disable(struct nvme_qpair *qpair) 657{ 658 struct nvme_tracker *tr; 659 660 qpair->is_enabled = FALSE; 661 mtx_lock(&qpair->lock); 662 TAILQ_FOREACH(tr, &qpair->outstanding_tr, tailq) 663 callout_stop(&tr->timer); 664 mtx_unlock(&qpair->lock); 665} 666 667void 668nvme_admin_qpair_disable(struct nvme_qpair *qpair) 669{ 670 671 nvme_qpair_disable(qpair); 672 nvme_admin_qpair_abort_aers(qpair); 673} 674 675void 676nvme_io_qpair_disable(struct nvme_qpair *qpair) 677{ 678 679 nvme_qpair_disable(qpair); 680} 681 682void 683nvme_qpair_fail(struct nvme_qpair *qpair) 684{ 685 struct nvme_tracker *tr; 686 struct nvme_request *req; 687 688 mtx_lock(&qpair->lock); 689 690 while (!STAILQ_EMPTY(&qpair->queued_req)) { 691 req = STAILQ_FIRST(&qpair->queued_req); 692 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); 693 device_printf(qpair->ctrlr->dev, 694 "failing queued i/o\n"); 695 mtx_unlock(&qpair->lock); 696 nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC, 697 NVME_SC_ABORTED_BY_REQUEST, TRUE); 698 mtx_lock(&qpair->lock); 699 } 700 701 /* Manually abort each outstanding I/O. */ 702 while (!TAILQ_EMPTY(&qpair->outstanding_tr)) { 703 tr = TAILQ_FIRST(&qpair->outstanding_tr); 704 /* 705 * Do not remove the tracker. The abort_tracker path will 706 * do that for us. 707 */ 708 device_printf(qpair->ctrlr->dev, 709 "failing outstanding i/o\n"); 710 mtx_unlock(&qpair->lock); 711 nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 712 NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, TRUE); 713 mtx_lock(&qpair->lock); 714 } 715 716 mtx_unlock(&qpair->lock); 717} 718 719