nvme_qpair.c revision 328677
1362593Sdim/*- 2362593Sdim * Copyright (C) 2012-2014 Intel Corporation 3362593Sdim * All rights reserved. 4362593Sdim * 5362593Sdim * Redistribution and use in source and binary forms, with or without 6362593Sdim * modification, are permitted provided that the following conditions 7362593Sdim * are met: 8362593Sdim * 1. Redistributions of source code must retain the above copyright 9362593Sdim * notice, this list of conditions and the following disclaimer. 10362593Sdim * 2. Redistributions in binary form must reproduce the above copyright 11362593Sdim * notice, this list of conditions and the following disclaimer in the 12362593Sdim * documentation and/or other materials provided with the distribution. 13362593Sdim * 14362593Sdim * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15362593Sdim * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16362593Sdim * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17362593Sdim * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18362593Sdim * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19362593Sdim * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20362593Sdim * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21362593Sdim * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22362593Sdim * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23362593Sdim * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24362593Sdim * SUCH DAMAGE. 25362593Sdim */ 26362593Sdim 27362593Sdim#include <sys/cdefs.h> 28362593Sdim__FBSDID("$FreeBSD: stable/11/sys/dev/nvme/nvme_qpair.c 328677 2018-02-01 16:27:10Z mav $"); 29362593Sdim 30362593Sdim#include <sys/param.h> 31362593Sdim#include <sys/bus.h> 32362593Sdim 33362593Sdim#include <dev/pci/pcivar.h> 34362593Sdim 35362593Sdim#include "nvme_private.h" 36362593Sdim 37362593Sdimstatic void _nvme_qpair_submit_request(struct nvme_qpair *qpair, 38362593Sdim struct nvme_request *req); 39362593Sdimstatic void nvme_qpair_destroy(struct nvme_qpair *qpair); 40362593Sdim 41362593Sdimstruct nvme_opcode_string { 42362593Sdim 43362593Sdim uint16_t opc; 44362593Sdim const char * str; 45362593Sdim}; 46362593Sdim 47362593Sdimstatic struct nvme_opcode_string admin_opcode[] = { 48362593Sdim { NVME_OPC_DELETE_IO_SQ, "DELETE IO SQ" }, 49362593Sdim { NVME_OPC_CREATE_IO_SQ, "CREATE IO SQ" }, 50362593Sdim { NVME_OPC_GET_LOG_PAGE, "GET LOG PAGE" }, 51362593Sdim { NVME_OPC_DELETE_IO_CQ, "DELETE IO CQ" }, 52362593Sdim { NVME_OPC_CREATE_IO_CQ, "CREATE IO CQ" }, 53362593Sdim { NVME_OPC_IDENTIFY, "IDENTIFY" }, 54362593Sdim { NVME_OPC_ABORT, "ABORT" }, 55362593Sdim { NVME_OPC_SET_FEATURES, "SET FEATURES" }, 56362593Sdim { NVME_OPC_GET_FEATURES, "GET FEATURES" }, 57362593Sdim { NVME_OPC_ASYNC_EVENT_REQUEST, "ASYNC EVENT REQUEST" }, 58362593Sdim { NVME_OPC_FIRMWARE_ACTIVATE, "FIRMWARE ACTIVATE" }, 59362593Sdim { NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD, "FIRMWARE IMAGE DOWNLOAD" }, 60362593Sdim { NVME_OPC_FORMAT_NVM, "FORMAT NVM" }, 61362593Sdim { NVME_OPC_SECURITY_SEND, "SECURITY SEND" }, 62362593Sdim { NVME_OPC_SECURITY_RECEIVE, "SECURITY RECEIVE" }, 63362593Sdim { 0xFFFF, "ADMIN COMMAND" } 64362593Sdim}; 65362593Sdim 66362593Sdimstatic struct nvme_opcode_string io_opcode[] = { 67362593Sdim { NVME_OPC_FLUSH, "FLUSH" }, 68362593Sdim { NVME_OPC_WRITE, "WRITE" }, 69362593Sdim { NVME_OPC_READ, "READ" }, 70362593Sdim { NVME_OPC_WRITE_UNCORRECTABLE, "WRITE UNCORRECTABLE" }, 71362593Sdim { NVME_OPC_COMPARE, "COMPARE" }, 72362593Sdim { NVME_OPC_DATASET_MANAGEMENT, "DATASET MANAGEMENT" }, 73362593Sdim { 0xFFFF, "IO COMMAND" } 74362593Sdim}; 75362593Sdim 76362593Sdimstatic const char * 77362593Sdimget_admin_opcode_string(uint16_t opc) 78362593Sdim{ 79362593Sdim struct nvme_opcode_string *entry; 80362593Sdim 81362593Sdim entry = admin_opcode; 82362593Sdim 83362593Sdim while (entry->opc != 0xFFFF) { 84362593Sdim if (entry->opc == opc) 85362593Sdim return (entry->str); 86362593Sdim entry++; 87362593Sdim } 88362593Sdim return (entry->str); 89362593Sdim} 90362593Sdim 91362593Sdimstatic const char * 92362593Sdimget_io_opcode_string(uint16_t opc) 93362593Sdim{ 94362593Sdim struct nvme_opcode_string *entry; 95362593Sdim 96362593Sdim entry = io_opcode; 97362593Sdim 98362593Sdim while (entry->opc != 0xFFFF) { 99362593Sdim if (entry->opc == opc) 100362593Sdim return (entry->str); 101362593Sdim entry++; 102362593Sdim } 103362593Sdim return (entry->str); 104362593Sdim} 105362593Sdim 106362593Sdim 107362593Sdimstatic void 108362593Sdimnvme_admin_qpair_print_command(struct nvme_qpair *qpair, 109362593Sdim struct nvme_command *cmd) 110362593Sdim{ 111362593Sdim 112362593Sdim nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%x " 113362593Sdim "cdw10:%08x cdw11:%08x\n", 114362593Sdim get_admin_opcode_string(cmd->opc), cmd->opc, qpair->id, cmd->cid, 115362593Sdim cmd->nsid, cmd->cdw10, cmd->cdw11); 116362593Sdim} 117362593Sdim 118362593Sdimstatic void 119362593Sdimnvme_io_qpair_print_command(struct nvme_qpair *qpair, 120362593Sdim struct nvme_command *cmd) 121362593Sdim{ 122362593Sdim 123362593Sdim switch (cmd->opc) { 124362593Sdim case NVME_OPC_WRITE: 125362593Sdim case NVME_OPC_READ: 126362593Sdim case NVME_OPC_WRITE_UNCORRECTABLE: 127362593Sdim case NVME_OPC_COMPARE: 128362593Sdim nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d " 129362593Sdim "lba:%llu len:%d\n", 130362593Sdim get_io_opcode_string(cmd->opc), qpair->id, cmd->cid, 131362593Sdim cmd->nsid, 132362593Sdim ((unsigned long long)cmd->cdw11 << 32) + cmd->cdw10, 133362593Sdim (cmd->cdw12 & 0xFFFF) + 1); 134362593Sdim break; 135362593Sdim case NVME_OPC_FLUSH: 136362593Sdim case NVME_OPC_DATASET_MANAGEMENT: 137362593Sdim nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d\n", 138362593Sdim get_io_opcode_string(cmd->opc), qpair->id, cmd->cid, 139362593Sdim cmd->nsid); 140362593Sdim break; 141362593Sdim default: 142362593Sdim nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%d\n", 143362593Sdim get_io_opcode_string(cmd->opc), cmd->opc, qpair->id, 144362593Sdim cmd->cid, cmd->nsid); 145362593Sdim break; 146362593Sdim } 147362593Sdim} 148362593Sdim 149362593Sdimstatic void 150362593Sdimnvme_qpair_print_command(struct nvme_qpair *qpair, struct nvme_command *cmd) 151362593Sdim{ 152362593Sdim if (qpair->id == 0) 153362593Sdim nvme_admin_qpair_print_command(qpair, cmd); 154362593Sdim else 155362593Sdim nvme_io_qpair_print_command(qpair, cmd); 156362593Sdim} 157362593Sdim 158362593Sdimstruct nvme_status_string { 159362593Sdim 160362593Sdim uint16_t sc; 161362593Sdim const char * str; 162362593Sdim}; 163362593Sdim 164362593Sdimstatic struct nvme_status_string generic_status[] = { 165362593Sdim { NVME_SC_SUCCESS, "SUCCESS" }, 166362593Sdim { NVME_SC_INVALID_OPCODE, "INVALID OPCODE" }, 167362593Sdim { NVME_SC_INVALID_FIELD, "INVALID_FIELD" }, 168362593Sdim { NVME_SC_COMMAND_ID_CONFLICT, "COMMAND ID CONFLICT" }, 169362593Sdim { NVME_SC_DATA_TRANSFER_ERROR, "DATA TRANSFER ERROR" }, 170362593Sdim { NVME_SC_ABORTED_POWER_LOSS, "ABORTED - POWER LOSS" }, 171362593Sdim { NVME_SC_INTERNAL_DEVICE_ERROR, "INTERNAL DEVICE ERROR" }, 172362593Sdim { NVME_SC_ABORTED_BY_REQUEST, "ABORTED - BY REQUEST" }, 173362593Sdim { NVME_SC_ABORTED_SQ_DELETION, "ABORTED - SQ DELETION" }, 174362593Sdim { NVME_SC_ABORTED_FAILED_FUSED, "ABORTED - FAILED FUSED" }, 175362593Sdim { NVME_SC_ABORTED_MISSING_FUSED, "ABORTED - MISSING FUSED" }, 176362593Sdim { NVME_SC_INVALID_NAMESPACE_OR_FORMAT, "INVALID NAMESPACE OR FORMAT" }, 177362593Sdim { NVME_SC_COMMAND_SEQUENCE_ERROR, "COMMAND SEQUENCE ERROR" }, 178362593Sdim { NVME_SC_LBA_OUT_OF_RANGE, "LBA OUT OF RANGE" }, 179362593Sdim { NVME_SC_CAPACITY_EXCEEDED, "CAPACITY EXCEEDED" }, 180362593Sdim { NVME_SC_NAMESPACE_NOT_READY, "NAMESPACE NOT READY" }, 181362593Sdim { 0xFFFF, "GENERIC" } 182362593Sdim}; 183362593Sdim 184362593Sdimstatic struct nvme_status_string command_specific_status[] = { 185362593Sdim { NVME_SC_COMPLETION_QUEUE_INVALID, "INVALID COMPLETION QUEUE" }, 186362593Sdim { NVME_SC_INVALID_QUEUE_IDENTIFIER, "INVALID QUEUE IDENTIFIER" }, 187362593Sdim { NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED, "MAX QUEUE SIZE EXCEEDED" }, 188362593Sdim { NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED, "ABORT CMD LIMIT EXCEEDED" }, 189362593Sdim { NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED, "ASYNC LIMIT EXCEEDED" }, 190362593Sdim { NVME_SC_INVALID_FIRMWARE_SLOT, "INVALID FIRMWARE SLOT" }, 191362593Sdim { NVME_SC_INVALID_FIRMWARE_IMAGE, "INVALID FIRMWARE IMAGE" }, 192362593Sdim { NVME_SC_INVALID_INTERRUPT_VECTOR, "INVALID INTERRUPT VECTOR" }, 193362593Sdim { NVME_SC_INVALID_LOG_PAGE, "INVALID LOG PAGE" }, 194362593Sdim { NVME_SC_INVALID_FORMAT, "INVALID FORMAT" }, 195362593Sdim { NVME_SC_FIRMWARE_REQUIRES_RESET, "FIRMWARE REQUIRES RESET" }, 196362593Sdim { NVME_SC_CONFLICTING_ATTRIBUTES, "CONFLICTING ATTRIBUTES" }, 197362593Sdim { NVME_SC_INVALID_PROTECTION_INFO, "INVALID PROTECTION INFO" }, 198362593Sdim { NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE, "WRITE TO RO PAGE" }, 199362593Sdim { 0xFFFF, "COMMAND SPECIFIC" } 200362593Sdim}; 201362593Sdim 202362593Sdimstatic struct nvme_status_string media_error_status[] = { 203362593Sdim { NVME_SC_WRITE_FAULTS, "WRITE FAULTS" }, 204362593Sdim { NVME_SC_UNRECOVERED_READ_ERROR, "UNRECOVERED READ ERROR" }, 205362593Sdim { NVME_SC_GUARD_CHECK_ERROR, "GUARD CHECK ERROR" }, 206362593Sdim { NVME_SC_APPLICATION_TAG_CHECK_ERROR, "APPLICATION TAG CHECK ERROR" }, 207362593Sdim { NVME_SC_REFERENCE_TAG_CHECK_ERROR, "REFERENCE TAG CHECK ERROR" }, 208362593Sdim { NVME_SC_COMPARE_FAILURE, "COMPARE FAILURE" }, 209362593Sdim { NVME_SC_ACCESS_DENIED, "ACCESS DENIED" }, 210362593Sdim { 0xFFFF, "MEDIA ERROR" } 211362593Sdim}; 212362593Sdim 213362593Sdimstatic const char * 214362593Sdimget_status_string(uint16_t sct, uint16_t sc) 215362593Sdim{ 216362593Sdim struct nvme_status_string *entry; 217362593Sdim 218362593Sdim switch (sct) { 219362593Sdim case NVME_SCT_GENERIC: 220362593Sdim entry = generic_status; 221362593Sdim break; 222362593Sdim case NVME_SCT_COMMAND_SPECIFIC: 223362593Sdim entry = command_specific_status; 224362593Sdim break; 225362593Sdim case NVME_SCT_MEDIA_ERROR: 226362593Sdim entry = media_error_status; 227362593Sdim break; 228362593Sdim case NVME_SCT_VENDOR_SPECIFIC: 229362593Sdim return ("VENDOR SPECIFIC"); 230362593Sdim default: 231362593Sdim return ("RESERVED"); 232362593Sdim } 233362593Sdim 234362593Sdim while (entry->sc != 0xFFFF) { 235362593Sdim if (entry->sc == sc) 236362593Sdim return (entry->str); 237362593Sdim entry++; 238362593Sdim } 239362593Sdim return (entry->str); 240362593Sdim} 241362593Sdim 242362593Sdimstatic void 243362593Sdimnvme_qpair_print_completion(struct nvme_qpair *qpair, 244362593Sdim struct nvme_completion *cpl) 245362593Sdim{ 246362593Sdim nvme_printf(qpair->ctrlr, "%s (%02x/%02x) sqid:%d cid:%d cdw0:%x\n", 247362593Sdim get_status_string(cpl->status.sct, cpl->status.sc), 248362593Sdim cpl->status.sct, cpl->status.sc, cpl->sqid, cpl->cid, cpl->cdw0); 249362593Sdim} 250362593Sdim 251362593Sdimstatic boolean_t 252362593Sdimnvme_completion_is_retry(const struct nvme_completion *cpl) 253362593Sdim{ 254362593Sdim /* 255362593Sdim * TODO: spec is not clear how commands that are aborted due 256362593Sdim * to TLER will be marked. So for now, it seems 257362593Sdim * NAMESPACE_NOT_READY is the only case where we should 258362593Sdim * look at the DNR bit. 259362593Sdim */ 260362593Sdim switch (cpl->status.sct) { 261362593Sdim case NVME_SCT_GENERIC: 262362593Sdim switch (cpl->status.sc) { 263362593Sdim case NVME_SC_ABORTED_BY_REQUEST: 264362593Sdim case NVME_SC_NAMESPACE_NOT_READY: 265362593Sdim if (cpl->status.dnr) 266362593Sdim return (0); 267362593Sdim else 268362593Sdim return (1); 269362593Sdim case NVME_SC_INVALID_OPCODE: 270362593Sdim case NVME_SC_INVALID_FIELD: 271362593Sdim case NVME_SC_COMMAND_ID_CONFLICT: 272362593Sdim case NVME_SC_DATA_TRANSFER_ERROR: 273362593Sdim case NVME_SC_ABORTED_POWER_LOSS: 274362593Sdim case NVME_SC_INTERNAL_DEVICE_ERROR: 275362593Sdim case NVME_SC_ABORTED_SQ_DELETION: 276362593Sdim case NVME_SC_ABORTED_FAILED_FUSED: 277362593Sdim case NVME_SC_ABORTED_MISSING_FUSED: 278362593Sdim case NVME_SC_INVALID_NAMESPACE_OR_FORMAT: 279362593Sdim case NVME_SC_COMMAND_SEQUENCE_ERROR: 280362593Sdim case NVME_SC_LBA_OUT_OF_RANGE: 281362593Sdim case NVME_SC_CAPACITY_EXCEEDED: 282362593Sdim default: 283362593Sdim return (0); 284362593Sdim } 285362593Sdim case NVME_SCT_COMMAND_SPECIFIC: 286362593Sdim case NVME_SCT_MEDIA_ERROR: 287362593Sdim case NVME_SCT_VENDOR_SPECIFIC: 288362593Sdim default: 289362593Sdim return (0); 290362593Sdim } 291362593Sdim} 292362593Sdim 293362593Sdimstatic void 294362593Sdimnvme_qpair_complete_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr, 295362593Sdim struct nvme_completion *cpl, boolean_t print_on_error) 296362593Sdim{ 297362593Sdim struct nvme_request *req; 298362593Sdim boolean_t retry, error; 299362593Sdim 300362593Sdim req = tr->req; 301362593Sdim error = nvme_completion_is_error(cpl); 302362593Sdim retry = error && nvme_completion_is_retry(cpl) && 303362593Sdim req->retries < nvme_retry_count; 304362593Sdim 305362593Sdim if (error && print_on_error) { 306362593Sdim nvme_qpair_print_command(qpair, &req->cmd); 307362593Sdim nvme_qpair_print_completion(qpair, cpl); 308362593Sdim } 309362593Sdim 310362593Sdim qpair->act_tr[cpl->cid] = NULL; 311362593Sdim 312362593Sdim KASSERT(cpl->cid == req->cmd.cid, ("cpl cid does not match cmd cid\n")); 313362593Sdim 314362593Sdim if (req->cb_fn && !retry) 315362593Sdim req->cb_fn(req->cb_arg, cpl); 316362593Sdim 317362593Sdim mtx_lock(&qpair->lock); 318362593Sdim callout_stop(&tr->timer); 319362593Sdim 320362593Sdim if (retry) { 321362593Sdim req->retries++; 322362593Sdim nvme_qpair_submit_tracker(qpair, tr); 323362593Sdim } else { 324362593Sdim if (req->type != NVME_REQUEST_NULL) 325362593Sdim bus_dmamap_unload(qpair->dma_tag_payload, 326362593Sdim tr->payload_dma_map); 327362593Sdim 328362593Sdim nvme_free_request(req); 329362593Sdim tr->req = NULL; 330362593Sdim 331362593Sdim TAILQ_REMOVE(&qpair->outstanding_tr, tr, tailq); 332362593Sdim TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq); 333362593Sdim 334362593Sdim /* 335362593Sdim * If the controller is in the middle of resetting, don't 336362593Sdim * try to submit queued requests here - let the reset logic 337362593Sdim * handle that instead. 338362593Sdim */ 339362593Sdim if (!STAILQ_EMPTY(&qpair->queued_req) && 340362593Sdim !qpair->ctrlr->is_resetting) { 341362593Sdim req = STAILQ_FIRST(&qpair->queued_req); 342362593Sdim STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); 343362593Sdim _nvme_qpair_submit_request(qpair, req); 344362593Sdim } 345362593Sdim } 346362593Sdim 347362593Sdim mtx_unlock(&qpair->lock); 348362593Sdim} 349362593Sdim 350362593Sdimstatic void 351362593Sdimnvme_qpair_manual_complete_tracker(struct nvme_qpair *qpair, 352362593Sdim struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr, 353362593Sdim boolean_t print_on_error) 354362593Sdim{ 355362593Sdim struct nvme_completion cpl; 356362593Sdim 357362593Sdim memset(&cpl, 0, sizeof(cpl)); 358362593Sdim cpl.sqid = qpair->id; 359362593Sdim cpl.cid = tr->cid; 360362593Sdim cpl.status.sct = sct; 361362593Sdim cpl.status.sc = sc; 362362593Sdim cpl.status.dnr = dnr; 363362593Sdim nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error); 364362593Sdim} 365362593Sdim 366362593Sdimvoid 367362593Sdimnvme_qpair_manual_complete_request(struct nvme_qpair *qpair, 368362593Sdim struct nvme_request *req, uint32_t sct, uint32_t sc, 369362593Sdim boolean_t print_on_error) 370362593Sdim{ 371362593Sdim struct nvme_completion cpl; 372362593Sdim boolean_t error; 373362593Sdim 374362593Sdim memset(&cpl, 0, sizeof(cpl)); 375362593Sdim cpl.sqid = qpair->id; 376362593Sdim cpl.status.sct = sct; 377362593Sdim cpl.status.sc = sc; 378362593Sdim 379362593Sdim error = nvme_completion_is_error(&cpl); 380362593Sdim 381362593Sdim if (error && print_on_error) { 382362593Sdim nvme_qpair_print_command(qpair, &req->cmd); 383362593Sdim nvme_qpair_print_completion(qpair, &cpl); 384362593Sdim } 385362593Sdim 386362593Sdim if (req->cb_fn) 387362593Sdim req->cb_fn(req->cb_arg, &cpl); 388362593Sdim 389362593Sdim nvme_free_request(req); 390362593Sdim} 391362593Sdim 392362593Sdimvoid 393362593Sdimnvme_qpair_process_completions(struct nvme_qpair *qpair) 394362593Sdim{ 395362593Sdim struct nvme_tracker *tr; 396362593Sdim struct nvme_completion *cpl; 397362593Sdim 398362593Sdim qpair->num_intr_handler_calls++; 399362593Sdim 400362593Sdim if (!qpair->is_enabled) 401362593Sdim /* 402362593Sdim * qpair is not enabled, likely because a controller reset is 403362593Sdim * is in progress. Ignore the interrupt - any I/O that was 404362593Sdim * associated with this interrupt will get retried when the 405362593Sdim * reset is complete. 406362593Sdim */ 407362593Sdim return; 408362593Sdim 409362593Sdim while (1) { 410362593Sdim cpl = &qpair->cpl[qpair->cq_head]; 411362593Sdim 412362593Sdim if (cpl->status.p != qpair->phase) 413362593Sdim break; 414362593Sdim 415362593Sdim tr = qpair->act_tr[cpl->cid]; 416362593Sdim 417362593Sdim if (tr != NULL) { 418362593Sdim nvme_qpair_complete_tracker(qpair, tr, cpl, TRUE); 419362593Sdim qpair->sq_head = cpl->sqhd; 420362593Sdim } else { 421362593Sdim nvme_printf(qpair->ctrlr, 422362593Sdim "cpl does not map to outstanding cmd\n"); 423362593Sdim nvme_dump_completion(cpl); 424362593Sdim KASSERT(0, ("received completion for unknown cmd\n")); 425362593Sdim } 426362593Sdim 427362593Sdim if (++qpair->cq_head == qpair->num_entries) { 428362593Sdim qpair->cq_head = 0; 429362593Sdim qpair->phase = !qpair->phase; 430362593Sdim } 431362593Sdim 432362593Sdim nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl, 433362593Sdim qpair->cq_head); 434362593Sdim } 435362593Sdim} 436362593Sdim 437362593Sdimstatic void 438362593Sdimnvme_qpair_msix_handler(void *arg) 439362593Sdim{ 440362593Sdim struct nvme_qpair *qpair = arg; 441362593Sdim 442362593Sdim nvme_qpair_process_completions(qpair); 443362593Sdim} 444362593Sdim 445362593Sdimint 446362593Sdimnvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id, 447362593Sdim uint16_t vector, uint32_t num_entries, uint32_t num_trackers, 448362593Sdim struct nvme_controller *ctrlr) 449362593Sdim{ 450362593Sdim struct nvme_tracker *tr; 451362593Sdim size_t cmdsz, cplsz, prpsz, allocsz, prpmemsz; 452362593Sdim uint64_t queuemem_phys, prpmem_phys, list_phys; 453362593Sdim uint8_t *queuemem, *prpmem, *prp_list; 454362593Sdim int i, err; 455362593Sdim 456362593Sdim qpair->id = id; 457362593Sdim qpair->vector = vector; 458362593Sdim qpair->num_entries = num_entries; 459362593Sdim qpair->num_trackers = num_trackers; 460362593Sdim qpair->ctrlr = ctrlr; 461362593Sdim 462362593Sdim if (ctrlr->msix_enabled) { 463362593Sdim 464362593Sdim /* 465362593Sdim * MSI-X vector resource IDs start at 1, so we add one to 466362593Sdim * the queue's vector to get the corresponding rid to use. 467362593Sdim */ 468362593Sdim qpair->rid = vector + 1; 469362593Sdim 470362593Sdim qpair->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ, 471362593Sdim &qpair->rid, RF_ACTIVE); 472362593Sdim bus_setup_intr(ctrlr->dev, qpair->res, 473362593Sdim INTR_TYPE_MISC | INTR_MPSAFE, NULL, 474362593Sdim nvme_qpair_msix_handler, qpair, &qpair->tag); 475362593Sdim } 476362593Sdim 477362593Sdim mtx_init(&qpair->lock, "nvme qpair lock", NULL, MTX_DEF); 478362593Sdim 479362593Sdim /* Note: NVMe PRP format is restricted to 4-byte alignment. */ 480362593Sdim err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), 481362593Sdim 4, PAGE_SIZE, BUS_SPACE_MAXADDR, 482362593Sdim BUS_SPACE_MAXADDR, NULL, NULL, NVME_MAX_XFER_SIZE, 483362593Sdim (NVME_MAX_XFER_SIZE/PAGE_SIZE)+1, PAGE_SIZE, 0, 484362593Sdim NULL, NULL, &qpair->dma_tag_payload); 485362593Sdim if (err != 0) { 486362593Sdim nvme_printf(ctrlr, "payload tag create failed %d\n", err); 487362593Sdim goto out; 488362593Sdim } 489362593Sdim 490362593Sdim /* 491362593Sdim * Each component must be page aligned, and individual PRP lists 492362593Sdim * cannot cross a page boundary. 493362593Sdim */ 494362593Sdim cmdsz = qpair->num_entries * sizeof(struct nvme_command); 495362593Sdim cmdsz = roundup2(cmdsz, PAGE_SIZE); 496362593Sdim cplsz = qpair->num_entries * sizeof(struct nvme_completion); 497362593Sdim cplsz = roundup2(cplsz, PAGE_SIZE); 498362593Sdim prpsz = sizeof(uint64_t) * NVME_MAX_PRP_LIST_ENTRIES;; 499362593Sdim prpmemsz = qpair->num_trackers * prpsz; 500362593Sdim allocsz = cmdsz + cplsz + prpmemsz; 501362593Sdim 502362593Sdim err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), 503362593Sdim PAGE_SIZE, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 504362593Sdim allocsz, 1, allocsz, 0, NULL, NULL, &qpair->dma_tag); 505362593Sdim if (err != 0) { 506362593Sdim nvme_printf(ctrlr, "tag create failed %d\n", err); 507362593Sdim goto out; 508362593Sdim } 509362593Sdim 510362593Sdim if (bus_dmamem_alloc(qpair->dma_tag, (void **)&queuemem, 511362593Sdim BUS_DMA_NOWAIT, &qpair->queuemem_map)) { 512362593Sdim nvme_printf(ctrlr, "failed to alloc qpair memory\n"); 513362593Sdim goto out; 514362593Sdim } 515362593Sdim 516362593Sdim if (bus_dmamap_load(qpair->dma_tag, qpair->queuemem_map, 517362593Sdim queuemem, allocsz, nvme_single_map, &queuemem_phys, 0) != 0) { 518362593Sdim nvme_printf(ctrlr, "failed to load qpair memory\n"); 519362593Sdim goto out; 520362593Sdim } 521362593Sdim 522362593Sdim qpair->num_cmds = 0; 523362593Sdim qpair->num_intr_handler_calls = 0; 524362593Sdim qpair->cmd = (struct nvme_command *)queuemem; 525362593Sdim qpair->cpl = (struct nvme_completion *)(queuemem + cmdsz); 526362593Sdim prpmem = (uint8_t *)(queuemem + cmdsz + cplsz); 527362593Sdim qpair->cmd_bus_addr = queuemem_phys; 528362593Sdim qpair->cpl_bus_addr = queuemem_phys + cmdsz; 529362593Sdim prpmem_phys = queuemem_phys + cmdsz + cplsz; 530362593Sdim 531362593Sdim qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl); 532362593Sdim qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl); 533362593Sdim 534362593Sdim TAILQ_INIT(&qpair->free_tr); 535362593Sdim TAILQ_INIT(&qpair->outstanding_tr); 536362593Sdim STAILQ_INIT(&qpair->queued_req); 537362593Sdim 538362593Sdim list_phys = prpmem_phys; 539362593Sdim prp_list = prpmem; 540362593Sdim for (i = 0; i < qpair->num_trackers; i++) { 541362593Sdim 542362593Sdim if (list_phys + prpsz > prpmem_phys + prpmemsz) { 543362593Sdim qpair->num_trackers = i; 544362593Sdim break; 545362593Sdim } 546362593Sdim 547362593Sdim /* 548362593Sdim * Make sure that the PRP list for this tracker doesn't 549362593Sdim * overflow to another page. 550362593Sdim */ 551362593Sdim if (trunc_page(list_phys) != 552362593Sdim trunc_page(list_phys + prpsz - 1)) { 553362593Sdim list_phys = roundup2(list_phys, PAGE_SIZE); 554362593Sdim prp_list = 555362593Sdim (uint8_t *)roundup2((uintptr_t)prp_list, PAGE_SIZE); 556362593Sdim } 557362593Sdim 558362593Sdim tr = malloc(sizeof(*tr), M_NVME, M_ZERO | M_WAITOK); 559362593Sdim bus_dmamap_create(qpair->dma_tag_payload, 0, 560362593Sdim &tr->payload_dma_map); 561362593Sdim callout_init(&tr->timer, 1); 562362593Sdim tr->cid = i; 563362593Sdim tr->qpair = qpair; 564362593Sdim tr->prp = (uint64_t *)prp_list; 565362593Sdim tr->prp_bus_addr = list_phys; 566362593Sdim TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq); 567362593Sdim list_phys += prpsz; 568362593Sdim prp_list += prpsz; 569362593Sdim } 570362593Sdim 571362593Sdim if (qpair->num_trackers == 0) { 572362593Sdim nvme_printf(ctrlr, "failed to allocate enough trackers\n"); 573362593Sdim goto out; 574362593Sdim } 575362593Sdim 576362593Sdim qpair->act_tr = malloc(sizeof(struct nvme_tracker *) * 577362593Sdim qpair->num_entries, M_NVME, M_ZERO | M_WAITOK); 578362593Sdim return (0); 579362593Sdim 580362593Sdimout: 581362593Sdim nvme_qpair_destroy(qpair); 582362593Sdim return (ENOMEM); 583362593Sdim} 584362593Sdim 585362593Sdimstatic void 586362593Sdimnvme_qpair_destroy(struct nvme_qpair *qpair) 587362593Sdim{ 588362593Sdim struct nvme_tracker *tr; 589362593Sdim 590362593Sdim if (qpair->tag) 591362593Sdim bus_teardown_intr(qpair->ctrlr->dev, qpair->res, qpair->tag); 592362593Sdim 593362593Sdim if (mtx_initialized(&qpair->lock)) 594362593Sdim mtx_destroy(&qpair->lock); 595362593Sdim 596362593Sdim if (qpair->res) 597362593Sdim bus_release_resource(qpair->ctrlr->dev, SYS_RES_IRQ, 598362593Sdim rman_get_rid(qpair->res), qpair->res); 599362593Sdim 600362593Sdim if (qpair->cmd != NULL) { 601362593Sdim bus_dmamap_unload(qpair->dma_tag, qpair->queuemem_map); 602362593Sdim bus_dmamem_free(qpair->dma_tag, qpair->cmd, 603362593Sdim qpair->queuemem_map); 604362593Sdim } 605362593Sdim 606362593Sdim if (qpair->dma_tag) 607362593Sdim bus_dma_tag_destroy(qpair->dma_tag); 608362593Sdim 609362593Sdim if (qpair->dma_tag_payload) 610362593Sdim bus_dma_tag_destroy(qpair->dma_tag_payload); 611362593Sdim 612362593Sdim if (qpair->act_tr) 613362593Sdim free(qpair->act_tr, M_NVME); 614362593Sdim 615362593Sdim while (!TAILQ_EMPTY(&qpair->free_tr)) { 616362593Sdim tr = TAILQ_FIRST(&qpair->free_tr); 617362593Sdim TAILQ_REMOVE(&qpair->free_tr, tr, tailq); 618362593Sdim bus_dmamap_destroy(qpair->dma_tag, tr->payload_dma_map); 619362593Sdim free(tr, M_NVME); 620362593Sdim } 621362593Sdim} 622362593Sdim 623362593Sdimstatic void 624362593Sdimnvme_admin_qpair_abort_aers(struct nvme_qpair *qpair) 625362593Sdim{ 626362593Sdim struct nvme_tracker *tr; 627362593Sdim 628362593Sdim tr = TAILQ_FIRST(&qpair->outstanding_tr); 629362593Sdim while (tr != NULL) { 630362593Sdim if (tr->req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST) { 631362593Sdim nvme_qpair_manual_complete_tracker(qpair, tr, 632362593Sdim NVME_SCT_GENERIC, NVME_SC_ABORTED_SQ_DELETION, 0, 633362593Sdim FALSE); 634362593Sdim tr = TAILQ_FIRST(&qpair->outstanding_tr); 635362593Sdim } else { 636362593Sdim tr = TAILQ_NEXT(tr, tailq); 637362593Sdim } 638362593Sdim } 639362593Sdim} 640362593Sdim 641362593Sdimvoid 642362593Sdimnvme_admin_qpair_destroy(struct nvme_qpair *qpair) 643362593Sdim{ 644362593Sdim 645362593Sdim nvme_admin_qpair_abort_aers(qpair); 646362593Sdim nvme_qpair_destroy(qpair); 647362593Sdim} 648362593Sdim 649362593Sdimvoid 650362593Sdimnvme_io_qpair_destroy(struct nvme_qpair *qpair) 651362593Sdim{ 652362593Sdim 653362593Sdim nvme_qpair_destroy(qpair); 654362593Sdim} 655362593Sdim 656362593Sdimstatic void 657362593Sdimnvme_abort_complete(void *arg, const struct nvme_completion *status) 658362593Sdim{ 659362593Sdim struct nvme_tracker *tr = arg; 660362593Sdim 661362593Sdim /* 662362593Sdim * If cdw0 == 1, the controller was not able to abort the command 663362593Sdim * we requested. We still need to check the active tracker array, 664362593Sdim * to cover race where I/O timed out at same time controller was 665362593Sdim * completing the I/O. 666362593Sdim */ 667362593Sdim if (status->cdw0 == 1 && tr->qpair->act_tr[tr->cid] != NULL) { 668362593Sdim /* 669362593Sdim * An I/O has timed out, and the controller was unable to 670362593Sdim * abort it for some reason. Construct a fake completion 671362593Sdim * status, and then complete the I/O's tracker manually. 672362593Sdim */ 673362593Sdim nvme_printf(tr->qpair->ctrlr, 674362593Sdim "abort command failed, aborting command manually\n"); 675362593Sdim nvme_qpair_manual_complete_tracker(tr->qpair, tr, 676362593Sdim NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST, 0, TRUE); 677362593Sdim } 678362593Sdim} 679362593Sdim 680362593Sdimstatic void 681362593Sdimnvme_timeout(void *arg) 682362593Sdim{ 683362593Sdim struct nvme_tracker *tr = arg; 684362593Sdim struct nvme_qpair *qpair = tr->qpair; 685362593Sdim struct nvme_controller *ctrlr = qpair->ctrlr; 686362593Sdim union csts_register csts; 687362593Sdim 688362593Sdim /* Read csts to get value of cfs - controller fatal status. */ 689362593Sdim csts.raw = nvme_mmio_read_4(ctrlr, csts); 690362593Sdim 691362593Sdim if (ctrlr->enable_aborts && csts.bits.cfs == 0) { 692362593Sdim /* 693362593Sdim * If aborts are enabled, only use them if the controller is 694362593Sdim * not reporting fatal status. 695362593Sdim */ 696362593Sdim nvme_ctrlr_cmd_abort(ctrlr, tr->cid, qpair->id, 697362593Sdim nvme_abort_complete, tr); 698362593Sdim } else 699362593Sdim nvme_ctrlr_reset(ctrlr); 700362593Sdim} 701362593Sdim 702362593Sdimvoid 703362593Sdimnvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr) 704362593Sdim{ 705362593Sdim struct nvme_request *req; 706362593Sdim struct nvme_controller *ctrlr; 707362593Sdim 708362593Sdim mtx_assert(&qpair->lock, MA_OWNED); 709362593Sdim 710362593Sdim req = tr->req; 711362593Sdim req->cmd.cid = tr->cid; 712362593Sdim qpair->act_tr[tr->cid] = tr; 713362593Sdim ctrlr = qpair->ctrlr; 714362593Sdim 715362593Sdim if (req->timeout) 716362593Sdim#if __FreeBSD_version >= 800030 717362593Sdim callout_reset_curcpu(&tr->timer, ctrlr->timeout_period * hz, 718362593Sdim nvme_timeout, tr); 719362593Sdim#else 720362593Sdim callout_reset(&tr->timer, ctrlr->timeout_period * hz, 721362593Sdim nvme_timeout, tr); 722362593Sdim#endif 723362593Sdim 724362593Sdim /* Copy the command from the tracker to the submission queue. */ 725362593Sdim memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd)); 726362593Sdim 727362593Sdim if (++qpair->sq_tail == qpair->num_entries) 728362593Sdim qpair->sq_tail = 0; 729362593Sdim 730362593Sdim wmb(); 731362593Sdim nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl, 732362593Sdim qpair->sq_tail); 733362593Sdim 734362593Sdim qpair->num_cmds++; 735362593Sdim} 736362593Sdim 737362593Sdimstatic void 738362593Sdimnvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) 739362593Sdim{ 740362593Sdim struct nvme_tracker *tr = arg; 741362593Sdim uint32_t cur_nseg; 742362593Sdim 743362593Sdim /* 744362593Sdim * If the mapping operation failed, return immediately. The caller 745362593Sdim * is responsible for detecting the error status and failing the 746362593Sdim * tracker manually. 747362593Sdim */ 748362593Sdim if (error != 0) { 749362593Sdim nvme_printf(tr->qpair->ctrlr, 750362593Sdim "nvme_payload_map err %d\n", error); 751362593Sdim return; 752362593Sdim } 753362593Sdim 754362593Sdim /* 755362593Sdim * Note that we specified PAGE_SIZE for alignment and max 756362593Sdim * segment size when creating the bus dma tags. So here 757362593Sdim * we can safely just transfer each segment to its 758362593Sdim * associated PRP entry. 759362593Sdim */ 760362593Sdim tr->req->cmd.prp1 = seg[0].ds_addr; 761362593Sdim 762362593Sdim if (nseg == 2) { 763362593Sdim tr->req->cmd.prp2 = seg[1].ds_addr; 764362593Sdim } else if (nseg > 2) { 765362593Sdim cur_nseg = 1; 766362593Sdim tr->req->cmd.prp2 = (uint64_t)tr->prp_bus_addr; 767362593Sdim while (cur_nseg < nseg) { 768362593Sdim tr->prp[cur_nseg-1] = 769362593Sdim (uint64_t)seg[cur_nseg].ds_addr; 770362593Sdim cur_nseg++; 771362593Sdim } 772362593Sdim } else { 773362593Sdim /* 774362593Sdim * prp2 should not be used by the controller 775362593Sdim * since there is only one segment, but set 776362593Sdim * to 0 just to be safe. 777362593Sdim */ 778362593Sdim tr->req->cmd.prp2 = 0; 779362593Sdim } 780362593Sdim 781362593Sdim nvme_qpair_submit_tracker(tr->qpair, tr); 782362593Sdim} 783362593Sdim 784362593Sdimstatic void 785362593Sdim_nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) 786362593Sdim{ 787362593Sdim struct nvme_tracker *tr; 788362593Sdim int err = 0; 789362593Sdim 790362593Sdim mtx_assert(&qpair->lock, MA_OWNED); 791362593Sdim 792362593Sdim tr = TAILQ_FIRST(&qpair->free_tr); 793362593Sdim req->qpair = qpair; 794362593Sdim 795362593Sdim if (tr == NULL || !qpair->is_enabled) { 796362593Sdim /* 797362593Sdim * No tracker is available, or the qpair is disabled due to 798362593Sdim * an in-progress controller-level reset or controller 799362593Sdim * failure. 800362593Sdim */ 801362593Sdim 802362593Sdim if (qpair->ctrlr->is_failed) { 803362593Sdim /* 804362593Sdim * The controller has failed. Post the request to a 805362593Sdim * task where it will be aborted, so that we do not 806362593Sdim * invoke the request's callback in the context 807362593Sdim * of the submission. 808362593Sdim */ 809362593Sdim nvme_ctrlr_post_failed_request(qpair->ctrlr, req); 810362593Sdim } else { 811362593Sdim /* 812362593Sdim * Put the request on the qpair's request queue to be 813362593Sdim * processed when a tracker frees up via a command 814362593Sdim * completion or when the controller reset is 815362593Sdim * completed. 816362593Sdim */ 817362593Sdim STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq); 818362593Sdim } 819362593Sdim return; 820362593Sdim } 821362593Sdim 822362593Sdim TAILQ_REMOVE(&qpair->free_tr, tr, tailq); 823362593Sdim TAILQ_INSERT_TAIL(&qpair->outstanding_tr, tr, tailq); 824362593Sdim tr->req = req; 825362593Sdim 826362593Sdim switch (req->type) { 827362593Sdim case NVME_REQUEST_VADDR: 828362593Sdim KASSERT(req->payload_size <= qpair->ctrlr->max_xfer_size, 829362593Sdim ("payload_size (%d) exceeds max_xfer_size (%d)\n", 830362593Sdim req->payload_size, qpair->ctrlr->max_xfer_size)); 831362593Sdim err = bus_dmamap_load(tr->qpair->dma_tag_payload, 832362593Sdim tr->payload_dma_map, req->u.payload, req->payload_size, 833362593Sdim nvme_payload_map, tr, 0); 834362593Sdim if (err != 0) 835362593Sdim nvme_printf(qpair->ctrlr, 836362593Sdim "bus_dmamap_load returned 0x%x!\n", err); 837362593Sdim break; 838362593Sdim case NVME_REQUEST_NULL: 839362593Sdim nvme_qpair_submit_tracker(tr->qpair, tr); 840362593Sdim break; 841362593Sdim#ifdef NVME_UNMAPPED_BIO_SUPPORT 842362593Sdim case NVME_REQUEST_BIO: 843362593Sdim KASSERT(req->u.bio->bio_bcount <= qpair->ctrlr->max_xfer_size, 844362593Sdim ("bio->bio_bcount (%jd) exceeds max_xfer_size (%d)\n", 845362593Sdim (intmax_t)req->u.bio->bio_bcount, 846362593Sdim qpair->ctrlr->max_xfer_size)); 847362593Sdim err = bus_dmamap_load_bio(tr->qpair->dma_tag_payload, 848362593Sdim tr->payload_dma_map, req->u.bio, nvme_payload_map, tr, 0); 849362593Sdim if (err != 0) 850362593Sdim nvme_printf(qpair->ctrlr, 851362593Sdim "bus_dmamap_load_bio returned 0x%x!\n", err); 852362593Sdim break; 853362593Sdim#endif 854362593Sdim default: 855362593Sdim panic("unknown nvme request type 0x%x\n", req->type); 856362593Sdim break; 857362593Sdim } 858362593Sdim 859362593Sdim if (err != 0) { 860362593Sdim /* 861362593Sdim * The dmamap operation failed, so we manually fail the 862362593Sdim * tracker here with DATA_TRANSFER_ERROR status. 863362593Sdim * 864362593Sdim * nvme_qpair_manual_complete_tracker must not be called 865362593Sdim * with the qpair lock held. 866362593Sdim */ 867362593Sdim mtx_unlock(&qpair->lock); 868362593Sdim nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 869362593Sdim NVME_SC_DATA_TRANSFER_ERROR, 1 /* do not retry */, TRUE); 870362593Sdim mtx_lock(&qpair->lock); 871362593Sdim } 872362593Sdim} 873362593Sdim 874362593Sdimvoid 875362593Sdimnvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) 876362593Sdim{ 877362593Sdim 878362593Sdim mtx_lock(&qpair->lock); 879362593Sdim _nvme_qpair_submit_request(qpair, req); 880362593Sdim mtx_unlock(&qpair->lock); 881362593Sdim} 882362593Sdim 883362593Sdimstatic void 884362593Sdimnvme_qpair_enable(struct nvme_qpair *qpair) 885362593Sdim{ 886362593Sdim 887362593Sdim qpair->is_enabled = TRUE; 888362593Sdim} 889362593Sdim 890362593Sdimvoid 891362593Sdimnvme_qpair_reset(struct nvme_qpair *qpair) 892362593Sdim{ 893362593Sdim 894362593Sdim qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0; 895362593Sdim 896362593Sdim /* 897362593Sdim * First time through the completion queue, HW will set phase 898362593Sdim * bit on completions to 1. So set this to 1 here, indicating 899362593Sdim * we're looking for a 1 to know which entries have completed. 900362593Sdim * we'll toggle the bit each time when the completion queue 901362593Sdim * rolls over. 902362593Sdim */ 903362593Sdim qpair->phase = 1; 904362593Sdim 905362593Sdim memset(qpair->cmd, 0, 906362593Sdim qpair->num_entries * sizeof(struct nvme_command)); 907362593Sdim memset(qpair->cpl, 0, 908362593Sdim qpair->num_entries * sizeof(struct nvme_completion)); 909362593Sdim} 910362593Sdim 911362593Sdimvoid 912362593Sdimnvme_admin_qpair_enable(struct nvme_qpair *qpair) 913362593Sdim{ 914362593Sdim struct nvme_tracker *tr; 915362593Sdim struct nvme_tracker *tr_temp; 916362593Sdim 917362593Sdim /* 918362593Sdim * Manually abort each outstanding admin command. Do not retry 919362593Sdim * admin commands found here, since they will be left over from 920362593Sdim * a controller reset and its likely the context in which the 921362593Sdim * command was issued no longer applies. 922362593Sdim */ 923362593Sdim TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) { 924362593Sdim nvme_printf(qpair->ctrlr, 925362593Sdim "aborting outstanding admin command\n"); 926362593Sdim nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 927362593Sdim NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, TRUE); 928362593Sdim } 929362593Sdim 930362593Sdim nvme_qpair_enable(qpair); 931362593Sdim} 932362593Sdim 933362593Sdimvoid 934362593Sdimnvme_io_qpair_enable(struct nvme_qpair *qpair) 935362593Sdim{ 936362593Sdim STAILQ_HEAD(, nvme_request) temp; 937362593Sdim struct nvme_tracker *tr; 938362593Sdim struct nvme_tracker *tr_temp; 939362593Sdim struct nvme_request *req; 940362593Sdim 941362593Sdim /* 942362593Sdim * Manually abort each outstanding I/O. This normally results in a 943362593Sdim * retry, unless the retry count on the associated request has 944362593Sdim * reached its limit. 945362593Sdim */ 946362593Sdim TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) { 947362593Sdim nvme_printf(qpair->ctrlr, "aborting outstanding i/o\n"); 948362593Sdim nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 949362593Sdim NVME_SC_ABORTED_BY_REQUEST, 0, TRUE); 950362593Sdim } 951362593Sdim 952362593Sdim mtx_lock(&qpair->lock); 953362593Sdim 954362593Sdim nvme_qpair_enable(qpair); 955362593Sdim 956362593Sdim STAILQ_INIT(&temp); 957362593Sdim STAILQ_SWAP(&qpair->queued_req, &temp, nvme_request); 958362593Sdim 959362593Sdim while (!STAILQ_EMPTY(&temp)) { 960362593Sdim req = STAILQ_FIRST(&temp); 961362593Sdim STAILQ_REMOVE_HEAD(&temp, stailq); 962362593Sdim nvme_printf(qpair->ctrlr, "resubmitting queued i/o\n"); 963362593Sdim nvme_qpair_print_command(qpair, &req->cmd); 964362593Sdim _nvme_qpair_submit_request(qpair, req); 965362593Sdim } 966362593Sdim 967362593Sdim mtx_unlock(&qpair->lock); 968362593Sdim} 969 970static void 971nvme_qpair_disable(struct nvme_qpair *qpair) 972{ 973 struct nvme_tracker *tr; 974 975 qpair->is_enabled = FALSE; 976 mtx_lock(&qpair->lock); 977 TAILQ_FOREACH(tr, &qpair->outstanding_tr, tailq) 978 callout_stop(&tr->timer); 979 mtx_unlock(&qpair->lock); 980} 981 982void 983nvme_admin_qpair_disable(struct nvme_qpair *qpair) 984{ 985 986 nvme_qpair_disable(qpair); 987 nvme_admin_qpair_abort_aers(qpair); 988} 989 990void 991nvme_io_qpair_disable(struct nvme_qpair *qpair) 992{ 993 994 nvme_qpair_disable(qpair); 995} 996 997void 998nvme_qpair_fail(struct nvme_qpair *qpair) 999{ 1000 struct nvme_tracker *tr; 1001 struct nvme_request *req; 1002 1003 if (!mtx_initialized(&qpair->lock)) 1004 return; 1005 1006 mtx_lock(&qpair->lock); 1007 1008 while (!STAILQ_EMPTY(&qpair->queued_req)) { 1009 req = STAILQ_FIRST(&qpair->queued_req); 1010 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); 1011 nvme_printf(qpair->ctrlr, "failing queued i/o\n"); 1012 mtx_unlock(&qpair->lock); 1013 nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC, 1014 NVME_SC_ABORTED_BY_REQUEST, TRUE); 1015 mtx_lock(&qpair->lock); 1016 } 1017 1018 /* Manually abort each outstanding I/O. */ 1019 while (!TAILQ_EMPTY(&qpair->outstanding_tr)) { 1020 tr = TAILQ_FIRST(&qpair->outstanding_tr); 1021 /* 1022 * Do not remove the tracker. The abort_tracker path will 1023 * do that for us. 1024 */ 1025 nvme_printf(qpair->ctrlr, "failing outstanding i/o\n"); 1026 mtx_unlock(&qpair->lock); 1027 nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 1028 NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, TRUE); 1029 mtx_lock(&qpair->lock); 1030 } 1031 1032 mtx_unlock(&qpair->lock); 1033} 1034 1035