1240616Sjimharris/*- 2253112Sjimharris * Copyright (C) 2012-2013 Intel Corporation 3240616Sjimharris * All rights reserved. 4240616Sjimharris * 5240616Sjimharris * Redistribution and use in source and binary forms, with or without 6240616Sjimharris * modification, are permitted provided that the following conditions 7240616Sjimharris * are met: 8240616Sjimharris * 1. Redistributions of source code must retain the above copyright 9240616Sjimharris * notice, this list of conditions and the following disclaimer. 10240616Sjimharris * 2. Redistributions in binary form must reproduce the above copyright 11240616Sjimharris * notice, this list of conditions and the following disclaimer in the 12240616Sjimharris * documentation and/or other materials provided with the distribution. 13240616Sjimharris * 14240616Sjimharris * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15240616Sjimharris * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16240616Sjimharris * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17240616Sjimharris * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18240616Sjimharris * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19240616Sjimharris * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20240616Sjimharris * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21240616Sjimharris * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22240616Sjimharris * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23240616Sjimharris * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24240616Sjimharris * SUCH DAMAGE. 25240616Sjimharris */ 26240616Sjimharris 27240616Sjimharris#include <sys/cdefs.h> 28240616Sjimharris__FBSDID("$FreeBSD$"); 29240616Sjimharris 30240616Sjimharris#include <sys/param.h> 31240616Sjimharris#include <sys/bus.h> 32240616Sjimharris 33241665Sjimharris#include <dev/pci/pcivar.h> 34241665Sjimharris 35240616Sjimharris#include "nvme_private.h" 36240616Sjimharris 37248731Sjimharrisstatic void _nvme_qpair_submit_request(struct nvme_qpair *qpair, 38248731Sjimharris struct nvme_request *req); 39248731Sjimharris 40248773Sjimharrisstruct nvme_opcode_string { 41248773Sjimharris 42248773Sjimharris uint16_t opc; 43248773Sjimharris const char * str; 44248773Sjimharris}; 45248773Sjimharris 46248773Sjimharrisstatic struct nvme_opcode_string admin_opcode[] = { 47248773Sjimharris { NVME_OPC_DELETE_IO_SQ, "DELETE IO SQ" }, 48248773Sjimharris { NVME_OPC_CREATE_IO_SQ, "CREATE IO SQ" }, 49248773Sjimharris { NVME_OPC_GET_LOG_PAGE, "GET LOG PAGE" }, 50248773Sjimharris { NVME_OPC_DELETE_IO_CQ, "DELETE IO CQ" }, 51248773Sjimharris { NVME_OPC_CREATE_IO_CQ, "CREATE IO CQ" }, 52248773Sjimharris { NVME_OPC_IDENTIFY, "IDENTIFY" }, 53248773Sjimharris { NVME_OPC_ABORT, "ABORT" }, 54248773Sjimharris { NVME_OPC_SET_FEATURES, "SET FEATURES" }, 55248773Sjimharris { NVME_OPC_GET_FEATURES, "GET FEATURES" }, 56248773Sjimharris { NVME_OPC_ASYNC_EVENT_REQUEST, "ASYNC EVENT REQUEST" }, 57248773Sjimharris { NVME_OPC_FIRMWARE_ACTIVATE, "FIRMWARE ACTIVATE" }, 58248773Sjimharris { NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD, "FIRMWARE IMAGE DOWNLOAD" }, 59248773Sjimharris { NVME_OPC_FORMAT_NVM, "FORMAT NVM" }, 60248773Sjimharris { NVME_OPC_SECURITY_SEND, "SECURITY SEND" }, 61248773Sjimharris { NVME_OPC_SECURITY_RECEIVE, "SECURITY RECEIVE" }, 62248773Sjimharris { 0xFFFF, "ADMIN COMMAND" } 63248773Sjimharris}; 64248773Sjimharris 65248773Sjimharrisstatic struct nvme_opcode_string io_opcode[] = { 66248773Sjimharris { NVME_OPC_FLUSH, "FLUSH" }, 67248773Sjimharris { NVME_OPC_WRITE, "WRITE" }, 68248773Sjimharris { NVME_OPC_READ, "READ" }, 69248773Sjimharris { NVME_OPC_WRITE_UNCORRECTABLE, "WRITE UNCORRECTABLE" }, 70248773Sjimharris { NVME_OPC_COMPARE, "COMPARE" }, 71248773Sjimharris { NVME_OPC_DATASET_MANAGEMENT, "DATASET MANAGEMENT" }, 72248773Sjimharris { 0xFFFF, "IO COMMAND" } 73248773Sjimharris}; 74248773Sjimharris 75248773Sjimharrisstatic const char * 76248773Sjimharrisget_admin_opcode_string(uint16_t opc) 77248773Sjimharris{ 78248773Sjimharris struct nvme_opcode_string *entry; 79248773Sjimharris 80248773Sjimharris entry = admin_opcode; 81248773Sjimharris 82248773Sjimharris while (entry->opc != 0xFFFF) { 83248773Sjimharris if (entry->opc == opc) 84248773Sjimharris return (entry->str); 85248773Sjimharris entry++; 86248773Sjimharris } 87248773Sjimharris return (entry->str); 88248773Sjimharris} 89248773Sjimharris 90248773Sjimharrisstatic const char * 91248773Sjimharrisget_io_opcode_string(uint16_t opc) 92248773Sjimharris{ 93248773Sjimharris struct nvme_opcode_string *entry; 94248773Sjimharris 95248773Sjimharris entry = io_opcode; 96248773Sjimharris 97248773Sjimharris while (entry->opc != 0xFFFF) { 98248773Sjimharris if (entry->opc == opc) 99248773Sjimharris return (entry->str); 100248773Sjimharris entry++; 101248773Sjimharris } 102248773Sjimharris return (entry->str); 103248773Sjimharris} 104248773Sjimharris 105248773Sjimharris 106248773Sjimharrisstatic void 107248773Sjimharrisnvme_admin_qpair_print_command(struct nvme_qpair *qpair, 108248773Sjimharris struct nvme_command *cmd) 109248773Sjimharris{ 110248773Sjimharris 111248773Sjimharris nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%x " 112248773Sjimharris "cdw10:%08x cdw11:%08x\n", 113248773Sjimharris get_admin_opcode_string(cmd->opc), cmd->opc, qpair->id, cmd->cid, 114248773Sjimharris cmd->nsid, cmd->cdw10, cmd->cdw11); 115248773Sjimharris} 116248773Sjimharris 117248773Sjimharrisstatic void 118248773Sjimharrisnvme_io_qpair_print_command(struct nvme_qpair *qpair, 119248773Sjimharris struct nvme_command *cmd) 120248773Sjimharris{ 121248773Sjimharris 122248773Sjimharris switch (cmd->opc) { 123248773Sjimharris case NVME_OPC_WRITE: 124248773Sjimharris case NVME_OPC_READ: 125248773Sjimharris case NVME_OPC_WRITE_UNCORRECTABLE: 126248773Sjimharris case NVME_OPC_COMPARE: 127248773Sjimharris nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d " 128248780Sjimharris "lba:%llu len:%d\n", 129248773Sjimharris get_io_opcode_string(cmd->opc), qpair->id, cmd->cid, 130248780Sjimharris cmd->nsid, 131248780Sjimharris ((unsigned long long)cmd->cdw11 << 32) + cmd->cdw10, 132248773Sjimharris (cmd->cdw12 & 0xFFFF) + 1); 133248773Sjimharris break; 134248773Sjimharris case NVME_OPC_FLUSH: 135248773Sjimharris case NVME_OPC_DATASET_MANAGEMENT: 136248773Sjimharris nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d\n", 137248773Sjimharris get_io_opcode_string(cmd->opc), qpair->id, cmd->cid, 138248773Sjimharris cmd->nsid); 139248773Sjimharris break; 140248773Sjimharris default: 141248773Sjimharris nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%d\n", 142248773Sjimharris get_io_opcode_string(cmd->opc), cmd->opc, qpair->id, 143248773Sjimharris cmd->cid, cmd->nsid); 144248773Sjimharris break; 145248773Sjimharris } 146248773Sjimharris} 147248773Sjimharris 148248773Sjimharrisstatic void 149248773Sjimharrisnvme_qpair_print_command(struct nvme_qpair *qpair, struct nvme_command *cmd) 150248773Sjimharris{ 151248773Sjimharris if (qpair->id == 0) 152248773Sjimharris nvme_admin_qpair_print_command(qpair, cmd); 153248773Sjimharris else 154248773Sjimharris nvme_io_qpair_print_command(qpair, cmd); 155248773Sjimharris} 156248773Sjimharris 157248773Sjimharrisstruct nvme_status_string { 158248773Sjimharris 159248773Sjimharris uint16_t sc; 160248773Sjimharris const char * str; 161248773Sjimharris}; 162248773Sjimharris 163248773Sjimharrisstatic struct nvme_status_string generic_status[] = { 164248773Sjimharris { NVME_SC_SUCCESS, "SUCCESS" }, 165248773Sjimharris { NVME_SC_INVALID_OPCODE, "INVALID OPCODE" }, 166248773Sjimharris { NVME_SC_INVALID_FIELD, "INVALID_FIELD" }, 167248773Sjimharris { NVME_SC_COMMAND_ID_CONFLICT, "COMMAND ID CONFLICT" }, 168248773Sjimharris { NVME_SC_DATA_TRANSFER_ERROR, "DATA TRANSFER ERROR" }, 169248773Sjimharris { NVME_SC_ABORTED_POWER_LOSS, "ABORTED - POWER LOSS" }, 170248773Sjimharris { NVME_SC_INTERNAL_DEVICE_ERROR, "INTERNAL DEVICE ERROR" }, 171248773Sjimharris { NVME_SC_ABORTED_BY_REQUEST, "ABORTED - BY REQUEST" }, 172248773Sjimharris { NVME_SC_ABORTED_SQ_DELETION, "ABORTED - SQ DELETION" }, 173248773Sjimharris { NVME_SC_ABORTED_FAILED_FUSED, "ABORTED - FAILED FUSED" }, 174248773Sjimharris { NVME_SC_ABORTED_MISSING_FUSED, "ABORTED - MISSING FUSED" }, 175248773Sjimharris { NVME_SC_INVALID_NAMESPACE_OR_FORMAT, "INVALID NAMESPACE OR FORMAT" }, 176248773Sjimharris { NVME_SC_COMMAND_SEQUENCE_ERROR, "COMMAND SEQUENCE ERROR" }, 177248773Sjimharris { NVME_SC_LBA_OUT_OF_RANGE, "LBA OUT OF RANGE" }, 178248773Sjimharris { NVME_SC_CAPACITY_EXCEEDED, "CAPACITY EXCEEDED" }, 179248773Sjimharris { NVME_SC_NAMESPACE_NOT_READY, "NAMESPACE NOT READY" }, 180248773Sjimharris { 0xFFFF, "GENERIC" } 181248773Sjimharris}; 182248773Sjimharris 183248773Sjimharrisstatic struct nvme_status_string command_specific_status[] = { 184248773Sjimharris { NVME_SC_COMPLETION_QUEUE_INVALID, "INVALID COMPLETION QUEUE" }, 185248773Sjimharris { NVME_SC_INVALID_QUEUE_IDENTIFIER, "INVALID QUEUE IDENTIFIER" }, 186248773Sjimharris { NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED, "MAX QUEUE SIZE EXCEEDED" }, 187248773Sjimharris { NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED, "ABORT CMD LIMIT EXCEEDED" }, 188248773Sjimharris { NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED, "ASYNC LIMIT EXCEEDED" }, 189248773Sjimharris { NVME_SC_INVALID_FIRMWARE_SLOT, "INVALID FIRMWARE SLOT" }, 190248773Sjimharris { NVME_SC_INVALID_FIRMWARE_IMAGE, "INVALID FIRMWARE IMAGE" }, 191248773Sjimharris { NVME_SC_INVALID_INTERRUPT_VECTOR, "INVALID INTERRUPT VECTOR" }, 192248773Sjimharris { NVME_SC_INVALID_LOG_PAGE, "INVALID LOG PAGE" }, 193248773Sjimharris { NVME_SC_INVALID_FORMAT, "INVALID FORMAT" }, 194248773Sjimharris { NVME_SC_FIRMWARE_REQUIRES_RESET, "FIRMWARE REQUIRES RESET" }, 195248773Sjimharris { NVME_SC_CONFLICTING_ATTRIBUTES, "CONFLICTING ATTRIBUTES" }, 196248773Sjimharris { NVME_SC_INVALID_PROTECTION_INFO, "INVALID PROTECTION INFO" }, 197248773Sjimharris { NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE, "WRITE TO RO PAGE" }, 198248773Sjimharris { 0xFFFF, "COMMAND SPECIFIC" } 199248773Sjimharris}; 200248773Sjimharris 201248773Sjimharrisstatic struct nvme_status_string media_error_status[] = { 202248773Sjimharris { NVME_SC_WRITE_FAULTS, "WRITE FAULTS" }, 203248773Sjimharris { NVME_SC_UNRECOVERED_READ_ERROR, "UNRECOVERED READ ERROR" }, 204248773Sjimharris { NVME_SC_GUARD_CHECK_ERROR, "GUARD CHECK ERROR" }, 205248773Sjimharris { NVME_SC_APPLICATION_TAG_CHECK_ERROR, "APPLICATION TAG CHECK ERROR" }, 206248773Sjimharris { NVME_SC_REFERENCE_TAG_CHECK_ERROR, "REFERENCE TAG CHECK ERROR" }, 207248773Sjimharris { NVME_SC_COMPARE_FAILURE, "COMPARE FAILURE" }, 208248773Sjimharris { NVME_SC_ACCESS_DENIED, "ACCESS DENIED" }, 209248773Sjimharris { 0xFFFF, "MEDIA ERROR" } 210248773Sjimharris}; 211248773Sjimharris 212248773Sjimharrisstatic const char * 213248773Sjimharrisget_status_string(uint16_t sct, uint16_t sc) 214248773Sjimharris{ 215248773Sjimharris struct nvme_status_string *entry; 216248773Sjimharris 217248773Sjimharris switch (sct) { 218248773Sjimharris case NVME_SCT_GENERIC: 219248773Sjimharris entry = generic_status; 220248773Sjimharris break; 221248773Sjimharris case NVME_SCT_COMMAND_SPECIFIC: 222248773Sjimharris entry = command_specific_status; 223248773Sjimharris break; 224248773Sjimharris case NVME_SCT_MEDIA_ERROR: 225248773Sjimharris entry = media_error_status; 226248773Sjimharris break; 227248773Sjimharris case NVME_SCT_VENDOR_SPECIFIC: 228248773Sjimharris return ("VENDOR SPECIFIC"); 229248773Sjimharris default: 230248773Sjimharris return ("RESERVED"); 231248773Sjimharris } 232248773Sjimharris 233248773Sjimharris while (entry->sc != 0xFFFF) { 234248773Sjimharris if (entry->sc == sc) 235248773Sjimharris return (entry->str); 236248773Sjimharris entry++; 237248773Sjimharris } 238248773Sjimharris return (entry->str); 239248773Sjimharris} 240248773Sjimharris 241248773Sjimharrisstatic void 242248773Sjimharrisnvme_qpair_print_completion(struct nvme_qpair *qpair, 243248773Sjimharris struct nvme_completion *cpl) 244248773Sjimharris{ 245248773Sjimharris nvme_printf(qpair->ctrlr, "%s (%02x/%02x) sqid:%d cid:%d cdw0:%x\n", 246248773Sjimharris get_status_string(cpl->status.sct, cpl->status.sc), 247248773Sjimharris cpl->status.sct, cpl->status.sc, cpl->sqid, cpl->cid, cpl->cdw0); 248248773Sjimharris} 249248773Sjimharris 250240616Sjimharrisstatic boolean_t 251248733Sjimharrisnvme_completion_is_retry(const struct nvme_completion *cpl) 252248733Sjimharris{ 253240616Sjimharris /* 254240616Sjimharris * TODO: spec is not clear how commands that are aborted due 255240616Sjimharris * to TLER will be marked. So for now, it seems 256240616Sjimharris * NAMESPACE_NOT_READY is the only case where we should 257240616Sjimharris * look at the DNR bit. 258240616Sjimharris */ 259248756Sjimharris switch (cpl->status.sct) { 260240616Sjimharris case NVME_SCT_GENERIC: 261248756Sjimharris switch (cpl->status.sc) { 262248732Sjimharris case NVME_SC_ABORTED_BY_REQUEST: 263240616Sjimharris case NVME_SC_NAMESPACE_NOT_READY: 264248756Sjimharris if (cpl->status.dnr) 265240616Sjimharris return (0); 266240616Sjimharris else 267240616Sjimharris return (1); 268240616Sjimharris case NVME_SC_INVALID_OPCODE: 269240616Sjimharris case NVME_SC_INVALID_FIELD: 270240616Sjimharris case NVME_SC_COMMAND_ID_CONFLICT: 271240616Sjimharris case NVME_SC_DATA_TRANSFER_ERROR: 272240616Sjimharris case NVME_SC_ABORTED_POWER_LOSS: 273240616Sjimharris case NVME_SC_INTERNAL_DEVICE_ERROR: 274240616Sjimharris case NVME_SC_ABORTED_SQ_DELETION: 275240616Sjimharris case NVME_SC_ABORTED_FAILED_FUSED: 276240616Sjimharris case NVME_SC_ABORTED_MISSING_FUSED: 277240616Sjimharris case NVME_SC_INVALID_NAMESPACE_OR_FORMAT: 278240616Sjimharris case NVME_SC_COMMAND_SEQUENCE_ERROR: 279240616Sjimharris case NVME_SC_LBA_OUT_OF_RANGE: 280240616Sjimharris case NVME_SC_CAPACITY_EXCEEDED: 281240616Sjimharris default: 282240616Sjimharris return (0); 283240616Sjimharris } 284240616Sjimharris case NVME_SCT_COMMAND_SPECIFIC: 285240616Sjimharris case NVME_SCT_MEDIA_ERROR: 286240616Sjimharris case NVME_SCT_VENDOR_SPECIFIC: 287240616Sjimharris default: 288240616Sjimharris return (0); 289240616Sjimharris } 290240616Sjimharris} 291240616Sjimharris 292241664Sjimharrisstatic void 293241664Sjimharrisnvme_qpair_construct_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr, 294241664Sjimharris uint16_t cid) 295240616Sjimharris{ 296240616Sjimharris 297241664Sjimharris bus_dmamap_create(qpair->dma_tag, 0, &tr->payload_dma_map); 298241664Sjimharris bus_dmamap_create(qpair->dma_tag, 0, &tr->prp_dma_map); 299241657Sjimharris 300241664Sjimharris bus_dmamap_load(qpair->dma_tag, tr->prp_dma_map, tr->prp, 301241664Sjimharris sizeof(tr->prp), nvme_single_map, &tr->prp_bus_addr, 0); 302240616Sjimharris 303248748Sjimharris callout_init(&tr->timer, 1); 304241664Sjimharris tr->cid = cid; 305241664Sjimharris tr->qpair = qpair; 306240616Sjimharris} 307240616Sjimharris 308248733Sjimharrisstatic void 309248733Sjimharrisnvme_qpair_complete_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr, 310248733Sjimharris struct nvme_completion *cpl, boolean_t print_on_error) 311240616Sjimharris{ 312241659Sjimharris struct nvme_request *req; 313240616Sjimharris boolean_t retry, error; 314240616Sjimharris 315248733Sjimharris req = tr->req; 316248733Sjimharris error = nvme_completion_is_error(cpl); 317248761Sjimharris retry = error && nvme_completion_is_retry(cpl) && 318248761Sjimharris req->retries < nvme_retry_count; 319241434Sjimharris 320248733Sjimharris if (error && print_on_error) { 321248773Sjimharris nvme_qpair_print_command(qpair, &req->cmd); 322248773Sjimharris nvme_qpair_print_completion(qpair, cpl); 323248733Sjimharris } 324240616Sjimharris 325248733Sjimharris qpair->act_tr[cpl->cid] = NULL; 326240616Sjimharris 327248733Sjimharris KASSERT(cpl->cid == req->cmd.cid, ("cpl cid does not match cmd cid\n")); 328241659Sjimharris 329248733Sjimharris if (req->cb_fn && !retry) 330248733Sjimharris req->cb_fn(req->cb_arg, cpl); 331240616Sjimharris 332248733Sjimharris mtx_lock(&qpair->lock); 333248733Sjimharris callout_stop(&tr->timer); 334240616Sjimharris 335248761Sjimharris if (retry) { 336248761Sjimharris req->retries++; 337248746Sjimharris nvme_qpair_submit_tracker(qpair, tr); 338248761Sjimharris } else { 339248913Sjimharris if (req->type != NVME_REQUEST_NULL) 340248733Sjimharris bus_dmamap_unload(qpair->dma_tag, 341248733Sjimharris tr->payload_dma_map); 342240616Sjimharris 343248733Sjimharris nvme_free_request(req); 344248737Sjimharris tr->req = NULL; 345240616Sjimharris 346248741Sjimharris TAILQ_REMOVE(&qpair->outstanding_tr, tr, tailq); 347248741Sjimharris TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq); 348240616Sjimharris 349248755Sjimharris /* 350248755Sjimharris * If the controller is in the middle of resetting, don't 351248755Sjimharris * try to submit queued requests here - let the reset logic 352248755Sjimharris * handle that instead. 353248755Sjimharris */ 354248755Sjimharris if (!STAILQ_EMPTY(&qpair->queued_req) && 355248755Sjimharris !qpair->ctrlr->is_resetting) { 356248733Sjimharris req = STAILQ_FIRST(&qpair->queued_req); 357248733Sjimharris STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); 358248733Sjimharris _nvme_qpair_submit_request(qpair, req); 359248733Sjimharris } 360248733Sjimharris } 361240616Sjimharris 362248733Sjimharris mtx_unlock(&qpair->lock); 363248733Sjimharris} 364240616Sjimharris 365248746Sjimharrisstatic void 366248746Sjimharrisnvme_qpair_manual_complete_tracker(struct nvme_qpair *qpair, 367248767Sjimharris struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr, 368248746Sjimharris boolean_t print_on_error) 369248746Sjimharris{ 370248746Sjimharris struct nvme_completion cpl; 371248746Sjimharris 372248746Sjimharris memset(&cpl, 0, sizeof(cpl)); 373248746Sjimharris cpl.sqid = qpair->id; 374248746Sjimharris cpl.cid = tr->cid; 375248756Sjimharris cpl.status.sct = sct; 376248756Sjimharris cpl.status.sc = sc; 377248767Sjimharris cpl.status.dnr = dnr; 378248746Sjimharris nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error); 379248746Sjimharris} 380248746Sjimharris 381248733Sjimharrisvoid 382248767Sjimharrisnvme_qpair_manual_complete_request(struct nvme_qpair *qpair, 383248767Sjimharris struct nvme_request *req, uint32_t sct, uint32_t sc, 384248767Sjimharris boolean_t print_on_error) 385248767Sjimharris{ 386248767Sjimharris struct nvme_completion cpl; 387248767Sjimharris boolean_t error; 388248767Sjimharris 389248767Sjimharris memset(&cpl, 0, sizeof(cpl)); 390248767Sjimharris cpl.sqid = qpair->id; 391248767Sjimharris cpl.status.sct = sct; 392248767Sjimharris cpl.status.sc = sc; 393248767Sjimharris 394248767Sjimharris error = nvme_completion_is_error(&cpl); 395248767Sjimharris 396248767Sjimharris if (error && print_on_error) { 397248773Sjimharris nvme_qpair_print_command(qpair, &req->cmd); 398248773Sjimharris nvme_qpair_print_completion(qpair, &cpl); 399248767Sjimharris } 400248767Sjimharris 401248767Sjimharris if (req->cb_fn) 402248767Sjimharris req->cb_fn(req->cb_arg, &cpl); 403248767Sjimharris 404248767Sjimharris nvme_free_request(req); 405248767Sjimharris} 406248767Sjimharris 407248767Sjimharrisvoid 408248733Sjimharrisnvme_qpair_process_completions(struct nvme_qpair *qpair) 409248733Sjimharris{ 410248733Sjimharris struct nvme_tracker *tr; 411248733Sjimharris struct nvme_completion *cpl; 412240616Sjimharris 413248733Sjimharris qpair->num_intr_handler_calls++; 414240616Sjimharris 415248746Sjimharris if (!qpair->is_enabled) 416248746Sjimharris /* 417248746Sjimharris * qpair is not enabled, likely because a controller reset is 418248746Sjimharris * is in progress. Ignore the interrupt - any I/O that was 419248746Sjimharris * associated with this interrupt will get retried when the 420248746Sjimharris * reset is complete. 421248746Sjimharris */ 422248746Sjimharris return; 423248746Sjimharris 424248733Sjimharris while (1) { 425248733Sjimharris cpl = &qpair->cpl[qpair->cq_head]; 426241664Sjimharris 427248756Sjimharris if (cpl->status.p != qpair->phase) 428248733Sjimharris break; 429241664Sjimharris 430248733Sjimharris tr = qpair->act_tr[cpl->cid]; 431248733Sjimharris 432248733Sjimharris if (tr != NULL) { 433248733Sjimharris nvme_qpair_complete_tracker(qpair, tr, cpl, TRUE); 434248733Sjimharris qpair->sq_head = cpl->sqhd; 435248733Sjimharris } else { 436248773Sjimharris nvme_printf(qpair->ctrlr, 437248773Sjimharris "cpl does not map to outstanding cmd\n"); 438248733Sjimharris nvme_dump_completion(cpl); 439248733Sjimharris KASSERT(0, ("received completion for unknown cmd\n")); 440240616Sjimharris } 441240616Sjimharris 442240616Sjimharris if (++qpair->cq_head == qpair->num_entries) { 443240616Sjimharris qpair->cq_head = 0; 444240616Sjimharris qpair->phase = !qpair->phase; 445240616Sjimharris } 446240616Sjimharris 447240616Sjimharris nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl, 448240616Sjimharris qpair->cq_head); 449240616Sjimharris } 450240616Sjimharris} 451240616Sjimharris 452240616Sjimharrisstatic void 453240616Sjimharrisnvme_qpair_msix_handler(void *arg) 454240616Sjimharris{ 455240616Sjimharris struct nvme_qpair *qpair = arg; 456240616Sjimharris 457240616Sjimharris nvme_qpair_process_completions(qpair); 458240616Sjimharris} 459240616Sjimharris 460240616Sjimharrisvoid 461240616Sjimharrisnvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id, 462241664Sjimharris uint16_t vector, uint32_t num_entries, uint32_t num_trackers, 463252271Sjimharris struct nvme_controller *ctrlr) 464240616Sjimharris{ 465241664Sjimharris struct nvme_tracker *tr; 466241664Sjimharris uint32_t i; 467240616Sjimharris 468240616Sjimharris qpair->id = id; 469240616Sjimharris qpair->vector = vector; 470240616Sjimharris qpair->num_entries = num_entries; 471241665Sjimharris#ifdef CHATHAM2 472241665Sjimharris /* 473241665Sjimharris * Chatham prototype board starts having issues at higher queue 474241665Sjimharris * depths. So use a conservative estimate here of no more than 64 475241665Sjimharris * outstanding I/O per queue at any one point. 476241665Sjimharris */ 477241665Sjimharris if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) 478241665Sjimharris num_trackers = min(num_trackers, 64); 479241665Sjimharris#endif 480241665Sjimharris qpair->num_trackers = num_trackers; 481240616Sjimharris qpair->ctrlr = ctrlr; 482240616Sjimharris 483240616Sjimharris if (ctrlr->msix_enabled) { 484240616Sjimharris 485240616Sjimharris /* 486240616Sjimharris * MSI-X vector resource IDs start at 1, so we add one to 487240616Sjimharris * the queue's vector to get the corresponding rid to use. 488240616Sjimharris */ 489240616Sjimharris qpair->rid = vector + 1; 490240616Sjimharris 491240616Sjimharris qpair->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ, 492240616Sjimharris &qpair->rid, RF_ACTIVE); 493240616Sjimharris 494240616Sjimharris bus_setup_intr(ctrlr->dev, qpair->res, 495240616Sjimharris INTR_TYPE_MISC | INTR_MPSAFE, NULL, 496240616Sjimharris nvme_qpair_msix_handler, qpair, &qpair->tag); 497240616Sjimharris } 498240616Sjimharris 499240616Sjimharris mtx_init(&qpair->lock, "nvme qpair lock", NULL, MTX_DEF); 500240616Sjimharris 501240616Sjimharris bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), 502240616Sjimharris sizeof(uint64_t), PAGE_SIZE, BUS_SPACE_MAXADDR, 503252271Sjimharris BUS_SPACE_MAXADDR, NULL, NULL, NVME_MAX_XFER_SIZE, 504252271Sjimharris (NVME_MAX_XFER_SIZE/PAGE_SIZE)+1, PAGE_SIZE, 0, 505240616Sjimharris NULL, NULL, &qpair->dma_tag); 506240616Sjimharris 507240616Sjimharris qpair->num_cmds = 0; 508241434Sjimharris qpair->num_intr_handler_calls = 0; 509240616Sjimharris 510240616Sjimharris qpair->cmd = contigmalloc(qpair->num_entries * 511248770Sjimharris sizeof(struct nvme_command), M_NVME, M_ZERO, 512240616Sjimharris 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); 513240616Sjimharris qpair->cpl = contigmalloc(qpair->num_entries * 514248770Sjimharris sizeof(struct nvme_completion), M_NVME, M_ZERO, 515240616Sjimharris 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); 516240616Sjimharris 517240616Sjimharris bus_dmamap_create(qpair->dma_tag, 0, &qpair->cmd_dma_map); 518240616Sjimharris bus_dmamap_create(qpair->dma_tag, 0, &qpair->cpl_dma_map); 519240616Sjimharris 520240616Sjimharris bus_dmamap_load(qpair->dma_tag, qpair->cmd_dma_map, 521240616Sjimharris qpair->cmd, qpair->num_entries * sizeof(struct nvme_command), 522240616Sjimharris nvme_single_map, &qpair->cmd_bus_addr, 0); 523240616Sjimharris bus_dmamap_load(qpair->dma_tag, qpair->cpl_dma_map, 524240616Sjimharris qpair->cpl, qpair->num_entries * sizeof(struct nvme_completion), 525240616Sjimharris nvme_single_map, &qpair->cpl_bus_addr, 0); 526240616Sjimharris 527240616Sjimharris qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl); 528240616Sjimharris qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl); 529240616Sjimharris 530248741Sjimharris TAILQ_INIT(&qpair->free_tr); 531248741Sjimharris TAILQ_INIT(&qpair->outstanding_tr); 532241665Sjimharris STAILQ_INIT(&qpair->queued_req); 533240616Sjimharris 534241665Sjimharris for (i = 0; i < qpair->num_trackers; i++) { 535248770Sjimharris tr = malloc(sizeof(*tr), M_NVME, M_ZERO | M_WAITOK); 536241664Sjimharris nvme_qpair_construct_tracker(qpair, tr, i); 537248741Sjimharris TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq); 538241664Sjimharris } 539241664Sjimharris 540240616Sjimharris qpair->act_tr = malloc(sizeof(struct nvme_tracker *) * qpair->num_entries, 541248770Sjimharris M_NVME, M_ZERO | M_WAITOK); 542240616Sjimharris} 543240616Sjimharris 544240616Sjimharrisstatic void 545240616Sjimharrisnvme_qpair_destroy(struct nvme_qpair *qpair) 546240616Sjimharris{ 547248737Sjimharris struct nvme_tracker *tr; 548240616Sjimharris 549240616Sjimharris if (qpair->tag) 550240616Sjimharris bus_teardown_intr(qpair->ctrlr->dev, qpair->res, qpair->tag); 551240616Sjimharris 552240616Sjimharris if (qpair->res) 553240616Sjimharris bus_release_resource(qpair->ctrlr->dev, SYS_RES_IRQ, 554240616Sjimharris rman_get_rid(qpair->res), qpair->res); 555240616Sjimharris 556248766Sjimharris if (qpair->cmd) { 557248766Sjimharris bus_dmamap_unload(qpair->dma_tag, qpair->cmd_dma_map); 558248766Sjimharris bus_dmamap_destroy(qpair->dma_tag, qpair->cmd_dma_map); 559248766Sjimharris contigfree(qpair->cmd, 560248766Sjimharris qpair->num_entries * sizeof(struct nvme_command), M_NVME); 561248766Sjimharris } 562248766Sjimharris 563248766Sjimharris if (qpair->cpl) { 564248766Sjimharris bus_dmamap_unload(qpair->dma_tag, qpair->cpl_dma_map); 565248766Sjimharris bus_dmamap_destroy(qpair->dma_tag, qpair->cpl_dma_map); 566248766Sjimharris contigfree(qpair->cpl, 567248766Sjimharris qpair->num_entries * sizeof(struct nvme_completion), 568248766Sjimharris M_NVME); 569248766Sjimharris } 570248766Sjimharris 571240616Sjimharris if (qpair->dma_tag) 572240616Sjimharris bus_dma_tag_destroy(qpair->dma_tag); 573240616Sjimharris 574240616Sjimharris if (qpair->act_tr) 575240616Sjimharris free(qpair->act_tr, M_NVME); 576240616Sjimharris 577248741Sjimharris while (!TAILQ_EMPTY(&qpair->free_tr)) { 578248741Sjimharris tr = TAILQ_FIRST(&qpair->free_tr); 579248741Sjimharris TAILQ_REMOVE(&qpair->free_tr, tr, tailq); 580241658Sjimharris bus_dmamap_destroy(qpair->dma_tag, tr->payload_dma_map); 581241658Sjimharris bus_dmamap_destroy(qpair->dma_tag, tr->prp_dma_map); 582240616Sjimharris free(tr, M_NVME); 583240616Sjimharris } 584240616Sjimharris} 585240616Sjimharris 586248746Sjimharrisstatic void 587248746Sjimharrisnvme_admin_qpair_abort_aers(struct nvme_qpair *qpair) 588248746Sjimharris{ 589248746Sjimharris struct nvme_tracker *tr; 590248746Sjimharris 591248746Sjimharris tr = TAILQ_FIRST(&qpair->outstanding_tr); 592248746Sjimharris while (tr != NULL) { 593248746Sjimharris if (tr->req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST) { 594248746Sjimharris nvme_qpair_manual_complete_tracker(qpair, tr, 595248767Sjimharris NVME_SCT_GENERIC, NVME_SC_ABORTED_SQ_DELETION, 0, 596248746Sjimharris FALSE); 597248746Sjimharris tr = TAILQ_FIRST(&qpair->outstanding_tr); 598248746Sjimharris } else { 599248746Sjimharris tr = TAILQ_NEXT(tr, tailq); 600248746Sjimharris } 601248746Sjimharris } 602248746Sjimharris} 603248746Sjimharris 604240616Sjimharrisvoid 605240616Sjimharrisnvme_admin_qpair_destroy(struct nvme_qpair *qpair) 606240616Sjimharris{ 607240616Sjimharris 608248746Sjimharris nvme_admin_qpair_abort_aers(qpair); 609240616Sjimharris nvme_qpair_destroy(qpair); 610240616Sjimharris} 611240616Sjimharris 612240616Sjimharrisvoid 613240616Sjimharrisnvme_io_qpair_destroy(struct nvme_qpair *qpair) 614240616Sjimharris{ 615240616Sjimharris 616248766Sjimharris nvme_qpair_destroy(qpair); 617240616Sjimharris} 618240616Sjimharris 619240616Sjimharrisstatic void 620248734Sjimharrisnvme_abort_complete(void *arg, const struct nvme_completion *status) 621248734Sjimharris{ 622248734Sjimharris struct nvme_tracker *tr = arg; 623248734Sjimharris 624248734Sjimharris /* 625248734Sjimharris * If cdw0 == 1, the controller was not able to abort the command 626248734Sjimharris * we requested. We still need to check the active tracker array, 627248734Sjimharris * to cover race where I/O timed out at same time controller was 628248734Sjimharris * completing the I/O. 629248734Sjimharris */ 630248734Sjimharris if (status->cdw0 == 1 && tr->qpair->act_tr[tr->cid] != NULL) { 631248734Sjimharris /* 632248734Sjimharris * An I/O has timed out, and the controller was unable to 633248734Sjimharris * abort it for some reason. Construct a fake completion 634248734Sjimharris * status, and then complete the I/O's tracker manually. 635248734Sjimharris */ 636248773Sjimharris nvme_printf(tr->qpair->ctrlr, 637248773Sjimharris "abort command failed, aborting command manually\n"); 638248746Sjimharris nvme_qpair_manual_complete_tracker(tr->qpair, tr, 639248767Sjimharris NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST, 0, TRUE); 640248734Sjimharris } 641248734Sjimharris} 642248734Sjimharris 643248734Sjimharrisstatic void 644240616Sjimharrisnvme_timeout(void *arg) 645240616Sjimharris{ 646248732Sjimharris struct nvme_tracker *tr = arg; 647248748Sjimharris struct nvme_qpair *qpair = tr->qpair; 648248748Sjimharris struct nvme_controller *ctrlr = qpair->ctrlr; 649248748Sjimharris union csts_register csts; 650248732Sjimharris 651248754Sjimharris /* Read csts to get value of cfs - controller fatal status. */ 652248748Sjimharris csts.raw = nvme_mmio_read_4(ctrlr, csts); 653248754Sjimharris 654248754Sjimharris if (ctrlr->enable_aborts && csts.bits.cfs == 0) { 655248748Sjimharris /* 656248754Sjimharris * If aborts are enabled, only use them if the controller is 657248754Sjimharris * not reporting fatal status. 658248748Sjimharris */ 659248754Sjimharris nvme_ctrlr_cmd_abort(ctrlr, tr->cid, qpair->id, 660248754Sjimharris nvme_abort_complete, tr); 661248754Sjimharris } else 662248748Sjimharris nvme_ctrlr_reset(ctrlr); 663240616Sjimharris} 664240616Sjimharris 665240616Sjimharrisvoid 666248746Sjimharrisnvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr) 667240616Sjimharris{ 668248749Sjimharris struct nvme_request *req; 669248749Sjimharris struct nvme_controller *ctrlr; 670240616Sjimharris 671248746Sjimharris mtx_assert(&qpair->lock, MA_OWNED); 672248746Sjimharris 673241659Sjimharris req = tr->req; 674241659Sjimharris req->cmd.cid = tr->cid; 675240616Sjimharris qpair->act_tr[tr->cid] = tr; 676248749Sjimharris ctrlr = qpair->ctrlr; 677240616Sjimharris 678248749Sjimharris if (req->timeout) 679242420Sjimharris#if __FreeBSD_version >= 800030 680248749Sjimharris callout_reset_curcpu(&tr->timer, ctrlr->timeout_period * hz, 681248735Sjimharris nvme_timeout, tr); 682242420Sjimharris#else 683248749Sjimharris callout_reset(&tr->timer, ctrlr->timeout_period * hz, 684248749Sjimharris nvme_timeout, tr); 685242420Sjimharris#endif 686240616Sjimharris 687240616Sjimharris /* Copy the command from the tracker to the submission queue. */ 688241659Sjimharris memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd)); 689240616Sjimharris 690240616Sjimharris if (++qpair->sq_tail == qpair->num_entries) 691240616Sjimharris qpair->sq_tail = 0; 692240616Sjimharris 693240616Sjimharris wmb(); 694240616Sjimharris nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl, 695240616Sjimharris qpair->sq_tail); 696240616Sjimharris 697240616Sjimharris qpair->num_cmds++; 698240616Sjimharris} 699241663Sjimharris 700248731Sjimharrisstatic void 701249420Sjimharrisnvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) 702249420Sjimharris{ 703249420Sjimharris struct nvme_tracker *tr = arg; 704249420Sjimharris uint32_t cur_nseg; 705249420Sjimharris 706249420Sjimharris /* 707249420Sjimharris * If the mapping operation failed, return immediately. The caller 708249420Sjimharris * is responsible for detecting the error status and failing the 709249420Sjimharris * tracker manually. 710249420Sjimharris */ 711249420Sjimharris if (error != 0) 712249420Sjimharris return; 713249420Sjimharris 714249420Sjimharris /* 715249420Sjimharris * Note that we specified PAGE_SIZE for alignment and max 716249420Sjimharris * segment size when creating the bus dma tags. So here 717249420Sjimharris * we can safely just transfer each segment to its 718249420Sjimharris * associated PRP entry. 719249420Sjimharris */ 720249420Sjimharris tr->req->cmd.prp1 = seg[0].ds_addr; 721249420Sjimharris 722249420Sjimharris if (nseg == 2) { 723249420Sjimharris tr->req->cmd.prp2 = seg[1].ds_addr; 724249420Sjimharris } else if (nseg > 2) { 725249420Sjimharris cur_nseg = 1; 726249420Sjimharris tr->req->cmd.prp2 = (uint64_t)tr->prp_bus_addr; 727249420Sjimharris while (cur_nseg < nseg) { 728249420Sjimharris tr->prp[cur_nseg-1] = 729249420Sjimharris (uint64_t)seg[cur_nseg].ds_addr; 730249420Sjimharris cur_nseg++; 731249420Sjimharris } 732249420Sjimharris } 733249420Sjimharris 734249420Sjimharris nvme_qpair_submit_tracker(tr->qpair, tr); 735249420Sjimharris} 736249420Sjimharris 737249420Sjimharrisstatic void 738248731Sjimharris_nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) 739241663Sjimharris{ 740241663Sjimharris struct nvme_tracker *tr; 741249416Sjimharris int err = 0; 742241663Sjimharris 743248731Sjimharris mtx_assert(&qpair->lock, MA_OWNED); 744241663Sjimharris 745248741Sjimharris tr = TAILQ_FIRST(&qpair->free_tr); 746248767Sjimharris req->qpair = qpair; 747241664Sjimharris 748248746Sjimharris if (tr == NULL || !qpair->is_enabled) { 749241665Sjimharris /* 750248746Sjimharris * No tracker is available, or the qpair is disabled due to 751248767Sjimharris * an in-progress controller-level reset or controller 752248767Sjimharris * failure. 753241665Sjimharris */ 754248767Sjimharris 755248767Sjimharris if (qpair->ctrlr->is_failed) { 756248767Sjimharris /* 757248767Sjimharris * The controller has failed. Post the request to a 758248767Sjimharris * task where it will be aborted, so that we do not 759248767Sjimharris * invoke the request's callback in the context 760248767Sjimharris * of the submission. 761248767Sjimharris */ 762248767Sjimharris nvme_ctrlr_post_failed_request(qpair->ctrlr, req); 763248767Sjimharris } else { 764248767Sjimharris /* 765248767Sjimharris * Put the request on the qpair's request queue to be 766248767Sjimharris * processed when a tracker frees up via a command 767248767Sjimharris * completion or when the controller reset is 768248767Sjimharris * completed. 769248767Sjimharris */ 770248767Sjimharris STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq); 771248767Sjimharris } 772248731Sjimharris return; 773241664Sjimharris } 774241664Sjimharris 775248741Sjimharris TAILQ_REMOVE(&qpair->free_tr, tr, tailq); 776248741Sjimharris TAILQ_INSERT_TAIL(&qpair->outstanding_tr, tr, tailq); 777241663Sjimharris tr->req = req; 778241663Sjimharris 779248913Sjimharris switch (req->type) { 780248913Sjimharris case NVME_REQUEST_VADDR: 781252272Sjimharris KASSERT(req->payload_size <= qpair->ctrlr->max_xfer_size, 782252272Sjimharris ("payload_size (%d) exceeds max_xfer_size (%d)\n", 783252272Sjimharris req->payload_size, qpair->ctrlr->max_xfer_size)); 784248913Sjimharris err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, 785248913Sjimharris req->u.payload, req->payload_size, nvme_payload_map, tr, 0); 786248913Sjimharris if (err != 0) 787249416Sjimharris nvme_printf(qpair->ctrlr, 788249416Sjimharris "bus_dmamap_load returned 0x%x!\n", err); 789248913Sjimharris break; 790248913Sjimharris case NVME_REQUEST_NULL: 791248913Sjimharris nvme_qpair_submit_tracker(tr->qpair, tr); 792248913Sjimharris break; 793248977Sjimharris#ifdef NVME_UNMAPPED_BIO_SUPPORT 794248977Sjimharris case NVME_REQUEST_BIO: 795252272Sjimharris KASSERT(req->u.bio->bio_bcount <= qpair->ctrlr->max_xfer_size, 796252272Sjimharris ("bio->bio_bcount (%jd) exceeds max_xfer_size (%d)\n", 797252272Sjimharris (intmax_t)req->u.bio->bio_bcount, 798252272Sjimharris qpair->ctrlr->max_xfer_size)); 799248977Sjimharris err = bus_dmamap_load_bio(tr->qpair->dma_tag, 800248977Sjimharris tr->payload_dma_map, req->u.bio, nvme_payload_map, tr, 0); 801248977Sjimharris if (err != 0) 802249416Sjimharris nvme_printf(qpair->ctrlr, 803249416Sjimharris "bus_dmamap_load_bio returned 0x%x!\n", err); 804248977Sjimharris break; 805248977Sjimharris#endif 806248913Sjimharris default: 807248913Sjimharris panic("unknown nvme request type 0x%x\n", req->type); 808248913Sjimharris break; 809241663Sjimharris } 810249416Sjimharris 811249416Sjimharris if (err != 0) { 812249416Sjimharris /* 813249416Sjimharris * The dmamap operation failed, so we manually fail the 814249416Sjimharris * tracker here with DATA_TRANSFER_ERROR status. 815249416Sjimharris * 816249416Sjimharris * nvme_qpair_manual_complete_tracker must not be called 817249416Sjimharris * with the qpair lock held. 818249416Sjimharris */ 819249416Sjimharris mtx_unlock(&qpair->lock); 820249416Sjimharris nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 821249416Sjimharris NVME_SC_DATA_TRANSFER_ERROR, 1 /* do not retry */, TRUE); 822249416Sjimharris mtx_lock(&qpair->lock); 823249416Sjimharris } 824248731Sjimharris} 825241663Sjimharris 826248731Sjimharrisvoid 827248731Sjimharrisnvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) 828248731Sjimharris{ 829248731Sjimharris 830248731Sjimharris mtx_lock(&qpair->lock); 831248731Sjimharris _nvme_qpair_submit_request(qpair, req); 832241663Sjimharris mtx_unlock(&qpair->lock); 833241663Sjimharris} 834248746Sjimharris 835248746Sjimharrisstatic void 836248746Sjimharrisnvme_qpair_enable(struct nvme_qpair *qpair) 837248746Sjimharris{ 838248746Sjimharris 839248746Sjimharris qpair->is_enabled = TRUE; 840248761Sjimharris} 841248761Sjimharris 842248761Sjimharrisvoid 843248761Sjimharrisnvme_qpair_reset(struct nvme_qpair *qpair) 844248761Sjimharris{ 845248761Sjimharris 846248746Sjimharris qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0; 847248746Sjimharris 848248746Sjimharris /* 849248746Sjimharris * First time through the completion queue, HW will set phase 850248746Sjimharris * bit on completions to 1. So set this to 1 here, indicating 851248746Sjimharris * we're looking for a 1 to know which entries have completed. 852248746Sjimharris * we'll toggle the bit each time when the completion queue 853248746Sjimharris * rolls over. 854248746Sjimharris */ 855248746Sjimharris qpair->phase = 1; 856248746Sjimharris 857248746Sjimharris memset(qpair->cmd, 0, 858248746Sjimharris qpair->num_entries * sizeof(struct nvme_command)); 859248746Sjimharris memset(qpair->cpl, 0, 860248746Sjimharris qpair->num_entries * sizeof(struct nvme_completion)); 861248746Sjimharris} 862248746Sjimharris 863248746Sjimharrisvoid 864248746Sjimharrisnvme_admin_qpair_enable(struct nvme_qpair *qpair) 865248746Sjimharris{ 866248768Sjimharris struct nvme_tracker *tr; 867248768Sjimharris struct nvme_tracker *tr_temp; 868248746Sjimharris 869248768Sjimharris /* 870248768Sjimharris * Manually abort each outstanding admin command. Do not retry 871248768Sjimharris * admin commands found here, since they will be left over from 872248768Sjimharris * a controller reset and its likely the context in which the 873248768Sjimharris * command was issued no longer applies. 874248768Sjimharris */ 875248768Sjimharris TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) { 876248773Sjimharris nvme_printf(qpair->ctrlr, 877248768Sjimharris "aborting outstanding admin command\n"); 878248768Sjimharris nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 879248768Sjimharris NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, TRUE); 880248768Sjimharris } 881248768Sjimharris 882248746Sjimharris nvme_qpair_enable(qpair); 883248746Sjimharris} 884248746Sjimharris 885248746Sjimharrisvoid 886248746Sjimharrisnvme_io_qpair_enable(struct nvme_qpair *qpair) 887248746Sjimharris{ 888248746Sjimharris STAILQ_HEAD(, nvme_request) temp; 889248746Sjimharris struct nvme_tracker *tr; 890248761Sjimharris struct nvme_tracker *tr_temp; 891248746Sjimharris struct nvme_request *req; 892248746Sjimharris 893248761Sjimharris /* 894248761Sjimharris * Manually abort each outstanding I/O. This normally results in a 895248761Sjimharris * retry, unless the retry count on the associated request has 896248761Sjimharris * reached its limit. 897248761Sjimharris */ 898248761Sjimharris TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) { 899248773Sjimharris nvme_printf(qpair->ctrlr, "aborting outstanding i/o\n"); 900248761Sjimharris nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 901248767Sjimharris NVME_SC_ABORTED_BY_REQUEST, 0, TRUE); 902248761Sjimharris } 903248761Sjimharris 904248746Sjimharris mtx_lock(&qpair->lock); 905248746Sjimharris 906248746Sjimharris nvme_qpair_enable(qpair); 907248746Sjimharris 908248746Sjimharris STAILQ_INIT(&temp); 909248746Sjimharris STAILQ_SWAP(&qpair->queued_req, &temp, nvme_request); 910248746Sjimharris 911248746Sjimharris while (!STAILQ_EMPTY(&temp)) { 912248746Sjimharris req = STAILQ_FIRST(&temp); 913248746Sjimharris STAILQ_REMOVE_HEAD(&temp, stailq); 914248773Sjimharris nvme_printf(qpair->ctrlr, "resubmitting queued i/o\n"); 915248773Sjimharris nvme_qpair_print_command(qpair, &req->cmd); 916248746Sjimharris _nvme_qpair_submit_request(qpair, req); 917248746Sjimharris } 918248746Sjimharris 919248746Sjimharris mtx_unlock(&qpair->lock); 920248746Sjimharris} 921248746Sjimharris 922248746Sjimharrisstatic void 923248746Sjimharrisnvme_qpair_disable(struct nvme_qpair *qpair) 924248746Sjimharris{ 925248746Sjimharris struct nvme_tracker *tr; 926248746Sjimharris 927248746Sjimharris qpair->is_enabled = FALSE; 928248746Sjimharris mtx_lock(&qpair->lock); 929248746Sjimharris TAILQ_FOREACH(tr, &qpair->outstanding_tr, tailq) 930248746Sjimharris callout_stop(&tr->timer); 931248746Sjimharris mtx_unlock(&qpair->lock); 932248746Sjimharris} 933248746Sjimharris 934248746Sjimharrisvoid 935248746Sjimharrisnvme_admin_qpair_disable(struct nvme_qpair *qpair) 936248746Sjimharris{ 937248746Sjimharris 938248746Sjimharris nvme_qpair_disable(qpair); 939248746Sjimharris nvme_admin_qpair_abort_aers(qpair); 940248746Sjimharris} 941248746Sjimharris 942248746Sjimharrisvoid 943248746Sjimharrisnvme_io_qpair_disable(struct nvme_qpair *qpair) 944248746Sjimharris{ 945248746Sjimharris 946248746Sjimharris nvme_qpair_disable(qpair); 947248746Sjimharris} 948248767Sjimharris 949248767Sjimharrisvoid 950248767Sjimharrisnvme_qpair_fail(struct nvme_qpair *qpair) 951248767Sjimharris{ 952248767Sjimharris struct nvme_tracker *tr; 953248767Sjimharris struct nvme_request *req; 954248767Sjimharris 955248767Sjimharris mtx_lock(&qpair->lock); 956248767Sjimharris 957248767Sjimharris while (!STAILQ_EMPTY(&qpair->queued_req)) { 958248767Sjimharris req = STAILQ_FIRST(&qpair->queued_req); 959248767Sjimharris STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); 960248773Sjimharris nvme_printf(qpair->ctrlr, "failing queued i/o\n"); 961248767Sjimharris mtx_unlock(&qpair->lock); 962248767Sjimharris nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC, 963248767Sjimharris NVME_SC_ABORTED_BY_REQUEST, TRUE); 964248767Sjimharris mtx_lock(&qpair->lock); 965248767Sjimharris } 966248767Sjimharris 967248767Sjimharris /* Manually abort each outstanding I/O. */ 968248767Sjimharris while (!TAILQ_EMPTY(&qpair->outstanding_tr)) { 969248767Sjimharris tr = TAILQ_FIRST(&qpair->outstanding_tr); 970248767Sjimharris /* 971248767Sjimharris * Do not remove the tracker. The abort_tracker path will 972248767Sjimharris * do that for us. 973248767Sjimharris */ 974248773Sjimharris nvme_printf(qpair->ctrlr, "failing outstanding i/o\n"); 975248767Sjimharris mtx_unlock(&qpair->lock); 976248767Sjimharris nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 977248767Sjimharris NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, TRUE); 978248767Sjimharris mtx_lock(&qpair->lock); 979248767Sjimharris } 980248767Sjimharris 981248767Sjimharris mtx_unlock(&qpair->lock); 982248767Sjimharris} 983248767Sjimharris 984