1240616Sjimharris/*- 2265566Sjimharris * Copyright (C) 2012-2014 Intel Corporation 3240616Sjimharris * All rights reserved. 4240616Sjimharris * 5240616Sjimharris * Redistribution and use in source and binary forms, with or without 6240616Sjimharris * modification, are permitted provided that the following conditions 7240616Sjimharris * are met: 8240616Sjimharris * 1. Redistributions of source code must retain the above copyright 9240616Sjimharris * notice, this list of conditions and the following disclaimer. 10240616Sjimharris * 2. Redistributions in binary form must reproduce the above copyright 11240616Sjimharris * notice, this list of conditions and the following disclaimer in the 12240616Sjimharris * documentation and/or other materials provided with the distribution. 13240616Sjimharris * 14240616Sjimharris * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15240616Sjimharris * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16240616Sjimharris * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17240616Sjimharris * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18240616Sjimharris * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19240616Sjimharris * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20240616Sjimharris * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21240616Sjimharris * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22240616Sjimharris * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23240616Sjimharris * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24240616Sjimharris * SUCH DAMAGE. 25240616Sjimharris */ 26240616Sjimharris 27240616Sjimharris#include <sys/cdefs.h> 28240616Sjimharris__FBSDID("$FreeBSD$"); 29240616Sjimharris 30240616Sjimharris#include <sys/param.h> 31240616Sjimharris#include <sys/bus.h> 32240616Sjimharris 33252222Sjimharris#include <dev/pci/pcivar.h> 34252222Sjimharris 35240616Sjimharris#include "nvme_private.h" 36240616Sjimharris 37252222Sjimharrisstatic void _nvme_qpair_submit_request(struct nvme_qpair *qpair, 38252222Sjimharris struct nvme_request *req); 39252222Sjimharris 40252222Sjimharrisstruct nvme_opcode_string { 41252222Sjimharris 42252222Sjimharris uint16_t opc; 43252222Sjimharris const char * str; 44252222Sjimharris}; 45252222Sjimharris 46252222Sjimharrisstatic struct nvme_opcode_string admin_opcode[] = { 47252222Sjimharris { NVME_OPC_DELETE_IO_SQ, "DELETE IO SQ" }, 48252222Sjimharris { NVME_OPC_CREATE_IO_SQ, "CREATE IO SQ" }, 49252222Sjimharris { NVME_OPC_GET_LOG_PAGE, "GET LOG PAGE" }, 50252222Sjimharris { NVME_OPC_DELETE_IO_CQ, "DELETE IO CQ" }, 51252222Sjimharris { NVME_OPC_CREATE_IO_CQ, "CREATE IO CQ" }, 52252222Sjimharris { NVME_OPC_IDENTIFY, "IDENTIFY" }, 53252222Sjimharris { NVME_OPC_ABORT, "ABORT" }, 54252222Sjimharris { NVME_OPC_SET_FEATURES, "SET FEATURES" }, 55252222Sjimharris { NVME_OPC_GET_FEATURES, "GET FEATURES" }, 56252222Sjimharris { NVME_OPC_ASYNC_EVENT_REQUEST, "ASYNC EVENT REQUEST" }, 57252222Sjimharris { NVME_OPC_FIRMWARE_ACTIVATE, "FIRMWARE ACTIVATE" }, 58252222Sjimharris { NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD, "FIRMWARE IMAGE DOWNLOAD" }, 59252222Sjimharris { NVME_OPC_FORMAT_NVM, "FORMAT NVM" }, 60252222Sjimharris { NVME_OPC_SECURITY_SEND, "SECURITY SEND" }, 61252222Sjimharris { NVME_OPC_SECURITY_RECEIVE, "SECURITY RECEIVE" }, 62252222Sjimharris { 0xFFFF, "ADMIN COMMAND" } 63252222Sjimharris}; 64252222Sjimharris 65252222Sjimharrisstatic struct nvme_opcode_string io_opcode[] = { 66252222Sjimharris { NVME_OPC_FLUSH, "FLUSH" }, 67252222Sjimharris { NVME_OPC_WRITE, "WRITE" }, 68252222Sjimharris { NVME_OPC_READ, "READ" }, 69252222Sjimharris { NVME_OPC_WRITE_UNCORRECTABLE, "WRITE UNCORRECTABLE" }, 70252222Sjimharris { NVME_OPC_COMPARE, "COMPARE" }, 71252222Sjimharris { NVME_OPC_DATASET_MANAGEMENT, "DATASET MANAGEMENT" }, 72252222Sjimharris { 0xFFFF, "IO COMMAND" } 73252222Sjimharris}; 74252222Sjimharris 75252222Sjimharrisstatic const char * 76252222Sjimharrisget_admin_opcode_string(uint16_t opc) 77252222Sjimharris{ 78252222Sjimharris struct nvme_opcode_string *entry; 79252222Sjimharris 80252222Sjimharris entry = admin_opcode; 81252222Sjimharris 82252222Sjimharris while (entry->opc != 0xFFFF) { 83252222Sjimharris if (entry->opc == opc) 84252222Sjimharris return (entry->str); 85252222Sjimharris entry++; 86252222Sjimharris } 87252222Sjimharris return (entry->str); 88252222Sjimharris} 89252222Sjimharris 90252222Sjimharrisstatic const char * 91252222Sjimharrisget_io_opcode_string(uint16_t opc) 92252222Sjimharris{ 93252222Sjimharris struct nvme_opcode_string *entry; 94252222Sjimharris 95252222Sjimharris entry = io_opcode; 96252222Sjimharris 97252222Sjimharris while (entry->opc != 0xFFFF) { 98252222Sjimharris if (entry->opc == opc) 99252222Sjimharris return (entry->str); 100252222Sjimharris entry++; 101252222Sjimharris } 102252222Sjimharris return (entry->str); 103252222Sjimharris} 104252222Sjimharris 105252222Sjimharris 106252222Sjimharrisstatic void 107252222Sjimharrisnvme_admin_qpair_print_command(struct nvme_qpair *qpair, 108252222Sjimharris struct nvme_command *cmd) 109252222Sjimharris{ 110252222Sjimharris 111252222Sjimharris nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%x " 112252222Sjimharris "cdw10:%08x cdw11:%08x\n", 113252222Sjimharris get_admin_opcode_string(cmd->opc), cmd->opc, qpair->id, cmd->cid, 114252222Sjimharris cmd->nsid, cmd->cdw10, cmd->cdw11); 115252222Sjimharris} 116252222Sjimharris 117252222Sjimharrisstatic void 118252222Sjimharrisnvme_io_qpair_print_command(struct nvme_qpair *qpair, 119252222Sjimharris struct nvme_command *cmd) 120252222Sjimharris{ 121252222Sjimharris 122252222Sjimharris switch (cmd->opc) { 123252222Sjimharris case NVME_OPC_WRITE: 124252222Sjimharris case NVME_OPC_READ: 125252222Sjimharris case NVME_OPC_WRITE_UNCORRECTABLE: 126252222Sjimharris case NVME_OPC_COMPARE: 127252222Sjimharris nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d " 128252222Sjimharris "lba:%llu len:%d\n", 129252222Sjimharris get_io_opcode_string(cmd->opc), qpair->id, cmd->cid, 130252222Sjimharris cmd->nsid, 131252222Sjimharris ((unsigned long long)cmd->cdw11 << 32) + cmd->cdw10, 132252222Sjimharris (cmd->cdw12 & 0xFFFF) + 1); 133252222Sjimharris break; 134252222Sjimharris case NVME_OPC_FLUSH: 135252222Sjimharris case NVME_OPC_DATASET_MANAGEMENT: 136252222Sjimharris nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d\n", 137252222Sjimharris get_io_opcode_string(cmd->opc), qpair->id, cmd->cid, 138252222Sjimharris cmd->nsid); 139252222Sjimharris break; 140252222Sjimharris default: 141252222Sjimharris nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%d\n", 142252222Sjimharris get_io_opcode_string(cmd->opc), cmd->opc, qpair->id, 143252222Sjimharris cmd->cid, cmd->nsid); 144252222Sjimharris break; 145252222Sjimharris } 146252222Sjimharris} 147252222Sjimharris 148252222Sjimharrisstatic void 149252222Sjimharrisnvme_qpair_print_command(struct nvme_qpair *qpair, struct nvme_command *cmd) 150252222Sjimharris{ 151252222Sjimharris if (qpair->id == 0) 152252222Sjimharris nvme_admin_qpair_print_command(qpair, cmd); 153252222Sjimharris else 154252222Sjimharris nvme_io_qpair_print_command(qpair, cmd); 155252222Sjimharris} 156252222Sjimharris 157252222Sjimharrisstruct nvme_status_string { 158252222Sjimharris 159252222Sjimharris uint16_t sc; 160252222Sjimharris const char * str; 161252222Sjimharris}; 162252222Sjimharris 163252222Sjimharrisstatic struct nvme_status_string generic_status[] = { 164252222Sjimharris { NVME_SC_SUCCESS, "SUCCESS" }, 165252222Sjimharris { NVME_SC_INVALID_OPCODE, "INVALID OPCODE" }, 166252222Sjimharris { NVME_SC_INVALID_FIELD, "INVALID_FIELD" }, 167252222Sjimharris { NVME_SC_COMMAND_ID_CONFLICT, "COMMAND ID CONFLICT" }, 168252222Sjimharris { NVME_SC_DATA_TRANSFER_ERROR, "DATA TRANSFER ERROR" }, 169252222Sjimharris { NVME_SC_ABORTED_POWER_LOSS, "ABORTED - POWER LOSS" }, 170252222Sjimharris { NVME_SC_INTERNAL_DEVICE_ERROR, "INTERNAL DEVICE ERROR" }, 171252222Sjimharris { NVME_SC_ABORTED_BY_REQUEST, "ABORTED - BY REQUEST" }, 172252222Sjimharris { NVME_SC_ABORTED_SQ_DELETION, "ABORTED - SQ DELETION" }, 173252222Sjimharris { NVME_SC_ABORTED_FAILED_FUSED, "ABORTED - FAILED FUSED" }, 174252222Sjimharris { NVME_SC_ABORTED_MISSING_FUSED, "ABORTED - MISSING FUSED" }, 175252222Sjimharris { NVME_SC_INVALID_NAMESPACE_OR_FORMAT, "INVALID NAMESPACE OR FORMAT" }, 176252222Sjimharris { NVME_SC_COMMAND_SEQUENCE_ERROR, "COMMAND SEQUENCE ERROR" }, 177252222Sjimharris { NVME_SC_LBA_OUT_OF_RANGE, "LBA OUT OF RANGE" }, 178252222Sjimharris { NVME_SC_CAPACITY_EXCEEDED, "CAPACITY EXCEEDED" }, 179252222Sjimharris { NVME_SC_NAMESPACE_NOT_READY, "NAMESPACE NOT READY" }, 180252222Sjimharris { 0xFFFF, "GENERIC" } 181252222Sjimharris}; 182252222Sjimharris 183252222Sjimharrisstatic struct nvme_status_string command_specific_status[] = { 184252222Sjimharris { NVME_SC_COMPLETION_QUEUE_INVALID, "INVALID COMPLETION QUEUE" }, 185252222Sjimharris { NVME_SC_INVALID_QUEUE_IDENTIFIER, "INVALID QUEUE IDENTIFIER" }, 186252222Sjimharris { NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED, "MAX QUEUE SIZE EXCEEDED" }, 187252222Sjimharris { NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED, "ABORT CMD LIMIT EXCEEDED" }, 188252222Sjimharris { NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED, "ASYNC LIMIT EXCEEDED" }, 189252222Sjimharris { NVME_SC_INVALID_FIRMWARE_SLOT, "INVALID FIRMWARE SLOT" }, 190252222Sjimharris { NVME_SC_INVALID_FIRMWARE_IMAGE, "INVALID FIRMWARE IMAGE" }, 191252222Sjimharris { NVME_SC_INVALID_INTERRUPT_VECTOR, "INVALID INTERRUPT VECTOR" }, 192252222Sjimharris { NVME_SC_INVALID_LOG_PAGE, "INVALID LOG PAGE" }, 193252222Sjimharris { NVME_SC_INVALID_FORMAT, "INVALID FORMAT" }, 194252222Sjimharris { NVME_SC_FIRMWARE_REQUIRES_RESET, "FIRMWARE REQUIRES RESET" }, 195252222Sjimharris { NVME_SC_CONFLICTING_ATTRIBUTES, "CONFLICTING ATTRIBUTES" }, 196252222Sjimharris { NVME_SC_INVALID_PROTECTION_INFO, "INVALID PROTECTION INFO" }, 197252222Sjimharris { NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE, "WRITE TO RO PAGE" }, 198252222Sjimharris { 0xFFFF, "COMMAND SPECIFIC" } 199252222Sjimharris}; 200252222Sjimharris 201252222Sjimharrisstatic struct nvme_status_string media_error_status[] = { 202252222Sjimharris { NVME_SC_WRITE_FAULTS, "WRITE FAULTS" }, 203252222Sjimharris { NVME_SC_UNRECOVERED_READ_ERROR, "UNRECOVERED READ ERROR" }, 204252222Sjimharris { NVME_SC_GUARD_CHECK_ERROR, "GUARD CHECK ERROR" }, 205252222Sjimharris { NVME_SC_APPLICATION_TAG_CHECK_ERROR, "APPLICATION TAG CHECK ERROR" }, 206252222Sjimharris { NVME_SC_REFERENCE_TAG_CHECK_ERROR, "REFERENCE TAG CHECK ERROR" }, 207252222Sjimharris { NVME_SC_COMPARE_FAILURE, "COMPARE FAILURE" }, 208252222Sjimharris { NVME_SC_ACCESS_DENIED, "ACCESS DENIED" }, 209252222Sjimharris { 0xFFFF, "MEDIA ERROR" } 210252222Sjimharris}; 211252222Sjimharris 212252222Sjimharrisstatic const char * 213252222Sjimharrisget_status_string(uint16_t sct, uint16_t sc) 214252222Sjimharris{ 215252222Sjimharris struct nvme_status_string *entry; 216252222Sjimharris 217252222Sjimharris switch (sct) { 218252222Sjimharris case NVME_SCT_GENERIC: 219252222Sjimharris entry = generic_status; 220252222Sjimharris break; 221252222Sjimharris case NVME_SCT_COMMAND_SPECIFIC: 222252222Sjimharris entry = command_specific_status; 223252222Sjimharris break; 224252222Sjimharris case NVME_SCT_MEDIA_ERROR: 225252222Sjimharris entry = media_error_status; 226252222Sjimharris break; 227252222Sjimharris case NVME_SCT_VENDOR_SPECIFIC: 228252222Sjimharris return ("VENDOR SPECIFIC"); 229252222Sjimharris default: 230252222Sjimharris return ("RESERVED"); 231252222Sjimharris } 232252222Sjimharris 233252222Sjimharris while (entry->sc != 0xFFFF) { 234252222Sjimharris if (entry->sc == sc) 235252222Sjimharris return (entry->str); 236252222Sjimharris entry++; 237252222Sjimharris } 238252222Sjimharris return (entry->str); 239252222Sjimharris} 240252222Sjimharris 241252222Sjimharrisstatic void 242252222Sjimharrisnvme_qpair_print_completion(struct nvme_qpair *qpair, 243252222Sjimharris struct nvme_completion *cpl) 244252222Sjimharris{ 245252222Sjimharris nvme_printf(qpair->ctrlr, "%s (%02x/%02x) sqid:%d cid:%d cdw0:%x\n", 246252222Sjimharris get_status_string(cpl->status.sct, cpl->status.sc), 247252222Sjimharris cpl->status.sct, cpl->status.sc, cpl->sqid, cpl->cid, cpl->cdw0); 248252222Sjimharris} 249252222Sjimharris 250240616Sjimharrisstatic boolean_t 251252222Sjimharrisnvme_completion_is_retry(const struct nvme_completion *cpl) 252240616Sjimharris{ 253240616Sjimharris /* 254240616Sjimharris * TODO: spec is not clear how commands that are aborted due 255240616Sjimharris * to TLER will be marked. So for now, it seems 256240616Sjimharris * NAMESPACE_NOT_READY is the only case where we should 257240616Sjimharris * look at the DNR bit. 258240616Sjimharris */ 259252222Sjimharris switch (cpl->status.sct) { 260240616Sjimharris case NVME_SCT_GENERIC: 261252222Sjimharris switch (cpl->status.sc) { 262252222Sjimharris case NVME_SC_ABORTED_BY_REQUEST: 263240616Sjimharris case NVME_SC_NAMESPACE_NOT_READY: 264252222Sjimharris if (cpl->status.dnr) 265240616Sjimharris return (0); 266240616Sjimharris else 267240616Sjimharris return (1); 268240616Sjimharris case NVME_SC_INVALID_OPCODE: 269240616Sjimharris case NVME_SC_INVALID_FIELD: 270240616Sjimharris case NVME_SC_COMMAND_ID_CONFLICT: 271240616Sjimharris case NVME_SC_DATA_TRANSFER_ERROR: 272240616Sjimharris case NVME_SC_ABORTED_POWER_LOSS: 273240616Sjimharris case NVME_SC_INTERNAL_DEVICE_ERROR: 274240616Sjimharris case NVME_SC_ABORTED_SQ_DELETION: 275240616Sjimharris case NVME_SC_ABORTED_FAILED_FUSED: 276240616Sjimharris case NVME_SC_ABORTED_MISSING_FUSED: 277240616Sjimharris case NVME_SC_INVALID_NAMESPACE_OR_FORMAT: 278240616Sjimharris case NVME_SC_COMMAND_SEQUENCE_ERROR: 279240616Sjimharris case NVME_SC_LBA_OUT_OF_RANGE: 280240616Sjimharris case NVME_SC_CAPACITY_EXCEEDED: 281240616Sjimharris default: 282240616Sjimharris return (0); 283240616Sjimharris } 284240616Sjimharris case NVME_SCT_COMMAND_SPECIFIC: 285240616Sjimharris case NVME_SCT_MEDIA_ERROR: 286240616Sjimharris case NVME_SCT_VENDOR_SPECIFIC: 287240616Sjimharris default: 288240616Sjimharris return (0); 289240616Sjimharris } 290240616Sjimharris} 291240616Sjimharris 292252222Sjimharrisstatic void 293252222Sjimharrisnvme_qpair_construct_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr, 294252222Sjimharris uint16_t cid) 295240616Sjimharris{ 296240616Sjimharris 297252222Sjimharris bus_dmamap_create(qpair->dma_tag, 0, &tr->payload_dma_map); 298252222Sjimharris bus_dmamap_create(qpair->dma_tag, 0, &tr->prp_dma_map); 299240616Sjimharris 300252222Sjimharris bus_dmamap_load(qpair->dma_tag, tr->prp_dma_map, tr->prp, 301252222Sjimharris sizeof(tr->prp), nvme_single_map, &tr->prp_bus_addr, 0); 302240616Sjimharris 303252222Sjimharris callout_init(&tr->timer, 1); 304252222Sjimharris tr->cid = cid; 305252222Sjimharris tr->qpair = qpair; 306252222Sjimharris} 307240616Sjimharris 308252222Sjimharrisstatic void 309252222Sjimharrisnvme_qpair_complete_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr, 310252222Sjimharris struct nvme_completion *cpl, boolean_t print_on_error) 311252222Sjimharris{ 312252222Sjimharris struct nvme_request *req; 313252222Sjimharris boolean_t retry, error; 314240616Sjimharris 315252222Sjimharris req = tr->req; 316252222Sjimharris error = nvme_completion_is_error(cpl); 317252222Sjimharris retry = error && nvme_completion_is_retry(cpl) && 318252222Sjimharris req->retries < nvme_retry_count; 319240616Sjimharris 320252222Sjimharris if (error && print_on_error) { 321252222Sjimharris nvme_qpair_print_command(qpair, &req->cmd); 322252222Sjimharris nvme_qpair_print_completion(qpair, cpl); 323252222Sjimharris } 324240616Sjimharris 325252222Sjimharris qpair->act_tr[cpl->cid] = NULL; 326240616Sjimharris 327252222Sjimharris KASSERT(cpl->cid == req->cmd.cid, ("cpl cid does not match cmd cid\n")); 328252222Sjimharris 329252222Sjimharris if (req->cb_fn && !retry) 330252222Sjimharris req->cb_fn(req->cb_arg, cpl); 331252222Sjimharris 332252222Sjimharris mtx_lock(&qpair->lock); 333252222Sjimharris callout_stop(&tr->timer); 334252222Sjimharris 335252222Sjimharris if (retry) { 336252222Sjimharris req->retries++; 337252222Sjimharris nvme_qpair_submit_tracker(qpair, tr); 338252222Sjimharris } else { 339252222Sjimharris if (req->type != NVME_REQUEST_NULL) 340252222Sjimharris bus_dmamap_unload(qpair->dma_tag, 341252222Sjimharris tr->payload_dma_map); 342252222Sjimharris 343252222Sjimharris nvme_free_request(req); 344252222Sjimharris tr->req = NULL; 345252222Sjimharris 346252222Sjimharris TAILQ_REMOVE(&qpair->outstanding_tr, tr, tailq); 347252222Sjimharris TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq); 348252222Sjimharris 349252222Sjimharris /* 350252222Sjimharris * If the controller is in the middle of resetting, don't 351252222Sjimharris * try to submit queued requests here - let the reset logic 352252222Sjimharris * handle that instead. 353252222Sjimharris */ 354252222Sjimharris if (!STAILQ_EMPTY(&qpair->queued_req) && 355252222Sjimharris !qpair->ctrlr->is_resetting) { 356252222Sjimharris req = STAILQ_FIRST(&qpair->queued_req); 357252222Sjimharris STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); 358252222Sjimharris _nvme_qpair_submit_request(qpair, req); 359240616Sjimharris } 360252222Sjimharris } 361240616Sjimharris 362252222Sjimharris mtx_unlock(&qpair->lock); 363252222Sjimharris} 364252222Sjimharris 365252222Sjimharrisstatic void 366252222Sjimharrisnvme_qpair_manual_complete_tracker(struct nvme_qpair *qpair, 367252222Sjimharris struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr, 368252222Sjimharris boolean_t print_on_error) 369252222Sjimharris{ 370252222Sjimharris struct nvme_completion cpl; 371252222Sjimharris 372252222Sjimharris memset(&cpl, 0, sizeof(cpl)); 373252222Sjimharris cpl.sqid = qpair->id; 374252222Sjimharris cpl.cid = tr->cid; 375252222Sjimharris cpl.status.sct = sct; 376252222Sjimharris cpl.status.sc = sc; 377252222Sjimharris cpl.status.dnr = dnr; 378252222Sjimharris nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error); 379252222Sjimharris} 380252222Sjimharris 381252222Sjimharrisvoid 382252222Sjimharrisnvme_qpair_manual_complete_request(struct nvme_qpair *qpair, 383252222Sjimharris struct nvme_request *req, uint32_t sct, uint32_t sc, 384252222Sjimharris boolean_t print_on_error) 385252222Sjimharris{ 386252222Sjimharris struct nvme_completion cpl; 387252222Sjimharris boolean_t error; 388252222Sjimharris 389252222Sjimharris memset(&cpl, 0, sizeof(cpl)); 390252222Sjimharris cpl.sqid = qpair->id; 391252222Sjimharris cpl.status.sct = sct; 392252222Sjimharris cpl.status.sc = sc; 393252222Sjimharris 394252222Sjimharris error = nvme_completion_is_error(&cpl); 395252222Sjimharris 396252222Sjimharris if (error && print_on_error) { 397252222Sjimharris nvme_qpair_print_command(qpair, &req->cmd); 398252222Sjimharris nvme_qpair_print_completion(qpair, &cpl); 399240616Sjimharris } 400240616Sjimharris 401252222Sjimharris if (req->cb_fn) 402252222Sjimharris req->cb_fn(req->cb_arg, &cpl); 403252222Sjimharris 404252222Sjimharris nvme_free_request(req); 405240616Sjimharris} 406240616Sjimharris 407240616Sjimharrisvoid 408240616Sjimharrisnvme_qpair_process_completions(struct nvme_qpair *qpair) 409240616Sjimharris{ 410240616Sjimharris struct nvme_tracker *tr; 411240616Sjimharris struct nvme_completion *cpl; 412240616Sjimharris 413252222Sjimharris qpair->num_intr_handler_calls++; 414252222Sjimharris 415252222Sjimharris if (!qpair->is_enabled) 416252222Sjimharris /* 417252222Sjimharris * qpair is not enabled, likely because a controller reset is 418252222Sjimharris * is in progress. Ignore the interrupt - any I/O that was 419252222Sjimharris * associated with this interrupt will get retried when the 420252222Sjimharris * reset is complete. 421252222Sjimharris */ 422252222Sjimharris return; 423252222Sjimharris 424240616Sjimharris while (1) { 425240616Sjimharris cpl = &qpair->cpl[qpair->cq_head]; 426240616Sjimharris 427252222Sjimharris if (cpl->status.p != qpair->phase) 428240616Sjimharris break; 429240616Sjimharris 430240616Sjimharris tr = qpair->act_tr[cpl->cid]; 431240616Sjimharris 432252222Sjimharris if (tr != NULL) { 433252222Sjimharris nvme_qpair_complete_tracker(qpair, tr, cpl, TRUE); 434252222Sjimharris qpair->sq_head = cpl->sqhd; 435252222Sjimharris } else { 436252222Sjimharris nvme_printf(qpair->ctrlr, 437252222Sjimharris "cpl does not map to outstanding cmd\n"); 438240616Sjimharris nvme_dump_completion(cpl); 439252222Sjimharris KASSERT(0, ("received completion for unknown cmd\n")); 440240616Sjimharris } 441240616Sjimharris 442240616Sjimharris if (++qpair->cq_head == qpair->num_entries) { 443240616Sjimharris qpair->cq_head = 0; 444240616Sjimharris qpair->phase = !qpair->phase; 445240616Sjimharris } 446240616Sjimharris 447240616Sjimharris nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl, 448240616Sjimharris qpair->cq_head); 449240616Sjimharris } 450240616Sjimharris} 451240616Sjimharris 452240616Sjimharrisstatic void 453240616Sjimharrisnvme_qpair_msix_handler(void *arg) 454240616Sjimharris{ 455240616Sjimharris struct nvme_qpair *qpair = arg; 456240616Sjimharris 457240616Sjimharris nvme_qpair_process_completions(qpair); 458240616Sjimharris} 459240616Sjimharris 460240616Sjimharrisvoid 461240616Sjimharrisnvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id, 462252222Sjimharris uint16_t vector, uint32_t num_entries, uint32_t num_trackers, 463252664Sjimharris struct nvme_controller *ctrlr) 464240616Sjimharris{ 465252222Sjimharris struct nvme_tracker *tr; 466252222Sjimharris uint32_t i; 467240616Sjimharris 468240616Sjimharris qpair->id = id; 469240616Sjimharris qpair->vector = vector; 470240616Sjimharris qpair->num_entries = num_entries; 471252222Sjimharris#ifdef CHATHAM2 472252222Sjimharris /* 473252222Sjimharris * Chatham prototype board starts having issues at higher queue 474252222Sjimharris * depths. So use a conservative estimate here of no more than 64 475252222Sjimharris * outstanding I/O per queue at any one point. 476252222Sjimharris */ 477252222Sjimharris if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) 478252222Sjimharris num_trackers = min(num_trackers, 64); 479252222Sjimharris#endif 480252222Sjimharris qpair->num_trackers = num_trackers; 481240616Sjimharris qpair->ctrlr = ctrlr; 482240616Sjimharris 483240616Sjimharris if (ctrlr->msix_enabled) { 484240616Sjimharris 485240616Sjimharris /* 486240616Sjimharris * MSI-X vector resource IDs start at 1, so we add one to 487240616Sjimharris * the queue's vector to get the corresponding rid to use. 488240616Sjimharris */ 489240616Sjimharris qpair->rid = vector + 1; 490265566Sjimharris qpair->res = ctrlr->msi_res[vector]; 491240616Sjimharris 492240616Sjimharris bus_setup_intr(ctrlr->dev, qpair->res, 493240616Sjimharris INTR_TYPE_MISC | INTR_MPSAFE, NULL, 494240616Sjimharris nvme_qpair_msix_handler, qpair, &qpair->tag); 495240616Sjimharris } 496240616Sjimharris 497240616Sjimharris mtx_init(&qpair->lock, "nvme qpair lock", NULL, MTX_DEF); 498240616Sjimharris 499265563Sjimharris /* Note: NVMe PRP format is restricted to 4-byte alignment. */ 500240616Sjimharris bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), 501265563Sjimharris 4, PAGE_SIZE, BUS_SPACE_MAXADDR, 502252664Sjimharris BUS_SPACE_MAXADDR, NULL, NULL, NVME_MAX_XFER_SIZE, 503252664Sjimharris (NVME_MAX_XFER_SIZE/PAGE_SIZE)+1, PAGE_SIZE, 0, 504240616Sjimharris NULL, NULL, &qpair->dma_tag); 505240616Sjimharris 506240616Sjimharris qpair->num_cmds = 0; 507252222Sjimharris qpair->num_intr_handler_calls = 0; 508240616Sjimharris 509240616Sjimharris qpair->cmd = contigmalloc(qpair->num_entries * 510252222Sjimharris sizeof(struct nvme_command), M_NVME, M_ZERO, 511240616Sjimharris 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); 512240616Sjimharris qpair->cpl = contigmalloc(qpair->num_entries * 513252222Sjimharris sizeof(struct nvme_completion), M_NVME, M_ZERO, 514240616Sjimharris 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); 515240616Sjimharris 516240616Sjimharris bus_dmamap_create(qpair->dma_tag, 0, &qpair->cmd_dma_map); 517240616Sjimharris bus_dmamap_create(qpair->dma_tag, 0, &qpair->cpl_dma_map); 518240616Sjimharris 519240616Sjimharris bus_dmamap_load(qpair->dma_tag, qpair->cmd_dma_map, 520240616Sjimharris qpair->cmd, qpair->num_entries * sizeof(struct nvme_command), 521240616Sjimharris nvme_single_map, &qpair->cmd_bus_addr, 0); 522240616Sjimharris bus_dmamap_load(qpair->dma_tag, qpair->cpl_dma_map, 523240616Sjimharris qpair->cpl, qpair->num_entries * sizeof(struct nvme_completion), 524240616Sjimharris nvme_single_map, &qpair->cpl_bus_addr, 0); 525240616Sjimharris 526240616Sjimharris qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl); 527240616Sjimharris qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl); 528240616Sjimharris 529252222Sjimharris TAILQ_INIT(&qpair->free_tr); 530252222Sjimharris TAILQ_INIT(&qpair->outstanding_tr); 531252222Sjimharris STAILQ_INIT(&qpair->queued_req); 532240616Sjimharris 533252222Sjimharris for (i = 0; i < qpair->num_trackers; i++) { 534252222Sjimharris tr = malloc(sizeof(*tr), M_NVME, M_ZERO | M_WAITOK); 535252222Sjimharris nvme_qpair_construct_tracker(qpair, tr, i); 536252222Sjimharris TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq); 537252222Sjimharris } 538252222Sjimharris 539240616Sjimharris qpair->act_tr = malloc(sizeof(struct nvme_tracker *) * qpair->num_entries, 540252222Sjimharris M_NVME, M_ZERO | M_WAITOK); 541240616Sjimharris} 542240616Sjimharris 543240616Sjimharrisstatic void 544240616Sjimharrisnvme_qpair_destroy(struct nvme_qpair *qpair) 545240616Sjimharris{ 546252222Sjimharris struct nvme_tracker *tr; 547240616Sjimharris 548240616Sjimharris if (qpair->tag) 549240616Sjimharris bus_teardown_intr(qpair->ctrlr->dev, qpair->res, qpair->tag); 550240616Sjimharris 551240616Sjimharris if (qpair->res) 552240616Sjimharris bus_release_resource(qpair->ctrlr->dev, SYS_RES_IRQ, 553240616Sjimharris rman_get_rid(qpair->res), qpair->res); 554240616Sjimharris 555252222Sjimharris if (qpair->cmd) { 556252222Sjimharris bus_dmamap_unload(qpair->dma_tag, qpair->cmd_dma_map); 557252222Sjimharris bus_dmamap_destroy(qpair->dma_tag, qpair->cmd_dma_map); 558252222Sjimharris contigfree(qpair->cmd, 559252222Sjimharris qpair->num_entries * sizeof(struct nvme_command), M_NVME); 560252222Sjimharris } 561252222Sjimharris 562252222Sjimharris if (qpair->cpl) { 563252222Sjimharris bus_dmamap_unload(qpair->dma_tag, qpair->cpl_dma_map); 564252222Sjimharris bus_dmamap_destroy(qpair->dma_tag, qpair->cpl_dma_map); 565252222Sjimharris contigfree(qpair->cpl, 566252222Sjimharris qpair->num_entries * sizeof(struct nvme_completion), 567252222Sjimharris M_NVME); 568252222Sjimharris } 569252222Sjimharris 570240616Sjimharris if (qpair->dma_tag) 571240616Sjimharris bus_dma_tag_destroy(qpair->dma_tag); 572240616Sjimharris 573240616Sjimharris if (qpair->act_tr) 574240616Sjimharris free(qpair->act_tr, M_NVME); 575240616Sjimharris 576252222Sjimharris while (!TAILQ_EMPTY(&qpair->free_tr)) { 577252222Sjimharris tr = TAILQ_FIRST(&qpair->free_tr); 578252222Sjimharris TAILQ_REMOVE(&qpair->free_tr, tr, tailq); 579252222Sjimharris bus_dmamap_destroy(qpair->dma_tag, tr->payload_dma_map); 580252222Sjimharris bus_dmamap_destroy(qpair->dma_tag, tr->prp_dma_map); 581240616Sjimharris free(tr, M_NVME); 582240616Sjimharris } 583252222Sjimharris} 584240616Sjimharris 585252222Sjimharrisstatic void 586252222Sjimharrisnvme_admin_qpair_abort_aers(struct nvme_qpair *qpair) 587252222Sjimharris{ 588252222Sjimharris struct nvme_tracker *tr; 589252222Sjimharris 590252222Sjimharris tr = TAILQ_FIRST(&qpair->outstanding_tr); 591252222Sjimharris while (tr != NULL) { 592252222Sjimharris if (tr->req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST) { 593252222Sjimharris nvme_qpair_manual_complete_tracker(qpair, tr, 594252222Sjimharris NVME_SCT_GENERIC, NVME_SC_ABORTED_SQ_DELETION, 0, 595252222Sjimharris FALSE); 596252222Sjimharris tr = TAILQ_FIRST(&qpair->outstanding_tr); 597252222Sjimharris } else { 598252222Sjimharris tr = TAILQ_NEXT(tr, tailq); 599252222Sjimharris } 600240616Sjimharris } 601240616Sjimharris} 602240616Sjimharris 603240616Sjimharrisvoid 604240616Sjimharrisnvme_admin_qpair_destroy(struct nvme_qpair *qpair) 605240616Sjimharris{ 606240616Sjimharris 607252222Sjimharris nvme_admin_qpair_abort_aers(qpair); 608252222Sjimharris nvme_qpair_destroy(qpair); 609252222Sjimharris} 610252222Sjimharris 611252222Sjimharrisvoid 612252222Sjimharrisnvme_io_qpair_destroy(struct nvme_qpair *qpair) 613252222Sjimharris{ 614252222Sjimharris 615252222Sjimharris nvme_qpair_destroy(qpair); 616252222Sjimharris} 617252222Sjimharris 618252222Sjimharrisstatic void 619252222Sjimharrisnvme_abort_complete(void *arg, const struct nvme_completion *status) 620252222Sjimharris{ 621252222Sjimharris struct nvme_tracker *tr = arg; 622252222Sjimharris 623240616Sjimharris /* 624252222Sjimharris * If cdw0 == 1, the controller was not able to abort the command 625252222Sjimharris * we requested. We still need to check the active tracker array, 626252222Sjimharris * to cover race where I/O timed out at same time controller was 627252222Sjimharris * completing the I/O. 628240616Sjimharris */ 629252222Sjimharris if (status->cdw0 == 1 && tr->qpair->act_tr[tr->cid] != NULL) { 630252222Sjimharris /* 631252222Sjimharris * An I/O has timed out, and the controller was unable to 632252222Sjimharris * abort it for some reason. Construct a fake completion 633252222Sjimharris * status, and then complete the I/O's tracker manually. 634252222Sjimharris */ 635252222Sjimharris nvme_printf(tr->qpair->ctrlr, 636252222Sjimharris "abort command failed, aborting command manually\n"); 637252222Sjimharris nvme_qpair_manual_complete_tracker(tr->qpair, tr, 638252222Sjimharris NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST, 0, TRUE); 639252222Sjimharris } 640252222Sjimharris} 641240616Sjimharris 642252222Sjimharrisstatic void 643252222Sjimharrisnvme_timeout(void *arg) 644252222Sjimharris{ 645252222Sjimharris struct nvme_tracker *tr = arg; 646252222Sjimharris struct nvme_qpair *qpair = tr->qpair; 647252222Sjimharris struct nvme_controller *ctrlr = qpair->ctrlr; 648252222Sjimharris union csts_register csts; 649240616Sjimharris 650252222Sjimharris /* Read csts to get value of cfs - controller fatal status. */ 651252222Sjimharris csts.raw = nvme_mmio_read_4(ctrlr, csts); 652240616Sjimharris 653252222Sjimharris if (ctrlr->enable_aborts && csts.bits.cfs == 0) { 654252222Sjimharris /* 655252222Sjimharris * If aborts are enabled, only use them if the controller is 656252222Sjimharris * not reporting fatal status. 657252222Sjimharris */ 658252222Sjimharris nvme_ctrlr_cmd_abort(ctrlr, tr->cid, qpair->id, 659252222Sjimharris nvme_abort_complete, tr); 660252222Sjimharris } else 661252222Sjimharris nvme_ctrlr_reset(ctrlr); 662240616Sjimharris} 663240616Sjimharris 664252222Sjimharrisvoid 665252222Sjimharrisnvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr) 666252222Sjimharris{ 667252222Sjimharris struct nvme_request *req; 668252222Sjimharris struct nvme_controller *ctrlr; 669252222Sjimharris 670252222Sjimharris mtx_assert(&qpair->lock, MA_OWNED); 671252222Sjimharris 672252222Sjimharris req = tr->req; 673252222Sjimharris req->cmd.cid = tr->cid; 674252222Sjimharris qpair->act_tr[tr->cid] = tr; 675252222Sjimharris ctrlr = qpair->ctrlr; 676252222Sjimharris 677252222Sjimharris if (req->timeout) 678252222Sjimharris#if __FreeBSD_version >= 800030 679252222Sjimharris callout_reset_curcpu(&tr->timer, ctrlr->timeout_period * hz, 680252222Sjimharris nvme_timeout, tr); 681252222Sjimharris#else 682252222Sjimharris callout_reset(&tr->timer, ctrlr->timeout_period * hz, 683252222Sjimharris nvme_timeout, tr); 684252222Sjimharris#endif 685252222Sjimharris 686252222Sjimharris /* Copy the command from the tracker to the submission queue. */ 687252222Sjimharris memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd)); 688252222Sjimharris 689252222Sjimharris if (++qpair->sq_tail == qpair->num_entries) 690252222Sjimharris qpair->sq_tail = 0; 691252222Sjimharris 692252222Sjimharris wmb(); 693252222Sjimharris nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl, 694252222Sjimharris qpair->sq_tail); 695252222Sjimharris 696252222Sjimharris qpair->num_cmds++; 697252222Sjimharris} 698252222Sjimharris 699240616Sjimharrisstatic void 700252222Sjimharrisnvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) 701240616Sjimharris{ 702252222Sjimharris struct nvme_tracker *tr = arg; 703252222Sjimharris uint32_t cur_nseg; 704240616Sjimharris 705252222Sjimharris /* 706252222Sjimharris * If the mapping operation failed, return immediately. The caller 707252222Sjimharris * is responsible for detecting the error status and failing the 708252222Sjimharris * tracker manually. 709252222Sjimharris */ 710252222Sjimharris if (error != 0) 711252222Sjimharris return; 712252222Sjimharris 713252222Sjimharris /* 714252222Sjimharris * Note that we specified PAGE_SIZE for alignment and max 715252222Sjimharris * segment size when creating the bus dma tags. So here 716252222Sjimharris * we can safely just transfer each segment to its 717252222Sjimharris * associated PRP entry. 718252222Sjimharris */ 719252222Sjimharris tr->req->cmd.prp1 = seg[0].ds_addr; 720252222Sjimharris 721252222Sjimharris if (nseg == 2) { 722252222Sjimharris tr->req->cmd.prp2 = seg[1].ds_addr; 723252222Sjimharris } else if (nseg > 2) { 724252222Sjimharris cur_nseg = 1; 725252222Sjimharris tr->req->cmd.prp2 = (uint64_t)tr->prp_bus_addr; 726252222Sjimharris while (cur_nseg < nseg) { 727252222Sjimharris tr->prp[cur_nseg-1] = 728252222Sjimharris (uint64_t)seg[cur_nseg].ds_addr; 729252222Sjimharris cur_nseg++; 730252222Sjimharris } 731252222Sjimharris } 732252222Sjimharris 733252222Sjimharris nvme_qpair_submit_tracker(tr->qpair, tr); 734240616Sjimharris} 735240616Sjimharris 736240616Sjimharrisstatic void 737252222Sjimharris_nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) 738240616Sjimharris{ 739252222Sjimharris struct nvme_tracker *tr; 740252222Sjimharris int err = 0; 741240616Sjimharris 742252222Sjimharris mtx_assert(&qpair->lock, MA_OWNED); 743240616Sjimharris 744252222Sjimharris tr = TAILQ_FIRST(&qpair->free_tr); 745252222Sjimharris req->qpair = qpair; 746240616Sjimharris 747252222Sjimharris if (tr == NULL || !qpair->is_enabled) { 748252222Sjimharris /* 749252222Sjimharris * No tracker is available, or the qpair is disabled due to 750252222Sjimharris * an in-progress controller-level reset or controller 751252222Sjimharris * failure. 752252222Sjimharris */ 753240616Sjimharris 754252222Sjimharris if (qpair->ctrlr->is_failed) { 755252222Sjimharris /* 756252222Sjimharris * The controller has failed. Post the request to a 757252222Sjimharris * task where it will be aborted, so that we do not 758252222Sjimharris * invoke the request's callback in the context 759252222Sjimharris * of the submission. 760252222Sjimharris */ 761252222Sjimharris nvme_ctrlr_post_failed_request(qpair->ctrlr, req); 762252222Sjimharris } else { 763252222Sjimharris /* 764252222Sjimharris * Put the request on the qpair's request queue to be 765252222Sjimharris * processed when a tracker frees up via a command 766252222Sjimharris * completion or when the controller reset is 767252222Sjimharris * completed. 768252222Sjimharris */ 769252222Sjimharris STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq); 770252222Sjimharris } 771252222Sjimharris return; 772240616Sjimharris } 773252222Sjimharris 774252222Sjimharris TAILQ_REMOVE(&qpair->free_tr, tr, tailq); 775252222Sjimharris TAILQ_INSERT_TAIL(&qpair->outstanding_tr, tr, tailq); 776252222Sjimharris tr->req = req; 777252222Sjimharris 778252222Sjimharris switch (req->type) { 779252222Sjimharris case NVME_REQUEST_VADDR: 780252665Sjimharris KASSERT(req->payload_size <= qpair->ctrlr->max_xfer_size, 781252665Sjimharris ("payload_size (%d) exceeds max_xfer_size (%d)\n", 782252665Sjimharris req->payload_size, qpair->ctrlr->max_xfer_size)); 783252222Sjimharris err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, 784252222Sjimharris req->u.payload, req->payload_size, nvme_payload_map, tr, 0); 785252222Sjimharris if (err != 0) 786252222Sjimharris nvme_printf(qpair->ctrlr, 787252222Sjimharris "bus_dmamap_load returned 0x%x!\n", err); 788252222Sjimharris break; 789252222Sjimharris case NVME_REQUEST_NULL: 790252222Sjimharris nvme_qpair_submit_tracker(tr->qpair, tr); 791252222Sjimharris break; 792252222Sjimharris#ifdef NVME_UNMAPPED_BIO_SUPPORT 793252222Sjimharris case NVME_REQUEST_BIO: 794252665Sjimharris KASSERT(req->u.bio->bio_bcount <= qpair->ctrlr->max_xfer_size, 795252665Sjimharris ("bio->bio_bcount (%jd) exceeds max_xfer_size (%d)\n", 796252665Sjimharris (intmax_t)req->u.bio->bio_bcount, 797252665Sjimharris qpair->ctrlr->max_xfer_size)); 798252222Sjimharris err = bus_dmamap_load_bio(tr->qpair->dma_tag, 799252222Sjimharris tr->payload_dma_map, req->u.bio, nvme_payload_map, tr, 0); 800252222Sjimharris if (err != 0) 801252222Sjimharris nvme_printf(qpair->ctrlr, 802252222Sjimharris "bus_dmamap_load_bio returned 0x%x!\n", err); 803252222Sjimharris break; 804252222Sjimharris#endif 805252222Sjimharris default: 806252222Sjimharris panic("unknown nvme request type 0x%x\n", req->type); 807252222Sjimharris break; 808252222Sjimharris } 809252222Sjimharris 810252222Sjimharris if (err != 0) { 811252222Sjimharris /* 812252222Sjimharris * The dmamap operation failed, so we manually fail the 813252222Sjimharris * tracker here with DATA_TRANSFER_ERROR status. 814252222Sjimharris * 815252222Sjimharris * nvme_qpair_manual_complete_tracker must not be called 816252222Sjimharris * with the qpair lock held. 817252222Sjimharris */ 818252222Sjimharris mtx_unlock(&qpair->lock); 819252222Sjimharris nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 820252222Sjimharris NVME_SC_DATA_TRANSFER_ERROR, 1 /* do not retry */, TRUE); 821252222Sjimharris mtx_lock(&qpair->lock); 822252222Sjimharris } 823240616Sjimharris} 824240616Sjimharris 825252222Sjimharrisvoid 826252222Sjimharrisnvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) 827252222Sjimharris{ 828252222Sjimharris 829252222Sjimharris mtx_lock(&qpair->lock); 830252222Sjimharris _nvme_qpair_submit_request(qpair, req); 831252222Sjimharris mtx_unlock(&qpair->lock); 832252222Sjimharris} 833252222Sjimharris 834240616Sjimharrisstatic void 835252222Sjimharrisnvme_qpair_enable(struct nvme_qpair *qpair) 836240616Sjimharris{ 837252222Sjimharris 838252222Sjimharris qpair->is_enabled = TRUE; 839252222Sjimharris} 840252222Sjimharris 841252222Sjimharrisvoid 842252222Sjimharrisnvme_qpair_reset(struct nvme_qpair *qpair) 843252222Sjimharris{ 844252222Sjimharris 845252222Sjimharris qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0; 846252222Sjimharris 847240616Sjimharris /* 848252222Sjimharris * First time through the completion queue, HW will set phase 849252222Sjimharris * bit on completions to 1. So set this to 1 here, indicating 850252222Sjimharris * we're looking for a 1 to know which entries have completed. 851252222Sjimharris * we'll toggle the bit each time when the completion queue 852252222Sjimharris * rolls over. 853240616Sjimharris */ 854252222Sjimharris qpair->phase = 1; 855252222Sjimharris 856252222Sjimharris memset(qpair->cmd, 0, 857252222Sjimharris qpair->num_entries * sizeof(struct nvme_command)); 858252222Sjimharris memset(qpair->cpl, 0, 859252222Sjimharris qpair->num_entries * sizeof(struct nvme_completion)); 860240616Sjimharris} 861240616Sjimharris 862240616Sjimharrisvoid 863252222Sjimharrisnvme_admin_qpair_enable(struct nvme_qpair *qpair) 864240616Sjimharris{ 865252222Sjimharris struct nvme_tracker *tr; 866252222Sjimharris struct nvme_tracker *tr_temp; 867240616Sjimharris 868252222Sjimharris /* 869252222Sjimharris * Manually abort each outstanding admin command. Do not retry 870252222Sjimharris * admin commands found here, since they will be left over from 871252222Sjimharris * a controller reset and its likely the context in which the 872252222Sjimharris * command was issued no longer applies. 873252222Sjimharris */ 874252222Sjimharris TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) { 875252222Sjimharris nvme_printf(qpair->ctrlr, 876252222Sjimharris "aborting outstanding admin command\n"); 877252222Sjimharris nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 878252222Sjimharris NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, TRUE); 879252222Sjimharris } 880240616Sjimharris 881252222Sjimharris nvme_qpair_enable(qpair); 882252222Sjimharris} 883252222Sjimharris 884252222Sjimharrisvoid 885252222Sjimharrisnvme_io_qpair_enable(struct nvme_qpair *qpair) 886252222Sjimharris{ 887252222Sjimharris STAILQ_HEAD(, nvme_request) temp; 888252222Sjimharris struct nvme_tracker *tr; 889252222Sjimharris struct nvme_tracker *tr_temp; 890252222Sjimharris struct nvme_request *req; 891252222Sjimharris 892240616Sjimharris /* 893252222Sjimharris * Manually abort each outstanding I/O. This normally results in a 894252222Sjimharris * retry, unless the retry count on the associated request has 895252222Sjimharris * reached its limit. 896240616Sjimharris */ 897252222Sjimharris TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) { 898252222Sjimharris nvme_printf(qpair->ctrlr, "aborting outstanding i/o\n"); 899252222Sjimharris nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 900252222Sjimharris NVME_SC_ABORTED_BY_REQUEST, 0, TRUE); 901240616Sjimharris } 902240616Sjimharris 903252222Sjimharris mtx_lock(&qpair->lock); 904240616Sjimharris 905252222Sjimharris nvme_qpair_enable(qpair); 906240616Sjimharris 907252222Sjimharris STAILQ_INIT(&temp); 908252222Sjimharris STAILQ_SWAP(&qpair->queued_req, &temp, nvme_request); 909240616Sjimharris 910252222Sjimharris while (!STAILQ_EMPTY(&temp)) { 911252222Sjimharris req = STAILQ_FIRST(&temp); 912252222Sjimharris STAILQ_REMOVE_HEAD(&temp, stailq); 913252222Sjimharris nvme_printf(qpair->ctrlr, "resubmitting queued i/o\n"); 914252222Sjimharris nvme_qpair_print_command(qpair, &req->cmd); 915252222Sjimharris _nvme_qpair_submit_request(qpair, req); 916252222Sjimharris } 917240616Sjimharris 918252222Sjimharris mtx_unlock(&qpair->lock); 919252222Sjimharris} 920240616Sjimharris 921252222Sjimharrisstatic void 922252222Sjimharrisnvme_qpair_disable(struct nvme_qpair *qpair) 923252222Sjimharris{ 924252222Sjimharris struct nvme_tracker *tr; 925252222Sjimharris 926252222Sjimharris qpair->is_enabled = FALSE; 927252222Sjimharris mtx_lock(&qpair->lock); 928252222Sjimharris TAILQ_FOREACH(tr, &qpair->outstanding_tr, tailq) 929252222Sjimharris callout_stop(&tr->timer); 930240616Sjimharris mtx_unlock(&qpair->lock); 931240616Sjimharris} 932252222Sjimharris 933252222Sjimharrisvoid 934252222Sjimharrisnvme_admin_qpair_disable(struct nvme_qpair *qpair) 935252222Sjimharris{ 936252222Sjimharris 937252222Sjimharris nvme_qpair_disable(qpair); 938252222Sjimharris nvme_admin_qpair_abort_aers(qpair); 939252222Sjimharris} 940252222Sjimharris 941252222Sjimharrisvoid 942252222Sjimharrisnvme_io_qpair_disable(struct nvme_qpair *qpair) 943252222Sjimharris{ 944252222Sjimharris 945252222Sjimharris nvme_qpair_disable(qpair); 946252222Sjimharris} 947252222Sjimharris 948252222Sjimharrisvoid 949252222Sjimharrisnvme_qpair_fail(struct nvme_qpair *qpair) 950252222Sjimharris{ 951252222Sjimharris struct nvme_tracker *tr; 952252222Sjimharris struct nvme_request *req; 953252222Sjimharris 954252222Sjimharris mtx_lock(&qpair->lock); 955252222Sjimharris 956252222Sjimharris while (!STAILQ_EMPTY(&qpair->queued_req)) { 957252222Sjimharris req = STAILQ_FIRST(&qpair->queued_req); 958252222Sjimharris STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); 959252222Sjimharris nvme_printf(qpair->ctrlr, "failing queued i/o\n"); 960252222Sjimharris mtx_unlock(&qpair->lock); 961252222Sjimharris nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC, 962252222Sjimharris NVME_SC_ABORTED_BY_REQUEST, TRUE); 963252222Sjimharris mtx_lock(&qpair->lock); 964252222Sjimharris } 965252222Sjimharris 966252222Sjimharris /* Manually abort each outstanding I/O. */ 967252222Sjimharris while (!TAILQ_EMPTY(&qpair->outstanding_tr)) { 968252222Sjimharris tr = TAILQ_FIRST(&qpair->outstanding_tr); 969252222Sjimharris /* 970252222Sjimharris * Do not remove the tracker. The abort_tracker path will 971252222Sjimharris * do that for us. 972252222Sjimharris */ 973252222Sjimharris nvme_printf(qpair->ctrlr, "failing outstanding i/o\n"); 974252222Sjimharris mtx_unlock(&qpair->lock); 975252222Sjimharris nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, 976252222Sjimharris NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, TRUE); 977252222Sjimharris mtx_lock(&qpair->lock); 978252222Sjimharris } 979252222Sjimharris 980252222Sjimharris mtx_unlock(&qpair->lock); 981252222Sjimharris} 982252222Sjimharris 983