1303126Simp/*- 2303126Simp * Copyright (c) 2016 Netflix, Inc 3303126Simp * All rights reserved. 4303126Simp * 5303126Simp * Redistribution and use in source and binary forms, with or without 6303126Simp * modification, are permitted provided that the following conditions 7303126Simp * are met: 8303126Simp * 1. Redistributions of source code must retain the above copyright 9303126Simp * notice, this list of conditions and the following disclaimer, 10303126Simp * without modification, immediately at the beginning of the file. 11303126Simp * 2. Redistributions in binary form must reproduce the above copyright 12303126Simp * notice, this list of conditions and the following disclaimer in the 13303126Simp * documentation and/or other materials provided with the distribution. 14303126Simp * 15303126Simp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16303126Simp * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17303126Simp * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18303126Simp * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19303126Simp * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20303126Simp * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21303126Simp * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22303126Simp * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23303126Simp * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24303126Simp * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25303126Simp */ 26303126Simp 27303126Simp#include <sys/cdefs.h> 28303126Simp__FBSDID("$FreeBSD: stable/11/sys/dev/nvme/nvme_sim.c 335166 2018-06-14 18:18:55Z mav $"); 29303126Simp 30303126Simp#include <sys/param.h> 31303126Simp#include <sys/systm.h> 32303126Simp#include <sys/buf.h> 33303126Simp#include <sys/bus.h> 34303126Simp#include <sys/conf.h> 35303126Simp#include <sys/ioccom.h> 36303126Simp#include <sys/malloc.h> 37303126Simp#include <sys/proc.h> 38303126Simp#include <sys/smp.h> 39303126Simp 40303126Simp#include <cam/cam.h> 41303126Simp#include <cam/cam_ccb.h> 42303126Simp#include <cam/cam_sim.h> 43303126Simp#include <cam/cam_xpt_sim.h> 44303126Simp#include <cam/cam_xpt_internal.h> // Yes, this is wrong. 45303126Simp#include <cam/cam_debug.h> 46303126Simp 47328751Smav#include <dev/pci/pcivar.h> 48328751Smav#include <dev/pci/pcireg.h> 49328751Smav 50303126Simp#include "nvme_private.h" 51303126Simp 52303126Simp#define ccb_accb_ptr spriv_ptr0 53303126Simp#define ccb_ctrlr_ptr spriv_ptr1 54303126Simpstatic void nvme_sim_action(struct cam_sim *sim, union ccb *ccb); 55303126Simpstatic void nvme_sim_poll(struct cam_sim *sim); 56303126Simp 57303126Simp#define sim2softc(sim) ((struct nvme_sim_softc *)cam_sim_softc(sim)) 58303126Simp#define sim2ns(sim) (sim2softc(sim)->s_ns) 59303126Simp#define sim2ctrlr(sim) (sim2softc(sim)->s_ctrlr) 60303126Simp 61303126Simpstruct nvme_sim_softc 62303126Simp{ 63303126Simp struct nvme_controller *s_ctrlr; 64303126Simp struct nvme_namespace *s_ns; 65303126Simp struct cam_sim *s_sim; 66303126Simp struct cam_path *s_path; 67303126Simp}; 68303126Simp 69303126Simpstatic void 70303126Simpnvme_sim_nvmeio_done(void *ccb_arg, const struct nvme_completion *cpl) 71303126Simp{ 72303126Simp union ccb *ccb = (union ccb *)ccb_arg; 73303126Simp 74303126Simp /* 75303126Simp * Let the periph know the completion, and let it sort out what 76303126Simp * it means. Make our best guess, though for the status code. 77303126Simp */ 78303126Simp memcpy(&ccb->nvmeio.cpl, cpl, sizeof(*cpl)); 79335142Smav ccb->ccb_h.status &= ~CAM_SIM_QUEUED; 80328699Smav if (nvme_completion_is_error(cpl)) { 81303126Simp ccb->ccb_h.status = CAM_REQ_CMP_ERR; 82328699Smav xpt_done(ccb); 83328699Smav } else { 84303126Simp ccb->ccb_h.status = CAM_REQ_CMP; 85328699Smav xpt_done_direct(ccb); 86328699Smav } 87303126Simp} 88303126Simp 89303126Simpstatic void 90303126Simpnvme_sim_nvmeio(struct cam_sim *sim, union ccb *ccb) 91303126Simp{ 92303126Simp struct ccb_nvmeio *nvmeio = &ccb->nvmeio; 93303126Simp struct nvme_request *req; 94303126Simp void *payload; 95303126Simp uint32_t size; 96303126Simp struct nvme_controller *ctrlr; 97303126Simp 98303126Simp ctrlr = sim2ctrlr(sim); 99303126Simp payload = nvmeio->data_ptr; 100303126Simp size = nvmeio->dxfer_len; 101303126Simp /* SG LIST ??? */ 102303126Simp if ((nvmeio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_BIO) 103303126Simp req = nvme_allocate_request_bio((struct bio *)payload, 104303126Simp nvme_sim_nvmeio_done, ccb); 105328696Smav else if ((nvmeio->ccb_h.flags & CAM_DATA_SG) == CAM_DATA_SG) 106328696Smav req = nvme_allocate_request_ccb(ccb, nvme_sim_nvmeio_done, ccb); 107303126Simp else if (payload == NULL) 108303126Simp req = nvme_allocate_request_null(nvme_sim_nvmeio_done, ccb); 109303126Simp else 110303126Simp req = nvme_allocate_request_vaddr(payload, size, 111303126Simp nvme_sim_nvmeio_done, ccb); 112303126Simp 113303126Simp if (req == NULL) { 114303126Simp nvmeio->ccb_h.status = CAM_RESRC_UNAVAIL; 115303126Simp xpt_done(ccb); 116303126Simp return; 117303126Simp } 118335142Smav ccb->ccb_h.status |= CAM_SIM_QUEUED; 119303126Simp 120303126Simp memcpy(&req->cmd, &ccb->nvmeio.cmd, sizeof(ccb->nvmeio.cmd)); 121303126Simp 122328680Smav if (ccb->ccb_h.func_code == XPT_NVME_IO) 123328680Smav nvme_ctrlr_submit_io_request(ctrlr, req); 124328680Smav else 125328680Smav nvme_ctrlr_submit_admin_request(ctrlr, req); 126303126Simp} 127303126Simp 128335166Smavstatic uint32_t 129335166Smavnvme_link_kBps(struct nvme_controller *ctrlr) 130335166Smav{ 131335166Smav uint32_t speed, lanes, link[] = { 1, 250000, 500000, 985000, 1970000 }; 132335166Smav uint32_t status; 133335166Smav 134335166Smav status = pcie_read_config(ctrlr->dev, PCIER_LINK_STA, 2); 135335166Smav speed = status & PCIEM_LINK_STA_SPEED; 136335166Smav lanes = (status & PCIEM_LINK_STA_WIDTH) >> 4; 137335166Smav /* 138335166Smav * Failsafe on link speed indicator. If it is insane report the number of 139335166Smav * lanes as the speed. Not 100% accurate, but may be diagnostic. 140335166Smav */ 141335166Smav if (speed >= nitems(link)) 142335166Smav speed = 0; 143335166Smav return link[speed] * lanes; 144335166Smav} 145335166Smav 146303126Simpstatic void 147303126Simpnvme_sim_action(struct cam_sim *sim, union ccb *ccb) 148303126Simp{ 149303126Simp struct nvme_controller *ctrlr; 150303126Simp struct nvme_namespace *ns; 151303126Simp 152303126Simp CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, 153303126Simp ("nvme_sim_action: func= %#x\n", 154303126Simp ccb->ccb_h.func_code)); 155303126Simp 156303126Simp /* 157303126Simp * XXX when we support multiple namespaces in the base driver we'll need 158303126Simp * to revisit how all this gets stored and saved in the periph driver's 159303126Simp * reserved areas. Right now we store all three in the softc of the sim. 160303126Simp */ 161303126Simp ns = sim2ns(sim); 162303126Simp ctrlr = sim2ctrlr(sim); 163303126Simp 164303126Simp mtx_assert(&ctrlr->lock, MA_OWNED); 165303126Simp 166303126Simp switch (ccb->ccb_h.func_code) { 167303126Simp case XPT_CALC_GEOMETRY: /* Calculate Geometry Totally nuts ? XXX */ 168303126Simp /* 169303126Simp * Only meaningful for old-school SCSI disks since only the SCSI 170303126Simp * da driver generates them. Reject all these that slip through. 171303126Simp */ 172303126Simp /*FALLTHROUGH*/ 173303126Simp case XPT_ABORT: /* Abort the specified CCB */ 174303126Simp ccb->ccb_h.status = CAM_REQ_INVALID; 175303126Simp break; 176303126Simp case XPT_SET_TRAN_SETTINGS: 177303126Simp /* 178303126Simp * NVMe doesn't really have different transfer settings, but 179303126Simp * other parts of CAM think failure here is a big deal. 180303126Simp */ 181303126Simp ccb->ccb_h.status = CAM_REQ_CMP; 182303126Simp break; 183303126Simp case XPT_PATH_INQ: /* Path routing inquiry */ 184303126Simp { 185328750Smav struct ccb_pathinq *cpi = &ccb->cpi; 186328750Smav device_t dev = ctrlr->dev; 187303126Simp 188303126Simp /* 189303126Simp * NVMe may have multiple LUNs on the same path. Current generation 190303126Simp * of NVMe devives support only a single name space. Multiple name 191303126Simp * space drives are coming, but it's unclear how we should report 192303126Simp * them up the stack. 193303126Simp */ 194303126Simp cpi->version_num = 1; 195303126Simp cpi->hba_inquiry = 0; 196303126Simp cpi->target_sprt = 0; 197303126Simp cpi->hba_misc = PIM_UNMAPPED /* | PIM_NOSCAN */; 198303126Simp cpi->hba_eng_cnt = 0; 199303126Simp cpi->max_target = 0; 200303126Simp cpi->max_lun = ctrlr->cdata.nn; 201303126Simp cpi->maxio = nvme_ns_get_max_io_xfer_size(ns); 202303126Simp cpi->initiator_id = 0; 203303126Simp cpi->bus_id = cam_sim_bus(sim); 204335166Smav cpi->base_transfer_speed = nvme_link_kBps(ctrlr); 205303126Simp strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); 206303126Simp strncpy(cpi->hba_vid, "NVMe", HBA_IDLEN); 207303126Simp strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); 208303126Simp cpi->unit_number = cam_sim_unit(sim); 209335138Smav cpi->transport = XPORT_NVME; /* XXX XPORT_PCIE ? */ 210335166Smav cpi->transport_version = nvme_mmio_read_4(ctrlr, vs); 211335138Smav cpi->protocol = PROTO_NVME; 212335166Smav cpi->protocol_version = nvme_mmio_read_4(ctrlr, vs); 213303126Simp cpi->xport_specific.nvme.nsid = ns->id; 214328750Smav cpi->xport_specific.nvme.domain = pci_get_domain(dev); 215328750Smav cpi->xport_specific.nvme.bus = pci_get_bus(dev); 216328750Smav cpi->xport_specific.nvme.slot = pci_get_slot(dev); 217328750Smav cpi->xport_specific.nvme.function = pci_get_function(dev); 218328750Smav cpi->xport_specific.nvme.extra = 0; 219303126Simp cpi->ccb_h.status = CAM_REQ_CMP; 220303126Simp break; 221303126Simp } 222303126Simp case XPT_GET_TRAN_SETTINGS: /* Get transport settings */ 223303126Simp { 224303126Simp struct ccb_trans_settings *cts; 225303126Simp struct ccb_trans_settings_nvme *nvmep; 226303126Simp struct ccb_trans_settings_nvme *nvmex; 227335166Smav device_t dev; 228335166Smav uint32_t status, caps; 229303126Simp 230335166Smav dev = ctrlr->dev; 231303126Simp cts = &ccb->cts; 232303126Simp nvmex = &cts->xport_specific.nvme; 233303126Simp nvmep = &cts->proto_specific.nvme; 234303126Simp 235335166Smav status = pcie_read_config(dev, PCIER_LINK_STA, 2); 236335166Smav caps = pcie_read_config(dev, PCIER_LINK_CAP, 2); 237335166Smav nvmex->valid = CTS_NVME_VALID_SPEC | CTS_NVME_VALID_LINK; 238335166Smav nvmex->spec = nvme_mmio_read_4(ctrlr, vs); 239335166Smav nvmex->speed = status & PCIEM_LINK_STA_SPEED; 240335166Smav nvmex->lanes = (status & PCIEM_LINK_STA_WIDTH) >> 4; 241335166Smav nvmex->max_speed = caps & PCIEM_LINK_CAP_MAX_SPEED; 242335166Smav nvmex->max_lanes = (caps & PCIEM_LINK_CAP_MAX_WIDTH) >> 4; 243303126Simp 244335166Smav /* XXX these should be something else maybe ? */ 245335166Smav nvmep->valid = 1; 246335166Smav nvmep->spec = nvmex->spec; 247335166Smav 248303126Simp cts->transport = XPORT_NVME; 249303126Simp cts->protocol = PROTO_NVME; 250303126Simp cts->ccb_h.status = CAM_REQ_CMP; 251303126Simp break; 252303126Simp } 253303126Simp case XPT_TERM_IO: /* Terminate the I/O process */ 254303126Simp /* 255303126Simp * every driver handles this, but nothing generates it. Assume 256303126Simp * it's OK to just say 'that worked'. 257303126Simp */ 258303126Simp /*FALLTHROUGH*/ 259303126Simp case XPT_RESET_DEV: /* Bus Device Reset the specified device */ 260303126Simp case XPT_RESET_BUS: /* Reset the specified bus */ 261303126Simp /* 262303126Simp * NVMe doesn't really support physically resetting the bus. It's part 263303126Simp * of the bus scanning dance, so return sucess to tell the process to 264303126Simp * proceed. 265303126Simp */ 266303126Simp ccb->ccb_h.status = CAM_REQ_CMP; 267303126Simp break; 268303126Simp case XPT_NVME_IO: /* Execute the requested I/O operation */ 269328680Smav case XPT_NVME_ADMIN: /* or Admin operation */ 270303126Simp nvme_sim_nvmeio(sim, ccb); 271303126Simp return; /* no done */ 272303126Simp default: 273303126Simp ccb->ccb_h.status = CAM_REQ_INVALID; 274303126Simp break; 275303126Simp } 276303126Simp xpt_done(ccb); 277303126Simp} 278303126Simp 279303126Simpstatic void 280303126Simpnvme_sim_poll(struct cam_sim *sim) 281303126Simp{ 282303126Simp 283328702Smav nvme_ctrlr_poll(sim2ctrlr(sim)); 284303126Simp} 285303126Simp 286303126Simpstatic void * 287303126Simpnvme_sim_new_controller(struct nvme_controller *ctrlr) 288303126Simp{ 289303126Simp struct cam_devq *devq; 290303126Simp int max_trans; 291303126Simp int unit; 292303126Simp struct nvme_sim_softc *sc = NULL; 293303126Simp 294328691Smav max_trans = ctrlr->max_hw_pend_io; 295303126Simp unit = device_get_unit(ctrlr->dev); 296303126Simp devq = cam_simq_alloc(max_trans); 297303126Simp if (devq == NULL) 298303126Simp return NULL; 299303126Simp 300303126Simp sc = malloc(sizeof(*sc), M_NVME, M_ZERO | M_WAITOK); 301303126Simp 302303126Simp sc->s_ctrlr = ctrlr; 303303126Simp 304303126Simp sc->s_sim = cam_sim_alloc(nvme_sim_action, nvme_sim_poll, 305303126Simp "nvme", sc, unit, &ctrlr->lock, max_trans, max_trans, devq); 306303126Simp if (sc->s_sim == NULL) { 307303126Simp printf("Failed to allocate a sim\n"); 308303126Simp cam_simq_free(devq); 309303126Simp free(sc, M_NVME); 310303126Simp return NULL; 311303126Simp } 312303126Simp 313303126Simp return sc; 314303126Simp} 315303126Simp 316303126Simpstatic void 317303126Simpnvme_sim_rescan_target(struct nvme_controller *ctrlr, struct cam_path *path) 318303126Simp{ 319303126Simp union ccb *ccb; 320303126Simp 321303126Simp ccb = xpt_alloc_ccb_nowait(); 322303126Simp if (ccb == NULL) { 323303126Simp printf("unable to alloc CCB for rescan\n"); 324303126Simp return; 325303126Simp } 326303126Simp 327303126Simp if (xpt_clone_path(&ccb->ccb_h.path, path) != CAM_REQ_CMP) { 328303126Simp printf("unable to copy path for rescan\n"); 329303126Simp xpt_free_ccb(ccb); 330303126Simp return; 331303126Simp } 332303126Simp 333303126Simp xpt_rescan(ccb); 334303126Simp} 335303126Simp 336303126Simpstatic void * 337303126Simpnvme_sim_new_ns(struct nvme_namespace *ns, void *sc_arg) 338303126Simp{ 339303126Simp struct nvme_sim_softc *sc = sc_arg; 340303126Simp struct nvme_controller *ctrlr = sc->s_ctrlr; 341303126Simp int i; 342303126Simp 343303126Simp sc->s_ns = ns; 344303126Simp 345303126Simp /* 346303126Simp * XXX this is creating one bus per ns, but it should be one 347303126Simp * XXX target per controller, and one LUN per namespace. 348303126Simp * XXX Current drives only support one NS, so there's time 349303126Simp * XXX to fix it later when new drives arrive. 350303126Simp * 351303126Simp * XXX I'm pretty sure the xpt_bus_register() call below is 352303126Simp * XXX like super lame and it really belongs in the sim_new_ctrlr 353303126Simp * XXX callback. Then the create_path below would be pretty close 354303126Simp * XXX to being right. Except we should be per-ns not per-ctrlr 355303126Simp * XXX data. 356303126Simp */ 357303126Simp 358303126Simp mtx_lock(&ctrlr->lock); 359303126Simp/* Create bus */ 360303126Simp 361303126Simp /* 362303126Simp * XXX do I need to lock ctrlr->lock ? 363303126Simp * XXX do I need to lock the path? 364303126Simp * ata and scsi seem to in their code, but their discovery is 365303126Simp * somewhat more asynchronous. We're only every called one at a 366303126Simp * time, and nothing is in parallel. 367303126Simp */ 368303126Simp 369303126Simp i = 0; 370303126Simp if (xpt_bus_register(sc->s_sim, ctrlr->dev, 0) != CAM_SUCCESS) 371303126Simp goto error; 372303126Simp i++; 373303126Simp if (xpt_create_path(&sc->s_path, /*periph*/NULL, cam_sim_path(sc->s_sim), 374303126Simp 1, ns->id) != CAM_REQ_CMP) 375303126Simp goto error; 376303126Simp i++; 377303126Simp 378303126Simp sc->s_path->device->nvme_data = nvme_ns_get_data(ns); 379303126Simp sc->s_path->device->nvme_cdata = nvme_ctrlr_get_data(ns->ctrlr); 380303126Simp 381303126Simp/* Scan bus */ 382303126Simp nvme_sim_rescan_target(ctrlr, sc->s_path); 383303126Simp 384303126Simp mtx_unlock(&ctrlr->lock); 385303126Simp 386303126Simp return ns; 387303126Simp 388303126Simperror: 389303126Simp switch (i) { 390303126Simp case 2: 391303126Simp xpt_free_path(sc->s_path); 392303126Simp case 1: 393303126Simp xpt_bus_deregister(cam_sim_path(sc->s_sim)); 394303126Simp case 0: 395303126Simp cam_sim_free(sc->s_sim, /*free_devq*/TRUE); 396303126Simp } 397303126Simp mtx_unlock(&ctrlr->lock); 398303126Simp return NULL; 399303126Simp} 400303126Simp 401303126Simpstatic void 402303126Simpnvme_sim_controller_fail(void *ctrlr_arg) 403303126Simp{ 404303126Simp /* XXX cleanup XXX */ 405303126Simp} 406303126Simp 407303126Simpstruct nvme_consumer *consumer_cookie; 408303126Simp 409303126Simpstatic void 410303126Simpnvme_sim_init(void) 411303126Simp{ 412328681Smav if (nvme_use_nvd) 413328681Smav return; 414303126Simp 415303126Simp consumer_cookie = nvme_register_consumer(nvme_sim_new_ns, 416303126Simp nvme_sim_new_controller, NULL, nvme_sim_controller_fail); 417303126Simp} 418303126Simp 419303126SimpSYSINIT(nvme_sim_register, SI_SUB_DRIVERS, SI_ORDER_ANY, 420303126Simp nvme_sim_init, NULL); 421303126Simp 422303126Simpstatic void 423303126Simpnvme_sim_uninit(void) 424303126Simp{ 425328681Smav if (nvme_use_nvd) 426328681Smav return; 427303126Simp /* XXX Cleanup */ 428303126Simp 429303126Simp nvme_unregister_consumer(consumer_cookie); 430303126Simp} 431303126Simp 432303126SimpSYSUNINIT(nvme_sim_unregister, SI_SUB_DRIVERS, SI_ORDER_ANY, 433303126Simp nvme_sim_uninit, NULL); 434