1240616Sjimharris/*- 2265576Sjimharris * Copyright (C) 2012-2014 Intel Corporation 3240616Sjimharris * All rights reserved. 4240616Sjimharris * 5240616Sjimharris * Redistribution and use in source and binary forms, with or without 6240616Sjimharris * modification, are permitted provided that the following conditions 7240616Sjimharris * are met: 8240616Sjimharris * 1. Redistributions of source code must retain the above copyright 9240616Sjimharris * notice, this list of conditions and the following disclaimer. 10240616Sjimharris * 2. Redistributions in binary form must reproduce the above copyright 11240616Sjimharris * notice, this list of conditions and the following disclaimer in the 12240616Sjimharris * documentation and/or other materials provided with the distribution. 13240616Sjimharris * 14240616Sjimharris * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15240616Sjimharris * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16240616Sjimharris * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17240616Sjimharris * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18240616Sjimharris * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19240616Sjimharris * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20240616Sjimharris * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21240616Sjimharris * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22240616Sjimharris * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23240616Sjimharris * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24240616Sjimharris * SUCH DAMAGE. 25240616Sjimharris */ 26240616Sjimharris 27240616Sjimharris#include <sys/cdefs.h> 28240616Sjimharris__FBSDID("$FreeBSD$"); 29240616Sjimharris 30240616Sjimharris#include <sys/param.h> 31240616Sjimharris#include <sys/bus.h> 32240616Sjimharris#include <sys/conf.h> 33240616Sjimharris#include <sys/module.h> 34240616Sjimharris 35241659Sjimharris#include <vm/uma.h> 36241659Sjimharris 37240697Sjimharris#include <dev/pci/pcireg.h> 38240616Sjimharris#include <dev/pci/pcivar.h> 39240616Sjimharris 40240616Sjimharris#include "nvme_private.h" 41240616Sjimharris 42240616Sjimharrisstruct nvme_consumer { 43248738Sjimharris uint32_t id; 44248738Sjimharris nvme_cons_ns_fn_t ns_fn; 45248738Sjimharris nvme_cons_ctrlr_fn_t ctrlr_fn; 46248738Sjimharris nvme_cons_async_fn_t async_fn; 47248767Sjimharris nvme_cons_fail_fn_t fail_fn; 48240616Sjimharris}; 49240616Sjimharris 50240616Sjimharrisstruct nvme_consumer nvme_consumer[NVME_MAX_CONSUMERS]; 51248738Sjimharris#define INVALID_CONSUMER_ID 0xFFFF 52240616Sjimharris 53248761Sjimharrisuma_zone_t nvme_request_zone; 54248761Sjimharrisint32_t nvme_retry_count; 55241659Sjimharris 56240616SjimharrisMALLOC_DEFINE(M_NVME, "nvme", "nvme(4) memory allocations"); 57240616Sjimharris 58240616Sjimharrisstatic int nvme_probe(device_t); 59240616Sjimharrisstatic int nvme_attach(device_t); 60240616Sjimharrisstatic int nvme_detach(device_t); 61244411Sjimharrisstatic int nvme_modevent(module_t mod, int type, void *arg); 62240616Sjimharris 63240616Sjimharrisstatic devclass_t nvme_devclass; 64240616Sjimharris 65240616Sjimharrisstatic device_method_t nvme_pci_methods[] = { 66240616Sjimharris /* Device interface */ 67240616Sjimharris DEVMETHOD(device_probe, nvme_probe), 68240616Sjimharris DEVMETHOD(device_attach, nvme_attach), 69240616Sjimharris DEVMETHOD(device_detach, nvme_detach), 70240616Sjimharris { 0, 0 } 71240616Sjimharris}; 72240616Sjimharris 73240616Sjimharrisstatic driver_t nvme_pci_driver = { 74240616Sjimharris "nvme", 75240616Sjimharris nvme_pci_methods, 76240616Sjimharris sizeof(struct nvme_controller), 77240616Sjimharris}; 78240616Sjimharris 79244411SjimharrisDRIVER_MODULE(nvme, pci, nvme_pci_driver, nvme_devclass, nvme_modevent, 0); 80240616SjimharrisMODULE_VERSION(nvme, 1); 81240616Sjimharris 82240616Sjimharrisstatic struct _pcsid 83240616Sjimharris{ 84240616Sjimharris u_int32_t type; 85240616Sjimharris const char *desc; 86240616Sjimharris} pci_ids[] = { 87240616Sjimharris { 0x01118086, "NVMe Controller" }, 88240616Sjimharris { CHATHAM_PCI_ID, "Chatham Prototype NVMe Controller" }, 89243951Sjimharris { IDT32_PCI_ID, "IDT NVMe Controller (32 channel)" }, 90243951Sjimharris { IDT8_PCI_ID, "IDT NVMe Controller (8 channel)" }, 91240616Sjimharris { 0x00000000, NULL } 92240616Sjimharris}; 93240616Sjimharris 94240616Sjimharrisstatic int 95240616Sjimharrisnvme_probe (device_t device) 96240616Sjimharris{ 97240697Sjimharris struct _pcsid *ep; 98240697Sjimharris u_int32_t type; 99240616Sjimharris 100240697Sjimharris type = pci_get_devid(device); 101240697Sjimharris ep = pci_ids; 102240697Sjimharris 103240616Sjimharris while (ep->type && ep->type != type) 104240616Sjimharris ++ep; 105240616Sjimharris 106240616Sjimharris if (ep->desc) { 107240616Sjimharris device_set_desc(device, ep->desc); 108240700Sjimharris return (BUS_PROBE_DEFAULT); 109240697Sjimharris } 110240697Sjimharris 111240700Sjimharris#if defined(PCIS_STORAGE_NVM) 112240700Sjimharris if (pci_get_class(device) == PCIC_STORAGE && 113240700Sjimharris pci_get_subclass(device) == PCIS_STORAGE_NVM && 114240700Sjimharris pci_get_progif(device) == PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0) { 115240700Sjimharris device_set_desc(device, "Generic NVMe Device"); 116240700Sjimharris return (BUS_PROBE_GENERIC); 117240700Sjimharris } 118240700Sjimharris#endif 119240700Sjimharris 120240700Sjimharris return (ENXIO); 121240616Sjimharris} 122240616Sjimharris 123240616Sjimharrisstatic void 124241659Sjimharrisnvme_init(void) 125241659Sjimharris{ 126248738Sjimharris uint32_t i; 127248738Sjimharris 128241659Sjimharris nvme_request_zone = uma_zcreate("nvme_request", 129241659Sjimharris sizeof(struct nvme_request), NULL, NULL, NULL, NULL, 0, 0); 130248738Sjimharris 131248738Sjimharris for (i = 0; i < NVME_MAX_CONSUMERS; i++) 132248738Sjimharris nvme_consumer[i].id = INVALID_CONSUMER_ID; 133241659Sjimharris} 134241659Sjimharris 135241659SjimharrisSYSINIT(nvme_register, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_init, NULL); 136241659Sjimharris 137241659Sjimharrisstatic void 138241659Sjimharrisnvme_uninit(void) 139241659Sjimharris{ 140241659Sjimharris uma_zdestroy(nvme_request_zone); 141241659Sjimharris} 142241659Sjimharris 143241659SjimharrisSYSUNINIT(nvme_unregister, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_uninit, NULL); 144241659Sjimharris 145241659Sjimharrisstatic void 146240616Sjimharrisnvme_load(void) 147240616Sjimharris{ 148240616Sjimharris} 149240616Sjimharris 150240616Sjimharrisstatic void 151240616Sjimharrisnvme_unload(void) 152240616Sjimharris{ 153240616Sjimharris} 154240616Sjimharris 155240616Sjimharrisstatic void 156240616Sjimharrisnvme_shutdown(void) 157240616Sjimharris{ 158240616Sjimharris device_t *devlist; 159240616Sjimharris struct nvme_controller *ctrlr; 160240616Sjimharris int dev, devcount; 161240616Sjimharris 162240616Sjimharris if (devclass_get_devices(nvme_devclass, &devlist, &devcount)) 163240616Sjimharris return; 164240616Sjimharris 165240616Sjimharris for (dev = 0; dev < devcount; dev++) { 166240616Sjimharris ctrlr = DEVICE2SOFTC(devlist[dev]); 167254302Sjimharris nvme_ctrlr_shutdown(ctrlr); 168240616Sjimharris } 169240616Sjimharris 170240616Sjimharris free(devlist, M_TEMP); 171240616Sjimharris} 172240616Sjimharris 173240616Sjimharrisstatic int 174240616Sjimharrisnvme_modevent(module_t mod, int type, void *arg) 175240616Sjimharris{ 176240616Sjimharris 177240616Sjimharris switch (type) { 178240616Sjimharris case MOD_LOAD: 179240616Sjimharris nvme_load(); 180240616Sjimharris break; 181240616Sjimharris case MOD_UNLOAD: 182240616Sjimharris nvme_unload(); 183240616Sjimharris break; 184240616Sjimharris case MOD_SHUTDOWN: 185240616Sjimharris nvme_shutdown(); 186240616Sjimharris break; 187240616Sjimharris default: 188240616Sjimharris break; 189240616Sjimharris } 190240616Sjimharris 191240616Sjimharris return (0); 192240616Sjimharris} 193240616Sjimharris 194240616Sjimharrisvoid 195240616Sjimharrisnvme_dump_command(struct nvme_command *cmd) 196240616Sjimharris{ 197247963Sobrien printf( 198247963Sobrien"opc:%x f:%x r1:%x cid:%x nsid:%x r2:%x r3:%x mptr:%jx prp1:%jx prp2:%jx cdw:%x %x %x %x %x %x\n", 199240616Sjimharris cmd->opc, cmd->fuse, cmd->rsvd1, cmd->cid, cmd->nsid, 200240616Sjimharris cmd->rsvd2, cmd->rsvd3, 201247963Sobrien (uintmax_t)cmd->mptr, (uintmax_t)cmd->prp1, (uintmax_t)cmd->prp2, 202240616Sjimharris cmd->cdw10, cmd->cdw11, cmd->cdw12, cmd->cdw13, cmd->cdw14, 203240616Sjimharris cmd->cdw15); 204240616Sjimharris} 205240616Sjimharris 206240616Sjimharrisvoid 207240616Sjimharrisnvme_dump_completion(struct nvme_completion *cpl) 208240616Sjimharris{ 209240616Sjimharris printf("cdw0:%08x sqhd:%04x sqid:%04x " 210240616Sjimharris "cid:%04x p:%x sc:%02x sct:%x m:%x dnr:%x\n", 211240616Sjimharris cpl->cdw0, cpl->sqhd, cpl->sqid, 212248756Sjimharris cpl->cid, cpl->status.p, cpl->status.sc, cpl->status.sct, 213248756Sjimharris cpl->status.m, cpl->status.dnr); 214240616Sjimharris} 215240616Sjimharris 216240616Sjimharrisstatic int 217240616Sjimharrisnvme_attach(device_t dev) 218240616Sjimharris{ 219240616Sjimharris struct nvme_controller *ctrlr = DEVICE2SOFTC(dev); 220240616Sjimharris int status; 221240616Sjimharris 222240616Sjimharris status = nvme_ctrlr_construct(ctrlr, dev); 223240616Sjimharris 224256155Sjimharris if (status != 0) { 225256155Sjimharris nvme_ctrlr_destruct(ctrlr, dev); 226240616Sjimharris return (status); 227256155Sjimharris } 228240616Sjimharris 229240616Sjimharris /* 230240616Sjimharris * Reset controller twice to ensure we do a transition from cc.en==1 231240616Sjimharris * to cc.en==0. This is because we don't really know what status 232240616Sjimharris * the controller was left in when boot handed off to OS. 233240616Sjimharris */ 234248746Sjimharris status = nvme_ctrlr_hw_reset(ctrlr); 235256155Sjimharris if (status != 0) { 236256155Sjimharris nvme_ctrlr_destruct(ctrlr, dev); 237240616Sjimharris return (status); 238256155Sjimharris } 239240616Sjimharris 240248746Sjimharris status = nvme_ctrlr_hw_reset(ctrlr); 241256155Sjimharris if (status != 0) { 242256155Sjimharris nvme_ctrlr_destruct(ctrlr, dev); 243240616Sjimharris return (status); 244256155Sjimharris } 245240616Sjimharris 246248763Sjimharris nvme_sysctl_initialize_ctrlr(ctrlr); 247248763Sjimharris 248253107Sjimharris pci_enable_busmaster(dev); 249253107Sjimharris 250248763Sjimharris ctrlr->config_hook.ich_func = nvme_ctrlr_start_config_hook; 251240616Sjimharris ctrlr->config_hook.ich_arg = ctrlr; 252240616Sjimharris 253240616Sjimharris config_intrhook_establish(&ctrlr->config_hook); 254240616Sjimharris 255240616Sjimharris return (0); 256240616Sjimharris} 257240616Sjimharris 258240616Sjimharrisstatic int 259240616Sjimharrisnvme_detach (device_t dev) 260240616Sjimharris{ 261240616Sjimharris struct nvme_controller *ctrlr = DEVICE2SOFTC(dev); 262240616Sjimharris 263248736Sjimharris nvme_ctrlr_destruct(ctrlr, dev); 264253107Sjimharris pci_disable_busmaster(dev); 265240616Sjimharris return (0); 266240616Sjimharris} 267240616Sjimharris 268240616Sjimharrisstatic void 269265576Sjimharrisnvme_notify(struct nvme_consumer *cons, 270265576Sjimharris struct nvme_controller *ctrlr) 271240616Sjimharris{ 272248738Sjimharris struct nvme_namespace *ns; 273248738Sjimharris void *ctrlr_cookie; 274265576Sjimharris int cmpset, ns_idx; 275240616Sjimharris 276265576Sjimharris /* 277265576Sjimharris * The consumer may register itself after the nvme devices 278265576Sjimharris * have registered with the kernel, but before the 279265576Sjimharris * driver has completed initialization. In that case, 280265576Sjimharris * return here, and when initialization completes, the 281265576Sjimharris * controller will make sure the consumer gets notified. 282265576Sjimharris */ 283265576Sjimharris if (!ctrlr->is_initialized) 284265576Sjimharris return; 285265576Sjimharris 286265576Sjimharris cmpset = atomic_cmpset_32(&ctrlr->notification_sent, 0, 1); 287265576Sjimharris 288265576Sjimharris if (cmpset == 0) 289265576Sjimharris return; 290265576Sjimharris 291265576Sjimharris if (cons->ctrlr_fn != NULL) 292265576Sjimharris ctrlr_cookie = (*cons->ctrlr_fn)(ctrlr); 293265576Sjimharris else 294265576Sjimharris ctrlr_cookie = NULL; 295265576Sjimharris ctrlr->cons_cookie[cons->id] = ctrlr_cookie; 296265576Sjimharris if (ctrlr->is_failed) { 297265576Sjimharris if (cons->fail_fn != NULL) 298265576Sjimharris (*cons->fail_fn)(ctrlr_cookie); 299265576Sjimharris /* 300265576Sjimharris * Do not notify consumers about the namespaces of a 301265576Sjimharris * failed controller. 302265576Sjimharris */ 303265576Sjimharris return; 304265576Sjimharris } 305265576Sjimharris for (ns_idx = 0; ns_idx < ctrlr->cdata.nn; ns_idx++) { 306265576Sjimharris ns = &ctrlr->ns[ns_idx]; 307265576Sjimharris if (cons->ns_fn != NULL) 308265576Sjimharris ns->cons_cookie[cons->id] = 309265576Sjimharris (*cons->ns_fn)(ns, ctrlr_cookie); 310265576Sjimharris } 311265576Sjimharris} 312265576Sjimharris 313265576Sjimharrisvoid 314265576Sjimharrisnvme_notify_new_controller(struct nvme_controller *ctrlr) 315265576Sjimharris{ 316265576Sjimharris int i; 317265576Sjimharris 318265576Sjimharris for (i = 0; i < NVME_MAX_CONSUMERS; i++) { 319265576Sjimharris if (nvme_consumer[i].id != INVALID_CONSUMER_ID) { 320265576Sjimharris nvme_notify(&nvme_consumer[i], ctrlr); 321265576Sjimharris } 322265576Sjimharris } 323265576Sjimharris} 324265576Sjimharris 325265576Sjimharrisstatic void 326265576Sjimharrisnvme_notify_new_consumer(struct nvme_consumer *cons) 327265576Sjimharris{ 328265576Sjimharris device_t *devlist; 329265576Sjimharris struct nvme_controller *ctrlr; 330265576Sjimharris int dev_idx, devcount; 331265576Sjimharris 332240616Sjimharris if (devclass_get_devices(nvme_devclass, &devlist, &devcount)) 333240616Sjimharris return; 334240616Sjimharris 335248738Sjimharris for (dev_idx = 0; dev_idx < devcount; dev_idx++) { 336248738Sjimharris ctrlr = DEVICE2SOFTC(devlist[dev_idx]); 337265576Sjimharris nvme_notify(cons, ctrlr); 338240616Sjimharris } 339240616Sjimharris 340240616Sjimharris free(devlist, M_TEMP); 341240616Sjimharris} 342240616Sjimharris 343248738Sjimharrisvoid 344248738Sjimharrisnvme_notify_async_consumers(struct nvme_controller *ctrlr, 345248760Sjimharris const struct nvme_completion *async_cpl, 346248760Sjimharris uint32_t log_page_id, void *log_page_buffer, 347248760Sjimharris uint32_t log_page_size) 348248738Sjimharris{ 349248738Sjimharris struct nvme_consumer *cons; 350248738Sjimharris uint32_t i; 351248738Sjimharris 352248738Sjimharris for (i = 0; i < NVME_MAX_CONSUMERS; i++) { 353248738Sjimharris cons = &nvme_consumer[i]; 354248738Sjimharris if (cons->id != INVALID_CONSUMER_ID && cons->async_fn != NULL) 355248760Sjimharris (*cons->async_fn)(ctrlr->cons_cookie[i], async_cpl, 356248760Sjimharris log_page_id, log_page_buffer, log_page_size); 357248738Sjimharris } 358248738Sjimharris} 359248738Sjimharris 360248767Sjimharrisvoid 361248767Sjimharrisnvme_notify_fail_consumers(struct nvme_controller *ctrlr) 362248767Sjimharris{ 363248767Sjimharris struct nvme_consumer *cons; 364248767Sjimharris uint32_t i; 365248767Sjimharris 366248767Sjimharris for (i = 0; i < NVME_MAX_CONSUMERS; i++) { 367248767Sjimharris cons = &nvme_consumer[i]; 368248767Sjimharris if (cons->id != INVALID_CONSUMER_ID && cons->fail_fn != NULL) 369248767Sjimharris cons->fail_fn(ctrlr->cons_cookie[i]); 370248767Sjimharris } 371248767Sjimharris} 372248767Sjimharris 373240616Sjimharrisstruct nvme_consumer * 374248738Sjimharrisnvme_register_consumer(nvme_cons_ns_fn_t ns_fn, nvme_cons_ctrlr_fn_t ctrlr_fn, 375248767Sjimharris nvme_cons_async_fn_t async_fn, 376248767Sjimharris nvme_cons_fail_fn_t fail_fn) 377240616Sjimharris{ 378240616Sjimharris int i; 379240616Sjimharris 380240616Sjimharris /* 381240616Sjimharris * TODO: add locking around consumer registration. Not an issue 382240616Sjimharris * right now since we only have one nvme consumer - nvd(4). 383240616Sjimharris */ 384240616Sjimharris for (i = 0; i < NVME_MAX_CONSUMERS; i++) 385248738Sjimharris if (nvme_consumer[i].id == INVALID_CONSUMER_ID) { 386248738Sjimharris nvme_consumer[i].id = i; 387248738Sjimharris nvme_consumer[i].ns_fn = ns_fn; 388248738Sjimharris nvme_consumer[i].ctrlr_fn = ctrlr_fn; 389248738Sjimharris nvme_consumer[i].async_fn = async_fn; 390248767Sjimharris nvme_consumer[i].fail_fn = fail_fn; 391240616Sjimharris 392265576Sjimharris nvme_notify_new_consumer(&nvme_consumer[i]); 393240616Sjimharris return (&nvme_consumer[i]); 394240616Sjimharris } 395240616Sjimharris 396240616Sjimharris printf("nvme(4): consumer not registered - no slots available\n"); 397240616Sjimharris return (NULL); 398240616Sjimharris} 399240616Sjimharris 400240616Sjimharrisvoid 401240616Sjimharrisnvme_unregister_consumer(struct nvme_consumer *consumer) 402240616Sjimharris{ 403240616Sjimharris 404248738Sjimharris consumer->id = INVALID_CONSUMER_ID; 405240616Sjimharris} 406240616Sjimharris 407248769Sjimharrisvoid 408248769Sjimharrisnvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl) 409248769Sjimharris{ 410248769Sjimharris struct nvme_completion_poll_status *status = arg; 411248769Sjimharris 412248769Sjimharris /* 413248769Sjimharris * Copy status into the argument passed by the caller, so that 414248769Sjimharris * the caller can check the status to determine if the 415248769Sjimharris * the request passed or failed. 416248769Sjimharris */ 417248769Sjimharris memcpy(&status->cpl, cpl, sizeof(*cpl)); 418248769Sjimharris wmb(); 419248769Sjimharris status->done = TRUE; 420248769Sjimharris} 421