1/*- 2 * Copyright (C) 2012-2014 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: stable/11/sys/dev/nvme/nvme.c 351586 2019-08-28 20:58:24Z mav $"); 29 30#include <sys/param.h> 31#include <sys/bus.h> 32#include <sys/conf.h> 33#include <sys/module.h> 34 35#include <vm/uma.h> 36 37#include <dev/pci/pcireg.h> 38#include <dev/pci/pcivar.h> 39 40#include "nvme_private.h" 41 42struct nvme_consumer { 43 uint32_t id; 44 nvme_cons_ns_fn_t ns_fn; 45 nvme_cons_ctrlr_fn_t ctrlr_fn; 46 nvme_cons_async_fn_t async_fn; 47 nvme_cons_fail_fn_t fail_fn; 48}; 49 50struct nvme_consumer nvme_consumer[NVME_MAX_CONSUMERS]; 51#define INVALID_CONSUMER_ID 0xFFFF 52 53uma_zone_t nvme_request_zone; 54int32_t nvme_retry_count; 55 56MALLOC_DEFINE(M_NVME, "nvme", "nvme(4) memory allocations"); 57 58static int nvme_probe(device_t); 59static int nvme_attach(device_t); 60static int nvme_detach(device_t); 61static int nvme_shutdown(device_t); 62 63static devclass_t nvme_devclass; 64 65static device_method_t nvme_pci_methods[] = { 66 /* Device interface */ 67 DEVMETHOD(device_probe, nvme_probe), 68 DEVMETHOD(device_attach, nvme_attach), 69 DEVMETHOD(device_detach, nvme_detach), 70 DEVMETHOD(device_shutdown, nvme_shutdown), 71 { 0, 0 } 72}; 73 74static driver_t nvme_pci_driver = { 75 "nvme", 76 nvme_pci_methods, 77 sizeof(struct nvme_controller), 78}; 79 80DRIVER_MODULE(nvme, pci, nvme_pci_driver, nvme_devclass, NULL, NULL); 81MODULE_VERSION(nvme, 1); 82MODULE_DEPEND(nvme, cam, 1, 1, 1); 83 84static struct _pcsid 85{ 86 uint32_t devid; 87 int match_subdevice; 88 uint16_t subdevice; 89 const char *desc; 90 uint32_t quirks; 91} pci_ids[] = { 92 { 0x01118086, 0, 0, "NVMe Controller" }, 93 { IDT32_PCI_ID, 0, 0, "IDT NVMe Controller (32 channel)" }, 94 { IDT8_PCI_ID, 0, 0, "IDT NVMe Controller (8 channel)" }, 95 { 0x09538086, 1, 0x3702, "DC P3700 SSD" }, 96 { 0x09538086, 1, 0x3703, "DC P3700 SSD [2.5\" SFF]" }, 97 { 0x09538086, 1, 0x3704, "DC P3500 SSD [Add-in Card]" }, 98 { 0x09538086, 1, 0x3705, "DC P3500 SSD [2.5\" SFF]" }, 99 { 0x09538086, 1, 0x3709, "DC P3600 SSD [Add-in Card]" }, 100 { 0x09538086, 1, 0x370a, "DC P3600 SSD [2.5\" SFF]" }, 101 { 0x00031c58, 0, 0, "HGST SN100", QUIRK_DELAY_B4_CHK_RDY }, 102 { 0x00231c58, 0, 0, "WDC SN200", QUIRK_DELAY_B4_CHK_RDY }, 103 { 0x05401c5f, 0, 0, "Memblaze Pblaze4", QUIRK_DELAY_B4_CHK_RDY }, 104 { 0xa821144d, 0, 0, "Samsung PM1725", QUIRK_DELAY_B4_CHK_RDY }, 105 { 0xa822144d, 0, 0, "Samsung PM1725a", QUIRK_DELAY_B4_CHK_RDY }, 106 { 0x01161179, 0, 0, "Toshiba XG5", QUIRK_DISABLE_TIMEOUT }, 107 { 0x00000000, 0, 0, NULL } 108}; 109 110static int 111nvme_match(uint32_t devid, uint16_t subdevice, struct _pcsid *ep) 112{ 113 if (devid != ep->devid) 114 return 0; 115 116 if (!ep->match_subdevice) 117 return 1; 118 119 if (subdevice == ep->subdevice) 120 return 1; 121 else 122 return 0; 123} 124 125static int 126nvme_probe (device_t device) 127{ 128 struct _pcsid *ep; 129 uint32_t devid; 130 uint16_t subdevice; 131 132 devid = pci_get_devid(device); 133 subdevice = pci_get_subdevice(device); 134 ep = pci_ids; 135 136 while (ep->devid) { 137 if (nvme_match(devid, subdevice, ep)) 138 break; 139 ++ep; 140 } 141 142 if (ep->desc) { 143 device_set_desc(device, ep->desc); 144 return (BUS_PROBE_DEFAULT); 145 } 146 147#if defined(PCIS_STORAGE_NVM) 148 if (pci_get_class(device) == PCIC_STORAGE && 149 pci_get_subclass(device) == PCIS_STORAGE_NVM && 150 pci_get_progif(device) == PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0) { 151 device_set_desc(device, "Generic NVMe Device"); 152 return (BUS_PROBE_GENERIC); 153 } 154#endif 155 156 return (ENXIO); 157} 158 159static void 160nvme_init(void) 161{ 162 uint32_t i; 163 164 nvme_request_zone = uma_zcreate("nvme_request", 165 sizeof(struct nvme_request), NULL, NULL, NULL, NULL, 0, 0); 166 167 for (i = 0; i < NVME_MAX_CONSUMERS; i++) 168 nvme_consumer[i].id = INVALID_CONSUMER_ID; 169} 170 171SYSINIT(nvme_register, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_init, NULL); 172 173static void 174nvme_uninit(void) 175{ 176 uma_zdestroy(nvme_request_zone); 177} 178 179SYSUNINIT(nvme_unregister, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_uninit, NULL); 180 181static int 182nvme_shutdown(device_t dev) 183{ 184 struct nvme_controller *ctrlr; 185 186 ctrlr = DEVICE2SOFTC(dev); 187 nvme_ctrlr_shutdown(ctrlr); 188 189 return (0); 190} 191 192void 193nvme_dump_command(struct nvme_command *cmd) 194{ 195 printf( 196"opc:%x f:%x r1:%x cid:%x nsid:%x r2:%x r3:%x mptr:%jx prp1:%jx prp2:%jx cdw:%x %x %x %x %x %x\n", 197 cmd->opc, cmd->fuse, cmd->rsvd1, cmd->cid, cmd->nsid, 198 cmd->rsvd2, cmd->rsvd3, 199 (uintmax_t)cmd->mptr, (uintmax_t)cmd->prp1, (uintmax_t)cmd->prp2, 200 cmd->cdw10, cmd->cdw11, cmd->cdw12, cmd->cdw13, cmd->cdw14, 201 cmd->cdw15); 202} 203 204void 205nvme_dump_completion(struct nvme_completion *cpl) 206{ 207 printf("cdw0:%08x sqhd:%04x sqid:%04x " 208 "cid:%04x p:%x sc:%02x sct:%x m:%x dnr:%x\n", 209 cpl->cdw0, cpl->sqhd, cpl->sqid, 210 cpl->cid, cpl->status.p, cpl->status.sc, cpl->status.sct, 211 cpl->status.m, cpl->status.dnr); 212} 213 214static int 215nvme_attach(device_t dev) 216{ 217 struct nvme_controller *ctrlr = DEVICE2SOFTC(dev); 218 int status; 219 struct _pcsid *ep; 220 uint32_t devid; 221 uint16_t subdevice; 222 223 devid = pci_get_devid(dev); 224 subdevice = pci_get_subdevice(dev); 225 ep = pci_ids; 226 while (ep->devid) { 227 if (nvme_match(devid, subdevice, ep)) 228 break; 229 ++ep; 230 } 231 ctrlr->quirks = ep->quirks; 232 233 status = nvme_ctrlr_construct(ctrlr, dev); 234 235 if (status != 0) { 236 nvme_ctrlr_destruct(ctrlr, dev); 237 return (status); 238 } 239 240 /* 241 * Some drives do not implement the completion timeout feature 242 * correctly. There's a WAR from the manufacturer to just disable it. 243 * The driver wouldn't respond correctly to a timeout anyway. 244 */ 245 if (ep->quirks & QUIRK_DISABLE_TIMEOUT) { 246 int ptr; 247 uint16_t devctl2; 248 249 status = pci_find_cap(dev, PCIY_EXPRESS, &ptr); 250 if (status) { 251 device_printf(dev, "Can't locate PCIe capability?"); 252 return (status); 253 } 254 devctl2 = pci_read_config(dev, ptr + PCIER_DEVICE_CTL2, sizeof(devctl2)); 255 devctl2 |= PCIEM_CTL2_COMP_TIMO_DISABLE; 256 pci_write_config(dev, ptr + PCIER_DEVICE_CTL2, devctl2, sizeof(devctl2)); 257 } 258 259 /* 260 * Enable busmastering so the completion status messages can 261 * be busmastered back to the host. 262 */ 263 pci_enable_busmaster(dev); 264 265 /* 266 * Reset controller twice to ensure we do a transition from cc.en==1 267 * to cc.en==0. This is because we don't really know what status 268 * the controller was left in when boot handed off to OS. 269 */ 270 status = nvme_ctrlr_hw_reset(ctrlr); 271 if (status != 0) { 272 nvme_ctrlr_destruct(ctrlr, dev); 273 return (status); 274 } 275 276 status = nvme_ctrlr_hw_reset(ctrlr); 277 if (status != 0) { 278 nvme_ctrlr_destruct(ctrlr, dev); 279 return (status); 280 } 281 282 ctrlr->config_hook.ich_func = nvme_ctrlr_start_config_hook; 283 ctrlr->config_hook.ich_arg = ctrlr; 284 285 config_intrhook_establish(&ctrlr->config_hook); 286 287 return (0); 288} 289 290static int 291nvme_detach (device_t dev) 292{ 293 struct nvme_controller *ctrlr = DEVICE2SOFTC(dev); 294 295 nvme_ctrlr_destruct(ctrlr, dev); 296 pci_disable_busmaster(dev); 297 return (0); 298} 299 300static void 301nvme_notify(struct nvme_consumer *cons, 302 struct nvme_controller *ctrlr) 303{ 304 struct nvme_namespace *ns; 305 void *ctrlr_cookie; 306 int cmpset, ns_idx; 307 308 /* 309 * The consumer may register itself after the nvme devices 310 * have registered with the kernel, but before the 311 * driver has completed initialization. In that case, 312 * return here, and when initialization completes, the 313 * controller will make sure the consumer gets notified. 314 */ 315 if (!ctrlr->is_initialized) 316 return; 317 318 cmpset = atomic_cmpset_32(&ctrlr->notification_sent, 0, 1); 319 if (cmpset == 0) 320 return; 321 322 if (cons->ctrlr_fn != NULL) 323 ctrlr_cookie = (*cons->ctrlr_fn)(ctrlr); 324 else 325 ctrlr_cookie = (void *)(uintptr_t)0xdeadc0dedeadc0de; 326 ctrlr->cons_cookie[cons->id] = ctrlr_cookie; 327 328 /* ctrlr_fn has failed. Nothing to notify here any more. */ 329 if (ctrlr_cookie == NULL) 330 return; 331 332 if (ctrlr->is_failed) { 333 ctrlr->cons_cookie[cons->id] = NULL; 334 if (cons->fail_fn != NULL) 335 (*cons->fail_fn)(ctrlr_cookie); 336 /* 337 * Do not notify consumers about the namespaces of a 338 * failed controller. 339 */ 340 return; 341 } 342 for (ns_idx = 0; ns_idx < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); ns_idx++) { 343 ns = &ctrlr->ns[ns_idx]; 344 if (ns->data.nsze == 0) 345 continue; 346 if (cons->ns_fn != NULL) 347 ns->cons_cookie[cons->id] = 348 (*cons->ns_fn)(ns, ctrlr_cookie); 349 } 350} 351 352void 353nvme_notify_new_controller(struct nvme_controller *ctrlr) 354{ 355 int i; 356 357 for (i = 0; i < NVME_MAX_CONSUMERS; i++) { 358 if (nvme_consumer[i].id != INVALID_CONSUMER_ID) { 359 nvme_notify(&nvme_consumer[i], ctrlr); 360 } 361 } 362} 363 364static void 365nvme_notify_new_consumer(struct nvme_consumer *cons) 366{ 367 device_t *devlist; 368 struct nvme_controller *ctrlr; 369 int dev_idx, devcount; 370 371 if (devclass_get_devices(nvme_devclass, &devlist, &devcount)) 372 return; 373 374 for (dev_idx = 0; dev_idx < devcount; dev_idx++) { 375 ctrlr = DEVICE2SOFTC(devlist[dev_idx]); 376 nvme_notify(cons, ctrlr); 377 } 378 379 free(devlist, M_TEMP); 380} 381 382void 383nvme_notify_async_consumers(struct nvme_controller *ctrlr, 384 const struct nvme_completion *async_cpl, 385 uint32_t log_page_id, void *log_page_buffer, 386 uint32_t log_page_size) 387{ 388 struct nvme_consumer *cons; 389 void *ctrlr_cookie; 390 uint32_t i; 391 392 for (i = 0; i < NVME_MAX_CONSUMERS; i++) { 393 cons = &nvme_consumer[i]; 394 if (cons->id != INVALID_CONSUMER_ID && cons->async_fn != NULL && 395 (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) { 396 (*cons->async_fn)(ctrlr_cookie, async_cpl, 397 log_page_id, log_page_buffer, log_page_size); 398 } 399 } 400} 401 402void 403nvme_notify_fail_consumers(struct nvme_controller *ctrlr) 404{ 405 struct nvme_consumer *cons; 406 void *ctrlr_cookie; 407 uint32_t i; 408 409 /* 410 * This controller failed during initialization (i.e. IDENTIFY 411 * command failed or timed out). Do not notify any nvme 412 * consumers of the failure here, since the consumer does not 413 * even know about the controller yet. 414 */ 415 if (!ctrlr->is_initialized) 416 return; 417 418 for (i = 0; i < NVME_MAX_CONSUMERS; i++) { 419 cons = &nvme_consumer[i]; 420 if (cons->id != INVALID_CONSUMER_ID && 421 (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) { 422 ctrlr->cons_cookie[i] = NULL; 423 if (cons->fail_fn != NULL) 424 cons->fail_fn(ctrlr_cookie); 425 } 426 } 427} 428 429struct nvme_consumer * 430nvme_register_consumer(nvme_cons_ns_fn_t ns_fn, nvme_cons_ctrlr_fn_t ctrlr_fn, 431 nvme_cons_async_fn_t async_fn, 432 nvme_cons_fail_fn_t fail_fn) 433{ 434 int i; 435 436 /* 437 * TODO: add locking around consumer registration. Not an issue 438 * right now since we only have one nvme consumer - nvd(4). 439 */ 440 for (i = 0; i < NVME_MAX_CONSUMERS; i++) 441 if (nvme_consumer[i].id == INVALID_CONSUMER_ID) { 442 nvme_consumer[i].id = i; 443 nvme_consumer[i].ns_fn = ns_fn; 444 nvme_consumer[i].ctrlr_fn = ctrlr_fn; 445 nvme_consumer[i].async_fn = async_fn; 446 nvme_consumer[i].fail_fn = fail_fn; 447 448 nvme_notify_new_consumer(&nvme_consumer[i]); 449 return (&nvme_consumer[i]); 450 } 451 452 printf("nvme(4): consumer not registered - no slots available\n"); 453 return (NULL); 454} 455 456void 457nvme_unregister_consumer(struct nvme_consumer *consumer) 458{ 459 460 consumer->id = INVALID_CONSUMER_ID; 461} 462 463void 464nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl) 465{ 466 struct nvme_completion_poll_status *status = arg; 467 468 /* 469 * Copy status into the argument passed by the caller, so that 470 * the caller can check the status to determine if the 471 * the request passed or failed. 472 */ 473 memcpy(&status->cpl, cpl, sizeof(*cpl)); 474 atomic_store_rel_int(&status->done, 1); 475} 476