140516Swpaul/*- 2117388Swpaul * SPDX-License-Identifier: BSD-2-Clause 340516Swpaul * 440516Swpaul * Copyright (C) 2012-2014 Intel Corporation 540516Swpaul * All rights reserved. 640516Swpaul * 740516Swpaul * Redistribution and use in source and binary forms, with or without 840516Swpaul * modification, are permitted provided that the following conditions 940516Swpaul * are met: 1040516Swpaul * 1. Redistributions of source code must retain the above copyright 1140516Swpaul * notice, this list of conditions and the following disclaimer. 1240516Swpaul * 2. Redistributions in binary form must reproduce the above copyright 1340516Swpaul * notice, this list of conditions and the following disclaimer in the 1440516Swpaul * documentation and/or other materials provided with the distribution. 1540516Swpaul * 1640516Swpaul * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1740516Swpaul * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1840516Swpaul * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1940516Swpaul * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2040516Swpaul * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2140516Swpaul * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2240516Swpaul * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2340516Swpaul * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2440516Swpaul * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2540516Swpaul * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2640516Swpaul * SUCH DAMAGE. 2740516Swpaul */ 2840516Swpaul 2940516Swpaul#include <sys/param.h> 3040516Swpaul#include <sys/bus.h> 3140516Swpaul#include <sys/conf.h> 3240516Swpaul#include <sys/module.h> 3340516Swpaul 34117388Swpaul#include <vm/uma.h> 3540516Swpaul 36117388Swpaul#include "nvme_private.h" 37117388Swpaul 3840516Swpaulstruct nvme_consumer { 3940516Swpaul uint32_t id; 40117388Swpaul nvme_cons_ns_fn_t ns_fn; 41117388Swpaul nvme_cons_ctrlr_fn_t ctrlr_fn; 42117388Swpaul nvme_cons_async_fn_t async_fn; 4340516Swpaul nvme_cons_fail_fn_t fail_fn; 4440516Swpaul}; 4540516Swpaul 4640516Swpaulstruct nvme_consumer nvme_consumer[NVME_MAX_CONSUMERS]; 4740516Swpaul#define INVALID_CONSUMER_ID 0xFFFF 4840516Swpaul 4940516Swpaulint32_t nvme_retry_count; 5040516Swpaul 5140516SwpaulMALLOC_DEFINE(M_NVME, "nvme", "nvme(4) memory allocations"); 5240516Swpaul 5340516Swpaulstatic void 5441569Swpaulnvme_init(void) 5540516Swpaul{ 5640516Swpaul uint32_t i; 5740516Swpaul 5840516Swpaul for (i = 0; i < NVME_MAX_CONSUMERS; i++) 5940516Swpaul nvme_consumer[i].id = INVALID_CONSUMER_ID; 6040516Swpaul} 6140516Swpaul 6240516SwpaulSYSINIT(nvme_register, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_init, NULL); 6340516Swpaul 6440516Swpaulstatic void 6540516Swpaulnvme_uninit(void) 6640516Swpaul{ 6740516Swpaul} 6840516Swpaul 6940516SwpaulSYSUNINIT(nvme_unregister, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_uninit, NULL); 7040516Swpaul 7140516Swpaulint 7240516Swpaulnvme_shutdown(device_t dev) 7340516Swpaul{ 7440516Swpaul struct nvme_controller *ctrlr; 7540516Swpaul 7640516Swpaul ctrlr = DEVICE2SOFTC(dev); 7740516Swpaul nvme_ctrlr_shutdown(ctrlr); 7840516Swpaul 7940516Swpaul return (0); 8040516Swpaul} 8140516Swpaul 82117388Swpaulint 83117388Swpaulnvme_attach(device_t dev) 84117388Swpaul{ 85117388Swpaul struct nvme_controller *ctrlr = DEVICE2SOFTC(dev); 86117388Swpaul int status; 87117388Swpaul 88117388Swpaul status = nvme_ctrlr_construct(ctrlr, dev); 89117388Swpaul if (status != 0) { 90117388Swpaul nvme_ctrlr_destruct(ctrlr, dev); 91117388Swpaul return (status); 92117388Swpaul } 93117388Swpaul 94117388Swpaul ctrlr->config_hook.ich_func = nvme_ctrlr_start_config_hook; 95117388Swpaul ctrlr->config_hook.ich_arg = ctrlr; 96117388Swpaul 97117388Swpaul if (config_intrhook_establish(&ctrlr->config_hook) != 0) 98117388Swpaul return (ENOMEM); 99117388Swpaul 100117388Swpaul return (0); 101117388Swpaul} 102117388Swpaul 103117388Swpaulint 104117388Swpaulnvme_detach(device_t dev) 105117388Swpaul{ 106117388Swpaul struct nvme_controller *ctrlr = DEVICE2SOFTC(dev); 107117388Swpaul 108117388Swpaul config_intrhook_drain(&ctrlr->config_hook); 109117388Swpaul 110117388Swpaul nvme_ctrlr_destruct(ctrlr, dev); 111117388Swpaul return (0); 112117388Swpaul} 113117388Swpaul 114117388Swpaulstatic void 115117388Swpaulnvme_notify(struct nvme_consumer *cons, 116117388Swpaul struct nvme_controller *ctrlr) 117117388Swpaul{ 118117388Swpaul struct nvme_namespace *ns; 119117388Swpaul void *ctrlr_cookie; 120117388Swpaul int cmpset, ns_idx; 121117388Swpaul 122117388Swpaul /* 123117388Swpaul * The consumer may register itself after the nvme devices 12440516Swpaul * have registered with the kernel, but before the 12540516Swpaul * driver has completed initialization. In that case, 126116192Sobrien * return here, and when initialization completes, the 127116192Sobrien * controller will make sure the consumer gets notified. 128116192Sobrien */ 12940516Swpaul if (!ctrlr->is_initialized) 130108729Sjake return; 13140516Swpaul 13240516Swpaul cmpset = atomic_cmpset_32(&ctrlr->notification_sent, 0, 1); 13340516Swpaul if (cmpset == 0) 13440516Swpaul return; 13540516Swpaul 13640516Swpaul if (cons->ctrlr_fn != NULL) 13740516Swpaul ctrlr_cookie = (*cons->ctrlr_fn)(ctrlr); 13840516Swpaul else 13940516Swpaul ctrlr_cookie = (void *)(uintptr_t)0xdeadc0dedeadc0de; 14040516Swpaul ctrlr->cons_cookie[cons->id] = ctrlr_cookie; 14140516Swpaul 14240516Swpaul /* ctrlr_fn has failed. Nothing to notify here any more. */ 143117388Swpaul if (ctrlr_cookie == NULL) { 14440516Swpaul (void)atomic_cmpset_32(&ctrlr->notification_sent, 1, 0); 14540516Swpaul return; 14640516Swpaul } 14741569Swpaul 14841569Swpaul if (ctrlr->is_failed) { 14941569Swpaul ctrlr->cons_cookie[cons->id] = NULL; 15050703Swpaul if (cons->fail_fn != NULL) 15150703Swpaul (*cons->fail_fn)(ctrlr_cookie); 15250703Swpaul /* 15340516Swpaul * Do not notify consumers about the namespaces of a 15450703Swpaul * failed controller. 15550703Swpaul */ 15650703Swpaul return; 15740516Swpaul } 15840516Swpaul for (ns_idx = 0; ns_idx < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); ns_idx++) { 15940516Swpaul ns = &ctrlr->ns[ns_idx]; 160113506Smdodd if (ns->data.nsze == 0) 161113506Smdodd continue; 16259758Speter if (cons->ns_fn != NULL) 16359758Speter ns->cons_cookie[cons->id] = 16451089Speter (*cons->ns_fn)(ns, ctrlr_cookie); 16550703Swpaul } 16650703Swpaul} 16740516Swpaul 16840516Swpaulvoid 16940516Swpaulnvme_notify_new_controller(struct nvme_controller *ctrlr) 17040516Swpaul{ 17140516Swpaul int i; 17240516Swpaul 17340516Swpaul for (i = 0; i < NVME_MAX_CONSUMERS; i++) { 17440516Swpaul if (nvme_consumer[i].id != INVALID_CONSUMER_ID) { 17540516Swpaul nvme_notify(&nvme_consumer[i], ctrlr); 17640516Swpaul } 17740516Swpaul } 17840516Swpaul} 179109109Sdes 18040516Swpaulstatic void 181117388Swpaulnvme_notify_new_consumer(struct nvme_consumer *cons) 182117388Swpaul{ 18340516Swpaul device_t *devlist; 18440516Swpaul struct nvme_controller *ctrlr; 18540516Swpaul int dev_idx, devcount; 18640516Swpaul 187117388Swpaul if (devclass_get_devices(devclass_find("nvme"), &devlist, &devcount)) 18840516Swpaul return; 189117388Swpaul 19040516Swpaul for (dev_idx = 0; dev_idx < devcount; dev_idx++) { 191117388Swpaul ctrlr = DEVICE2SOFTC(devlist[dev_idx]); 19267771Swpaul nvme_notify(cons, ctrlr); 193117388Swpaul } 19441243Swpaul 195117388Swpaul free(devlist, M_TEMP); 19644238Swpaul} 197117388Swpaul 19844238Swpaulvoid 199117388Swpaulnvme_notify_async_consumers(struct nvme_controller *ctrlr, 20072813Swpaul const struct nvme_completion *async_cpl, 201117388Swpaul uint32_t log_page_id, void *log_page_buffer, 20296112Sjhb uint32_t log_page_size) 203117388Swpaul{ 20494400Swpaul struct nvme_consumer *cons; 205117388Swpaul void *ctrlr_cookie; 206103020Siwasaki uint32_t i; 207117388Swpaul 208109095Ssanpei for (i = 0; i < NVME_MAX_CONSUMERS; i++) { 209117388Swpaul cons = &nvme_consumer[i]; 210117388Swpaul if (cons->id != INVALID_CONSUMER_ID && cons->async_fn != NULL && 211111381Sdan (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) { 212117388Swpaul (*cons->async_fn)(ctrlr_cookie, async_cpl, 213112379Ssanpei log_page_id, log_page_buffer, log_page_size); 214117388Swpaul } 215117388Swpaul } 216117388Swpaul} 217117388Swpaul 218117388Swpaulvoid 219117388Swpaulnvme_notify_fail_consumers(struct nvme_controller *ctrlr) 220117388Swpaul{ 221117388Swpaul struct nvme_consumer *cons; 222117388Swpaul void *ctrlr_cookie; 223117388Swpaul uint32_t i; 224117388Swpaul 225117388Swpaul /* 226117388Swpaul * This controller failed during initialization (i.e. IDENTIFY 227117388Swpaul * command failed or timed out). Do not notify any nvme 228117388Swpaul * consumers of the failure here, since the consumer does not 229117388Swpaul * even know about the controller yet. 230117388Swpaul */ 231117388Swpaul if (!ctrlr->is_initialized) 23240516Swpaul return; 23340516Swpaul 23440516Swpaul for (i = 0; i < NVME_MAX_CONSUMERS; i++) { 23592739Salfred cons = &nvme_consumer[i]; 23692739Salfred if (cons->id != INVALID_CONSUMER_ID && 23792739Salfred (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) { 23840516Swpaul ctrlr->cons_cookie[i] = NULL; 239117388Swpaul if (cons->fail_fn != NULL) 240117388Swpaul cons->fail_fn(ctrlr_cookie); 24140516Swpaul } 242117388Swpaul } 243117388Swpaul} 244117388Swpaul 245117388Swpaulvoid 246117388Swpaulnvme_notify_ns(struct nvme_controller *ctrlr, int nsid) 247117388Swpaul{ 248117388Swpaul struct nvme_consumer *cons; 249117388Swpaul struct nvme_namespace *ns; 25092739Salfred void *ctrlr_cookie; 251117388Swpaul uint32_t i; 25292739Salfred 253117388Swpaul KASSERT(nsid <= NVME_MAX_NAMESPACES, 25492739Salfred ("%s: Namespace notification to nsid %d exceeds range\n", 255117388Swpaul device_get_nameunit(ctrlr->dev), nsid)); 25692739Salfred 25792739Salfred if (!ctrlr->is_initialized) 258117388Swpaul return; 25992739Salfred 26092739Salfred ns = &ctrlr->ns[nsid - 1]; 26192739Salfred for (i = 0; i < NVME_MAX_CONSUMERS; i++) { 26292739Salfred cons = &nvme_consumer[i]; 26392739Salfred if (cons->id != INVALID_CONSUMER_ID && cons->ns_fn != NULL && 26492739Salfred (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) 26592739Salfred ns->cons_cookie[i] = (*cons->ns_fn)(ns, ctrlr_cookie); 26692739Salfred } 26792739Salfred} 26840516Swpaul 26992739Salfredstruct nvme_consumer * 27092739Salfrednvme_register_consumer(nvme_cons_ns_fn_t ns_fn, nvme_cons_ctrlr_fn_t ctrlr_fn, 27192739Salfred nvme_cons_async_fn_t async_fn, 27292739Salfred nvme_cons_fail_fn_t fail_fn) 27392739Salfred{ 27492739Salfred int i; 27592739Salfred 27640516Swpaul /* 27792739Salfred * TODO: add locking around consumer registration. 27892739Salfred */ 27992739Salfred for (i = 0; i < NVME_MAX_CONSUMERS; i++) 28040516Swpaul if (nvme_consumer[i].id == INVALID_CONSUMER_ID) { 28192739Salfred nvme_consumer[i].id = i; 28292739Salfred nvme_consumer[i].ns_fn = ns_fn; 28392739Salfred nvme_consumer[i].ctrlr_fn = ctrlr_fn; 28492739Salfred nvme_consumer[i].async_fn = async_fn; 28540516Swpaul nvme_consumer[i].fail_fn = fail_fn; 28692739Salfred 28792739Salfred nvme_notify_new_consumer(&nvme_consumer[i]); 28881713Swpaul return (&nvme_consumer[i]); 28950703Swpaul } 29050703Swpaul 29150703Swpaul printf("nvme(4): consumer not registered - no slots available\n"); 29250703Swpaul return (NULL); 29350703Swpaul} 29450703Swpaul 29550703Swpaulvoid 29650703Swpaulnvme_unregister_consumer(struct nvme_consumer *consumer) 29750703Swpaul{ 29850703Swpaul 29950703Swpaul consumer->id = INVALID_CONSUMER_ID; 30050703Swpaul} 30150703Swpaul 30286822Siwasakivoid 30386822Siwasakinvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl) 30450703Swpaul{ 30550703Swpaul struct nvme_completion_poll_status *status = arg; 30650703Swpaul 30750703Swpaul /* 30850703Swpaul * Copy status into the argument passed by the caller, so that 30950703Swpaul * the caller can check the status to determine if the 31050703Swpaul * the request passed or failed. 31150703Swpaul */ 31250703Swpaul memcpy(&status->cpl, cpl, sizeof(*cpl)); 31350703Swpaul atomic_store_rel_int(&status->done, 1); 31450703Swpaul} 31550703Swpaul 31650703Swpaulstatic int 31750703Swpaulnvme_modevent(module_t mod __unused, int type __unused, void *argp __unused) 31850703Swpaul{ 31951455Swpaul return (0); 32050703Swpaul} 32150703Swpaul 32250703Swpaulstatic moduledata_t nvme_mod = { 32350703Swpaul "nvme", 32450703Swpaul nvme_modevent, 32550703Swpaul 0 326113506Smdodd}; 327113506Smdodd 32851473SwpaulDECLARE_MODULE(nvme, nvme_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST); 32950703SwpaulMODULE_VERSION(nvme, 1); 33040516SwpaulMODULE_DEPEND(nvme, cam, 1, 1, 1); 33140516Swpaul