nvme.c revision 254302 
1/*-
2 * Copyright (C) 2012-2013 Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/dev/nvme/nvme.c 254302 2013-08-13 21:47:08Z jimharris $");
29
30#include <sys/param.h>
31#include <sys/bus.h>
32#include <sys/conf.h>
33#include <sys/module.h>
34
35#include <vm/uma.h>
36
37#include <dev/pci/pcireg.h>
38#include <dev/pci/pcivar.h>
39
40#include "nvme_private.h"
41
42struct nvme_consumer {
43	uint32_t		id;
44	nvme_cons_ns_fn_t	ns_fn;
45	nvme_cons_ctrlr_fn_t	ctrlr_fn;
46	nvme_cons_async_fn_t	async_fn;
47	nvme_cons_fail_fn_t	fail_fn;
48};
49
50struct nvme_consumer nvme_consumer[NVME_MAX_CONSUMERS];
51#define	INVALID_CONSUMER_ID	0xFFFF
52
53uma_zone_t	nvme_request_zone;
54int32_t		nvme_retry_count;
55
56MALLOC_DEFINE(M_NVME, "nvme", "nvme(4) memory allocations");
57
58static int    nvme_probe(device_t);
59static int    nvme_attach(device_t);
60static int    nvme_detach(device_t);
61static int    nvme_modevent(module_t mod, int type, void *arg);
62
63static devclass_t nvme_devclass;
64
65static device_method_t nvme_pci_methods[] = {
66	/* Device interface */
67	DEVMETHOD(device_probe,     nvme_probe),
68	DEVMETHOD(device_attach,    nvme_attach),
69	DEVMETHOD(device_detach,    nvme_detach),
70	{ 0, 0 }
71};
72
73static driver_t nvme_pci_driver = {
74	"nvme",
75	nvme_pci_methods,
76	sizeof(struct nvme_controller),
77};
78
79DRIVER_MODULE(nvme, pci, nvme_pci_driver, nvme_devclass, nvme_modevent, 0);
80MODULE_VERSION(nvme, 1);
81
82static struct _pcsid
83{
84	u_int32_t   type;
85	const char  *desc;
86} pci_ids[] = {
87	{ 0x01118086,		"NVMe Controller"  },
88	{ CHATHAM_PCI_ID,	"Chatham Prototype NVMe Controller"  },
89	{ IDT32_PCI_ID,		"IDT NVMe Controller (32 channel)"  },
90	{ IDT8_PCI_ID,		"IDT NVMe Controller (8 channel)" },
91	{ 0x00000000,		NULL  }
92};
93
94static int
95nvme_probe (device_t device)
96{
97	struct _pcsid	*ep;
98	u_int32_t	type;
99
100	type = pci_get_devid(device);
101	ep = pci_ids;
102
103	while (ep->type && ep->type != type)
104		++ep;
105
106	if (ep->desc) {
107		device_set_desc(device, ep->desc);
108		return (BUS_PROBE_DEFAULT);
109	}
110
111#if defined(PCIS_STORAGE_NVM)
112	if (pci_get_class(device)    == PCIC_STORAGE &&
113	    pci_get_subclass(device) == PCIS_STORAGE_NVM &&
114	    pci_get_progif(device)   == PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0) {
115		device_set_desc(device, "Generic NVMe Device");
116		return (BUS_PROBE_GENERIC);
117	}
118#endif
119
120	return (ENXIO);
121}
122
123static void
124nvme_init(void)
125{
126	uint32_t	i;
127
128	nvme_request_zone = uma_zcreate("nvme_request",
129	    sizeof(struct nvme_request), NULL, NULL, NULL, NULL, 0, 0);
130
131	for (i = 0; i < NVME_MAX_CONSUMERS; i++)
132		nvme_consumer[i].id = INVALID_CONSUMER_ID;
133}
134
135SYSINIT(nvme_register, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_init, NULL);
136
137static void
138nvme_uninit(void)
139{
140	uma_zdestroy(nvme_request_zone);
141}
142
143SYSUNINIT(nvme_unregister, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_uninit, NULL);
144
145static void
146nvme_load(void)
147{
148}
149
150static void
151nvme_unload(void)
152{
153}
154
155static void
156nvme_shutdown(void)
157{
158	device_t		*devlist;
159	struct nvme_controller	*ctrlr;
160	int			dev, devcount;
161
162	if (devclass_get_devices(nvme_devclass, &devlist, &devcount))
163		return;
164
165	for (dev = 0; dev < devcount; dev++) {
166		ctrlr = DEVICE2SOFTC(devlist[dev]);
167		nvme_ctrlr_shutdown(ctrlr);
168	}
169
170	free(devlist, M_TEMP);
171}
172
173static int
174nvme_modevent(module_t mod, int type, void *arg)
175{
176
177	switch (type) {
178	case MOD_LOAD:
179		nvme_load();
180		break;
181	case MOD_UNLOAD:
182		nvme_unload();
183		break;
184	case MOD_SHUTDOWN:
185		nvme_shutdown();
186		break;
187	default:
188		break;
189	}
190
191	return (0);
192}
193
194void
195nvme_dump_command(struct nvme_command *cmd)
196{
197	printf(
198"opc:%x f:%x r1:%x cid:%x nsid:%x r2:%x r3:%x mptr:%jx prp1:%jx prp2:%jx cdw:%x %x %x %x %x %x\n",
199	    cmd->opc, cmd->fuse, cmd->rsvd1, cmd->cid, cmd->nsid,
200	    cmd->rsvd2, cmd->rsvd3,
201	    (uintmax_t)cmd->mptr, (uintmax_t)cmd->prp1, (uintmax_t)cmd->prp2,
202	    cmd->cdw10, cmd->cdw11, cmd->cdw12, cmd->cdw13, cmd->cdw14,
203	    cmd->cdw15);
204}
205
206void
207nvme_dump_completion(struct nvme_completion *cpl)
208{
209	printf("cdw0:%08x sqhd:%04x sqid:%04x "
210	    "cid:%04x p:%x sc:%02x sct:%x m:%x dnr:%x\n",
211	    cpl->cdw0, cpl->sqhd, cpl->sqid,
212	    cpl->cid, cpl->status.p, cpl->status.sc, cpl->status.sct,
213	    cpl->status.m, cpl->status.dnr);
214}
215
216static int
217nvme_attach(device_t dev)
218{
219	struct nvme_controller	*ctrlr = DEVICE2SOFTC(dev);
220	int			status;
221
222	status = nvme_ctrlr_construct(ctrlr, dev);
223
224	if (status != 0)
225		return (status);
226
227	/*
228	 * Reset controller twice to ensure we do a transition from cc.en==1
229	 *  to cc.en==0.  This is because we don't really know what status
230	 *  the controller was left in when boot handed off to OS.
231	 */
232	status = nvme_ctrlr_hw_reset(ctrlr);
233	if (status != 0)
234		return (status);
235
236	status = nvme_ctrlr_hw_reset(ctrlr);
237	if (status != 0)
238		return (status);
239
240	nvme_sysctl_initialize_ctrlr(ctrlr);
241
242	pci_enable_busmaster(dev);
243
244	ctrlr->config_hook.ich_func = nvme_ctrlr_start_config_hook;
245	ctrlr->config_hook.ich_arg = ctrlr;
246
247	config_intrhook_establish(&ctrlr->config_hook);
248
249	return (0);
250}
251
252static int
253nvme_detach (device_t dev)
254{
255	struct nvme_controller	*ctrlr = DEVICE2SOFTC(dev);
256
257	nvme_ctrlr_destruct(ctrlr, dev);
258	pci_disable_busmaster(dev);
259	return (0);
260}
261
262static void
263nvme_notify_consumer(struct nvme_consumer *cons)
264{
265	device_t		*devlist;
266	struct nvme_controller	*ctrlr;
267	struct nvme_namespace	*ns;
268	void			*ctrlr_cookie;
269	int			dev_idx, ns_idx, devcount;
270
271	if (devclass_get_devices(nvme_devclass, &devlist, &devcount))
272		return;
273
274	for (dev_idx = 0; dev_idx < devcount; dev_idx++) {
275		ctrlr = DEVICE2SOFTC(devlist[dev_idx]);
276		if (cons->ctrlr_fn != NULL)
277			ctrlr_cookie = (*cons->ctrlr_fn)(ctrlr);
278		else
279			ctrlr_cookie = NULL;
280		ctrlr->cons_cookie[cons->id] = ctrlr_cookie;
281		for (ns_idx = 0; ns_idx < ctrlr->cdata.nn; ns_idx++) {
282			ns = &ctrlr->ns[ns_idx];
283			if (cons->ns_fn != NULL)
284				ns->cons_cookie[cons->id] =
285				    (*cons->ns_fn)(ns, ctrlr_cookie);
286		}
287	}
288
289	free(devlist, M_TEMP);
290}
291
292void
293nvme_notify_async_consumers(struct nvme_controller *ctrlr,
294			    const struct nvme_completion *async_cpl,
295			    uint32_t log_page_id, void *log_page_buffer,
296			    uint32_t log_page_size)
297{
298	struct nvme_consumer	*cons;
299	uint32_t		i;
300
301	for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
302		cons = &nvme_consumer[i];
303		if (cons->id != INVALID_CONSUMER_ID && cons->async_fn != NULL)
304			(*cons->async_fn)(ctrlr->cons_cookie[i], async_cpl,
305			    log_page_id, log_page_buffer, log_page_size);
306	}
307}
308
309void
310nvme_notify_fail_consumers(struct nvme_controller *ctrlr)
311{
312	struct nvme_consumer	*cons;
313	uint32_t		i;
314
315	for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
316		cons = &nvme_consumer[i];
317		if (cons->id != INVALID_CONSUMER_ID && cons->fail_fn != NULL)
318			cons->fail_fn(ctrlr->cons_cookie[i]);
319	}
320}
321
322struct nvme_consumer *
323nvme_register_consumer(nvme_cons_ns_fn_t ns_fn, nvme_cons_ctrlr_fn_t ctrlr_fn,
324		       nvme_cons_async_fn_t async_fn,
325		       nvme_cons_fail_fn_t fail_fn)
326{
327	int i;
328
329	/*
330	 * TODO: add locking around consumer registration.  Not an issue
331	 *  right now since we only have one nvme consumer - nvd(4).
332	 */
333	for (i = 0; i < NVME_MAX_CONSUMERS; i++)
334		if (nvme_consumer[i].id == INVALID_CONSUMER_ID) {
335			nvme_consumer[i].id = i;
336			nvme_consumer[i].ns_fn = ns_fn;
337			nvme_consumer[i].ctrlr_fn = ctrlr_fn;
338			nvme_consumer[i].async_fn = async_fn;
339			nvme_consumer[i].fail_fn = fail_fn;
340
341			nvme_notify_consumer(&nvme_consumer[i]);
342			return (&nvme_consumer[i]);
343		}
344
345	printf("nvme(4): consumer not registered - no slots available\n");
346	return (NULL);
347}
348
349void
350nvme_unregister_consumer(struct nvme_consumer *consumer)
351{
352
353	consumer->id = INVALID_CONSUMER_ID;
354}
355
356void
357nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl)
358{
359	struct nvme_completion_poll_status	*status = arg;
360
361	/*
362	 * Copy status into the argument passed by the caller, so that
363	 *  the caller can check the status to determine if the
364	 *  the request passed or failed.
365	 */
366	memcpy(&status->cpl, cpl, sizeof(*cpl));
367	wmb();
368	status->done = TRUE;
369}
370