nvme.c revision 241658 
1/*-
2 * Copyright (C) 2012 Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/dev/nvme/nvme.c 241658 2012-10-18 00:37:11Z jimharris $");
29
30#include <sys/param.h>
31#include <sys/bus.h>
32#include <sys/conf.h>
33#include <sys/module.h>
34
35#include <dev/pci/pcireg.h>
36#include <dev/pci/pcivar.h>
37
38#include "nvme_private.h"
39
40struct nvme_consumer {
41	nvme_consumer_cb_fn_t		cb_fn;
42	void				*cb_arg;
43};
44
45struct nvme_consumer nvme_consumer[NVME_MAX_CONSUMERS];
46
47MALLOC_DEFINE(M_NVME, "nvme", "nvme(4) memory allocations");
48
49static int    nvme_probe(device_t);
50static int    nvme_attach(device_t);
51static int    nvme_detach(device_t);
52
53static devclass_t nvme_devclass;
54
55static device_method_t nvme_pci_methods[] = {
56	/* Device interface */
57	DEVMETHOD(device_probe,     nvme_probe),
58	DEVMETHOD(device_attach,    nvme_attach),
59	DEVMETHOD(device_detach,    nvme_detach),
60	{ 0, 0 }
61};
62
63static driver_t nvme_pci_driver = {
64	"nvme",
65	nvme_pci_methods,
66	sizeof(struct nvme_controller),
67};
68
69DRIVER_MODULE(nvme, pci, nvme_pci_driver, nvme_devclass, 0, 0);
70MODULE_VERSION(nvme, 1);
71
72static struct _pcsid
73{
74	u_int32_t   type;
75	const char  *desc;
76} pci_ids[] = {
77	{ 0x01118086,		"NVMe Controller"  },
78	{ CHATHAM_PCI_ID,	"Chatham Prototype NVMe Controller"  },
79	{ IDT_PCI_ID,		"IDT NVMe Controller"  },
80	{ 0x00000000,		NULL  }
81};
82
83static int
84nvme_probe (device_t device)
85{
86	struct _pcsid	*ep;
87	u_int32_t	type;
88
89	type = pci_get_devid(device);
90	ep = pci_ids;
91
92	while (ep->type && ep->type != type)
93		++ep;
94
95	if (ep->desc) {
96		device_set_desc(device, ep->desc);
97		return (BUS_PROBE_DEFAULT);
98	}
99
100#if defined(PCIS_STORAGE_NVM)
101	if (pci_get_class(device)    == PCIC_STORAGE &&
102	    pci_get_subclass(device) == PCIS_STORAGE_NVM &&
103	    pci_get_progif(device)   == PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0) {
104		device_set_desc(device, "Generic NVMe Device");
105		return (BUS_PROBE_GENERIC);
106	}
107#endif
108
109	return (ENXIO);
110}
111
112static void
113nvme_load(void)
114{
115}
116
117static void
118nvme_unload(void)
119{
120}
121
122static void
123nvme_shutdown(void)
124{
125	device_t		*devlist;
126	struct nvme_controller	*ctrlr;
127	union cc_register	cc;
128	union csts_register	csts;
129	int			dev, devcount;
130
131	if (devclass_get_devices(nvme_devclass, &devlist, &devcount))
132		return;
133
134	for (dev = 0; dev < devcount; dev++) {
135		/*
136		 * Only notify controller of shutdown when a real shutdown is
137		 *  in process, not when a module unload occurs.  It seems at
138		 *  least some controllers (Chatham at least) don't let you
139		 *  re-enable the controller after shutdown notification has
140		 *  been received.
141		 */
142		ctrlr = DEVICE2SOFTC(devlist[dev]);
143		cc.raw = nvme_mmio_read_4(ctrlr, cc);
144		cc.bits.shn = NVME_SHN_NORMAL;
145		nvme_mmio_write_4(ctrlr, cc, cc.raw);
146		csts.raw = nvme_mmio_read_4(ctrlr, csts);
147		while (csts.bits.shst != NVME_SHST_COMPLETE) {
148			DELAY(5);
149			csts.raw = nvme_mmio_read_4(ctrlr, csts);
150		}
151	}
152
153	free(devlist, M_TEMP);
154}
155
156static int
157nvme_modevent(module_t mod, int type, void *arg)
158{
159
160	switch (type) {
161	case MOD_LOAD:
162		nvme_load();
163		break;
164	case MOD_UNLOAD:
165		nvme_unload();
166		break;
167	case MOD_SHUTDOWN:
168		nvme_shutdown();
169		break;
170	default:
171		break;
172	}
173
174	return (0);
175}
176
177moduledata_t nvme_mod = {
178	"nvme",
179	(modeventhand_t)nvme_modevent,
180	0
181};
182
183DECLARE_MODULE(nvme, nvme_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST);
184
185void
186nvme_dump_command(struct nvme_command *cmd)
187{
188	printf("opc:%x f:%x r1:%x cid:%x nsid:%x r2:%x r3:%x "
189	    "mptr:%qx prp1:%qx prp2:%qx cdw:%x %x %x %x %x %x\n",
190	    cmd->opc, cmd->fuse, cmd->rsvd1, cmd->cid, cmd->nsid,
191	    cmd->rsvd2, cmd->rsvd3,
192	    (long long unsigned int)cmd->mptr,
193	    (long long unsigned int)cmd->prp1,
194	    (long long unsigned int)cmd->prp2,
195	    cmd->cdw10, cmd->cdw11, cmd->cdw12, cmd->cdw13, cmd->cdw14,
196	    cmd->cdw15);
197}
198
199void
200nvme_dump_completion(struct nvme_completion *cpl)
201{
202	printf("cdw0:%08x sqhd:%04x sqid:%04x "
203	    "cid:%04x p:%x sc:%02x sct:%x m:%x dnr:%x\n",
204	    cpl->cdw0, cpl->sqhd, cpl->sqid,
205	    cpl->cid, cpl->p, cpl->sf_sc, cpl->sf_sct, cpl->sf_m,
206	    cpl->sf_dnr);
207}
208
209void
210nvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
211{
212	struct nvme_tracker 	*tr;
213	struct nvme_qpair 	*qpair;
214	uint32_t		cur_nseg;
215
216	KASSERT(error == 0, ("nvme_payload_map error != 0\n"));
217
218	tr = (struct nvme_tracker *)arg;
219	qpair = tr->qpair;
220
221	/*
222	 * Note that we specified PAGE_SIZE for alignment and max
223	 *  segment size when creating the bus dma tags.  So here
224	 *  we can safely just transfer each segment to its
225	 *  associated PRP entry.
226	 */
227	tr->cmd.prp1 = seg[0].ds_addr;
228
229	if (nseg == 2) {
230		tr->cmd.prp2 = seg[1].ds_addr;
231	} else if (nseg > 2) {
232		cur_nseg = 1;
233		tr->cmd.prp2 = (uint64_t)tr->prp_bus_addr;
234		while (cur_nseg < nseg) {
235			tr->prp[cur_nseg-1] =
236			    (uint64_t)seg[cur_nseg].ds_addr;
237			cur_nseg++;
238		}
239	}
240
241	nvme_qpair_submit_cmd(qpair, tr);
242}
243
244struct nvme_tracker *
245nvme_allocate_tracker(struct nvme_controller *ctrlr, boolean_t is_admin,
246    nvme_cb_fn_t cb_fn, void *cb_arg, uint32_t payload_size, void *payload)
247{
248	struct nvme_tracker 	*tr;
249	struct nvme_qpair	*qpair;
250
251	if (is_admin) {
252		qpair = &ctrlr->adminq;
253	} else {
254		if (ctrlr->per_cpu_io_queues)
255			qpair = &ctrlr->ioq[curcpu];
256		else
257			qpair = &ctrlr->ioq[0];
258	}
259
260	tr = nvme_qpair_allocate_tracker(qpair);
261
262	if (tr == NULL)
263		return (NULL);
264
265	memset(&tr->cmd, 0, sizeof(tr->cmd));
266
267	tr->qpair = qpair;
268	tr->cb_fn = cb_fn;
269	tr->cb_arg = cb_arg;
270	tr->payload_size = payload_size;
271
272	return (tr);
273}
274
275static int
276nvme_attach(device_t dev)
277{
278	struct nvme_controller	*ctrlr = DEVICE2SOFTC(dev);
279	int			status;
280
281	status = nvme_ctrlr_construct(ctrlr, dev);
282
283	if (status != 0)
284		return (status);
285
286	/*
287	 * Reset controller twice to ensure we do a transition from cc.en==1
288	 *  to cc.en==0.  This is because we don't really know what status
289	 *  the controller was left in when boot handed off to OS.
290	 */
291	status = nvme_ctrlr_reset(ctrlr);
292	if (status != 0)
293		return (status);
294
295	status = nvme_ctrlr_reset(ctrlr);
296	if (status != 0)
297		return (status);
298
299	ctrlr->config_hook.ich_func = nvme_ctrlr_start;
300	ctrlr->config_hook.ich_arg = ctrlr;
301
302	config_intrhook_establish(&ctrlr->config_hook);
303
304	return (0);
305}
306
307static int
308nvme_detach (device_t dev)
309{
310	struct nvme_controller	*ctrlr = DEVICE2SOFTC(dev);
311	struct nvme_namespace	*ns;
312	int			i;
313
314	if (ctrlr->taskqueue) {
315		taskqueue_drain(ctrlr->taskqueue, &ctrlr->task);
316		taskqueue_free(ctrlr->taskqueue);
317	}
318
319	for (i = 0; i < NVME_MAX_NAMESPACES; i++) {
320		ns = &ctrlr->ns[i];
321		if (ns->cdev)
322			destroy_dev(ns->cdev);
323	}
324
325	if (ctrlr->cdev)
326		destroy_dev(ctrlr->cdev);
327
328	for (i = 0; i < ctrlr->num_io_queues; i++) {
329		nvme_io_qpair_destroy(&ctrlr->ioq[i]);
330	}
331
332	free(ctrlr->ioq, M_NVME);
333
334	nvme_admin_qpair_destroy(&ctrlr->adminq);
335
336	if (ctrlr->resource != NULL) {
337		bus_release_resource(dev, SYS_RES_MEMORY,
338		    ctrlr->resource_id, ctrlr->resource);
339	}
340
341#ifdef CHATHAM2
342	if (ctrlr->chatham_resource != NULL) {
343		bus_release_resource(dev, SYS_RES_MEMORY,
344		    ctrlr->chatham_resource_id, ctrlr->chatham_resource);
345	}
346#endif
347
348	if (ctrlr->tag)
349		bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag);
350
351	if (ctrlr->res)
352		bus_release_resource(ctrlr->dev, SYS_RES_IRQ,
353		    rman_get_rid(ctrlr->res), ctrlr->res);
354
355	if (ctrlr->msix_enabled)
356		pci_release_msi(dev);
357
358	return (0);
359}
360
361static void
362nvme_notify_consumer(struct nvme_consumer *consumer)
363{
364	device_t		*devlist;
365	struct nvme_controller	*ctrlr;
366	int			dev, ns, devcount;
367
368	if (devclass_get_devices(nvme_devclass, &devlist, &devcount))
369		return;
370
371	for (dev = 0; dev < devcount; dev++) {
372		ctrlr = DEVICE2SOFTC(devlist[dev]);
373		for (ns = 0; ns < ctrlr->cdata.nn; ns++)
374			(*consumer->cb_fn)(consumer->cb_arg, &ctrlr->ns[ns]);
375	}
376
377	free(devlist, M_TEMP);
378}
379
380struct nvme_consumer *
381nvme_register_consumer(nvme_consumer_cb_fn_t cb_fn, void *cb_arg)
382{
383	int i;
384
385	/*
386	 * TODO: add locking around consumer registration.  Not an issue
387	 *  right now since we only have one nvme consumer - nvd(4).
388	 */
389	for (i = 0; i < NVME_MAX_CONSUMERS; i++)
390		if (nvme_consumer[i].cb_fn == NULL) {
391			nvme_consumer[i].cb_fn = cb_fn;
392			nvme_consumer[i].cb_arg = cb_arg;
393
394			nvme_notify_consumer(&nvme_consumer[i]);
395			return (&nvme_consumer[i]);
396		}
397
398	printf("nvme(4): consumer not registered - no slots available\n");
399	return (NULL);
400}
401
402void
403nvme_unregister_consumer(struct nvme_consumer *consumer)
404{
405
406	consumer->cb_fn = NULL;
407	consumer->cb_arg = NULL;
408}
409
410