1/*-
2 * Copyright (C) 2012-2014 Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: stable/11/sys/dev/nvme/nvme.c 351586 2019-08-28 20:58:24Z mav $");
29
30#include <sys/param.h>
31#include <sys/bus.h>
32#include <sys/conf.h>
33#include <sys/module.h>
34
35#include <vm/uma.h>
36
37#include <dev/pci/pcireg.h>
38#include <dev/pci/pcivar.h>
39
40#include "nvme_private.h"
41
42struct nvme_consumer {
43	uint32_t		id;
44	nvme_cons_ns_fn_t	ns_fn;
45	nvme_cons_ctrlr_fn_t	ctrlr_fn;
46	nvme_cons_async_fn_t	async_fn;
47	nvme_cons_fail_fn_t	fail_fn;
48};
49
50struct nvme_consumer nvme_consumer[NVME_MAX_CONSUMERS];
51#define	INVALID_CONSUMER_ID	0xFFFF
52
53uma_zone_t	nvme_request_zone;
54int32_t		nvme_retry_count;
55
56MALLOC_DEFINE(M_NVME, "nvme", "nvme(4) memory allocations");
57
58static int    nvme_probe(device_t);
59static int    nvme_attach(device_t);
60static int    nvme_detach(device_t);
61static int    nvme_shutdown(device_t);
62
63static devclass_t nvme_devclass;
64
65static device_method_t nvme_pci_methods[] = {
66	/* Device interface */
67	DEVMETHOD(device_probe,     nvme_probe),
68	DEVMETHOD(device_attach,    nvme_attach),
69	DEVMETHOD(device_detach,    nvme_detach),
70	DEVMETHOD(device_shutdown,  nvme_shutdown),
71	{ 0, 0 }
72};
73
74static driver_t nvme_pci_driver = {
75	"nvme",
76	nvme_pci_methods,
77	sizeof(struct nvme_controller),
78};
79
80DRIVER_MODULE(nvme, pci, nvme_pci_driver, nvme_devclass, NULL, NULL);
81MODULE_VERSION(nvme, 1);
82MODULE_DEPEND(nvme, cam, 1, 1, 1);
83
84static struct _pcsid
85{
86	uint32_t	devid;
87	int		match_subdevice;
88	uint16_t	subdevice;
89	const char	*desc;
90	uint32_t	quirks;
91} pci_ids[] = {
92	{ 0x01118086,		0, 0, "NVMe Controller"  },
93	{ IDT32_PCI_ID,		0, 0, "IDT NVMe Controller (32 channel)"  },
94	{ IDT8_PCI_ID,		0, 0, "IDT NVMe Controller (8 channel)" },
95	{ 0x09538086,		1, 0x3702, "DC P3700 SSD" },
96	{ 0x09538086,		1, 0x3703, "DC P3700 SSD [2.5\" SFF]" },
97	{ 0x09538086,		1, 0x3704, "DC P3500 SSD [Add-in Card]" },
98	{ 0x09538086,		1, 0x3705, "DC P3500 SSD [2.5\" SFF]" },
99	{ 0x09538086,		1, 0x3709, "DC P3600 SSD [Add-in Card]" },
100	{ 0x09538086,		1, 0x370a, "DC P3600 SSD [2.5\" SFF]" },
101	{ 0x00031c58,		0, 0, "HGST SN100",	QUIRK_DELAY_B4_CHK_RDY },
102	{ 0x00231c58,		0, 0, "WDC SN200",	QUIRK_DELAY_B4_CHK_RDY },
103	{ 0x05401c5f,		0, 0, "Memblaze Pblaze4", QUIRK_DELAY_B4_CHK_RDY },
104	{ 0xa821144d,		0, 0, "Samsung PM1725", QUIRK_DELAY_B4_CHK_RDY },
105	{ 0xa822144d,		0, 0, "Samsung PM1725a", QUIRK_DELAY_B4_CHK_RDY },
106	{ 0x01161179,		0, 0, "Toshiba XG5", QUIRK_DISABLE_TIMEOUT },
107	{ 0x00000000,		0, 0, NULL  }
108};
109
110static int
111nvme_match(uint32_t devid, uint16_t subdevice, struct _pcsid *ep)
112{
113	if (devid != ep->devid)
114		return 0;
115
116	if (!ep->match_subdevice)
117		return 1;
118
119	if (subdevice == ep->subdevice)
120		return 1;
121	else
122		return 0;
123}
124
125static int
126nvme_probe (device_t device)
127{
128	struct _pcsid	*ep;
129	uint32_t	devid;
130	uint16_t	subdevice;
131
132	devid = pci_get_devid(device);
133	subdevice = pci_get_subdevice(device);
134	ep = pci_ids;
135
136	while (ep->devid) {
137		if (nvme_match(devid, subdevice, ep))
138			break;
139		++ep;
140	}
141
142	if (ep->desc) {
143		device_set_desc(device, ep->desc);
144		return (BUS_PROBE_DEFAULT);
145	}
146
147#if defined(PCIS_STORAGE_NVM)
148	if (pci_get_class(device)    == PCIC_STORAGE &&
149	    pci_get_subclass(device) == PCIS_STORAGE_NVM &&
150	    pci_get_progif(device)   == PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0) {
151		device_set_desc(device, "Generic NVMe Device");
152		return (BUS_PROBE_GENERIC);
153	}
154#endif
155
156	return (ENXIO);
157}
158
159static void
160nvme_init(void)
161{
162	uint32_t	i;
163
164	nvme_request_zone = uma_zcreate("nvme_request",
165	    sizeof(struct nvme_request), NULL, NULL, NULL, NULL, 0, 0);
166
167	for (i = 0; i < NVME_MAX_CONSUMERS; i++)
168		nvme_consumer[i].id = INVALID_CONSUMER_ID;
169}
170
171SYSINIT(nvme_register, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_init, NULL);
172
173static void
174nvme_uninit(void)
175{
176	uma_zdestroy(nvme_request_zone);
177}
178
179SYSUNINIT(nvme_unregister, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_uninit, NULL);
180
181static int
182nvme_shutdown(device_t dev)
183{
184	struct nvme_controller	*ctrlr;
185
186	ctrlr = DEVICE2SOFTC(dev);
187	nvme_ctrlr_shutdown(ctrlr);
188
189	return (0);
190}
191
192void
193nvme_dump_command(struct nvme_command *cmd)
194{
195	printf(
196"opc:%x f:%x r1:%x cid:%x nsid:%x r2:%x r3:%x mptr:%jx prp1:%jx prp2:%jx cdw:%x %x %x %x %x %x\n",
197	    cmd->opc, cmd->fuse, cmd->rsvd1, cmd->cid, cmd->nsid,
198	    cmd->rsvd2, cmd->rsvd3,
199	    (uintmax_t)cmd->mptr, (uintmax_t)cmd->prp1, (uintmax_t)cmd->prp2,
200	    cmd->cdw10, cmd->cdw11, cmd->cdw12, cmd->cdw13, cmd->cdw14,
201	    cmd->cdw15);
202}
203
204void
205nvme_dump_completion(struct nvme_completion *cpl)
206{
207	printf("cdw0:%08x sqhd:%04x sqid:%04x "
208	    "cid:%04x p:%x sc:%02x sct:%x m:%x dnr:%x\n",
209	    cpl->cdw0, cpl->sqhd, cpl->sqid,
210	    cpl->cid, cpl->status.p, cpl->status.sc, cpl->status.sct,
211	    cpl->status.m, cpl->status.dnr);
212}
213
214static int
215nvme_attach(device_t dev)
216{
217	struct nvme_controller	*ctrlr = DEVICE2SOFTC(dev);
218	int			status;
219	struct _pcsid		*ep;
220	uint32_t		devid;
221	uint16_t		subdevice;
222
223	devid = pci_get_devid(dev);
224	subdevice = pci_get_subdevice(dev);
225	ep = pci_ids;
226	while (ep->devid) {
227		if (nvme_match(devid, subdevice, ep))
228			break;
229		++ep;
230	}
231	ctrlr->quirks = ep->quirks;
232
233	status = nvme_ctrlr_construct(ctrlr, dev);
234
235	if (status != 0) {
236		nvme_ctrlr_destruct(ctrlr, dev);
237		return (status);
238	}
239
240	/*
241	 * Some drives do not implement the completion timeout feature
242	 * correctly. There's a WAR from the manufacturer to just disable it.
243	 * The driver wouldn't respond correctly to a timeout anyway.
244	 */
245	if (ep->quirks & QUIRK_DISABLE_TIMEOUT) {
246		int ptr;
247		uint16_t devctl2;
248
249		status = pci_find_cap(dev, PCIY_EXPRESS, &ptr);
250		if (status) {
251			device_printf(dev, "Can't locate PCIe capability?");
252			return (status);
253		}
254		devctl2 = pci_read_config(dev, ptr + PCIER_DEVICE_CTL2, sizeof(devctl2));
255		devctl2 |= PCIEM_CTL2_COMP_TIMO_DISABLE;
256		pci_write_config(dev, ptr + PCIER_DEVICE_CTL2, devctl2, sizeof(devctl2));
257	}
258
259	/*
260	 * Enable busmastering so the completion status messages can
261	 * be busmastered back to the host.
262	 */
263	pci_enable_busmaster(dev);
264
265	/*
266	 * Reset controller twice to ensure we do a transition from cc.en==1
267	 *  to cc.en==0.  This is because we don't really know what status
268	 *  the controller was left in when boot handed off to OS.
269	 */
270	status = nvme_ctrlr_hw_reset(ctrlr);
271	if (status != 0) {
272		nvme_ctrlr_destruct(ctrlr, dev);
273		return (status);
274	}
275
276	status = nvme_ctrlr_hw_reset(ctrlr);
277	if (status != 0) {
278		nvme_ctrlr_destruct(ctrlr, dev);
279		return (status);
280	}
281
282	ctrlr->config_hook.ich_func = nvme_ctrlr_start_config_hook;
283	ctrlr->config_hook.ich_arg = ctrlr;
284
285	config_intrhook_establish(&ctrlr->config_hook);
286
287	return (0);
288}
289
290static int
291nvme_detach (device_t dev)
292{
293	struct nvme_controller	*ctrlr = DEVICE2SOFTC(dev);
294
295	nvme_ctrlr_destruct(ctrlr, dev);
296	pci_disable_busmaster(dev);
297	return (0);
298}
299
300static void
301nvme_notify(struct nvme_consumer *cons,
302	    struct nvme_controller *ctrlr)
303{
304	struct nvme_namespace	*ns;
305	void			*ctrlr_cookie;
306	int			cmpset, ns_idx;
307
308	/*
309	 * The consumer may register itself after the nvme devices
310	 *  have registered with the kernel, but before the
311	 *  driver has completed initialization.  In that case,
312	 *  return here, and when initialization completes, the
313	 *  controller will make sure the consumer gets notified.
314	 */
315	if (!ctrlr->is_initialized)
316		return;
317
318	cmpset = atomic_cmpset_32(&ctrlr->notification_sent, 0, 1);
319	if (cmpset == 0)
320		return;
321
322	if (cons->ctrlr_fn != NULL)
323		ctrlr_cookie = (*cons->ctrlr_fn)(ctrlr);
324	else
325		ctrlr_cookie = (void *)(uintptr_t)0xdeadc0dedeadc0de;
326	ctrlr->cons_cookie[cons->id] = ctrlr_cookie;
327
328	/* ctrlr_fn has failed.  Nothing to notify here any more. */
329	if (ctrlr_cookie == NULL)
330		return;
331
332	if (ctrlr->is_failed) {
333		ctrlr->cons_cookie[cons->id] = NULL;
334		if (cons->fail_fn != NULL)
335			(*cons->fail_fn)(ctrlr_cookie);
336		/*
337		 * Do not notify consumers about the namespaces of a
338		 *  failed controller.
339		 */
340		return;
341	}
342	for (ns_idx = 0; ns_idx < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); ns_idx++) {
343		ns = &ctrlr->ns[ns_idx];
344		if (ns->data.nsze == 0)
345			continue;
346		if (cons->ns_fn != NULL)
347			ns->cons_cookie[cons->id] =
348			    (*cons->ns_fn)(ns, ctrlr_cookie);
349	}
350}
351
352void
353nvme_notify_new_controller(struct nvme_controller *ctrlr)
354{
355	int i;
356
357	for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
358		if (nvme_consumer[i].id != INVALID_CONSUMER_ID) {
359			nvme_notify(&nvme_consumer[i], ctrlr);
360		}
361	}
362}
363
364static void
365nvme_notify_new_consumer(struct nvme_consumer *cons)
366{
367	device_t		*devlist;
368	struct nvme_controller	*ctrlr;
369	int			dev_idx, devcount;
370
371	if (devclass_get_devices(nvme_devclass, &devlist, &devcount))
372		return;
373
374	for (dev_idx = 0; dev_idx < devcount; dev_idx++) {
375		ctrlr = DEVICE2SOFTC(devlist[dev_idx]);
376		nvme_notify(cons, ctrlr);
377	}
378
379	free(devlist, M_TEMP);
380}
381
382void
383nvme_notify_async_consumers(struct nvme_controller *ctrlr,
384			    const struct nvme_completion *async_cpl,
385			    uint32_t log_page_id, void *log_page_buffer,
386			    uint32_t log_page_size)
387{
388	struct nvme_consumer	*cons;
389	void			*ctrlr_cookie;
390	uint32_t		i;
391
392	for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
393		cons = &nvme_consumer[i];
394		if (cons->id != INVALID_CONSUMER_ID && cons->async_fn != NULL &&
395		    (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) {
396			(*cons->async_fn)(ctrlr_cookie, async_cpl,
397			    log_page_id, log_page_buffer, log_page_size);
398		}
399	}
400}
401
402void
403nvme_notify_fail_consumers(struct nvme_controller *ctrlr)
404{
405	struct nvme_consumer	*cons;
406	void			*ctrlr_cookie;
407	uint32_t		i;
408
409	/*
410	 * This controller failed during initialization (i.e. IDENTIFY
411	 *  command failed or timed out).  Do not notify any nvme
412	 *  consumers of the failure here, since the consumer does not
413	 *  even know about the controller yet.
414	 */
415	if (!ctrlr->is_initialized)
416		return;
417
418	for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
419		cons = &nvme_consumer[i];
420		if (cons->id != INVALID_CONSUMER_ID &&
421		    (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) {
422			ctrlr->cons_cookie[i] = NULL;
423			if (cons->fail_fn != NULL)
424				cons->fail_fn(ctrlr_cookie);
425		}
426	}
427}
428
429struct nvme_consumer *
430nvme_register_consumer(nvme_cons_ns_fn_t ns_fn, nvme_cons_ctrlr_fn_t ctrlr_fn,
431		       nvme_cons_async_fn_t async_fn,
432		       nvme_cons_fail_fn_t fail_fn)
433{
434	int i;
435
436	/*
437	 * TODO: add locking around consumer registration.  Not an issue
438	 *  right now since we only have one nvme consumer - nvd(4).
439	 */
440	for (i = 0; i < NVME_MAX_CONSUMERS; i++)
441		if (nvme_consumer[i].id == INVALID_CONSUMER_ID) {
442			nvme_consumer[i].id = i;
443			nvme_consumer[i].ns_fn = ns_fn;
444			nvme_consumer[i].ctrlr_fn = ctrlr_fn;
445			nvme_consumer[i].async_fn = async_fn;
446			nvme_consumer[i].fail_fn = fail_fn;
447
448			nvme_notify_new_consumer(&nvme_consumer[i]);
449			return (&nvme_consumer[i]);
450		}
451
452	printf("nvme(4): consumer not registered - no slots available\n");
453	return (NULL);
454}
455
456void
457nvme_unregister_consumer(struct nvme_consumer *consumer)
458{
459
460	consumer->id = INVALID_CONSUMER_ID;
461}
462
463void
464nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl)
465{
466	struct nvme_completion_poll_status	*status = arg;
467
468	/*
469	 * Copy status into the argument passed by the caller, so that
470	 *  the caller can check the status to determine if the
471	 *  the request passed or failed.
472	 */
473	memcpy(&status->cpl, cpl, sizeof(*cpl));
474	atomic_store_rel_int(&status->done, 1);
475}
476