1240616Sjimharris/*-
2253112Sjimharris * Copyright (C) 2012-2013 Intel Corporation
3240616Sjimharris * All rights reserved.
4240616Sjimharris *
5240616Sjimharris * Redistribution and use in source and binary forms, with or without
6240616Sjimharris * modification, are permitted provided that the following conditions
7240616Sjimharris * are met:
8240616Sjimharris * 1. Redistributions of source code must retain the above copyright
9240616Sjimharris *    notice, this list of conditions and the following disclaimer.
10240616Sjimharris * 2. Redistributions in binary form must reproduce the above copyright
11240616Sjimharris *    notice, this list of conditions and the following disclaimer in the
12240616Sjimharris *    documentation and/or other materials provided with the distribution.
13240616Sjimharris *
14240616Sjimharris * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15240616Sjimharris * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16240616Sjimharris * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17240616Sjimharris * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18240616Sjimharris * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19240616Sjimharris * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20240616Sjimharris * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21240616Sjimharris * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22240616Sjimharris * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23240616Sjimharris * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24240616Sjimharris * SUCH DAMAGE.
25240616Sjimharris *
26240616Sjimharris * $FreeBSD: releng/10.3/sys/dev/nvme/nvme.h 294711 2016-01-25 09:40:25Z smh $
27240616Sjimharris */
28240616Sjimharris
29240616Sjimharris#ifndef __NVME_H__
30240616Sjimharris#define __NVME_H__
31240616Sjimharris
32240616Sjimharris#ifdef _KERNEL
33240616Sjimharris#include <sys/types.h>
34240616Sjimharris#endif
35240616Sjimharris
36252271Sjimharris#include <sys/param.h>
37252271Sjimharris
38249422Sjimharris#define	NVME_PASSTHROUGH_CMD		_IOWR('n', 0, struct nvme_pt_command)
39249422Sjimharris#define	NVME_RESET_CONTROLLER		_IO('n', 1)
40240616Sjimharris
41249422Sjimharris#define	NVME_IO_TEST			_IOWR('n', 100, struct nvme_io_test)
42249422Sjimharris#define	NVME_BIO_TEST			_IOWR('n', 101, struct nvme_io_test)
43249422Sjimharris
44240616Sjimharris/*
45240616Sjimharris * Use to mark a command to apply to all namespaces, or to retrieve global
46240616Sjimharris *  log pages.
47240616Sjimharris */
48240616Sjimharris#define NVME_GLOBAL_NAMESPACE_TAG	((uint32_t)0xFFFFFFFF)
49240616Sjimharris
50252271Sjimharris#define NVME_MAX_XFER_SIZE		MAXPHYS
51252271Sjimharris
52240616Sjimharrisunion cap_lo_register {
53240616Sjimharris	uint32_t	raw;
54240616Sjimharris	struct {
55240616Sjimharris		/** maximum queue entries supported */
56240616Sjimharris		uint32_t mqes		: 16;
57240616Sjimharris
58240616Sjimharris		/** contiguous queues required */
59240616Sjimharris		uint32_t cqr		: 1;
60240616Sjimharris
61240616Sjimharris		/** arbitration mechanism supported */
62240616Sjimharris		uint32_t ams		: 2;
63240616Sjimharris
64240616Sjimharris		uint32_t reserved1	: 5;
65240616Sjimharris
66240616Sjimharris		/** timeout */
67240616Sjimharris		uint32_t to		: 8;
68240616Sjimharris	} bits __packed;
69240616Sjimharris} __packed;
70240616Sjimharris
71240616Sjimharrisunion cap_hi_register {
72240616Sjimharris	uint32_t	raw;
73240616Sjimharris	struct {
74240616Sjimharris		/** doorbell stride */
75240616Sjimharris		uint32_t dstrd		: 4;
76240616Sjimharris
77240616Sjimharris		uint32_t reserved3	: 1;
78240616Sjimharris
79240616Sjimharris		/** command sets supported */
80240616Sjimharris		uint32_t css_nvm	: 1;
81240616Sjimharris
82240616Sjimharris		uint32_t css_reserved	: 3;
83240616Sjimharris		uint32_t reserved2	: 7;
84240616Sjimharris
85240616Sjimharris		/** memory page size minimum */
86240616Sjimharris		uint32_t mpsmin		: 4;
87240616Sjimharris
88240616Sjimharris		/** memory page size maximum */
89240616Sjimharris		uint32_t mpsmax		: 4;
90240616Sjimharris
91240616Sjimharris		uint32_t reserved1	: 8;
92240616Sjimharris	} bits __packed;
93240616Sjimharris} __packed;
94240616Sjimharris
95240616Sjimharrisunion cc_register {
96240616Sjimharris	uint32_t	raw;
97240616Sjimharris	struct {
98240616Sjimharris		/** enable */
99240616Sjimharris		uint32_t en		: 1;
100240616Sjimharris
101240616Sjimharris		uint32_t reserved1	: 3;
102240616Sjimharris
103240616Sjimharris		/** i/o command set selected */
104240616Sjimharris		uint32_t css		: 3;
105240616Sjimharris
106240616Sjimharris		/** memory page size */
107240616Sjimharris		uint32_t mps		: 4;
108240616Sjimharris
109240616Sjimharris		/** arbitration mechanism selected */
110240616Sjimharris		uint32_t ams		: 3;
111240616Sjimharris
112240616Sjimharris		/** shutdown notification */
113240616Sjimharris		uint32_t shn		: 2;
114240616Sjimharris
115240616Sjimharris		/** i/o submission queue entry size */
116240616Sjimharris		uint32_t iosqes		: 4;
117240616Sjimharris
118240616Sjimharris		/** i/o completion queue entry size */
119240616Sjimharris		uint32_t iocqes		: 4;
120240616Sjimharris
121240616Sjimharris		uint32_t reserved2	: 8;
122240616Sjimharris	} bits __packed;
123240616Sjimharris} __packed;
124240616Sjimharris
125240616Sjimharrisenum shn_value {
126240616Sjimharris	NVME_SHN_NORMAL		= 0x1,
127240616Sjimharris	NVME_SHN_ABRUPT		= 0x2,
128240616Sjimharris};
129240616Sjimharris
130240616Sjimharrisunion csts_register {
131240616Sjimharris	uint32_t	raw;
132240616Sjimharris	struct {
133240616Sjimharris		/** ready */
134240616Sjimharris		uint32_t rdy		: 1;
135240616Sjimharris
136240616Sjimharris		/** controller fatal status */
137240616Sjimharris		uint32_t cfs		: 1;
138240616Sjimharris
139240616Sjimharris		/** shutdown status */
140240616Sjimharris		uint32_t shst		: 2;
141240616Sjimharris
142240616Sjimharris		uint32_t reserved1	: 28;
143240616Sjimharris	} bits __packed;
144240616Sjimharris} __packed;
145240616Sjimharris
146240616Sjimharrisenum shst_value {
147240616Sjimharris	NVME_SHST_NORMAL	= 0x0,
148240616Sjimharris	NVME_SHST_OCCURRING	= 0x1,
149240616Sjimharris	NVME_SHST_COMPLETE	= 0x2,
150240616Sjimharris};
151240616Sjimharris
152240616Sjimharrisunion aqa_register {
153240616Sjimharris	uint32_t	raw;
154240616Sjimharris	struct {
155240616Sjimharris		/** admin submission queue size */
156240616Sjimharris		uint32_t asqs		: 12;
157240616Sjimharris
158240616Sjimharris		uint32_t reserved1	: 4;
159240616Sjimharris
160240616Sjimharris		/** admin completion queue size */
161240616Sjimharris		uint32_t acqs		: 12;
162240616Sjimharris
163240616Sjimharris		uint32_t reserved2	: 4;
164240616Sjimharris	} bits __packed;
165240616Sjimharris} __packed;
166240616Sjimharris
167240616Sjimharrisstruct nvme_registers
168240616Sjimharris{
169240616Sjimharris	/** controller capabilities */
170240616Sjimharris	union cap_lo_register	cap_lo;
171240616Sjimharris	union cap_hi_register	cap_hi;
172240616Sjimharris
173254302Sjimharris	uint32_t		vs;	/* version */
174254302Sjimharris	uint32_t		intms;	/* interrupt mask set */
175254302Sjimharris	uint32_t		intmc;	/* interrupt mask clear */
176240616Sjimharris
177240616Sjimharris	/** controller configuration */
178240616Sjimharris	union cc_register	cc;
179240616Sjimharris
180254302Sjimharris	uint32_t		reserved1;
181240616Sjimharris
182254302Sjimharris	/** controller status */
183254302Sjimharris	union csts_register	csts;
184254302Sjimharris
185254302Sjimharris	uint32_t		reserved2;
186254302Sjimharris
187240616Sjimharris	/** admin queue attributes */
188240616Sjimharris	union aqa_register	aqa;
189240616Sjimharris
190254302Sjimharris	uint64_t		asq;	/* admin submission queue base addr */
191254302Sjimharris	uint64_t		acq;	/* admin completion queue base addr */
192254302Sjimharris	uint32_t		reserved3[0x3f2];
193240616Sjimharris
194240616Sjimharris	struct {
195254302Sjimharris	    uint32_t		sq_tdbl; /* submission queue tail doorbell */
196254302Sjimharris	    uint32_t		cq_hdbl; /* completion queue head doorbell */
197240616Sjimharris	} doorbell[1] __packed;
198240616Sjimharris} __packed;
199240616Sjimharris
200240616Sjimharrisstruct nvme_command
201240616Sjimharris{
202240616Sjimharris	/* dword 0 */
203240616Sjimharris	uint16_t opc	:  8;	/* opcode */
204240616Sjimharris	uint16_t fuse	:  2;	/* fused operation */
205240616Sjimharris	uint16_t rsvd1	:  6;
206240616Sjimharris	uint16_t cid;		/* command identifier */
207240616Sjimharris
208240616Sjimharris	/* dword 1 */
209240616Sjimharris	uint32_t nsid;		/* namespace identifier */
210240616Sjimharris
211240616Sjimharris	/* dword 2-3 */
212240616Sjimharris	uint32_t rsvd2;
213240616Sjimharris	uint32_t rsvd3;
214240616Sjimharris
215240616Sjimharris	/* dword 4-5 */
216240616Sjimharris	uint64_t mptr;		/* metadata pointer */
217240616Sjimharris
218240616Sjimharris	/* dword 6-7 */
219240616Sjimharris	uint64_t prp1;		/* prp entry 1 */
220240616Sjimharris
221240616Sjimharris	/* dword 8-9 */
222240616Sjimharris	uint64_t prp2;		/* prp entry 2 */
223240616Sjimharris
224240616Sjimharris	/* dword 10-15 */
225240616Sjimharris	uint32_t cdw10;		/* command-specific */
226240616Sjimharris	uint32_t cdw11;		/* command-specific */
227240616Sjimharris	uint32_t cdw12;		/* command-specific */
228240616Sjimharris	uint32_t cdw13;		/* command-specific */
229240616Sjimharris	uint32_t cdw14;		/* command-specific */
230240616Sjimharris	uint32_t cdw15;		/* command-specific */
231240616Sjimharris} __packed;
232240616Sjimharris
233248756Sjimharrisstruct nvme_status {
234248756Sjimharris
235248756Sjimharris	uint16_t p	:  1;	/* phase tag */
236248756Sjimharris	uint16_t sc	:  8;	/* status code */
237248756Sjimharris	uint16_t sct	:  3;	/* status code type */
238248756Sjimharris	uint16_t rsvd2	:  2;
239248756Sjimharris	uint16_t m	:  1;	/* more */
240248756Sjimharris	uint16_t dnr	:  1;	/* do not retry */
241248756Sjimharris} __packed;
242248756Sjimharris
243240616Sjimharrisstruct nvme_completion {
244240616Sjimharris
245240616Sjimharris	/* dword 0 */
246248756Sjimharris	uint32_t		cdw0;	/* command-specific */
247240616Sjimharris
248240616Sjimharris	/* dword 1 */
249248756Sjimharris	uint32_t		rsvd1;
250240616Sjimharris
251240616Sjimharris	/* dword 2 */
252248756Sjimharris	uint16_t		sqhd;	/* submission queue head pointer */
253248756Sjimharris	uint16_t		sqid;	/* submission queue identifier */
254240616Sjimharris
255240616Sjimharris	/* dword 3 */
256248756Sjimharris	uint16_t		cid;	/* command identifier */
257248756Sjimharris	struct nvme_status	status;
258240616Sjimharris} __packed;
259240616Sjimharris
260240616Sjimharrisstruct nvme_dsm_range {
261240616Sjimharris
262240616Sjimharris	uint32_t attributes;
263240616Sjimharris	uint32_t length;
264240616Sjimharris	uint64_t starting_lba;
265240616Sjimharris} __packed;
266240616Sjimharris
267240616Sjimharris/* status code types */
268240616Sjimharrisenum nvme_status_code_type {
269240616Sjimharris	NVME_SCT_GENERIC		= 0x0,
270240616Sjimharris	NVME_SCT_COMMAND_SPECIFIC	= 0x1,
271240616Sjimharris	NVME_SCT_MEDIA_ERROR		= 0x2,
272240616Sjimharris	/* 0x3-0x6 - reserved */
273240616Sjimharris	NVME_SCT_VENDOR_SPECIFIC	= 0x7,
274240616Sjimharris};
275240616Sjimharris
276240616Sjimharris/* generic command status codes */
277240616Sjimharrisenum nvme_generic_command_status_code {
278240616Sjimharris	NVME_SC_SUCCESS				= 0x00,
279240616Sjimharris	NVME_SC_INVALID_OPCODE			= 0x01,
280240616Sjimharris	NVME_SC_INVALID_FIELD			= 0x02,
281240616Sjimharris	NVME_SC_COMMAND_ID_CONFLICT		= 0x03,
282240616Sjimharris	NVME_SC_DATA_TRANSFER_ERROR		= 0x04,
283240616Sjimharris	NVME_SC_ABORTED_POWER_LOSS		= 0x05,
284240616Sjimharris	NVME_SC_INTERNAL_DEVICE_ERROR		= 0x06,
285240616Sjimharris	NVME_SC_ABORTED_BY_REQUEST		= 0x07,
286240616Sjimharris	NVME_SC_ABORTED_SQ_DELETION		= 0x08,
287240616Sjimharris	NVME_SC_ABORTED_FAILED_FUSED		= 0x09,
288240616Sjimharris	NVME_SC_ABORTED_MISSING_FUSED		= 0x0a,
289240616Sjimharris	NVME_SC_INVALID_NAMESPACE_OR_FORMAT	= 0x0b,
290240616Sjimharris	NVME_SC_COMMAND_SEQUENCE_ERROR		= 0x0c,
291240616Sjimharris
292240616Sjimharris	NVME_SC_LBA_OUT_OF_RANGE		= 0x80,
293240616Sjimharris	NVME_SC_CAPACITY_EXCEEDED		= 0x81,
294240616Sjimharris	NVME_SC_NAMESPACE_NOT_READY		= 0x82,
295240616Sjimharris};
296240616Sjimharris
297240616Sjimharris/* command specific status codes */
298240616Sjimharrisenum nvme_command_specific_status_code {
299240616Sjimharris	NVME_SC_COMPLETION_QUEUE_INVALID	= 0x00,
300240616Sjimharris	NVME_SC_INVALID_QUEUE_IDENTIFIER	= 0x01,
301240616Sjimharris	NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED	= 0x02,
302240616Sjimharris	NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED	= 0x03,
303240616Sjimharris	/* 0x04 - reserved */
304240616Sjimharris	NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED = 0x05,
305240616Sjimharris	NVME_SC_INVALID_FIRMWARE_SLOT		= 0x06,
306240616Sjimharris	NVME_SC_INVALID_FIRMWARE_IMAGE		= 0x07,
307240616Sjimharris	NVME_SC_INVALID_INTERRUPT_VECTOR	= 0x08,
308240616Sjimharris	NVME_SC_INVALID_LOG_PAGE		= 0x09,
309240616Sjimharris	NVME_SC_INVALID_FORMAT			= 0x0a,
310240616Sjimharris	NVME_SC_FIRMWARE_REQUIRES_RESET		= 0x0b,
311240616Sjimharris
312240616Sjimharris	NVME_SC_CONFLICTING_ATTRIBUTES		= 0x80,
313240616Sjimharris	NVME_SC_INVALID_PROTECTION_INFO		= 0x81,
314240616Sjimharris	NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE	= 0x82,
315240616Sjimharris};
316240616Sjimharris
317240616Sjimharris/* media error status codes */
318240616Sjimharrisenum nvme_media_error_status_code {
319240616Sjimharris	NVME_SC_WRITE_FAULTS			= 0x80,
320240616Sjimharris	NVME_SC_UNRECOVERED_READ_ERROR		= 0x81,
321240616Sjimharris	NVME_SC_GUARD_CHECK_ERROR		= 0x82,
322240616Sjimharris	NVME_SC_APPLICATION_TAG_CHECK_ERROR	= 0x83,
323240616Sjimharris	NVME_SC_REFERENCE_TAG_CHECK_ERROR	= 0x84,
324240616Sjimharris	NVME_SC_COMPARE_FAILURE			= 0x85,
325240616Sjimharris	NVME_SC_ACCESS_DENIED			= 0x86,
326240616Sjimharris};
327240616Sjimharris
328240616Sjimharris/* admin opcodes */
329240616Sjimharrisenum nvme_admin_opcode {
330240616Sjimharris	NVME_OPC_DELETE_IO_SQ			= 0x00,
331240616Sjimharris	NVME_OPC_CREATE_IO_SQ			= 0x01,
332240616Sjimharris	NVME_OPC_GET_LOG_PAGE			= 0x02,
333240616Sjimharris	/* 0x03 - reserved */
334240616Sjimharris	NVME_OPC_DELETE_IO_CQ			= 0x04,
335240616Sjimharris	NVME_OPC_CREATE_IO_CQ			= 0x05,
336240616Sjimharris	NVME_OPC_IDENTIFY			= 0x06,
337240616Sjimharris	/* 0x07 - reserved */
338240616Sjimharris	NVME_OPC_ABORT				= 0x08,
339240616Sjimharris	NVME_OPC_SET_FEATURES			= 0x09,
340240616Sjimharris	NVME_OPC_GET_FEATURES			= 0x0a,
341240616Sjimharris	/* 0x0b - reserved */
342240616Sjimharris	NVME_OPC_ASYNC_EVENT_REQUEST		= 0x0c,
343240616Sjimharris	/* 0x0d-0x0f - reserved */
344240616Sjimharris	NVME_OPC_FIRMWARE_ACTIVATE		= 0x10,
345240616Sjimharris	NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD	= 0x11,
346240616Sjimharris
347240616Sjimharris	NVME_OPC_FORMAT_NVM			= 0x80,
348240616Sjimharris	NVME_OPC_SECURITY_SEND			= 0x81,
349240616Sjimharris	NVME_OPC_SECURITY_RECEIVE		= 0x82,
350240616Sjimharris};
351240616Sjimharris
352240616Sjimharris/* nvme nvm opcodes */
353240616Sjimharrisenum nvme_nvm_opcode {
354240616Sjimharris	NVME_OPC_FLUSH				= 0x00,
355240616Sjimharris	NVME_OPC_WRITE				= 0x01,
356240616Sjimharris	NVME_OPC_READ				= 0x02,
357240616Sjimharris	/* 0x03 - reserved */
358240616Sjimharris	NVME_OPC_WRITE_UNCORRECTABLE		= 0x04,
359240616Sjimharris	NVME_OPC_COMPARE			= 0x05,
360240616Sjimharris	/* 0x06-0x07 - reserved */
361240616Sjimharris	NVME_OPC_DATASET_MANAGEMENT		= 0x09,
362240616Sjimharris};
363240616Sjimharris
364240616Sjimharrisenum nvme_feature {
365240616Sjimharris	/* 0x00 - reserved */
366240616Sjimharris	NVME_FEAT_ARBITRATION			= 0x01,
367240616Sjimharris	NVME_FEAT_POWER_MANAGEMENT		= 0x02,
368240616Sjimharris	NVME_FEAT_LBA_RANGE_TYPE		= 0x03,
369240616Sjimharris	NVME_FEAT_TEMPERATURE_THRESHOLD		= 0x04,
370240616Sjimharris	NVME_FEAT_ERROR_RECOVERY		= 0x05,
371240616Sjimharris	NVME_FEAT_VOLATILE_WRITE_CACHE		= 0x06,
372240616Sjimharris	NVME_FEAT_NUMBER_OF_QUEUES		= 0x07,
373240616Sjimharris	NVME_FEAT_INTERRUPT_COALESCING		= 0x08,
374240616Sjimharris	NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION = 0x09,
375240616Sjimharris	NVME_FEAT_WRITE_ATOMICITY		= 0x0A,
376248737Sjimharris	NVME_FEAT_ASYNC_EVENT_CONFIGURATION	= 0x0B,
377240616Sjimharris	/* 0x0C-0x7F - reserved */
378240616Sjimharris	NVME_FEAT_SOFTWARE_PROGRESS_MARKER	= 0x80,
379240616Sjimharris	/* 0x81-0xBF - command set specific (reserved) */
380240616Sjimharris	/* 0xC0-0xFF - vendor specific */
381240616Sjimharris};
382240616Sjimharris
383240616Sjimharrisenum nvme_dsm_attribute {
384240616Sjimharris	NVME_DSM_ATTR_INTEGRAL_READ		= 0x1,
385240616Sjimharris	NVME_DSM_ATTR_INTEGRAL_WRITE		= 0x2,
386240616Sjimharris	NVME_DSM_ATTR_DEALLOCATE		= 0x4,
387240616Sjimharris};
388240616Sjimharris
389252278Sjimharrisenum nvme_activate_action {
390252278Sjimharris	NVME_AA_REPLACE_NO_ACTIVATE		= 0x0,
391252278Sjimharris	NVME_AA_REPLACE_ACTIVATE		= 0x1,
392252278Sjimharris	NVME_AA_ACTIVATE			= 0x2,
393252278Sjimharris};
394252278Sjimharris
395253437Sjimharris#define NVME_SERIAL_NUMBER_LENGTH	20
396253437Sjimharris#define NVME_MODEL_NUMBER_LENGTH	40
397253437Sjimharris#define NVME_FIRMWARE_REVISION_LENGTH	8
398253437Sjimharris
399240616Sjimharrisstruct nvme_controller_data {
400240616Sjimharris
401240616Sjimharris	/* bytes 0-255: controller capabilities and features */
402240616Sjimharris
403240616Sjimharris	/** pci vendor id */
404240616Sjimharris	uint16_t		vid;
405240616Sjimharris
406240616Sjimharris	/** pci subsystem vendor id */
407240616Sjimharris	uint16_t		ssvid;
408240616Sjimharris
409240616Sjimharris	/** serial number */
410253476Sjimharris	uint8_t			sn[NVME_SERIAL_NUMBER_LENGTH];
411240616Sjimharris
412240616Sjimharris	/** model number */
413253476Sjimharris	uint8_t			mn[NVME_MODEL_NUMBER_LENGTH];
414240616Sjimharris
415240616Sjimharris	/** firmware revision */
416253437Sjimharris	uint8_t			fr[NVME_FIRMWARE_REVISION_LENGTH];
417240616Sjimharris
418240616Sjimharris	/** recommended arbitration burst */
419240616Sjimharris	uint8_t			rab;
420240616Sjimharris
421240616Sjimharris	/** ieee oui identifier */
422240616Sjimharris	uint8_t			ieee[3];
423240616Sjimharris
424240616Sjimharris	/** multi-interface capabilities */
425240616Sjimharris	uint8_t			mic;
426240616Sjimharris
427240616Sjimharris	/** maximum data transfer size */
428240616Sjimharris	uint8_t			mdts;
429240616Sjimharris
430240616Sjimharris	uint8_t			reserved1[178];
431240616Sjimharris
432240616Sjimharris	/* bytes 256-511: admin command set attributes */
433240616Sjimharris
434240616Sjimharris	/** optional admin command support */
435240616Sjimharris	struct {
436240616Sjimharris		/* supports security send/receive commands */
437240616Sjimharris		uint16_t	security  : 1;
438240616Sjimharris
439240616Sjimharris		/* supports format nvm command */
440240616Sjimharris		uint16_t	format    : 1;
441240616Sjimharris
442240616Sjimharris		/* supports firmware activate/download commands */
443240616Sjimharris		uint16_t	firmware  : 1;
444240616Sjimharris
445240616Sjimharris		uint16_t	oacs_rsvd : 13;
446240616Sjimharris	} __packed oacs;
447240616Sjimharris
448240616Sjimharris	/** abort command limit */
449240616Sjimharris	uint8_t			acl;
450240616Sjimharris
451240616Sjimharris	/** asynchronous event request limit */
452240616Sjimharris	uint8_t			aerl;
453240616Sjimharris
454240616Sjimharris	/** firmware updates */
455240616Sjimharris	struct {
456240616Sjimharris		/* first slot is read-only */
457240616Sjimharris		uint8_t		slot1_ro  : 1;
458240616Sjimharris
459240616Sjimharris		/* number of firmware slots */
460240616Sjimharris		uint8_t		num_slots : 3;
461240616Sjimharris
462240616Sjimharris		uint8_t		frmw_rsvd : 4;
463240616Sjimharris	} __packed frmw;
464240616Sjimharris
465240616Sjimharris	/** log page attributes */
466240616Sjimharris	struct {
467240616Sjimharris		/* per namespace smart/health log page */
468240616Sjimharris		uint8_t		ns_smart : 1;
469240616Sjimharris
470240616Sjimharris		uint8_t		lpa_rsvd : 7;
471240616Sjimharris	} __packed lpa;
472240616Sjimharris
473240616Sjimharris	/** error log page entries */
474240616Sjimharris	uint8_t			elpe;
475240616Sjimharris
476240616Sjimharris	/** number of power states supported */
477240616Sjimharris	uint8_t			npss;
478240616Sjimharris
479240616Sjimharris	/** admin vendor specific command configuration */
480240616Sjimharris	struct {
481240616Sjimharris		/* admin vendor specific commands use spec format */
482240616Sjimharris		uint8_t		spec_format : 1;
483240616Sjimharris
484240616Sjimharris		uint8_t		avscc_rsvd  : 7;
485240616Sjimharris	} __packed avscc;
486240616Sjimharris
487240616Sjimharris	uint8_t			reserved2[247];
488240616Sjimharris
489240616Sjimharris	/* bytes 512-703: nvm command set attributes */
490240616Sjimharris
491240616Sjimharris	/** submission queue entry size */
492240616Sjimharris	struct {
493240616Sjimharris		uint8_t		min : 4;
494240616Sjimharris		uint8_t		max : 4;
495240616Sjimharris	} __packed sqes;
496240616Sjimharris
497240616Sjimharris	/** completion queue entry size */
498240616Sjimharris	struct {
499240616Sjimharris		uint8_t		min : 4;
500240616Sjimharris		uint8_t		max : 4;
501240616Sjimharris	} __packed cqes;
502240616Sjimharris
503240616Sjimharris	uint8_t			reserved3[2];
504240616Sjimharris
505240616Sjimharris	/** number of namespaces */
506240616Sjimharris	uint32_t		nn;
507240616Sjimharris
508240616Sjimharris	/** optional nvm command support */
509240616Sjimharris	struct {
510240616Sjimharris		uint16_t	compare : 1;
511240616Sjimharris		uint16_t	write_unc : 1;
512240616Sjimharris		uint16_t	dsm: 1;
513240616Sjimharris		uint16_t	reserved: 13;
514240616Sjimharris	} __packed oncs;
515240616Sjimharris
516240616Sjimharris	/** fused operation support */
517240616Sjimharris	uint16_t		fuses;
518240616Sjimharris
519240616Sjimharris	/** format nvm attributes */
520240616Sjimharris	uint8_t			fna;
521240616Sjimharris
522240616Sjimharris	/** volatile write cache */
523240616Sjimharris	struct {
524240616Sjimharris		uint8_t		present : 1;
525240616Sjimharris		uint8_t		reserved : 7;
526240616Sjimharris	} __packed vwc;
527240616Sjimharris
528240616Sjimharris	/* TODO: flesh out remaining nvm command set attributes */
529240616Sjimharris	uint8_t			reserved4[178];
530240616Sjimharris
531240616Sjimharris	/* bytes 704-2047: i/o command set attributes */
532240616Sjimharris	uint8_t			reserved5[1344];
533240616Sjimharris
534240616Sjimharris	/* bytes 2048-3071: power state descriptors */
535240616Sjimharris	uint8_t			reserved6[1024];
536240616Sjimharris
537240616Sjimharris	/* bytes 3072-4095: vendor specific */
538256151Sjimharris	uint8_t			vs[1024];
539240671Sjimharris} __packed __aligned(4);
540240616Sjimharris
541240616Sjimharrisstruct nvme_namespace_data {
542240616Sjimharris
543240616Sjimharris	/** namespace size */
544240616Sjimharris	uint64_t		nsze;
545240616Sjimharris
546240616Sjimharris	/** namespace capacity */
547240616Sjimharris	uint64_t		ncap;
548240616Sjimharris
549240616Sjimharris	/** namespace utilization */
550240616Sjimharris	uint64_t		nuse;
551240616Sjimharris
552240616Sjimharris	/** namespace features */
553240616Sjimharris	struct {
554240616Sjimharris		/** thin provisioning */
555240616Sjimharris		uint8_t		thin_prov : 1;
556240616Sjimharris		uint8_t		reserved1 : 7;
557240616Sjimharris	} __packed nsfeat;
558240616Sjimharris
559240616Sjimharris	/** number of lba formats */
560240616Sjimharris	uint8_t			nlbaf;
561240616Sjimharris
562240616Sjimharris	/** formatted lba size */
563240616Sjimharris	struct {
564240616Sjimharris		uint8_t		format    : 4;
565240616Sjimharris		uint8_t		extended  : 1;
566240616Sjimharris		uint8_t		reserved2 : 3;
567240616Sjimharris	} __packed flbas;
568240616Sjimharris
569240616Sjimharris	/** metadata capabilities */
570240616Sjimharris	struct {
571240616Sjimharris		/* metadata can be transferred as part of data prp list */
572240616Sjimharris		uint8_t		extended  : 1;
573240616Sjimharris
574240616Sjimharris		/* metadata can be transferred with separate metadata pointer */
575240616Sjimharris		uint8_t		pointer   : 1;
576240616Sjimharris
577240616Sjimharris		uint8_t		reserved3 : 6;
578240616Sjimharris	} __packed mc;
579240616Sjimharris
580240616Sjimharris	/** end-to-end data protection capabilities */
581240616Sjimharris	struct {
582240616Sjimharris		/* protection information type 1 */
583240616Sjimharris		uint8_t		pit1     : 1;
584240616Sjimharris
585240616Sjimharris		/* protection information type 2 */
586240616Sjimharris		uint8_t		pit2     : 1;
587240616Sjimharris
588240616Sjimharris		/* protection information type 3 */
589240616Sjimharris		uint8_t		pit3     : 1;
590240616Sjimharris
591240616Sjimharris		/* first eight bytes of metadata */
592240616Sjimharris		uint8_t		md_start : 1;
593240616Sjimharris
594240616Sjimharris		/* last eight bytes of metadata */
595240616Sjimharris		uint8_t		md_end   : 1;
596240616Sjimharris	} __packed dpc;
597240616Sjimharris
598240616Sjimharris	/** end-to-end data protection type settings */
599240616Sjimharris	struct {
600240616Sjimharris		/* protection information type */
601240616Sjimharris		uint8_t		pit       : 3;
602240616Sjimharris
603240616Sjimharris		/* 1 == protection info transferred at start of metadata */
604240616Sjimharris		/* 0 == protection info transferred at end of metadata */
605240616Sjimharris		uint8_t		md_start  : 1;
606240616Sjimharris
607240616Sjimharris		uint8_t		reserved4 : 4;
608240616Sjimharris	} __packed dps;
609240616Sjimharris
610240616Sjimharris	uint8_t			reserved5[98];
611240616Sjimharris
612240616Sjimharris	/** lba format support */
613240616Sjimharris	struct {
614240616Sjimharris		/** metadata size */
615240616Sjimharris		uint32_t	ms	  : 16;
616240616Sjimharris
617240616Sjimharris		/** lba data size */
618240616Sjimharris		uint32_t	lbads	  : 8;
619240616Sjimharris
620240616Sjimharris		/** relative performance */
621240616Sjimharris		uint32_t	rp	  : 2;
622240616Sjimharris
623240616Sjimharris		uint32_t	reserved6 : 6;
624240616Sjimharris	} __packed lbaf[16];
625240616Sjimharris
626240616Sjimharris	uint8_t			reserved6[192];
627240616Sjimharris
628240616Sjimharris	uint8_t			vendor_specific[3712];
629240671Sjimharris} __packed __aligned(4);
630240616Sjimharris
631240616Sjimharrisenum nvme_log_page {
632240616Sjimharris
633240616Sjimharris	/* 0x00 - reserved */
634240616Sjimharris	NVME_LOG_ERROR			= 0x01,
635240616Sjimharris	NVME_LOG_HEALTH_INFORMATION	= 0x02,
636240616Sjimharris	NVME_LOG_FIRMWARE_SLOT		= 0x03,
637240616Sjimharris	/* 0x04-0x7F - reserved */
638240616Sjimharris	/* 0x80-0xBF - I/O command set specific */
639240616Sjimharris	/* 0xC0-0xFF - vendor specific */
640240616Sjimharris};
641240616Sjimharris
642248757Sjimharrisstruct nvme_error_information_entry {
643248757Sjimharris
644248757Sjimharris	uint64_t		error_count;
645248757Sjimharris	uint16_t		sqid;
646248757Sjimharris	uint16_t		cid;
647248757Sjimharris	struct nvme_status	status;
648248757Sjimharris	uint16_t		error_location;
649248757Sjimharris	uint64_t		lba;
650248757Sjimharris	uint32_t		nsid;
651248757Sjimharris	uint8_t			vendor_specific;
652248757Sjimharris	uint8_t			reserved[35];
653248757Sjimharris} __packed __aligned(4);
654248757Sjimharris
655240616Sjimharrisunion nvme_critical_warning_state {
656240616Sjimharris
657240616Sjimharris	uint8_t		raw;
658240616Sjimharris
659240616Sjimharris	struct {
660240616Sjimharris		uint8_t	available_spare		: 1;
661240616Sjimharris		uint8_t	temperature		: 1;
662240616Sjimharris		uint8_t	device_reliability	: 1;
663240616Sjimharris		uint8_t	read_only		: 1;
664240616Sjimharris		uint8_t	volatile_memory_backup	: 1;
665240616Sjimharris		uint8_t	reserved		: 3;
666240616Sjimharris	} __packed bits;
667240616Sjimharris} __packed;
668240616Sjimharris
669240616Sjimharrisstruct nvme_health_information_page {
670240616Sjimharris
671240616Sjimharris	union nvme_critical_warning_state	critical_warning;
672240616Sjimharris
673240616Sjimharris	uint16_t		temperature;
674240616Sjimharris	uint8_t			available_spare;
675240616Sjimharris	uint8_t			available_spare_threshold;
676240616Sjimharris	uint8_t			percentage_used;
677240616Sjimharris
678240616Sjimharris	uint8_t			reserved[26];
679240616Sjimharris
680240616Sjimharris	/*
681240616Sjimharris	 * Note that the following are 128-bit values, but are
682240616Sjimharris	 *  defined as an array of 2 64-bit values.
683240616Sjimharris	 */
684240616Sjimharris	/* Data Units Read is always in 512-byte units. */
685240616Sjimharris	uint64_t		data_units_read[2];
686240616Sjimharris	/* Data Units Written is always in 512-byte units. */
687240616Sjimharris	uint64_t		data_units_written[2];
688240616Sjimharris	/* For NVM command set, this includes Compare commands. */
689240616Sjimharris	uint64_t		host_read_commands[2];
690240616Sjimharris	uint64_t		host_write_commands[2];
691240616Sjimharris	/* Controller Busy Time is reported in minutes. */
692240616Sjimharris	uint64_t		controller_busy_time[2];
693240616Sjimharris	uint64_t		power_cycles[2];
694240616Sjimharris	uint64_t		power_on_hours[2];
695240616Sjimharris	uint64_t		unsafe_shutdowns[2];
696240616Sjimharris	uint64_t		media_errors[2];
697240616Sjimharris	uint64_t		num_error_info_log_entries[2];
698240616Sjimharris
699240616Sjimharris	uint8_t			reserved2[320];
700240671Sjimharris} __packed __aligned(4);
701240616Sjimharris
702248758Sjimharrisstruct nvme_firmware_page {
703248758Sjimharris
704248758Sjimharris	struct {
705248758Sjimharris		uint8_t	slot		: 3; /* slot for current FW */
706248758Sjimharris		uint8_t	reserved	: 5;
707248758Sjimharris	} __packed afi;
708248758Sjimharris
709248758Sjimharris	uint8_t			reserved[7];
710248758Sjimharris	uint64_t		revision[7]; /* revisions for 7 slots */
711248758Sjimharris	uint8_t			reserved2[448];
712248758Sjimharris} __packed __aligned(4);
713248758Sjimharris
714240616Sjimharris#define NVME_TEST_MAX_THREADS	128
715240616Sjimharris
716240616Sjimharrisstruct nvme_io_test {
717240616Sjimharris
718240616Sjimharris	enum nvme_nvm_opcode	opc;
719240616Sjimharris	uint32_t		size;
720240616Sjimharris	uint32_t		time;	/* in seconds */
721240616Sjimharris	uint32_t		num_threads;
722240616Sjimharris	uint32_t		flags;
723256152Sjimharris	uint64_t		io_completed[NVME_TEST_MAX_THREADS];
724240616Sjimharris};
725240616Sjimharris
726240616Sjimharrisenum nvme_io_test_flags {
727240616Sjimharris
728240616Sjimharris	/*
729240616Sjimharris	 * Specifies whether dev_refthread/dev_relthread should be
730240616Sjimharris	 *  called during NVME_BIO_TEST.  Ignored for other test
731240616Sjimharris	 *  types.
732240616Sjimharris	 */
733240616Sjimharris	NVME_TEST_FLAG_REFTHREAD =	0x1,
734240616Sjimharris};
735240616Sjimharris
736249421Sjimharrisstruct nvme_pt_command {
737249421Sjimharris
738249421Sjimharris	/*
739249421Sjimharris	 * cmd is used to specify a passthrough command to a controller or
740249421Sjimharris	 *  namespace.
741249421Sjimharris	 *
742249421Sjimharris	 * The following fields from cmd may be specified by the caller:
743249421Sjimharris	 *	* opc  (opcode)
744249421Sjimharris	 *	* nsid (namespace id) - for admin commands only
745249421Sjimharris	 *	* cdw10-cdw15
746249421Sjimharris	 *
747249421Sjimharris	 * Remaining fields must be set to 0 by the caller.
748249421Sjimharris	 */
749249421Sjimharris	struct nvme_command	cmd;
750249421Sjimharris
751249421Sjimharris	/*
752249421Sjimharris	 * cpl returns completion status for the passthrough command
753249421Sjimharris	 *  specified by cmd.
754249421Sjimharris	 *
755249421Sjimharris	 * The following fields will be filled out by the driver, for
756249421Sjimharris	 *  consumption by the caller:
757249421Sjimharris	 *	* cdw0
758249421Sjimharris	 *	* status (except for phase)
759249421Sjimharris	 *
760249421Sjimharris	 * Remaining fields will be set to 0 by the driver.
761249421Sjimharris	 */
762249421Sjimharris	struct nvme_completion	cpl;
763249421Sjimharris
764249421Sjimharris	/* buf is the data buffer associated with this passthrough command. */
765249421Sjimharris	void *			buf;
766249421Sjimharris
767249421Sjimharris	/*
768249421Sjimharris	 * len is the length of the data buffer associated with this
769249421Sjimharris	 *  passthrough command.
770249421Sjimharris	 */
771249421Sjimharris	uint32_t		len;
772249421Sjimharris
773249421Sjimharris	/*
774249421Sjimharris	 * is_read = 1 if the passthrough command will read data into the
775253209Sjimharris	 *  supplied buffer from the controller.
776249421Sjimharris	 *
777253209Sjimharris	 * is_read = 0 if the passthrough command will write data from the
778253209Sjimharris	 *  supplied buffer to the controller.
779249421Sjimharris	 */
780249421Sjimharris	uint32_t		is_read;
781249421Sjimharris
782249421Sjimharris	/*
783249421Sjimharris	 * driver_lock is used by the driver only.  It must be set to 0
784249421Sjimharris	 *  by the caller.
785249421Sjimharris	 */
786249421Sjimharris	struct mtx *		driver_lock;
787249421Sjimharris};
788249421Sjimharris
789248756Sjimharris#define nvme_completion_is_error(cpl)					\
790248756Sjimharris	((cpl)->status.sc != 0 || (cpl)->status.sct != 0)
791248756Sjimharris
792253476Sjimharrisvoid	nvme_strvis(uint8_t *dst, const uint8_t *src, int dstlen, int srclen);
793253476Sjimharris
794240616Sjimharris#ifdef _KERNEL
795240616Sjimharris
796240616Sjimharrisstruct bio;
797240616Sjimharris
798240616Sjimharrisstruct nvme_namespace;
799248738Sjimharrisstruct nvme_controller;
800240616Sjimharrisstruct nvme_consumer;
801240616Sjimharris
802240616Sjimharristypedef void (*nvme_cb_fn_t)(void *, const struct nvme_completion *);
803240616Sjimharris
804248738Sjimharristypedef void *(*nvme_cons_ns_fn_t)(struct nvme_namespace *, void *);
805248738Sjimharristypedef void *(*nvme_cons_ctrlr_fn_t)(struct nvme_controller *);
806248760Sjimharristypedef void (*nvme_cons_async_fn_t)(void *, const struct nvme_completion *,
807248760Sjimharris				     uint32_t, void *, uint32_t);
808248767Sjimharristypedef void (*nvme_cons_fail_fn_t)(void *);
809248738Sjimharris
810240616Sjimharrisenum nvme_namespace_flags {
811240616Sjimharris	NVME_NS_DEALLOCATE_SUPPORTED	= 0x1,
812240616Sjimharris	NVME_NS_FLUSH_SUPPORTED		= 0x2,
813240616Sjimharris};
814240616Sjimharris
815249421Sjimharrisint	nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr,
816249421Sjimharris				   struct nvme_pt_command *pt,
817249421Sjimharris				   uint32_t nsid, int is_user_buffer,
818249421Sjimharris				   int is_admin_cmd);
819249421Sjimharris
820248739Sjimharris/* Admin functions */
821248739Sjimharrisvoid	nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr,
822248739Sjimharris				   uint8_t feature, uint32_t cdw11,
823248739Sjimharris				   void *payload, uint32_t payload_size,
824248739Sjimharris				   nvme_cb_fn_t cb_fn, void *cb_arg);
825248739Sjimharrisvoid	nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr,
826248739Sjimharris				   uint8_t feature, uint32_t cdw11,
827248739Sjimharris				   void *payload, uint32_t payload_size,
828248739Sjimharris				   nvme_cb_fn_t cb_fn, void *cb_arg);
829248740Sjimharrisvoid	nvme_ctrlr_cmd_get_log_page(struct nvme_controller *ctrlr,
830248740Sjimharris				    uint8_t log_page, uint32_t nsid,
831248740Sjimharris				    void *payload, uint32_t payload_size,
832248740Sjimharris				    nvme_cb_fn_t cb_fn, void *cb_arg);
833248739Sjimharris
834240616Sjimharris/* NVM I/O functions */
835241657Sjimharrisint	nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload,
836240616Sjimharris			  uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn,
837240616Sjimharris			  void *cb_arg);
838248977Sjimharrisint	nvme_ns_cmd_write_bio(struct nvme_namespace *ns, struct bio *bp,
839248977Sjimharris			      nvme_cb_fn_t cb_fn, void *cb_arg);
840241657Sjimharrisint	nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload,
841240616Sjimharris			 uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn,
842240616Sjimharris			 void *cb_arg);
843248977Sjimharrisint	nvme_ns_cmd_read_bio(struct nvme_namespace *ns, struct bio *bp,
844248977Sjimharris			      nvme_cb_fn_t cb_fn, void *cb_arg);
845241657Sjimharrisint	nvme_ns_cmd_deallocate(struct nvme_namespace *ns, void *payload,
846240616Sjimharris			       uint8_t num_ranges, nvme_cb_fn_t cb_fn,
847240616Sjimharris			       void *cb_arg);
848241657Sjimharrisint	nvme_ns_cmd_flush(struct nvme_namespace *ns, nvme_cb_fn_t cb_fn,
849240616Sjimharris			  void *cb_arg);
850240616Sjimharris
851240616Sjimharris/* Registration functions */
852248738Sjimharrisstruct nvme_consumer *	nvme_register_consumer(nvme_cons_ns_fn_t    ns_fn,
853248738Sjimharris					       nvme_cons_ctrlr_fn_t ctrlr_fn,
854248767Sjimharris					       nvme_cons_async_fn_t async_fn,
855248767Sjimharris					       nvme_cons_fail_fn_t  fail_fn);
856240616Sjimharrisvoid		nvme_unregister_consumer(struct nvme_consumer *consumer);
857240616Sjimharris
858248738Sjimharris/* Controller helper functions */
859248738Sjimharrisdevice_t	nvme_ctrlr_get_device(struct nvme_controller *ctrlr);
860248747Sjimharrisconst struct nvme_controller_data *
861248747Sjimharris		nvme_ctrlr_get_data(struct nvme_controller *ctrlr);
862248738Sjimharris
863240616Sjimharris/* Namespace helper functions */
864240616Sjimharrisuint32_t	nvme_ns_get_max_io_xfer_size(struct nvme_namespace *ns);
865240616Sjimharrisuint32_t	nvme_ns_get_sector_size(struct nvme_namespace *ns);
866240616Sjimharrisuint64_t	nvme_ns_get_num_sectors(struct nvme_namespace *ns);
867240616Sjimharrisuint64_t	nvme_ns_get_size(struct nvme_namespace *ns);
868240616Sjimharrisuint32_t	nvme_ns_get_flags(struct nvme_namespace *ns);
869240616Sjimharrisconst char *	nvme_ns_get_serial_number(struct nvme_namespace *ns);
870240616Sjimharrisconst char *	nvme_ns_get_model_number(struct nvme_namespace *ns);
871248747Sjimharrisconst struct nvme_namespace_data *
872248747Sjimharris		nvme_ns_get_data(struct nvme_namespace *ns);
873294711Ssmhuint32_t	nvme_ns_get_optimal_sector_size(struct nvme_namespace *ns);
874291214Sjimharrisuint32_t	nvme_ns_get_stripesize(struct nvme_namespace *ns);
875240616Sjimharris
876240616Sjimharrisint	nvme_ns_bio_process(struct nvme_namespace *ns, struct bio *bp,
877240616Sjimharris			    nvme_cb_fn_t cb_fn);
878240616Sjimharris
879240616Sjimharris#endif /* _KERNEL */
880240616Sjimharris
881240616Sjimharris#endif /* __NVME_H__ */
882