27 */
28
29#ifndef __NVME_PRIVATE_H__
30#define __NVME_PRIVATE_H__
31
32#include <sys/param.h>
33#include <sys/bio.h>
34#include <sys/bus.h>
35#include <sys/kernel.h>
36#include <sys/lock.h>
37#include <sys/malloc.h>
38#include <sys/mutex.h>
39#include <sys/rman.h>
40#include <sys/systm.h>
41#include <sys/taskqueue.h>
42
43#include <vm/uma.h>
44
45#include <machine/bus.h>
46
47#include "nvme.h"
48
49#define DEVICE2SOFTC(dev) ((struct nvme_controller *) device_get_softc(dev))
50
51MALLOC_DECLARE(M_NVME);
52
53#define CHATHAM2
54
55#ifdef CHATHAM2
56#define CHATHAM_PCI_ID 0x20118086
57#define CHATHAM_CONTROL_BAR 0
58#endif
59
60#define IDT32_PCI_ID 0x80d0111d /* 32 channel board */
61#define IDT8_PCI_ID 0x80d2111d /* 8 channel board */
62
63/*
64 * For commands requiring more than 2 PRP entries, one PRP will be
65 * embedded in the command (prp1), and the rest of the PRP entries
66 * will be in a list pointed to by the command (prp2). This means
67 * that real max number of PRP entries we support is 32+1, which
68 * results in a max xfer size of 32*PAGE_SIZE.
69 */
70#define NVME_MAX_PRP_LIST_ENTRIES (NVME_MAX_XFER_SIZE / PAGE_SIZE)
71
72#define NVME_ADMIN_TRACKERS (16)
73#define NVME_ADMIN_ENTRIES (128)
74/* min and max are defined in admin queue attributes section of spec */
75#define NVME_MIN_ADMIN_ENTRIES (2)
76#define NVME_MAX_ADMIN_ENTRIES (4096)
77
78/*
79 * NVME_IO_ENTRIES defines the size of an I/O qpair's submission and completion
80 * queues, while NVME_IO_TRACKERS defines the maximum number of I/O that we
81 * will allow outstanding on an I/O qpair at any time. The only advantage in
82 * having IO_ENTRIES > IO_TRACKERS is for debugging purposes - when dumping
83 * the contents of the submission and completion queues, it will show a longer
84 * history of data.
85 */
86#define NVME_IO_ENTRIES (256)
87#define NVME_IO_TRACKERS (128)
88#define NVME_MIN_IO_TRACKERS (4)
89#define NVME_MAX_IO_TRACKERS (1024)
90
91/*
92 * NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES
93 * for each controller.
94 */
95
96#define NVME_INT_COAL_TIME (0) /* disabled */
97#define NVME_INT_COAL_THRESHOLD (0) /* 0-based */
98
99#define NVME_MAX_NAMESPACES (16)
100#define NVME_MAX_CONSUMERS (2)
101#define NVME_MAX_ASYNC_EVENTS (8)
102
103#define NVME_DEFAULT_TIMEOUT_PERIOD (30) /* in seconds */
104#define NVME_MIN_TIMEOUT_PERIOD (5)
105#define NVME_MAX_TIMEOUT_PERIOD (120)
106
107#define NVME_DEFAULT_RETRY_COUNT (4)
108
109/* Maximum log page size to fetch for AERs. */
110#define NVME_MAX_AER_LOG_SIZE (4096)
111
112#ifndef CACHE_LINE_SIZE
113#define CACHE_LINE_SIZE (64)
114#endif
115
116/*
117 * Use presence of the BIO_UNMAPPED flag to determine whether unmapped I/O
118 * support and the bus_dmamap_load_bio API are available on the target
119 * kernel. This will ease porting back to earlier stable branches at a
120 * later point.
121 */
122#ifdef BIO_UNMAPPED
123#define NVME_UNMAPPED_BIO_SUPPORT
124#endif
125
126extern uma_zone_t nvme_request_zone;
127extern int32_t nvme_retry_count;
128
129struct nvme_completion_poll_status {
130
131 struct nvme_completion cpl;
132 boolean_t done;
133};
134
135#define NVME_REQUEST_VADDR 1
136#define NVME_REQUEST_NULL 2 /* For requests with no payload. */
137#define NVME_REQUEST_UIO 3
138#ifdef NVME_UNMAPPED_BIO_SUPPORT
139#define NVME_REQUEST_BIO 4
140#endif
141
142struct nvme_request {
143
144 struct nvme_command cmd;
145 struct nvme_qpair *qpair;
146 union {
147 void *payload;
148 struct bio *bio;
149 } u;
150 uint32_t type;
151 uint32_t payload_size;
152 boolean_t timeout;
153 nvme_cb_fn_t cb_fn;
154 void *cb_arg;
155 int32_t retries;
156 STAILQ_ENTRY(nvme_request) stailq;
157};
158
159struct nvme_async_event_request {
160
161 struct nvme_controller *ctrlr;
162 struct nvme_request *req;
163 struct nvme_completion cpl;
164 uint32_t log_page_id;
165 uint32_t log_page_size;
166 uint8_t log_page_buffer[NVME_MAX_AER_LOG_SIZE];
167};
168
169struct nvme_tracker {
170
171 TAILQ_ENTRY(nvme_tracker) tailq;
172 struct nvme_request *req;
173 struct nvme_qpair *qpair;
174 struct callout timer;
175 bus_dmamap_t payload_dma_map;
176 uint16_t cid;
177
178 uint64_t prp[NVME_MAX_PRP_LIST_ENTRIES];
179 bus_addr_t prp_bus_addr;
180 bus_dmamap_t prp_dma_map;
181};
182
183struct nvme_qpair {
184
185 struct nvme_controller *ctrlr;
186 uint32_t id;
187 uint32_t phase;
188
189 uint16_t vector;
190 int rid;
191 struct resource *res;
192 void *tag;
193
194 uint32_t num_entries;
195 uint32_t num_trackers;
196 uint32_t sq_tdbl_off;
197 uint32_t cq_hdbl_off;
198
199 uint32_t sq_head;
200 uint32_t sq_tail;
201 uint32_t cq_head;
202
203 int64_t num_cmds;
204 int64_t num_intr_handler_calls;
205
206 struct nvme_command *cmd;
207 struct nvme_completion *cpl;
208
209 bus_dma_tag_t dma_tag;
210
211 bus_dmamap_t cmd_dma_map;
212 uint64_t cmd_bus_addr;
213
214 bus_dmamap_t cpl_dma_map;
215 uint64_t cpl_bus_addr;
216
217 TAILQ_HEAD(, nvme_tracker) free_tr;
218 TAILQ_HEAD(, nvme_tracker) outstanding_tr;
219 STAILQ_HEAD(, nvme_request) queued_req;
220
221 struct nvme_tracker **act_tr;
222
223 boolean_t is_enabled;
224
225 struct mtx lock __aligned(CACHE_LINE_SIZE);
226
227} __aligned(CACHE_LINE_SIZE);
228
229struct nvme_namespace {
230
231 struct nvme_controller *ctrlr;
232 struct nvme_namespace_data data;
233 uint16_t id;
234 uint16_t flags;
235 struct cdev *cdev;
236 void *cons_cookie[NVME_MAX_CONSUMERS];
237 struct mtx lock;
238};
239
240/*
241 * One of these per allocated PCI device.
242 */
243struct nvme_controller {
244
245 device_t dev;
246
247 struct mtx lock;
248
249 uint32_t ready_timeout_in_ms;
250
251 bus_space_tag_t bus_tag;
252 bus_space_handle_t bus_handle;
253 int resource_id;
254 struct resource *resource;
255
256 /*
257 * The NVMe spec allows for the MSI-X table to be placed in BAR 4/5,
258 * separate from the control registers which are in BAR 0/1. These
259 * members track the mapping of BAR 4/5 for that reason.
260 */
261 int bar4_resource_id;
262 struct resource *bar4_resource;
263
264#ifdef CHATHAM2
265 bus_space_tag_t chatham_bus_tag;
266 bus_space_handle_t chatham_bus_handle;
267 int chatham_resource_id;
268 struct resource *chatham_resource;
269#endif
270
271 uint32_t msix_enabled;
272 uint32_t force_intx;
273 uint32_t enable_aborts;
274
275 uint32_t num_io_queues;
276 boolean_t per_cpu_io_queues;
277
278 /* Fields for tracking progress during controller initialization. */
279 struct intr_config_hook config_hook;
280 uint32_t ns_identified;
281 uint32_t queues_created;
282
283 struct task reset_task;
284 struct task fail_req_task;
285 struct taskqueue *taskqueue;
286
287 /* For shared legacy interrupt. */
288 int rid;
289 struct resource *res;
290 void *tag;
291
292 bus_dma_tag_t hw_desc_tag;
293 bus_dmamap_t hw_desc_map;
294
295 /** maximum i/o size in bytes */
296 uint32_t max_xfer_size;
297
298 /** minimum page size supported by this controller in bytes */
299 uint32_t min_page_size;
300
301 /** interrupt coalescing time period (in microseconds) */
302 uint32_t int_coal_time;
303
304 /** interrupt coalescing threshold */
305 uint32_t int_coal_threshold;
306
307 /** timeout period in seconds */
308 uint32_t timeout_period;
309
310 struct nvme_qpair adminq;
311 struct nvme_qpair *ioq;
312
313 struct nvme_registers *regs;
314
315 struct nvme_controller_data cdata;
316 struct nvme_namespace ns[NVME_MAX_NAMESPACES];
317
318 struct cdev *cdev;
319
320 uint32_t num_aers;
321 struct nvme_async_event_request aer[NVME_MAX_ASYNC_EVENTS];
322
323 void *cons_cookie[NVME_MAX_CONSUMERS];
324
325 uint32_t is_resetting;
326
327 boolean_t is_failed;
328 STAILQ_HEAD(, nvme_request) fail_req;
329
330#ifdef CHATHAM2
331 uint64_t chatham_size;
332 uint64_t chatham_lbas;
333#endif
334};
335
336#define nvme_mmio_offsetof(reg) \
337 offsetof(struct nvme_registers, reg)
338
339#define nvme_mmio_read_4(sc, reg) \
340 bus_space_read_4((sc)->bus_tag, (sc)->bus_handle, \
341 nvme_mmio_offsetof(reg))
342
343#define nvme_mmio_write_4(sc, reg, val) \
344 bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \
345 nvme_mmio_offsetof(reg), val)
346
347#define nvme_mmio_write_8(sc, reg, val) \
348 do { \
349 bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \
350 nvme_mmio_offsetof(reg), val & 0xFFFFFFFF); \
351 bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \
352 nvme_mmio_offsetof(reg)+4, \
353 (val & 0xFFFFFFFF00000000UL) >> 32); \
354 } while (0);
355
356#ifdef CHATHAM2
357#define chatham_read_4(softc, reg) \
358 bus_space_read_4((softc)->chatham_bus_tag, \
359 (softc)->chatham_bus_handle, reg)
360
361#define chatham_write_8(sc, reg, val) \
362 do { \
363 bus_space_write_4((sc)->chatham_bus_tag, \
364 (sc)->chatham_bus_handle, reg, val & 0xffffffff); \
365 bus_space_write_4((sc)->chatham_bus_tag, \
366 (sc)->chatham_bus_handle, reg+4, \
367 (val & 0xFFFFFFFF00000000UL) >> 32); \
368 } while (0);
369
370#endif /* CHATHAM2 */
371
372#if __FreeBSD_version < 800054
373#define wmb() __asm volatile("sfence" ::: "memory")
374#define mb() __asm volatile("mfence" ::: "memory")
375#endif
376
377#define nvme_printf(ctrlr, fmt, args...) \
378 device_printf(ctrlr->dev, fmt, ##args)
379
380void nvme_ns_test(struct nvme_namespace *ns, u_long cmd, caddr_t arg);
381
382void nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr,
383 void *payload,
384 nvme_cb_fn_t cb_fn, void *cb_arg);
385void nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr,
386 uint16_t nsid, void *payload,
387 nvme_cb_fn_t cb_fn, void *cb_arg);
388void nvme_ctrlr_cmd_set_interrupt_coalescing(struct nvme_controller *ctrlr,
389 uint32_t microseconds,
390 uint32_t threshold,
391 nvme_cb_fn_t cb_fn,
392 void *cb_arg);
393void nvme_ctrlr_cmd_get_error_page(struct nvme_controller *ctrlr,
394 struct nvme_error_information_entry *payload,
395 uint32_t num_entries, /* 0 = max */
396 nvme_cb_fn_t cb_fn,
397 void *cb_arg);
398void nvme_ctrlr_cmd_get_health_information_page(struct nvme_controller *ctrlr,
399 uint32_t nsid,
400 struct nvme_health_information_page *payload,
401 nvme_cb_fn_t cb_fn,
402 void *cb_arg);
403void nvme_ctrlr_cmd_get_firmware_page(struct nvme_controller *ctrlr,
404 struct nvme_firmware_page *payload,
405 nvme_cb_fn_t cb_fn,
406 void *cb_arg);
407void nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr,
408 struct nvme_qpair *io_que, uint16_t vector,
409 nvme_cb_fn_t cb_fn, void *cb_arg);
410void nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr,
411 struct nvme_qpair *io_que,
412 nvme_cb_fn_t cb_fn, void *cb_arg);
413void nvme_ctrlr_cmd_delete_io_cq(struct nvme_controller *ctrlr,
414 struct nvme_qpair *io_que,
415 nvme_cb_fn_t cb_fn, void *cb_arg);
416void nvme_ctrlr_cmd_delete_io_sq(struct nvme_controller *ctrlr,
417 struct nvme_qpair *io_que,
418 nvme_cb_fn_t cb_fn, void *cb_arg);
419void nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr,
420 uint32_t num_queues, nvme_cb_fn_t cb_fn,
421 void *cb_arg);
422void nvme_ctrlr_cmd_set_async_event_config(struct nvme_controller *ctrlr,
423 union nvme_critical_warning_state state,
424 nvme_cb_fn_t cb_fn, void *cb_arg);
425void nvme_ctrlr_cmd_abort(struct nvme_controller *ctrlr, uint16_t cid,
426 uint16_t sqid, nvme_cb_fn_t cb_fn, void *cb_arg);
427
428void nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl);
429
430int nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev);
431void nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev);
432int nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr);
433void nvme_ctrlr_reset(struct nvme_controller *ctrlr);
434/* ctrlr defined as void * to allow use with config_intrhook. */
435void nvme_ctrlr_start_config_hook(void *ctrlr_arg);
436void nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,
437 struct nvme_request *req);
438void nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
439 struct nvme_request *req);
440void nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr,
441 struct nvme_request *req);
442
443void nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id,
444 uint16_t vector, uint32_t num_entries,
445 uint32_t num_trackers,
446 struct nvme_controller *ctrlr);
447void nvme_qpair_submit_tracker(struct nvme_qpair *qpair,
448 struct nvme_tracker *tr);
449void nvme_qpair_process_completions(struct nvme_qpair *qpair);
450void nvme_qpair_submit_request(struct nvme_qpair *qpair,
451 struct nvme_request *req);
452void nvme_qpair_reset(struct nvme_qpair *qpair);
453void nvme_qpair_fail(struct nvme_qpair *qpair);
454void nvme_qpair_manual_complete_request(struct nvme_qpair *qpair,
455 struct nvme_request *req,
456 uint32_t sct, uint32_t sc,
457 boolean_t print_on_error);
458
459void nvme_admin_qpair_enable(struct nvme_qpair *qpair);
460void nvme_admin_qpair_disable(struct nvme_qpair *qpair);
461void nvme_admin_qpair_destroy(struct nvme_qpair *qpair);
462
463void nvme_io_qpair_enable(struct nvme_qpair *qpair);
464void nvme_io_qpair_disable(struct nvme_qpair *qpair);
465void nvme_io_qpair_destroy(struct nvme_qpair *qpair);
466
467int nvme_ns_construct(struct nvme_namespace *ns, uint16_t id,
468 struct nvme_controller *ctrlr);
469void nvme_ns_destruct(struct nvme_namespace *ns);
470
471void nvme_sysctl_initialize_ctrlr(struct nvme_controller *ctrlr);
472
473void nvme_dump_command(struct nvme_command *cmd);
474void nvme_dump_completion(struct nvme_completion *cpl);
475
476static __inline void
477nvme_single_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
478{
479 uint64_t *bus_addr = (uint64_t *)arg;
480
481 *bus_addr = seg[0].ds_addr;
482}
483
484static __inline struct nvme_request *
485_nvme_allocate_request(nvme_cb_fn_t cb_fn, void *cb_arg)
486{
487 struct nvme_request *req;
488
489 req = uma_zalloc(nvme_request_zone, M_NOWAIT | M_ZERO);
490 if (req != NULL) {
491 req->cb_fn = cb_fn;
492 req->cb_arg = cb_arg;
493 req->timeout = TRUE;
494 }
495 return (req);
496}
497
498static __inline struct nvme_request *
499nvme_allocate_request_vaddr(void *payload, uint32_t payload_size,
500 nvme_cb_fn_t cb_fn, void *cb_arg)
501{
502 struct nvme_request *req;
503
504 req = _nvme_allocate_request(cb_fn, cb_arg);
505 if (req != NULL) {
506 req->type = NVME_REQUEST_VADDR;
507 req->u.payload = payload;
508 req->payload_size = payload_size;
509 }
510 return (req);
511}
512
513static __inline struct nvme_request *
514nvme_allocate_request_null(nvme_cb_fn_t cb_fn, void *cb_arg)
515{
516 struct nvme_request *req;
517
518 req = _nvme_allocate_request(cb_fn, cb_arg);
519 if (req != NULL)
520 req->type = NVME_REQUEST_NULL;
521 return (req);
522}
523
524static __inline struct nvme_request *
525nvme_allocate_request_bio(struct bio *bio, nvme_cb_fn_t cb_fn, void *cb_arg)
526{
527 struct nvme_request *req;
528
529 req = _nvme_allocate_request(cb_fn, cb_arg);
530 if (req != NULL) {
531#ifdef NVME_UNMAPPED_BIO_SUPPORT
532 req->type = NVME_REQUEST_BIO;
533 req->u.bio = bio;
534#else
535 req->type = NVME_REQUEST_VADDR;
536 req->u.payload = bio->bio_data;
537 req->payload_size = bio->bio_bcount;
538#endif
539 }
540 return (req);
541}
542
543#define nvme_free_request(req) uma_zfree(nvme_request_zone, req)
544
545void nvme_notify_async_consumers(struct nvme_controller *ctrlr,
546 const struct nvme_completion *async_cpl,
547 uint32_t log_page_id, void *log_page_buffer,
548 uint32_t log_page_size);
549void nvme_notify_fail_consumers(struct nvme_controller *ctrlr);
550
551#endif /* __NVME_PRIVATE_H__ */
| 27 */
28
29#ifndef __NVME_PRIVATE_H__
30#define __NVME_PRIVATE_H__
31
32#include <sys/param.h>
33#include <sys/bio.h>
34#include <sys/bus.h>
35#include <sys/kernel.h>
36#include <sys/lock.h>
37#include <sys/malloc.h>
38#include <sys/mutex.h>
39#include <sys/rman.h>
40#include <sys/systm.h>
41#include <sys/taskqueue.h>
42
43#include <vm/uma.h>
44
45#include <machine/bus.h>
46
47#include "nvme.h"
48
49#define DEVICE2SOFTC(dev) ((struct nvme_controller *) device_get_softc(dev))
50
51MALLOC_DECLARE(M_NVME);
52
53#define CHATHAM2
54
55#ifdef CHATHAM2
56#define CHATHAM_PCI_ID 0x20118086
57#define CHATHAM_CONTROL_BAR 0
58#endif
59
60#define IDT32_PCI_ID 0x80d0111d /* 32 channel board */
61#define IDT8_PCI_ID 0x80d2111d /* 8 channel board */
62
63/*
64 * For commands requiring more than 2 PRP entries, one PRP will be
65 * embedded in the command (prp1), and the rest of the PRP entries
66 * will be in a list pointed to by the command (prp2). This means
67 * that real max number of PRP entries we support is 32+1, which
68 * results in a max xfer size of 32*PAGE_SIZE.
69 */
70#define NVME_MAX_PRP_LIST_ENTRIES (NVME_MAX_XFER_SIZE / PAGE_SIZE)
71
72#define NVME_ADMIN_TRACKERS (16)
73#define NVME_ADMIN_ENTRIES (128)
74/* min and max are defined in admin queue attributes section of spec */
75#define NVME_MIN_ADMIN_ENTRIES (2)
76#define NVME_MAX_ADMIN_ENTRIES (4096)
77
78/*
79 * NVME_IO_ENTRIES defines the size of an I/O qpair's submission and completion
80 * queues, while NVME_IO_TRACKERS defines the maximum number of I/O that we
81 * will allow outstanding on an I/O qpair at any time. The only advantage in
82 * having IO_ENTRIES > IO_TRACKERS is for debugging purposes - when dumping
83 * the contents of the submission and completion queues, it will show a longer
84 * history of data.
85 */
86#define NVME_IO_ENTRIES (256)
87#define NVME_IO_TRACKERS (128)
88#define NVME_MIN_IO_TRACKERS (4)
89#define NVME_MAX_IO_TRACKERS (1024)
90
91/*
92 * NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES
93 * for each controller.
94 */
95
96#define NVME_INT_COAL_TIME (0) /* disabled */
97#define NVME_INT_COAL_THRESHOLD (0) /* 0-based */
98
99#define NVME_MAX_NAMESPACES (16)
100#define NVME_MAX_CONSUMERS (2)
101#define NVME_MAX_ASYNC_EVENTS (8)
102
103#define NVME_DEFAULT_TIMEOUT_PERIOD (30) /* in seconds */
104#define NVME_MIN_TIMEOUT_PERIOD (5)
105#define NVME_MAX_TIMEOUT_PERIOD (120)
106
107#define NVME_DEFAULT_RETRY_COUNT (4)
108
109/* Maximum log page size to fetch for AERs. */
110#define NVME_MAX_AER_LOG_SIZE (4096)
111
112#ifndef CACHE_LINE_SIZE
113#define CACHE_LINE_SIZE (64)
114#endif
115
116/*
117 * Use presence of the BIO_UNMAPPED flag to determine whether unmapped I/O
118 * support and the bus_dmamap_load_bio API are available on the target
119 * kernel. This will ease porting back to earlier stable branches at a
120 * later point.
121 */
122#ifdef BIO_UNMAPPED
123#define NVME_UNMAPPED_BIO_SUPPORT
124#endif
125
126extern uma_zone_t nvme_request_zone;
127extern int32_t nvme_retry_count;
128
129struct nvme_completion_poll_status {
130
131 struct nvme_completion cpl;
132 boolean_t done;
133};
134
135#define NVME_REQUEST_VADDR 1
136#define NVME_REQUEST_NULL 2 /* For requests with no payload. */
137#define NVME_REQUEST_UIO 3
138#ifdef NVME_UNMAPPED_BIO_SUPPORT
139#define NVME_REQUEST_BIO 4
140#endif
141
142struct nvme_request {
143
144 struct nvme_command cmd;
145 struct nvme_qpair *qpair;
146 union {
147 void *payload;
148 struct bio *bio;
149 } u;
150 uint32_t type;
151 uint32_t payload_size;
152 boolean_t timeout;
153 nvme_cb_fn_t cb_fn;
154 void *cb_arg;
155 int32_t retries;
156 STAILQ_ENTRY(nvme_request) stailq;
157};
158
159struct nvme_async_event_request {
160
161 struct nvme_controller *ctrlr;
162 struct nvme_request *req;
163 struct nvme_completion cpl;
164 uint32_t log_page_id;
165 uint32_t log_page_size;
166 uint8_t log_page_buffer[NVME_MAX_AER_LOG_SIZE];
167};
168
169struct nvme_tracker {
170
171 TAILQ_ENTRY(nvme_tracker) tailq;
172 struct nvme_request *req;
173 struct nvme_qpair *qpair;
174 struct callout timer;
175 bus_dmamap_t payload_dma_map;
176 uint16_t cid;
177
178 uint64_t prp[NVME_MAX_PRP_LIST_ENTRIES];
179 bus_addr_t prp_bus_addr;
180 bus_dmamap_t prp_dma_map;
181};
182
183struct nvme_qpair {
184
185 struct nvme_controller *ctrlr;
186 uint32_t id;
187 uint32_t phase;
188
189 uint16_t vector;
190 int rid;
191 struct resource *res;
192 void *tag;
193
194 uint32_t num_entries;
195 uint32_t num_trackers;
196 uint32_t sq_tdbl_off;
197 uint32_t cq_hdbl_off;
198
199 uint32_t sq_head;
200 uint32_t sq_tail;
201 uint32_t cq_head;
202
203 int64_t num_cmds;
204 int64_t num_intr_handler_calls;
205
206 struct nvme_command *cmd;
207 struct nvme_completion *cpl;
208
209 bus_dma_tag_t dma_tag;
210
211 bus_dmamap_t cmd_dma_map;
212 uint64_t cmd_bus_addr;
213
214 bus_dmamap_t cpl_dma_map;
215 uint64_t cpl_bus_addr;
216
217 TAILQ_HEAD(, nvme_tracker) free_tr;
218 TAILQ_HEAD(, nvme_tracker) outstanding_tr;
219 STAILQ_HEAD(, nvme_request) queued_req;
220
221 struct nvme_tracker **act_tr;
222
223 boolean_t is_enabled;
224
225 struct mtx lock __aligned(CACHE_LINE_SIZE);
226
227} __aligned(CACHE_LINE_SIZE);
228
229struct nvme_namespace {
230
231 struct nvme_controller *ctrlr;
232 struct nvme_namespace_data data;
233 uint16_t id;
234 uint16_t flags;
235 struct cdev *cdev;
236 void *cons_cookie[NVME_MAX_CONSUMERS];
237 struct mtx lock;
238};
239
240/*
241 * One of these per allocated PCI device.
242 */
243struct nvme_controller {
244
245 device_t dev;
246
247 struct mtx lock;
248
249 uint32_t ready_timeout_in_ms;
250
251 bus_space_tag_t bus_tag;
252 bus_space_handle_t bus_handle;
253 int resource_id;
254 struct resource *resource;
255
256 /*
257 * The NVMe spec allows for the MSI-X table to be placed in BAR 4/5,
258 * separate from the control registers which are in BAR 0/1. These
259 * members track the mapping of BAR 4/5 for that reason.
260 */
261 int bar4_resource_id;
262 struct resource *bar4_resource;
263
264#ifdef CHATHAM2
265 bus_space_tag_t chatham_bus_tag;
266 bus_space_handle_t chatham_bus_handle;
267 int chatham_resource_id;
268 struct resource *chatham_resource;
269#endif
270
271 uint32_t msix_enabled;
272 uint32_t force_intx;
273 uint32_t enable_aborts;
274
275 uint32_t num_io_queues;
276 boolean_t per_cpu_io_queues;
277
278 /* Fields for tracking progress during controller initialization. */
279 struct intr_config_hook config_hook;
280 uint32_t ns_identified;
281 uint32_t queues_created;
282
283 struct task reset_task;
284 struct task fail_req_task;
285 struct taskqueue *taskqueue;
286
287 /* For shared legacy interrupt. */
288 int rid;
289 struct resource *res;
290 void *tag;
291
292 bus_dma_tag_t hw_desc_tag;
293 bus_dmamap_t hw_desc_map;
294
295 /** maximum i/o size in bytes */
296 uint32_t max_xfer_size;
297
298 /** minimum page size supported by this controller in bytes */
299 uint32_t min_page_size;
300
301 /** interrupt coalescing time period (in microseconds) */
302 uint32_t int_coal_time;
303
304 /** interrupt coalescing threshold */
305 uint32_t int_coal_threshold;
306
307 /** timeout period in seconds */
308 uint32_t timeout_period;
309
310 struct nvme_qpair adminq;
311 struct nvme_qpair *ioq;
312
313 struct nvme_registers *regs;
314
315 struct nvme_controller_data cdata;
316 struct nvme_namespace ns[NVME_MAX_NAMESPACES];
317
318 struct cdev *cdev;
319
320 uint32_t num_aers;
321 struct nvme_async_event_request aer[NVME_MAX_ASYNC_EVENTS];
322
323 void *cons_cookie[NVME_MAX_CONSUMERS];
324
325 uint32_t is_resetting;
326
327 boolean_t is_failed;
328 STAILQ_HEAD(, nvme_request) fail_req;
329
330#ifdef CHATHAM2
331 uint64_t chatham_size;
332 uint64_t chatham_lbas;
333#endif
334};
335
336#define nvme_mmio_offsetof(reg) \
337 offsetof(struct nvme_registers, reg)
338
339#define nvme_mmio_read_4(sc, reg) \
340 bus_space_read_4((sc)->bus_tag, (sc)->bus_handle, \
341 nvme_mmio_offsetof(reg))
342
343#define nvme_mmio_write_4(sc, reg, val) \
344 bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \
345 nvme_mmio_offsetof(reg), val)
346
347#define nvme_mmio_write_8(sc, reg, val) \
348 do { \
349 bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \
350 nvme_mmio_offsetof(reg), val & 0xFFFFFFFF); \
351 bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \
352 nvme_mmio_offsetof(reg)+4, \
353 (val & 0xFFFFFFFF00000000UL) >> 32); \
354 } while (0);
355
356#ifdef CHATHAM2
357#define chatham_read_4(softc, reg) \
358 bus_space_read_4((softc)->chatham_bus_tag, \
359 (softc)->chatham_bus_handle, reg)
360
361#define chatham_write_8(sc, reg, val) \
362 do { \
363 bus_space_write_4((sc)->chatham_bus_tag, \
364 (sc)->chatham_bus_handle, reg, val & 0xffffffff); \
365 bus_space_write_4((sc)->chatham_bus_tag, \
366 (sc)->chatham_bus_handle, reg+4, \
367 (val & 0xFFFFFFFF00000000UL) >> 32); \
368 } while (0);
369
370#endif /* CHATHAM2 */
371
372#if __FreeBSD_version < 800054
373#define wmb() __asm volatile("sfence" ::: "memory")
374#define mb() __asm volatile("mfence" ::: "memory")
375#endif
376
377#define nvme_printf(ctrlr, fmt, args...) \
378 device_printf(ctrlr->dev, fmt, ##args)
379
380void nvme_ns_test(struct nvme_namespace *ns, u_long cmd, caddr_t arg);
381
382void nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr,
383 void *payload,
384 nvme_cb_fn_t cb_fn, void *cb_arg);
385void nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr,
386 uint16_t nsid, void *payload,
387 nvme_cb_fn_t cb_fn, void *cb_arg);
388void nvme_ctrlr_cmd_set_interrupt_coalescing(struct nvme_controller *ctrlr,
389 uint32_t microseconds,
390 uint32_t threshold,
391 nvme_cb_fn_t cb_fn,
392 void *cb_arg);
393void nvme_ctrlr_cmd_get_error_page(struct nvme_controller *ctrlr,
394 struct nvme_error_information_entry *payload,
395 uint32_t num_entries, /* 0 = max */
396 nvme_cb_fn_t cb_fn,
397 void *cb_arg);
398void nvme_ctrlr_cmd_get_health_information_page(struct nvme_controller *ctrlr,
399 uint32_t nsid,
400 struct nvme_health_information_page *payload,
401 nvme_cb_fn_t cb_fn,
402 void *cb_arg);
403void nvme_ctrlr_cmd_get_firmware_page(struct nvme_controller *ctrlr,
404 struct nvme_firmware_page *payload,
405 nvme_cb_fn_t cb_fn,
406 void *cb_arg);
407void nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr,
408 struct nvme_qpair *io_que, uint16_t vector,
409 nvme_cb_fn_t cb_fn, void *cb_arg);
410void nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr,
411 struct nvme_qpair *io_que,
412 nvme_cb_fn_t cb_fn, void *cb_arg);
413void nvme_ctrlr_cmd_delete_io_cq(struct nvme_controller *ctrlr,
414 struct nvme_qpair *io_que,
415 nvme_cb_fn_t cb_fn, void *cb_arg);
416void nvme_ctrlr_cmd_delete_io_sq(struct nvme_controller *ctrlr,
417 struct nvme_qpair *io_que,
418 nvme_cb_fn_t cb_fn, void *cb_arg);
419void nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr,
420 uint32_t num_queues, nvme_cb_fn_t cb_fn,
421 void *cb_arg);
422void nvme_ctrlr_cmd_set_async_event_config(struct nvme_controller *ctrlr,
423 union nvme_critical_warning_state state,
424 nvme_cb_fn_t cb_fn, void *cb_arg);
425void nvme_ctrlr_cmd_abort(struct nvme_controller *ctrlr, uint16_t cid,
426 uint16_t sqid, nvme_cb_fn_t cb_fn, void *cb_arg);
427
428void nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl);
429
430int nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev);
431void nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev);
432int nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr);
433void nvme_ctrlr_reset(struct nvme_controller *ctrlr);
434/* ctrlr defined as void * to allow use with config_intrhook. */
435void nvme_ctrlr_start_config_hook(void *ctrlr_arg);
436void nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,
437 struct nvme_request *req);
438void nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
439 struct nvme_request *req);
440void nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr,
441 struct nvme_request *req);
442
443void nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id,
444 uint16_t vector, uint32_t num_entries,
445 uint32_t num_trackers,
446 struct nvme_controller *ctrlr);
447void nvme_qpair_submit_tracker(struct nvme_qpair *qpair,
448 struct nvme_tracker *tr);
449void nvme_qpair_process_completions(struct nvme_qpair *qpair);
450void nvme_qpair_submit_request(struct nvme_qpair *qpair,
451 struct nvme_request *req);
452void nvme_qpair_reset(struct nvme_qpair *qpair);
453void nvme_qpair_fail(struct nvme_qpair *qpair);
454void nvme_qpair_manual_complete_request(struct nvme_qpair *qpair,
455 struct nvme_request *req,
456 uint32_t sct, uint32_t sc,
457 boolean_t print_on_error);
458
459void nvme_admin_qpair_enable(struct nvme_qpair *qpair);
460void nvme_admin_qpair_disable(struct nvme_qpair *qpair);
461void nvme_admin_qpair_destroy(struct nvme_qpair *qpair);
462
463void nvme_io_qpair_enable(struct nvme_qpair *qpair);
464void nvme_io_qpair_disable(struct nvme_qpair *qpair);
465void nvme_io_qpair_destroy(struct nvme_qpair *qpair);
466
467int nvme_ns_construct(struct nvme_namespace *ns, uint16_t id,
468 struct nvme_controller *ctrlr);
469void nvme_ns_destruct(struct nvme_namespace *ns);
470
471void nvme_sysctl_initialize_ctrlr(struct nvme_controller *ctrlr);
472
473void nvme_dump_command(struct nvme_command *cmd);
474void nvme_dump_completion(struct nvme_completion *cpl);
475
476static __inline void
477nvme_single_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
478{
479 uint64_t *bus_addr = (uint64_t *)arg;
480
481 *bus_addr = seg[0].ds_addr;
482}
483
484static __inline struct nvme_request *
485_nvme_allocate_request(nvme_cb_fn_t cb_fn, void *cb_arg)
486{
487 struct nvme_request *req;
488
489 req = uma_zalloc(nvme_request_zone, M_NOWAIT | M_ZERO);
490 if (req != NULL) {
491 req->cb_fn = cb_fn;
492 req->cb_arg = cb_arg;
493 req->timeout = TRUE;
494 }
495 return (req);
496}
497
498static __inline struct nvme_request *
499nvme_allocate_request_vaddr(void *payload, uint32_t payload_size,
500 nvme_cb_fn_t cb_fn, void *cb_arg)
501{
502 struct nvme_request *req;
503
504 req = _nvme_allocate_request(cb_fn, cb_arg);
505 if (req != NULL) {
506 req->type = NVME_REQUEST_VADDR;
507 req->u.payload = payload;
508 req->payload_size = payload_size;
509 }
510 return (req);
511}
512
513static __inline struct nvme_request *
514nvme_allocate_request_null(nvme_cb_fn_t cb_fn, void *cb_arg)
515{
516 struct nvme_request *req;
517
518 req = _nvme_allocate_request(cb_fn, cb_arg);
519 if (req != NULL)
520 req->type = NVME_REQUEST_NULL;
521 return (req);
522}
523
524static __inline struct nvme_request *
525nvme_allocate_request_bio(struct bio *bio, nvme_cb_fn_t cb_fn, void *cb_arg)
526{
527 struct nvme_request *req;
528
529 req = _nvme_allocate_request(cb_fn, cb_arg);
530 if (req != NULL) {
531#ifdef NVME_UNMAPPED_BIO_SUPPORT
532 req->type = NVME_REQUEST_BIO;
533 req->u.bio = bio;
534#else
535 req->type = NVME_REQUEST_VADDR;
536 req->u.payload = bio->bio_data;
537 req->payload_size = bio->bio_bcount;
538#endif
539 }
540 return (req);
541}
542
543#define nvme_free_request(req) uma_zfree(nvme_request_zone, req)
544
545void nvme_notify_async_consumers(struct nvme_controller *ctrlr,
546 const struct nvme_completion *async_cpl,
547 uint32_t log_page_id, void *log_page_buffer,
548 uint32_t log_page_size);
549void nvme_notify_fail_consumers(struct nvme_controller *ctrlr);
550
551#endif /* __NVME_PRIVATE_H__ */
|