nvme_private.h revision 241434
1253544Shselasky/*- 2253544Shselasky * Copyright (C) 2012 Intel Corporation 3253544Shselasky * All rights reserved. 4253544Shselasky * 5253544Shselasky * Redistribution and use in source and binary forms, with or without 6253544Shselasky * modification, are permitted provided that the following conditions 7253544Shselasky * are met: 8253544Shselasky * 1. Redistributions of source code must retain the above copyright 9253544Shselasky * notice, this list of conditions and the following disclaimer. 10253544Shselasky * 2. Redistributions in binary form must reproduce the above copyright 11253544Shselasky * notice, this list of conditions and the following disclaimer in the 12253544Shselasky * documentation and/or other materials provided with the distribution. 13253544Shselasky * 14253544Shselasky * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15253544Shselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16253544Shselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17253544Shselasky * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18253544Shselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19253544Shselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20253544Shselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21253544Shselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22253544Shselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23253544Shselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24253544Shselasky * SUCH DAMAGE. 25253544Shselasky * 26253544Shselasky * $FreeBSD: head/sys/dev/nvme/nvme_private.h 241434 2012-10-10 23:35:16Z jimharris $ 27253544Shselasky */ 28253544Shselasky 29253544Shselasky#ifndef __NVME_PRIVATE_H__ 30253544Shselasky#define __NVME_PRIVATE_H__ 31253544Shselasky 32253544Shselasky#include <sys/param.h> 33253544Shselasky#include <sys/kernel.h> 34253544Shselasky#include <sys/lock.h> 35253544Shselasky#include <sys/malloc.h> 36#include <sys/mutex.h> 37#include <sys/rman.h> 38#include <sys/systm.h> 39#include <sys/taskqueue.h> 40 41#include <machine/bus.h> 42 43#include "nvme.h" 44 45#define DEVICE2SOFTC(dev) ((struct nvme_controller *) device_get_softc(dev)) 46 47MALLOC_DECLARE(M_NVME); 48 49#define CHATHAM2 50 51#ifdef CHATHAM2 52#define CHATHAM_PCI_ID 0x20118086 53#define CHATHAM_CONTROL_BAR 0 54#endif 55 56#define IDT_PCI_ID 0x80d0111d 57 58#define NVME_MAX_PRP_LIST_ENTRIES (128) 59 60/* 61 * For commands requiring more than 2 PRP entries, one PRP will be 62 * embedded in the command (prp1), and the rest of the PRP entries 63 * will be in a list pointed to by the command (prp2). This means 64 * that real max number of PRP entries we support is 128+1, which 65 * results in a max xfer size of 128*PAGE_SIZE. 66 */ 67#define NVME_MAX_XFER_SIZE NVME_MAX_PRP_LIST_ENTRIES * PAGE_SIZE 68 69#define NVME_ADMIN_ENTRIES (128) 70/* min and max are defined in admin queue attributes section of spec */ 71#define NVME_MIN_ADMIN_ENTRIES (2) 72#define NVME_MAX_ADMIN_ENTRIES (4096) 73 74#define NVME_IO_ENTRIES (1024) 75/* min is a reasonable value picked for the nvme(4) driver */ 76#define NVME_MIN_IO_ENTRIES (128) 77/* 78 * NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES 79 * for each controller. 80 */ 81 82#define NVME_INT_COAL_TIME (0) /* disabled */ 83#define NVME_INT_COAL_THRESHOLD (0) /* 0-based */ 84 85#define NVME_MAX_NAMESPACES (16) 86#define NVME_MAX_CONSUMERS (2) 87#define NVME_MAX_ASYNC_EVENTS (4) 88 89#define NVME_TIMEOUT_IN_SEC (30) 90 91#ifndef CACHE_LINE_SIZE 92#define CACHE_LINE_SIZE (64) 93#endif 94 95struct nvme_prp_list { 96 uint64_t prp[NVME_MAX_PRP_LIST_ENTRIES]; 97 SLIST_ENTRY(nvme_prp_list) slist; 98 bus_addr_t bus_addr; 99 bus_dmamap_t dma_map; 100}; 101 102struct nvme_tracker { 103 104 SLIST_ENTRY(nvme_tracker) slist; 105 struct nvme_qpair *qpair; 106 struct nvme_command cmd; 107 struct callout timer; 108 bus_dmamap_t dma_map; 109 nvme_cb_fn_t cb_fn; 110 void *cb_arg; 111 uint32_t payload_size; 112 struct nvme_prp_list *prp_list; 113 uint16_t cid; 114}; 115 116struct nvme_qpair { 117 118 struct nvme_controller *ctrlr; 119 uint32_t id; 120 uint32_t phase; 121 122 uint16_t vector; 123 int rid; 124 struct resource *res; 125 void *tag; 126 127 uint32_t max_xfer_size; 128 uint32_t num_entries; 129 uint32_t sq_tdbl_off; 130 uint32_t cq_hdbl_off; 131 132 uint32_t sq_head; 133 uint32_t sq_tail; 134 uint32_t cq_head; 135 136 int64_t num_cmds; 137 int64_t num_intr_handler_calls; 138 139 struct nvme_command *cmd; 140 struct nvme_completion *cpl; 141 142 bus_dma_tag_t dma_tag; 143 144 bus_dmamap_t cmd_dma_map; 145 uint64_t cmd_bus_addr; 146 147 bus_dmamap_t cpl_dma_map; 148 uint64_t cpl_bus_addr; 149 150 uint32_t num_tr; 151 uint32_t num_prp_list; 152 153 SLIST_HEAD(, nvme_tracker) free_tr; 154 155 struct nvme_tracker **act_tr; 156 157 SLIST_HEAD(, nvme_prp_list) free_prp_list; 158 159 struct mtx lock __aligned(CACHE_LINE_SIZE); 160 161} __aligned(CACHE_LINE_SIZE); 162 163struct nvme_namespace { 164 165 struct nvme_controller *ctrlr; 166 struct nvme_namespace_data data; 167 uint16_t id; 168 uint16_t flags; 169 struct cdev *cdev; 170}; 171 172/* 173 * One of these per allocated PCI device. 174 */ 175struct nvme_controller { 176 177 device_t dev; 178 179 uint32_t ready_timeout_in_ms; 180 181 bus_space_tag_t bus_tag; 182 bus_space_handle_t bus_handle; 183 int resource_id; 184 struct resource *resource; 185 186#ifdef CHATHAM2 187 bus_space_tag_t chatham_bus_tag; 188 bus_space_handle_t chatham_bus_handle; 189 int chatham_resource_id; 190 struct resource *chatham_resource; 191#endif 192 193 uint32_t msix_enabled; 194 uint32_t force_intx; 195 196 uint32_t num_io_queues; 197 boolean_t per_cpu_io_queues; 198 199 /* Fields for tracking progress during controller initialization. */ 200 struct intr_config_hook config_hook; 201 uint32_t ns_identified; 202 uint32_t queues_created; 203 204 /* For shared legacy interrupt. */ 205 int rid; 206 struct resource *res; 207 void *tag; 208 struct task task; 209 struct taskqueue *taskqueue; 210 211 bus_dma_tag_t hw_desc_tag; 212 bus_dmamap_t hw_desc_map; 213 214 /** maximum i/o size in bytes */ 215 uint32_t max_xfer_size; 216 217 /** interrupt coalescing time period (in microseconds) */ 218 uint32_t int_coal_time; 219 220 /** interrupt coalescing threshold */ 221 uint32_t int_coal_threshold; 222 223 struct nvme_qpair adminq; 224 struct nvme_qpair *ioq; 225 226 struct nvme_registers *regs; 227 228 struct nvme_controller_data cdata; 229 struct nvme_namespace ns[NVME_MAX_NAMESPACES]; 230 231 struct cdev *cdev; 232 233 boolean_t is_started; 234 235#ifdef CHATHAM2 236 uint64_t chatham_size; 237 uint64_t chatham_lbas; 238#endif 239}; 240 241#define nvme_mmio_offsetof(reg) \ 242 offsetof(struct nvme_registers, reg) 243 244#define nvme_mmio_read_4(sc, reg) \ 245 bus_space_read_4((sc)->bus_tag, (sc)->bus_handle, \ 246 nvme_mmio_offsetof(reg)) 247 248#define nvme_mmio_write_4(sc, reg, val) \ 249 bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \ 250 nvme_mmio_offsetof(reg), val) 251 252#define nvme_mmio_write_8(sc, reg, val) \ 253 do { \ 254 bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \ 255 nvme_mmio_offsetof(reg), val & 0xFFFFFFFF); \ 256 bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \ 257 nvme_mmio_offsetof(reg)+4, \ 258 (val & 0xFFFFFFFF00000000UL) >> 32); \ 259 } while (0); 260 261#ifdef CHATHAM2 262#define chatham_read_4(softc, reg) \ 263 bus_space_read_4((softc)->chatham_bus_tag, \ 264 (softc)->chatham_bus_handle, reg) 265 266#define chatham_write_8(sc, reg, val) \ 267 do { \ 268 bus_space_write_4((sc)->chatham_bus_tag, \ 269 (sc)->chatham_bus_handle, reg, val & 0xffffffff); \ 270 bus_space_write_4((sc)->chatham_bus_tag, \ 271 (sc)->chatham_bus_handle, reg+4, \ 272 (val & 0xFFFFFFFF00000000UL) >> 32); \ 273 } while (0); 274 275#endif /* CHATHAM2 */ 276 277#if __FreeBSD_version < 800054 278#define wmb() __asm volatile("sfence" ::: "memory") 279#define mb() __asm volatile("mfence" ::: "memory") 280#endif 281 282void nvme_ns_test(struct nvme_namespace *ns, u_long cmd, caddr_t arg); 283 284void nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr, 285 uint8_t feature, uint32_t cdw11, 286 void *payload, uint32_t payload_size, 287 nvme_cb_fn_t cb_fn, void *cb_arg); 288void nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr, 289 uint8_t feature, uint32_t cdw11, 290 void *payload, uint32_t payload_size, 291 nvme_cb_fn_t cb_fn, void *cb_arg); 292void nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr, 293 void *payload, 294 nvme_cb_fn_t cb_fn, void *cb_arg); 295void nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr, 296 uint16_t nsid, void *payload, 297 nvme_cb_fn_t cb_fn, void *cb_arg); 298void nvme_ctrlr_cmd_set_interrupt_coalescing(struct nvme_controller *ctrlr, 299 uint32_t microseconds, 300 uint32_t threshold, 301 nvme_cb_fn_t cb_fn, 302 void *cb_arg); 303void nvme_ctrlr_cmd_get_health_information_page(struct nvme_controller *ctrlr, 304 uint32_t nsid, 305 struct nvme_health_information_page *payload, 306 nvme_cb_fn_t cb_fn, 307 void *cb_arg); 308void nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr, 309 struct nvme_qpair *io_que, uint16_t vector, 310 nvme_cb_fn_t cb_fn, void *cb_arg); 311void nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr, 312 struct nvme_qpair *io_que, 313 nvme_cb_fn_t cb_fn, void *cb_arg); 314void nvme_ctrlr_cmd_delete_io_cq(struct nvme_controller *ctrlr, 315 struct nvme_qpair *io_que, 316 nvme_cb_fn_t cb_fn, void *cb_arg); 317void nvme_ctrlr_cmd_delete_io_sq(struct nvme_controller *ctrlr, 318 struct nvme_qpair *io_que, 319 nvme_cb_fn_t cb_fn, void *cb_arg); 320void nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr, 321 uint32_t num_queues, nvme_cb_fn_t cb_fn, 322 void *cb_arg); 323void nvme_ctrlr_cmd_set_asynchronous_event_config(struct nvme_controller *ctrlr, 324 union nvme_critical_warning_state state, 325 nvme_cb_fn_t cb_fn, void *cb_arg); 326void nvme_ctrlr_cmd_asynchronous_event_request(struct nvme_controller *ctrlr, 327 nvme_cb_fn_t cb_fn, 328 void *cb_arg); 329 330struct nvme_tracker * nvme_allocate_tracker(struct nvme_controller *ctrlr, 331 boolean_t is_admin, 332 nvme_cb_fn_t cb_fn, void *cb_arg, 333 uint32_t payload_size, 334 void *payload); 335void nvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, 336 int error); 337 338int nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev); 339int nvme_ctrlr_reset(struct nvme_controller *ctrlr); 340/* ctrlr defined as void * to allow use with config_intrhook. */ 341void nvme_ctrlr_start(void *ctrlr_arg); 342 343void nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id, 344 uint16_t vector, uint32_t num_entries, 345 uint32_t max_xfer_size, 346 struct nvme_controller *ctrlr); 347void nvme_qpair_submit_cmd(struct nvme_qpair *qpair, 348 struct nvme_tracker *tr); 349void nvme_qpair_process_completions(struct nvme_qpair *qpair); 350struct nvme_tracker * nvme_qpair_allocate_tracker(struct nvme_qpair *qpair, 351 boolean_t alloc_prp_list); 352 353void nvme_admin_qpair_destroy(struct nvme_qpair *qpair); 354 355void nvme_io_qpair_destroy(struct nvme_qpair *qpair); 356 357int nvme_ns_construct(struct nvme_namespace *ns, uint16_t id, 358 struct nvme_controller *ctrlr); 359 360int nvme_ns_physio(struct cdev *dev, struct uio *uio, int ioflag); 361 362void nvme_sysctl_initialize_ctrlr(struct nvme_controller *ctrlr); 363 364void nvme_dump_command(struct nvme_command *cmd); 365void nvme_dump_completion(struct nvme_completion *cpl); 366 367static __inline void 368nvme_single_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) 369{ 370 uint64_t *bus_addr = (uint64_t *)arg; 371 372 *bus_addr = seg[0].ds_addr; 373} 374 375#endif /* __NVME_PRIVATE_H__ */ 376