1/* $NetBSD: nvmevar.h,v 1.28 2022/08/14 12:08:57 jmcneill Exp $ */ 2/* $OpenBSD: nvmevar.h,v 1.8 2016/04/14 11:18:32 dlg Exp $ */ 3 4/* 5 * Copyright (c) 2014 David Gwynne <dlg@openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20#include <sys/bus.h> 21#include <sys/cpu.h> 22#include <sys/device.h> 23#include <sys/mutex.h> 24#include <sys/pool.h> 25#include <sys/queue.h> 26#include <sys/buf.h> 27 28struct nvme_dmamem { 29 bus_dmamap_t ndm_map; 30 bus_dma_segment_t ndm_seg; 31 size_t ndm_size; 32 void *ndm_kva; 33}; 34#define NVME_DMA_MAP(_ndm) ((_ndm)->ndm_map) 35#define NVME_DMA_LEN(_ndm) ((_ndm)->ndm_map->dm_segs[0].ds_len) 36#define NVME_DMA_DVA(_ndm) ((uint64_t)(_ndm)->ndm_map->dm_segs[0].ds_addr) 37#define NVME_DMA_KVA(_ndm) ((void *)(_ndm)->ndm_kva) 38 39struct nvme_softc; 40struct nvme_queue; 41 42typedef void (*nvme_nnc_done)(void *, struct buf *, uint16_t, uint32_t); 43 44struct nvme_ccb { 45 SIMPLEQ_ENTRY(nvme_ccb) ccb_entry; 46 47 /* DMA handles */ 48 bus_dmamap_t ccb_dmamap; 49 50 bus_addr_t ccb_prpl_off; 51 uint64_t ccb_prpl_dva; 52 uint64_t *ccb_prpl; 53 54 /* command context */ 55 uint16_t ccb_id; 56 void *ccb_cookie; 57#define NVME_CCB_FREE 0xbeefdeed 58 void (*ccb_done)(struct nvme_queue *, 59 struct nvme_ccb *, struct nvme_cqe *); 60 61 /* namespace context */ 62 void *nnc_cookie; 63 nvme_nnc_done nnc_done; 64 uint16_t nnc_nsid; 65 uint16_t nnc_flags; 66#define NVME_NS_CTX_F_READ __BIT(0) 67#define NVME_NS_CTX_F_POLL __BIT(1) 68#define NVME_NS_CTX_F_FUA __BIT(2) 69 70 struct buf *nnc_buf; 71 daddr_t nnc_blkno; 72 size_t nnc_datasize; 73 int nnc_secsize; 74}; 75 76struct nvme_queue { 77 struct nvme_softc *q_sc; 78 kmutex_t q_sq_mtx; 79 kmutex_t q_cq_mtx; 80 struct nvme_dmamem *q_sq_dmamem; 81 struct nvme_dmamem *q_cq_dmamem; 82 struct nvme_dmamem *q_nvmmu_dmamem; /* for apple m1 nvme */ 83 84 bus_size_t q_sqtdbl; /* submission queue tail doorbell */ 85 bus_size_t q_cqhdbl; /* completion queue head doorbell */ 86 uint16_t q_id; 87 uint32_t q_entries; 88 uint32_t q_sq_tail; 89 uint32_t q_cq_head; 90 uint16_t q_cq_phase; 91 92 kmutex_t q_ccb_mtx; 93 kcondvar_t q_ccb_wait; /* wait for ccb avail/finish */ 94 bool q_ccb_waiting; /* whether there are waiters */ 95 uint16_t q_nccbs; /* total number of ccbs */ 96 struct nvme_ccb *q_ccbs; 97 SIMPLEQ_HEAD(, nvme_ccb) q_ccb_list; 98 struct nvme_dmamem *q_ccb_prpls; 99}; 100 101struct nvme_namespace { 102 struct nvm_identify_namespace *ident; 103 device_t dev; 104 uint32_t flags; 105#define NVME_NS_F_OPEN __BIT(0) 106}; 107 108struct nvme_ops { 109 void (*op_enable)(struct nvme_softc *); 110 111 int (*op_q_alloc)(struct nvme_softc *, 112 struct nvme_queue *); 113 void (*op_q_free)(struct nvme_softc *, 114 struct nvme_queue *); 115 116 uint32_t (*op_sq_enter)(struct nvme_softc *, 117 struct nvme_queue *, struct nvme_ccb *); 118 void (*op_sq_leave)(struct nvme_softc *, 119 struct nvme_queue *, struct nvme_ccb *); 120 uint32_t (*op_sq_enter_locked)(struct nvme_softc *, 121 struct nvme_queue *, struct nvme_ccb *); 122 void (*op_sq_leave_locked)(struct nvme_softc *, 123 struct nvme_queue *, struct nvme_ccb *); 124 125 void (*op_cq_done)(struct nvme_softc *, 126 struct nvme_queue *, struct nvme_ccb *); 127}; 128 129struct nvme_softc { 130 device_t sc_dev; 131 132 const struct nvme_ops *sc_ops; 133 134 bus_space_tag_t sc_iot; 135 bus_space_handle_t sc_ioh; 136 bus_size_t sc_ios; 137 bus_dma_tag_t sc_dmat; 138 139 int (*sc_intr_establish)(struct nvme_softc *, 140 uint16_t qid, struct nvme_queue *); 141 int (*sc_intr_disestablish)(struct nvme_softc *, 142 uint16_t qid); 143 void **sc_ih; /* interrupt handlers */ 144 void **sc_softih; /* softintr handlers */ 145 146 u_int sc_rdy_to; /* RDY timeout */ 147 size_t sc_mps; /* memory page size */ 148 size_t sc_mdts; /* max data trasfer size */ 149 u_int sc_max_sgl; /* max S/G segments */ 150 u_int sc_dstrd; 151 152 struct nvm_identify_controller 153 sc_identify; 154 155 u_int sc_nn; /* namespace count */ 156 struct nvme_namespace *sc_namespaces; 157 158 bool sc_use_mq; 159 u_int sc_nq; /* # of io queue (sc_q) */ 160 struct nvme_queue *sc_admin_q; 161 struct nvme_queue **sc_q; 162 163 uint32_t sc_flags; 164#define NVME_F_ATTACHED __BIT(0) 165#define NVME_F_OPEN __BIT(1) 166 167 uint32_t sc_quirks; 168#define NVME_QUIRK_DELAY_B4_CHK_RDY __BIT(0) 169#define NVME_QUIRK_NOMSI __BIT(1) 170 171 char sc_modelname[81]; 172}; 173 174#define lemtoh16(p) le16toh(*((uint16_t *)(p))) 175#define lemtoh32(p) le32toh(*((uint32_t *)(p))) 176#define lemtoh64(p) le64toh(*((uint64_t *)(p))) 177#define htolem16(p, x) (*((uint16_t *)(p)) = htole16(x)) 178#define htolem32(p, x) (*((uint32_t *)(p)) = htole32(x)) 179#define htolem64(p, x) (*((uint64_t *)(p)) = htole64(x)) 180 181struct nvme_attach_args { 182 uint16_t naa_nsid; 183 uint32_t naa_qentries; /* total number of queue slots */ 184 uint32_t naa_maxphys; /* maximum device transfer size */ 185 const char *naa_typename; /* identifier */ 186}; 187 188int nvme_attach(struct nvme_softc *); 189int nvme_detach(struct nvme_softc *, int flags); 190int nvme_rescan(device_t, const char *, const int *); 191void nvme_childdet(device_t, device_t); 192int nvme_suspend(struct nvme_softc *); 193int nvme_resume(struct nvme_softc *); 194int nvme_intr(void *); 195void nvme_softintr_intx(void *); 196int nvme_intr_msi(void *); 197void nvme_softintr_msi(void *); 198 199static __inline struct nvme_queue * 200nvme_get_q(struct nvme_softc *sc) 201{ 202 return sc->sc_q[cpu_index(curcpu()) % sc->sc_nq]; 203} 204 205/* 206 * namespace 207 */ 208static __inline struct nvme_namespace * 209nvme_ns_get(struct nvme_softc *sc, uint16_t nsid) 210{ 211 if (nsid == 0 || nsid - 1 >= sc->sc_nn) 212 return NULL; 213 return &sc->sc_namespaces[nsid - 1]; 214} 215 216#define nvme_read4(_s, _r) \ 217 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 218#define nvme_write4(_s, _r, _v) \ 219 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 220uint64_t 221 nvme_read8(struct nvme_softc *, bus_size_t); 222void nvme_write8(struct nvme_softc *, bus_size_t, uint64_t); 223 224#define nvme_barrier(_s, _r, _l, _f) \ 225 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 226 227struct nvme_dmamem * 228 nvme_dmamem_alloc(struct nvme_softc *, size_t); 229void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 230void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, int); 231 232int nvme_ns_identify(struct nvme_softc *, uint16_t); 233void nvme_ns_free(struct nvme_softc *, uint16_t); 234int nvme_ns_dobio(struct nvme_softc *, uint16_t, void *, 235 struct buf *, void *, size_t, int, daddr_t, int, nvme_nnc_done); 236int nvme_ns_sync(struct nvme_softc *, uint16_t, int); 237int nvme_admin_getcache(struct nvme_softc *, int *); 238int nvme_admin_setcache(struct nvme_softc *, int); 239