1301771Simp/*- 2301771Simp * Copyright (c) 2015 Netflix, Inc 3301771Simp * All rights reserved. 4301771Simp * 5301771Simp * Redistribution and use in source and binary forms, with or without 6301771Simp * modification, are permitted provided that the following conditions 7301771Simp * are met: 8301771Simp * 1. Redistributions of source code must retain the above copyright 9301771Simp * notice, this list of conditions and the following disclaimer, 10301771Simp * without modification, immediately at the beginning of the file. 11301771Simp * 2. Redistributions in binary form must reproduce the above copyright 12301771Simp * notice, this list of conditions and the following disclaimer in the 13301771Simp * documentation and/or other materials provided with the distribution. 14301771Simp * 15301771Simp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16301771Simp * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17301771Simp * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18301771Simp * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19301771Simp * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20301771Simp * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21301771Simp * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22301771Simp * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23301771Simp * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24301771Simp * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25301771Simp * 26301771Simp * Derived from ata_da.c: 27301771Simp * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org> 28301771Simp */ 29301771Simp 30301771Simp#include <sys/cdefs.h> 31301771Simp__FBSDID("$FreeBSD: stable/11/sys/cam/nvme/nvme_da.c 359237 2020-03-23 12:39:18Z dab $"); 32301771Simp 33301771Simp#include <sys/param.h> 34301771Simp 35301771Simp#ifdef _KERNEL 36301771Simp#include <sys/systm.h> 37301771Simp#include <sys/kernel.h> 38301771Simp#include <sys/bio.h> 39301771Simp#include <sys/sysctl.h> 40301771Simp#include <sys/taskqueue.h> 41301771Simp#include <sys/lock.h> 42301771Simp#include <sys/mutex.h> 43301771Simp#include <sys/conf.h> 44301771Simp#include <sys/devicestat.h> 45301771Simp#include <sys/eventhandler.h> 46301771Simp#include <sys/malloc.h> 47301771Simp#include <sys/cons.h> 48301771Simp#include <sys/proc.h> 49301771Simp#include <sys/reboot.h> 50301771Simp#include <geom/geom_disk.h> 51301771Simp#endif /* _KERNEL */ 52301771Simp 53301771Simp#ifndef _KERNEL 54301771Simp#include <stdio.h> 55301771Simp#include <string.h> 56301771Simp#endif /* _KERNEL */ 57301771Simp 58301771Simp#include <cam/cam.h> 59301771Simp#include <cam/cam_ccb.h> 60301771Simp#include <cam/cam_periph.h> 61301771Simp#include <cam/cam_xpt_periph.h> 62301771Simp#include <cam/cam_sim.h> 63301771Simp#include <cam/cam_iosched.h> 64301771Simp 65301771Simp#include <cam/nvme/nvme_all.h> 66301771Simp 67301771Simptypedef enum { 68301771Simp NDA_STATE_NORMAL 69301771Simp} nda_state; 70301771Simp 71301771Simptypedef enum { 72301771Simp NDA_FLAG_OPEN = 0x0001, 73301771Simp NDA_FLAG_DIRTY = 0x0002, 74301771Simp NDA_FLAG_SCTX_INIT = 0x0004, 75301771Simp} nda_flags; 76301771Simp 77301771Simptypedef enum { 78301771Simp NDA_Q_4K = 0x01, 79301771Simp NDA_Q_NONE = 0x00, 80301771Simp} nda_quirks; 81301771Simp 82301771Simp#define NDA_Q_BIT_STRING \ 83301771Simp "\020" \ 84301771Simp "\001Bit 0" 85301771Simp 86301771Simptypedef enum { 87301771Simp NDA_CCB_BUFFER_IO = 0x01, 88301771Simp NDA_CCB_DUMP = 0x02, 89301771Simp NDA_CCB_TRIM = 0x03, 90301771Simp NDA_CCB_TYPE_MASK = 0x0F, 91301771Simp} nda_ccb_state; 92301771Simp 93301771Simp/* Offsets into our private area for storing information */ 94301771Simp#define ccb_state ppriv_field0 95301771Simp#define ccb_bp ppriv_ptr1 96301771Simp 97301771Simpstruct trim_request { 98301771Simp TAILQ_HEAD(, bio) bps; 99301771Simp}; 100301771Simpstruct nda_softc { 101301771Simp struct cam_iosched_softc *cam_iosched; 102301771Simp int outstanding_cmds; /* Number of active commands */ 103301771Simp int refcount; /* Active xpt_action() calls */ 104301771Simp nda_state state; 105301771Simp nda_flags flags; 106301771Simp nda_quirks quirks; 107301771Simp int unmappedio; 108301771Simp uint32_t nsid; /* Namespace ID for this nda device */ 109301771Simp struct disk *disk; 110301771Simp struct task sysctl_task; 111301771Simp struct sysctl_ctx_list sysctl_ctx; 112301771Simp struct sysctl_oid *sysctl_tree; 113301771Simp struct trim_request trim_req; 114301771Simp#ifdef CAM_IO_STATS 115301771Simp struct sysctl_ctx_list sysctl_stats_ctx; 116301771Simp struct sysctl_oid *sysctl_stats_tree; 117301771Simp u_int timeouts; 118301771Simp u_int errors; 119301771Simp u_int invalidations; 120301771Simp#endif 121301771Simp}; 122301771Simp 123301771Simp/* Need quirk table */ 124301771Simp 125301771Simpstatic disk_strategy_t ndastrategy; 126301771Simpstatic dumper_t ndadump; 127301771Simpstatic periph_init_t ndainit; 128301771Simpstatic void ndaasync(void *callback_arg, u_int32_t code, 129301771Simp struct cam_path *path, void *arg); 130301771Simpstatic void ndasysctlinit(void *context, int pending); 131301771Simpstatic periph_ctor_t ndaregister; 132301771Simpstatic periph_dtor_t ndacleanup; 133301771Simpstatic periph_start_t ndastart; 134301771Simpstatic periph_oninv_t ndaoninvalidate; 135301771Simpstatic void ndadone(struct cam_periph *periph, 136301771Simp union ccb *done_ccb); 137301771Simpstatic int ndaerror(union ccb *ccb, u_int32_t cam_flags, 138301771Simp u_int32_t sense_flags); 139301771Simpstatic void ndashutdown(void *arg, int howto); 140301771Simpstatic void ndasuspend(void *arg); 141301771Simp 142301771Simp#ifndef NDA_DEFAULT_SEND_ORDERED 143301771Simp#define NDA_DEFAULT_SEND_ORDERED 1 144301771Simp#endif 145301771Simp#ifndef NDA_DEFAULT_TIMEOUT 146301771Simp#define NDA_DEFAULT_TIMEOUT 30 /* Timeout in seconds */ 147301771Simp#endif 148301771Simp#ifndef NDA_DEFAULT_RETRY 149301771Simp#define NDA_DEFAULT_RETRY 4 150301771Simp#endif 151301771Simp 152301771Simp 153301771Simp//static int nda_retry_count = NDA_DEFAULT_RETRY; 154301771Simpstatic int nda_send_ordered = NDA_DEFAULT_SEND_ORDERED; 155301771Simpstatic int nda_default_timeout = NDA_DEFAULT_TIMEOUT; 156301771Simp 157301771Simp/* 158301771Simp * All NVMe media is non-rotational, so all nvme device instances 159301771Simp * share this to implement the sysctl. 160301771Simp */ 161301771Simpstatic int nda_rotating_media = 0; 162301771Simp 163301771Simpstatic SYSCTL_NODE(_kern_cam, OID_AUTO, nda, CTLFLAG_RD, 0, 164301771Simp "CAM Direct Access Disk driver"); 165301771Simp 166301771Simpstatic struct periph_driver ndadriver = 167301771Simp{ 168301771Simp ndainit, "nda", 169301771Simp TAILQ_HEAD_INITIALIZER(ndadriver.units), /* generation */ 0 170301771Simp}; 171301771Simp 172301771SimpPERIPHDRIVER_DECLARE(nda, ndadriver); 173301771Simp 174301771Simpstatic MALLOC_DEFINE(M_NVMEDA, "nvme_da", "nvme_da buffers"); 175301771Simp 176301771Simp/* 177301771Simp * nice wrappers. Maybe these belong in nvme_all.c instead of 178301771Simp * here, but this is the only place that uses these. Should 179301771Simp * we ever grow another NVME periph, we should move them 180301771Simp * all there wholesale. 181301771Simp */ 182301771Simp 183301771Simpstatic void 184301771Simpnda_nvme_flush(struct nda_softc *softc, struct ccb_nvmeio *nvmeio) 185301771Simp{ 186301771Simp cam_fill_nvmeio(nvmeio, 187301771Simp 0, /* retries */ 188301771Simp ndadone, /* cbfcnp */ 189301771Simp CAM_DIR_NONE, /* flags */ 190301771Simp NULL, /* data_ptr */ 191301771Simp 0, /* dxfer_len */ 192328737Smav nda_default_timeout * 1000); /* timeout 30s */ 193301771Simp nvme_ns_flush_cmd(&nvmeio->cmd, softc->nsid); 194301771Simp} 195301771Simp 196301771Simpstatic void 197301771Simpnda_nvme_trim(struct nda_softc *softc, struct ccb_nvmeio *nvmeio, 198301771Simp void *payload, uint32_t num_ranges) 199301771Simp{ 200301771Simp cam_fill_nvmeio(nvmeio, 201301771Simp 0, /* retries */ 202301771Simp ndadone, /* cbfcnp */ 203301771Simp CAM_DIR_OUT, /* flags */ 204301771Simp payload, /* data_ptr */ 205301771Simp num_ranges * sizeof(struct nvme_dsm_range), /* dxfer_len */ 206328737Smav nda_default_timeout * 1000); /* timeout 30s */ 207301771Simp nvme_ns_trim_cmd(&nvmeio->cmd, softc->nsid, num_ranges); 208301771Simp} 209301771Simp 210301771Simpstatic void 211301771Simpnda_nvme_write(struct nda_softc *softc, struct ccb_nvmeio *nvmeio, 212301771Simp void *payload, uint64_t lba, uint32_t len, uint32_t count) 213301771Simp{ 214301771Simp cam_fill_nvmeio(nvmeio, 215301771Simp 0, /* retries */ 216301771Simp ndadone, /* cbfcnp */ 217301771Simp CAM_DIR_OUT, /* flags */ 218301771Simp payload, /* data_ptr */ 219301771Simp len, /* dxfer_len */ 220328737Smav nda_default_timeout * 1000); /* timeout 30s */ 221301771Simp nvme_ns_write_cmd(&nvmeio->cmd, softc->nsid, lba, count); 222301771Simp} 223301771Simp 224301771Simpstatic void 225301771Simpnda_nvme_rw_bio(struct nda_softc *softc, struct ccb_nvmeio *nvmeio, 226301771Simp struct bio *bp, uint32_t rwcmd) 227301771Simp{ 228301771Simp int flags = rwcmd == NVME_OPC_READ ? CAM_DIR_IN : CAM_DIR_OUT; 229301771Simp void *payload; 230301771Simp uint64_t lba; 231301771Simp uint32_t count; 232301771Simp 233301771Simp if (bp->bio_flags & BIO_UNMAPPED) { 234301771Simp flags |= CAM_DATA_BIO; 235301771Simp payload = bp; 236301771Simp } else { 237301771Simp payload = bp->bio_data; 238301771Simp } 239301771Simp 240301771Simp lba = bp->bio_pblkno; 241301771Simp count = bp->bio_bcount / softc->disk->d_sectorsize; 242301771Simp 243301771Simp cam_fill_nvmeio(nvmeio, 244301771Simp 0, /* retries */ 245301771Simp ndadone, /* cbfcnp */ 246301771Simp flags, /* flags */ 247301771Simp payload, /* data_ptr */ 248301771Simp bp->bio_bcount, /* dxfer_len */ 249328737Smav nda_default_timeout * 1000); /* timeout 30s */ 250301771Simp nvme_ns_rw_cmd(&nvmeio->cmd, rwcmd, softc->nsid, lba, count); 251301771Simp} 252301771Simp 253301771Simpstatic int 254301771Simpndaopen(struct disk *dp) 255301771Simp{ 256301771Simp struct cam_periph *periph; 257301771Simp struct nda_softc *softc; 258301771Simp int error; 259301771Simp 260301771Simp periph = (struct cam_periph *)dp->d_drv1; 261301771Simp if (cam_periph_acquire(periph) != CAM_REQ_CMP) { 262301771Simp return(ENXIO); 263301771Simp } 264301771Simp 265301771Simp cam_periph_lock(periph); 266301771Simp if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) { 267301771Simp cam_periph_unlock(periph); 268301771Simp cam_periph_release(periph); 269301771Simp return (error); 270301771Simp } 271301771Simp 272301771Simp CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH, 273301771Simp ("ndaopen\n")); 274301771Simp 275301771Simp softc = (struct nda_softc *)periph->softc; 276301771Simp softc->flags |= NDA_FLAG_OPEN; 277301771Simp 278301771Simp cam_periph_unhold(periph); 279301771Simp cam_periph_unlock(periph); 280301771Simp return (0); 281301771Simp} 282301771Simp 283301771Simpstatic int 284301771Simpndaclose(struct disk *dp) 285301771Simp{ 286301771Simp struct cam_periph *periph; 287301771Simp struct nda_softc *softc; 288301771Simp union ccb *ccb; 289301771Simp int error; 290301771Simp 291301771Simp periph = (struct cam_periph *)dp->d_drv1; 292301771Simp softc = (struct nda_softc *)periph->softc; 293301771Simp cam_periph_lock(periph); 294301771Simp 295301771Simp CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH, 296301771Simp ("ndaclose\n")); 297301771Simp 298301771Simp if ((softc->flags & NDA_FLAG_DIRTY) != 0 && 299301771Simp (periph->flags & CAM_PERIPH_INVALID) == 0 && 300301771Simp cam_periph_hold(periph, PRIBIO) == 0) { 301301771Simp 302301771Simp ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL); 303301771Simp nda_nvme_flush(softc, &ccb->nvmeio); 304301771Simp error = cam_periph_runccb(ccb, ndaerror, /*cam_flags*/0, 305301771Simp /*sense_flags*/0, softc->disk->d_devstat); 306301771Simp 307301771Simp if (error != 0) 308301771Simp xpt_print(periph->path, "Synchronize cache failed\n"); 309301771Simp else 310301771Simp softc->flags &= ~NDA_FLAG_DIRTY; 311301771Simp xpt_release_ccb(ccb); 312301771Simp cam_periph_unhold(periph); 313301771Simp } 314301771Simp 315301771Simp softc->flags &= ~NDA_FLAG_OPEN; 316301771Simp 317301771Simp while (softc->refcount != 0) 318301771Simp cam_periph_sleep(periph, &softc->refcount, PRIBIO, "ndaclose", 1); 319301771Simp cam_periph_unlock(periph); 320301771Simp cam_periph_release(periph); 321301771Simp return (0); 322301771Simp} 323301771Simp 324301771Simpstatic void 325301771Simpndaschedule(struct cam_periph *periph) 326301771Simp{ 327301771Simp struct nda_softc *softc = (struct nda_softc *)periph->softc; 328301771Simp 329301771Simp if (softc->state != NDA_STATE_NORMAL) 330301771Simp return; 331301771Simp 332301771Simp cam_iosched_schedule(softc->cam_iosched, periph); 333301771Simp} 334301771Simp 335301771Simp/* 336301771Simp * Actually translate the requested transfer into one the physical driver 337301771Simp * can understand. The transfer is described by a buf and will include 338301771Simp * only one physical transfer. 339301771Simp */ 340301771Simpstatic void 341301771Simpndastrategy(struct bio *bp) 342301771Simp{ 343301771Simp struct cam_periph *periph; 344301771Simp struct nda_softc *softc; 345301771Simp 346301771Simp periph = (struct cam_periph *)bp->bio_disk->d_drv1; 347301771Simp softc = (struct nda_softc *)periph->softc; 348301771Simp 349301771Simp cam_periph_lock(periph); 350301771Simp 351301771Simp CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("ndastrategy(%p)\n", bp)); 352301771Simp 353301771Simp /* 354301771Simp * If the device has been made invalid, error out 355301771Simp */ 356301771Simp if ((periph->flags & CAM_PERIPH_INVALID) != 0) { 357301771Simp cam_periph_unlock(periph); 358301771Simp biofinish(bp, NULL, ENXIO); 359301771Simp return; 360301771Simp } 361301771Simp 362301771Simp /* 363301771Simp * Place it in the queue of disk activities for this disk 364301771Simp */ 365301771Simp cam_iosched_queue_work(softc->cam_iosched, bp); 366301771Simp 367301771Simp /* 368301771Simp * Schedule ourselves for performing the work. 369301771Simp */ 370301771Simp ndaschedule(periph); 371301771Simp cam_periph_unlock(periph); 372301771Simp 373301771Simp return; 374301771Simp} 375301771Simp 376301771Simpstatic int 377301771Simpndadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length) 378301771Simp{ 379301771Simp struct cam_periph *periph; 380301771Simp struct nda_softc *softc; 381301771Simp u_int secsize; 382328704Smav struct ccb_nvmeio nvmeio; 383301771Simp struct disk *dp; 384301771Simp uint64_t lba; 385301771Simp uint32_t count; 386301771Simp int error = 0; 387301771Simp 388301771Simp dp = arg; 389301771Simp periph = dp->d_drv1; 390301771Simp softc = (struct nda_softc *)periph->softc; 391301771Simp cam_periph_lock(periph); 392301771Simp secsize = softc->disk->d_sectorsize; 393301771Simp lba = offset / secsize; 394301771Simp count = length / secsize; 395301771Simp 396301771Simp if ((periph->flags & CAM_PERIPH_INVALID) != 0) { 397301771Simp cam_periph_unlock(periph); 398301771Simp return (ENXIO); 399301771Simp } 400301771Simp 401328703Smav /* xpt_get_ccb returns a zero'd allocation for the ccb, mimic that here */ 402328703Smav memset(&nvmeio, 0, sizeof(nvmeio)); 403301771Simp if (length > 0) { 404328704Smav xpt_setup_ccb(&nvmeio.ccb_h, periph->path, CAM_PRIORITY_NORMAL); 405328704Smav nvmeio.ccb_h.ccb_state = NDA_CCB_DUMP; 406328704Smav nda_nvme_write(softc, &nvmeio, virtual, lba, length, count); 407328704Smav xpt_polled_action((union ccb *)&nvmeio); 408301771Simp 409328704Smav error = cam_periph_error((union ccb *)&nvmeio, 410301771Simp 0, SF_NO_RECOVERY | SF_NO_RETRY, NULL); 411328704Smav if ((nvmeio.ccb_h.status & CAM_DEV_QFRZN) != 0) 412328704Smav cam_release_devq(nvmeio.ccb_h.path, /*relsim_flags*/0, 413301771Simp /*reduction*/0, /*timeout*/0, /*getcount_only*/0); 414301771Simp if (error != 0) 415301771Simp printf("Aborting dump due to I/O error.\n"); 416301771Simp 417301771Simp cam_periph_unlock(periph); 418301771Simp return (error); 419301771Simp } 420301771Simp 421301771Simp /* Flush */ 422328704Smav xpt_setup_ccb(&nvmeio.ccb_h, periph->path, CAM_PRIORITY_NORMAL); 423301771Simp 424328704Smav nvmeio.ccb_h.ccb_state = NDA_CCB_DUMP; 425328704Smav nda_nvme_flush(softc, &nvmeio); 426328704Smav xpt_polled_action((union ccb *)&nvmeio); 427301771Simp 428328704Smav error = cam_periph_error((union ccb *)&nvmeio, 429301771Simp 0, SF_NO_RECOVERY | SF_NO_RETRY, NULL); 430328704Smav if ((nvmeio.ccb_h.status & CAM_DEV_QFRZN) != 0) 431328704Smav cam_release_devq(nvmeio.ccb_h.path, /*relsim_flags*/0, 432301771Simp /*reduction*/0, /*timeout*/0, /*getcount_only*/0); 433301771Simp if (error != 0) 434301771Simp xpt_print(periph->path, "flush cmd failed\n"); 435301771Simp cam_periph_unlock(periph); 436301771Simp return (error); 437301771Simp} 438301771Simp 439301771Simpstatic void 440301771Simpndainit(void) 441301771Simp{ 442301771Simp cam_status status; 443301771Simp 444301771Simp /* 445301771Simp * Install a global async callback. This callback will 446301771Simp * receive async callbacks like "new device found". 447301771Simp */ 448301771Simp status = xpt_register_async(AC_FOUND_DEVICE, ndaasync, NULL, NULL); 449301771Simp 450301771Simp if (status != CAM_REQ_CMP) { 451301771Simp printf("nda: Failed to attach master async callback " 452301771Simp "due to status 0x%x!\n", status); 453301771Simp } else if (nda_send_ordered) { 454301771Simp 455301771Simp /* Register our event handlers */ 456301771Simp if ((EVENTHANDLER_REGISTER(power_suspend, ndasuspend, 457301771Simp NULL, EVENTHANDLER_PRI_LAST)) == NULL) 458301771Simp printf("ndainit: power event registration failed!\n"); 459301771Simp if ((EVENTHANDLER_REGISTER(shutdown_post_sync, ndashutdown, 460301771Simp NULL, SHUTDOWN_PRI_DEFAULT)) == NULL) 461301771Simp printf("ndainit: shutdown event registration failed!\n"); 462301771Simp } 463301771Simp} 464301771Simp 465301771Simp/* 466301771Simp * Callback from GEOM, called when it has finished cleaning up its 467301771Simp * resources. 468301771Simp */ 469301771Simpstatic void 470301771Simpndadiskgonecb(struct disk *dp) 471301771Simp{ 472301771Simp struct cam_periph *periph; 473301771Simp 474301771Simp periph = (struct cam_periph *)dp->d_drv1; 475301771Simp 476301771Simp cam_periph_release(periph); 477301771Simp} 478301771Simp 479301771Simpstatic void 480301771Simpndaoninvalidate(struct cam_periph *periph) 481301771Simp{ 482301771Simp struct nda_softc *softc; 483301771Simp 484301771Simp softc = (struct nda_softc *)periph->softc; 485301771Simp 486301771Simp /* 487301771Simp * De-register any async callbacks. 488301771Simp */ 489301771Simp xpt_register_async(0, ndaasync, periph, periph->path); 490301771Simp#ifdef CAM_IO_STATS 491301771Simp softc->invalidations++; 492301771Simp#endif 493301771Simp 494301771Simp /* 495301771Simp * Return all queued I/O with ENXIO. 496301771Simp * XXX Handle any transactions queued to the card 497301771Simp * with XPT_ABORT_CCB. 498301771Simp */ 499301771Simp cam_iosched_flush(softc->cam_iosched, NULL, ENXIO); 500301771Simp 501301771Simp disk_gone(softc->disk); 502301771Simp} 503301771Simp 504301771Simpstatic void 505301771Simpndacleanup(struct cam_periph *periph) 506301771Simp{ 507301771Simp struct nda_softc *softc; 508301771Simp 509301771Simp softc = (struct nda_softc *)periph->softc; 510301771Simp 511301771Simp cam_periph_unlock(periph); 512301771Simp 513301771Simp cam_iosched_fini(softc->cam_iosched); 514301771Simp 515301771Simp /* 516301771Simp * If we can't free the sysctl tree, oh well... 517301771Simp */ 518301771Simp if ((softc->flags & NDA_FLAG_SCTX_INIT) != 0) { 519301771Simp#ifdef CAM_IO_STATS 520301771Simp if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0) 521301771Simp xpt_print(periph->path, 522301771Simp "can't remove sysctl stats context\n"); 523301771Simp#endif 524301771Simp if (sysctl_ctx_free(&softc->sysctl_ctx) != 0) 525301771Simp xpt_print(periph->path, 526301771Simp "can't remove sysctl context\n"); 527301771Simp } 528301771Simp 529301771Simp disk_destroy(softc->disk); 530301771Simp free(softc, M_DEVBUF); 531301771Simp cam_periph_lock(periph); 532301771Simp} 533301771Simp 534301771Simpstatic void 535301771Simpndaasync(void *callback_arg, u_int32_t code, 536301771Simp struct cam_path *path, void *arg) 537301771Simp{ 538301771Simp struct cam_periph *periph; 539301771Simp 540301771Simp periph = (struct cam_periph *)callback_arg; 541301771Simp switch (code) { 542301771Simp case AC_FOUND_DEVICE: 543301771Simp { 544301771Simp struct ccb_getdev *cgd; 545301771Simp cam_status status; 546301771Simp 547301771Simp cgd = (struct ccb_getdev *)arg; 548301771Simp if (cgd == NULL) 549301771Simp break; 550301771Simp 551301771Simp if (cgd->protocol != PROTO_NVME) 552301771Simp break; 553301771Simp 554301771Simp /* 555301771Simp * Allocate a peripheral instance for 556301771Simp * this device and start the probe 557301771Simp * process. 558301771Simp */ 559301771Simp status = cam_periph_alloc(ndaregister, ndaoninvalidate, 560301771Simp ndacleanup, ndastart, 561301771Simp "nda", CAM_PERIPH_BIO, 562301771Simp path, ndaasync, 563301771Simp AC_FOUND_DEVICE, cgd); 564301771Simp 565301771Simp if (status != CAM_REQ_CMP 566301771Simp && status != CAM_REQ_INPROG) 567301771Simp printf("ndaasync: Unable to attach to new device " 568301771Simp "due to status 0x%x\n", status); 569301771Simp break; 570301771Simp } 571301771Simp case AC_ADVINFO_CHANGED: 572301771Simp { 573301771Simp uintptr_t buftype; 574301771Simp 575301771Simp buftype = (uintptr_t)arg; 576301771Simp if (buftype == CDAI_TYPE_PHYS_PATH) { 577301771Simp struct nda_softc *softc; 578301771Simp 579301771Simp softc = periph->softc; 580301771Simp disk_attr_changed(softc->disk, "GEOM::physpath", 581301771Simp M_NOWAIT); 582301771Simp } 583301771Simp break; 584301771Simp } 585301771Simp case AC_LOST_DEVICE: 586301771Simp default: 587301771Simp cam_periph_async(periph, code, path, arg); 588301771Simp break; 589301771Simp } 590301771Simp} 591301771Simp 592301771Simpstatic void 593301771Simpndasysctlinit(void *context, int pending) 594301771Simp{ 595301771Simp struct cam_periph *periph; 596301771Simp struct nda_softc *softc; 597327228Smav char tmpstr[32], tmpstr2[16]; 598301771Simp 599301771Simp periph = (struct cam_periph *)context; 600301771Simp 601301771Simp /* periph was held for us when this task was enqueued */ 602301771Simp if ((periph->flags & CAM_PERIPH_INVALID) != 0) { 603301771Simp cam_periph_release(periph); 604301771Simp return; 605301771Simp } 606301771Simp 607301771Simp softc = (struct nda_softc *)periph->softc; 608301771Simp snprintf(tmpstr, sizeof(tmpstr), "CAM NDA unit %d", periph->unit_number); 609301771Simp snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number); 610301771Simp 611301771Simp sysctl_ctx_init(&softc->sysctl_ctx); 612301771Simp softc->flags |= NDA_FLAG_SCTX_INIT; 613301771Simp softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx, 614301771Simp SYSCTL_STATIC_CHILDREN(_kern_cam_nda), OID_AUTO, tmpstr2, 615301771Simp CTLFLAG_RD, 0, tmpstr); 616301771Simp if (softc->sysctl_tree == NULL) { 617301771Simp printf("ndasysctlinit: unable to allocate sysctl tree\n"); 618301771Simp cam_periph_release(periph); 619301771Simp return; 620301771Simp } 621301771Simp 622301771Simp SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), 623301771Simp OID_AUTO, "unmapped_io", CTLFLAG_RD | CTLFLAG_MPSAFE, 624301771Simp &softc->unmappedio, 0, "Unmapped I/O leaf"); 625301771Simp 626301771Simp SYSCTL_ADD_INT(&softc->sysctl_ctx, 627301771Simp SYSCTL_CHILDREN(softc->sysctl_tree), 628301771Simp OID_AUTO, 629301771Simp "rotating", 630301771Simp CTLFLAG_RD | CTLFLAG_MPSAFE, 631301771Simp &nda_rotating_media, 632301771Simp 0, 633301771Simp "Rotating media"); 634301771Simp 635301771Simp#ifdef CAM_IO_STATS 636301771Simp softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx, 637301771Simp SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats", 638301771Simp CTLFLAG_RD, 0, "Statistics"); 639301771Simp if (softc->sysctl_stats_tree == NULL) { 640301771Simp printf("ndasysctlinit: unable to allocate sysctl tree for stats\n"); 641301771Simp cam_periph_release(periph); 642301771Simp return; 643301771Simp } 644301771Simp SYSCTL_ADD_INT(&softc->sysctl_stats_ctx, 645301771Simp SYSCTL_CHILDREN(softc->sysctl_stats_tree), 646301771Simp OID_AUTO, "timeouts", CTLFLAG_RD | CTLFLAG_MPSAFE, 647301771Simp &softc->timeouts, 0, 648301771Simp "Device timeouts reported by the SIM"); 649301771Simp SYSCTL_ADD_INT(&softc->sysctl_stats_ctx, 650301771Simp SYSCTL_CHILDREN(softc->sysctl_stats_tree), 651301771Simp OID_AUTO, "errors", CTLFLAG_RD | CTLFLAG_MPSAFE, 652301771Simp &softc->errors, 0, 653301771Simp "Transport errors reported by the SIM."); 654301771Simp SYSCTL_ADD_INT(&softc->sysctl_stats_ctx, 655301771Simp SYSCTL_CHILDREN(softc->sysctl_stats_tree), 656301771Simp OID_AUTO, "pack_invalidations", CTLFLAG_RD | CTLFLAG_MPSAFE, 657301771Simp &softc->invalidations, 0, 658301771Simp "Device pack invalidations."); 659301771Simp#endif 660301771Simp 661301771Simp cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx, 662301771Simp softc->sysctl_tree); 663301771Simp 664301771Simp cam_periph_release(periph); 665301771Simp} 666301771Simp 667301771Simpstatic int 668301771Simpndagetattr(struct bio *bp) 669301771Simp{ 670301771Simp int ret; 671301771Simp struct cam_periph *periph; 672301771Simp 673301771Simp periph = (struct cam_periph *)bp->bio_disk->d_drv1; 674301771Simp cam_periph_lock(periph); 675301771Simp ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute, 676301771Simp periph->path); 677301771Simp cam_periph_unlock(periph); 678301771Simp if (ret == 0) 679301771Simp bp->bio_completed = bp->bio_length; 680301771Simp return ret; 681301771Simp} 682301771Simp 683301771Simpstatic cam_status 684301771Simpndaregister(struct cam_periph *periph, void *arg) 685301771Simp{ 686301771Simp struct nda_softc *softc; 687301771Simp struct disk *disk; 688301771Simp struct ccb_pathinq cpi; 689301771Simp const struct nvme_namespace_data *nsd; 690301771Simp const struct nvme_controller_data *cd; 691301771Simp char announce_buf[80]; 692301771Simp u_int maxio; 693301771Simp int quirks; 694301771Simp 695328735Smav nsd = nvme_get_identify_ns(periph); 696328735Smav cd = nvme_get_identify_cntrl(periph); 697301771Simp 698301771Simp softc = (struct nda_softc *)malloc(sizeof(*softc), M_DEVBUF, 699301771Simp M_NOWAIT | M_ZERO); 700301771Simp 701301771Simp if (softc == NULL) { 702301771Simp printf("ndaregister: Unable to probe new device. " 703301771Simp "Unable to allocate softc\n"); 704301771Simp return(CAM_REQ_CMP_ERR); 705301771Simp } 706301771Simp 707301771Simp if (cam_iosched_init(&softc->cam_iosched, periph) != 0) { 708301771Simp printf("ndaregister: Unable to probe new device. " 709301771Simp "Unable to allocate iosched memory\n"); 710301771Simp return(CAM_REQ_CMP_ERR); 711301771Simp } 712301771Simp 713301771Simp /* ident_data parsing */ 714301771Simp 715301771Simp periph->softc = softc; 716301771Simp 717328735Smav softc->quirks = NDA_Q_NONE; 718301771Simp 719350804Smav xpt_path_inq(&cpi, periph->path); 720301771Simp 721301771Simp TASK_INIT(&softc->sysctl_task, 0, ndasysctlinit, periph); 722301771Simp 723301771Simp /* 724301771Simp * The name space ID is the lun, save it for later I/O 725301771Simp */ 726328727Smav softc->nsid = (uint32_t)xpt_path_lun_id(periph->path); 727301771Simp 728301771Simp /* 729301771Simp * Register this media as a disk 730301771Simp */ 731301771Simp (void)cam_periph_hold(periph, PRIBIO); 732301771Simp cam_periph_unlock(periph); 733301771Simp snprintf(announce_buf, sizeof(announce_buf), 734301771Simp "kern.cam.nda.%d.quirks", periph->unit_number); 735301771Simp quirks = softc->quirks; 736301771Simp TUNABLE_INT_FETCH(announce_buf, &quirks); 737301771Simp softc->quirks = quirks; 738301771Simp cam_iosched_set_sort_queue(softc->cam_iosched, 0); 739301771Simp softc->disk = disk = disk_alloc(); 740301771Simp strlcpy(softc->disk->d_descr, cd->mn, 741301771Simp MIN(sizeof(softc->disk->d_descr), sizeof(cd->mn))); 742301771Simp strlcpy(softc->disk->d_ident, cd->sn, 743301771Simp MIN(sizeof(softc->disk->d_ident), sizeof(cd->sn))); 744312405Smav disk->d_rotation_rate = DISK_RR_NON_ROTATING; 745301771Simp disk->d_open = ndaopen; 746301771Simp disk->d_close = ndaclose; 747301771Simp disk->d_strategy = ndastrategy; 748301771Simp disk->d_getattr = ndagetattr; 749301771Simp disk->d_dump = ndadump; 750301771Simp disk->d_gone = ndadiskgonecb; 751301771Simp disk->d_name = "nda"; 752301771Simp disk->d_drv1 = periph; 753301771Simp disk->d_unit = periph->unit_number; 754301771Simp maxio = cpi.maxio; /* Honor max I/O size of SIM */ 755301771Simp if (maxio == 0) 756301771Simp maxio = DFLTPHYS; /* traditional default */ 757301771Simp else if (maxio > MAXPHYS) 758301771Simp maxio = MAXPHYS; /* for safety */ 759301771Simp disk->d_maxsize = maxio; 760301771Simp disk->d_sectorsize = 1 << nsd->lbaf[nsd->flbas.format].lbads; 761301771Simp disk->d_mediasize = (off_t)(disk->d_sectorsize * nsd->nsze); 762301771Simp disk->d_delmaxsize = disk->d_mediasize; 763301771Simp disk->d_flags = DISKFLAG_DIRECT_COMPLETION; 764301771Simp// if (cd->oncs.dsm) // XXX broken? 765301771Simp disk->d_flags |= DISKFLAG_CANDELETE; 766301771Simp if (cd->vwc.present) 767301771Simp disk->d_flags |= DISKFLAG_CANFLUSHCACHE; 768301771Simp if ((cpi.hba_misc & PIM_UNMAPPED) != 0) { 769301771Simp disk->d_flags |= DISKFLAG_UNMAPPED_BIO; 770301771Simp softc->unmappedio = 1; 771301771Simp } 772301771Simp /* 773301771Simp * d_ident and d_descr are both far bigger than the length of either 774301771Simp * the serial or model number strings. 775301771Simp */ 776301771Simp nvme_strvis(disk->d_descr, cd->mn, 777359237Sdab NVME_MODEL_NUMBER_LENGTH, sizeof(disk->d_descr)); 778301771Simp nvme_strvis(disk->d_ident, cd->sn, 779359237Sdab NVME_SERIAL_NUMBER_LENGTH, sizeof(disk->d_ident)); 780301771Simp disk->d_hba_vendor = cpi.hba_vendor; 781301771Simp disk->d_hba_device = cpi.hba_device; 782301771Simp disk->d_hba_subvendor = cpi.hba_subvendor; 783301771Simp disk->d_hba_subdevice = cpi.hba_subdevice; 784301771Simp disk->d_stripesize = disk->d_sectorsize; 785301771Simp disk->d_stripeoffset = 0; 786301771Simp disk->d_devstat = devstat_new_entry(periph->periph_name, 787301771Simp periph->unit_number, disk->d_sectorsize, 788301771Simp DEVSTAT_ALL_SUPPORTED, 789301771Simp DEVSTAT_TYPE_DIRECT | XPORT_DEVSTAT_TYPE(cpi.transport), 790301771Simp DEVSTAT_PRIORITY_DISK); 791301771Simp 792301771Simp /* 793301771Simp * Acquire a reference to the periph before we register with GEOM. 794301771Simp * We'll release this reference once GEOM calls us back (via 795301771Simp * ndadiskgonecb()) telling us that our provider has been freed. 796301771Simp */ 797301771Simp if (cam_periph_acquire(periph) != CAM_REQ_CMP) { 798301771Simp xpt_print(periph->path, "%s: lost periph during " 799301771Simp "registration!\n", __func__); 800301771Simp cam_periph_lock(periph); 801301771Simp return (CAM_REQ_CMP_ERR); 802301771Simp } 803301771Simp disk_create(softc->disk, DISK_VERSION); 804301771Simp cam_periph_lock(periph); 805301771Simp cam_periph_unhold(periph); 806301771Simp 807301771Simp snprintf(announce_buf, sizeof(announce_buf), 808301771Simp "%juMB (%ju %u byte sectors)", 809301771Simp (uintmax_t)((uintmax_t)disk->d_mediasize / (1024*1024)), 810301771Simp (uintmax_t)disk->d_mediasize / disk->d_sectorsize, 811301771Simp disk->d_sectorsize); 812301771Simp xpt_announce_periph(periph, announce_buf); 813301771Simp xpt_announce_quirks(periph, softc->quirks, NDA_Q_BIT_STRING); 814301771Simp 815301771Simp /* 816301771Simp * Create our sysctl variables, now that we know 817301771Simp * we have successfully attached. 818301771Simp */ 819301771Simp if (cam_periph_acquire(periph) == CAM_REQ_CMP) 820301771Simp taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task); 821301771Simp 822301771Simp /* 823301771Simp * Register for device going away and info about the drive 824301771Simp * changing (though with NVMe, it can't) 825301771Simp */ 826301771Simp xpt_register_async(AC_LOST_DEVICE | AC_ADVINFO_CHANGED, 827301771Simp ndaasync, periph, periph->path); 828301771Simp 829301771Simp softc->state = NDA_STATE_NORMAL; 830301771Simp return(CAM_REQ_CMP); 831301771Simp} 832301771Simp 833301771Simpstatic void 834301771Simpndastart(struct cam_periph *periph, union ccb *start_ccb) 835301771Simp{ 836301771Simp struct nda_softc *softc = (struct nda_softc *)periph->softc; 837301771Simp struct ccb_nvmeio *nvmeio = &start_ccb->nvmeio; 838301771Simp 839301771Simp CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("ndastart\n")); 840301771Simp 841301771Simp switch (softc->state) { 842301771Simp case NDA_STATE_NORMAL: 843301771Simp { 844301771Simp struct bio *bp; 845301771Simp 846301771Simp bp = cam_iosched_next_bio(softc->cam_iosched); 847301771Simp CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("ndastart: bio %p\n", bp)); 848301771Simp if (bp == NULL) { 849301771Simp xpt_release_ccb(start_ccb); 850301771Simp break; 851301771Simp } 852301771Simp 853301771Simp switch (bp->bio_cmd) { 854301771Simp case BIO_WRITE: 855301771Simp softc->flags |= NDA_FLAG_DIRTY; 856301771Simp /* FALLTHROUGH */ 857301771Simp case BIO_READ: 858301771Simp { 859301771Simp#ifdef NDA_TEST_FAILURE 860301771Simp int fail = 0; 861301771Simp 862301771Simp /* 863301771Simp * Support the failure ioctls. If the command is a 864301771Simp * read, and there are pending forced read errors, or 865301771Simp * if a write and pending write errors, then fail this 866301771Simp * operation with EIO. This is useful for testing 867301771Simp * purposes. Also, support having every Nth read fail. 868301771Simp * 869301771Simp * This is a rather blunt tool. 870301771Simp */ 871301771Simp if (bp->bio_cmd == BIO_READ) { 872301771Simp if (softc->force_read_error) { 873301771Simp softc->force_read_error--; 874301771Simp fail = 1; 875301771Simp } 876301771Simp if (softc->periodic_read_error > 0) { 877301771Simp if (++softc->periodic_read_count >= 878301771Simp softc->periodic_read_error) { 879301771Simp softc->periodic_read_count = 0; 880301771Simp fail = 1; 881301771Simp } 882301771Simp } 883301771Simp } else { 884301771Simp if (softc->force_write_error) { 885301771Simp softc->force_write_error--; 886301771Simp fail = 1; 887301771Simp } 888301771Simp } 889301771Simp if (fail) { 890301771Simp biofinish(bp, NULL, EIO); 891301771Simp xpt_release_ccb(start_ccb); 892301771Simp ndaschedule(periph); 893301771Simp return; 894301771Simp } 895301771Simp#endif 896301771Simp KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 || 897301771Simp round_page(bp->bio_bcount + bp->bio_ma_offset) / 898301771Simp PAGE_SIZE == bp->bio_ma_n, 899301771Simp ("Short bio %p", bp)); 900301771Simp nda_nvme_rw_bio(softc, &start_ccb->nvmeio, bp, bp->bio_cmd == BIO_READ ? 901301771Simp NVME_OPC_READ : NVME_OPC_WRITE); 902301771Simp break; 903301771Simp } 904301771Simp case BIO_DELETE: 905301771Simp { 906301771Simp struct nvme_dsm_range *dsm_range; 907301771Simp 908301771Simp dsm_range = 909301771Simp malloc(sizeof(*dsm_range), M_NVMEDA, M_ZERO | M_WAITOK); 910301771Simp dsm_range->length = 911301771Simp bp->bio_bcount / softc->disk->d_sectorsize; 912301771Simp dsm_range->starting_lba = 913301771Simp bp->bio_offset / softc->disk->d_sectorsize; 914301771Simp bp->bio_driver2 = dsm_range; 915301771Simp nda_nvme_trim(softc, &start_ccb->nvmeio, dsm_range, 1); 916301771Simp start_ccb->ccb_h.ccb_state = NDA_CCB_TRIM; 917301771Simp start_ccb->ccb_h.flags |= CAM_UNLOCKED; 918328736Smav /* 919328736Smav * Note: We can have multiple TRIMs in flight, so we don't call 920328736Smav * cam_iosched_submit_trim(softc->cam_iosched); 921328736Smav * since that forces the I/O scheduler to only schedule one at a time. 922328736Smav * On NVMe drives, this is a performance disaster. 923328736Smav */ 924301771Simp goto out; 925301771Simp } 926301771Simp case BIO_FLUSH: 927301771Simp nda_nvme_flush(softc, nvmeio); 928301771Simp break; 929301771Simp } 930301771Simp start_ccb->ccb_h.ccb_state = NDA_CCB_BUFFER_IO; 931301771Simp start_ccb->ccb_h.flags |= CAM_UNLOCKED; 932301771Simpout: 933301771Simp start_ccb->ccb_h.ccb_bp = bp; 934301771Simp softc->outstanding_cmds++; 935301771Simp softc->refcount++; 936301771Simp cam_periph_unlock(periph); 937301771Simp xpt_action(start_ccb); 938301771Simp cam_periph_lock(periph); 939301771Simp softc->refcount--; 940301771Simp 941301771Simp /* May have more work to do, so ensure we stay scheduled */ 942301771Simp ndaschedule(periph); 943301771Simp break; 944301771Simp } 945301771Simp } 946301771Simp} 947301771Simp 948301771Simpstatic void 949301771Simpndadone(struct cam_periph *periph, union ccb *done_ccb) 950301771Simp{ 951301771Simp struct nda_softc *softc; 952301771Simp struct ccb_nvmeio *nvmeio = &done_ccb->nvmeio; 953301771Simp struct cam_path *path; 954301771Simp int state; 955301771Simp 956301771Simp softc = (struct nda_softc *)periph->softc; 957301771Simp path = done_ccb->ccb_h.path; 958301771Simp 959301771Simp CAM_DEBUG(path, CAM_DEBUG_TRACE, ("ndadone\n")); 960301771Simp 961301771Simp state = nvmeio->ccb_h.ccb_state & NDA_CCB_TYPE_MASK; 962301771Simp switch (state) { 963301771Simp case NDA_CCB_BUFFER_IO: 964301771Simp case NDA_CCB_TRIM: 965301771Simp { 966301771Simp struct bio *bp; 967301771Simp int error; 968301771Simp 969301771Simp cam_periph_lock(periph); 970301771Simp bp = (struct bio *)done_ccb->ccb_h.ccb_bp; 971301771Simp if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { 972301771Simp error = ndaerror(done_ccb, 0, 0); 973301771Simp if (error == ERESTART) { 974301771Simp /* A retry was scheduled, so just return. */ 975301771Simp cam_periph_unlock(periph); 976301771Simp return; 977301771Simp } 978301771Simp if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) 979301771Simp cam_release_devq(path, 980301771Simp /*relsim_flags*/0, 981301771Simp /*reduction*/0, 982301771Simp /*timeout*/0, 983301771Simp /*getcount_only*/0); 984301771Simp } else { 985301771Simp if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) 986301771Simp panic("REQ_CMP with QFRZN"); 987301771Simp error = 0; 988301771Simp } 989301771Simp bp->bio_error = error; 990301771Simp if (error != 0) { 991301771Simp bp->bio_resid = bp->bio_bcount; 992301771Simp bp->bio_flags |= BIO_ERROR; 993301771Simp } else { 994328696Smav bp->bio_resid = 0; 995301771Simp } 996301771Simp if (state == NDA_CCB_TRIM) 997301771Simp free(bp->bio_driver2, M_NVMEDA); 998301771Simp softc->outstanding_cmds--; 999301771Simp 1000301771Simp cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb); 1001301771Simp xpt_release_ccb(done_ccb); 1002301771Simp if (state == NDA_CCB_TRIM) { 1003301771Simp#ifdef notyet 1004301771Simp TAILQ_HEAD(, bio) queue; 1005301771Simp struct bio *bp1; 1006301771Simp 1007301771Simp TAILQ_INIT(&queue); 1008301771Simp TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue); 1009301771Simp#endif 1010328736Smav /* 1011328736Smav * Since we can have multiple trims in flight, we don't 1012328736Smav * need to call this here. 1013328736Smav * cam_iosched_trim_done(softc->cam_iosched); 1014328736Smav */ 1015301771Simp ndaschedule(periph); 1016301771Simp cam_periph_unlock(periph); 1017301771Simp#ifdef notyet 1018301771Simp/* Not yet collapsing several BIO_DELETE requests into one TRIM */ 1019301771Simp while ((bp1 = TAILQ_FIRST(&queue)) != NULL) { 1020301771Simp TAILQ_REMOVE(&queue, bp1, bio_queue); 1021301771Simp bp1->bio_error = error; 1022301771Simp if (error != 0) { 1023301771Simp bp1->bio_flags |= BIO_ERROR; 1024301771Simp bp1->bio_resid = bp1->bio_bcount; 1025301771Simp } else 1026301771Simp bp1->bio_resid = 0; 1027301771Simp biodone(bp1); 1028301771Simp } 1029301771Simp#else 1030301771Simp biodone(bp); 1031301771Simp#endif 1032301771Simp } else { 1033301771Simp ndaschedule(periph); 1034301771Simp cam_periph_unlock(periph); 1035301771Simp biodone(bp); 1036301771Simp } 1037301771Simp return; 1038301771Simp } 1039301771Simp case NDA_CCB_DUMP: 1040301771Simp /* No-op. We're polling */ 1041301771Simp return; 1042301771Simp default: 1043301771Simp break; 1044301771Simp } 1045301771Simp xpt_release_ccb(done_ccb); 1046301771Simp} 1047301771Simp 1048301771Simpstatic int 1049301771Simpndaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags) 1050301771Simp{ 1051301771Simp struct nda_softc *softc; 1052301771Simp struct cam_periph *periph; 1053301771Simp 1054301771Simp periph = xpt_path_periph(ccb->ccb_h.path); 1055301771Simp softc = (struct nda_softc *)periph->softc; 1056301771Simp 1057301771Simp switch (ccb->ccb_h.status & CAM_STATUS_MASK) { 1058301771Simp case CAM_CMD_TIMEOUT: 1059301771Simp#ifdef CAM_IO_STATS 1060301771Simp softc->timeouts++; 1061301771Simp#endif 1062301771Simp break; 1063301771Simp case CAM_REQ_ABORTED: 1064301771Simp case CAM_REQ_CMP_ERR: 1065301771Simp case CAM_REQ_TERMIO: 1066301771Simp case CAM_UNREC_HBA_ERROR: 1067301771Simp case CAM_DATA_RUN_ERR: 1068301771Simp case CAM_ATA_STATUS_ERROR: 1069301771Simp#ifdef CAM_IO_STATS 1070301771Simp softc->errors++; 1071301771Simp#endif 1072301771Simp break; 1073301771Simp default: 1074301771Simp break; 1075301771Simp } 1076301771Simp 1077301771Simp return(cam_periph_error(ccb, cam_flags, sense_flags, NULL)); 1078301771Simp} 1079301771Simp 1080301771Simp/* 1081301771Simp * Step through all NDA peripheral drivers, and if the device is still open, 1082301771Simp * sync the disk cache to physical media. 1083301771Simp */ 1084301771Simpstatic void 1085301771Simpndaflush(void) 1086301771Simp{ 1087301771Simp struct cam_periph *periph; 1088301771Simp struct nda_softc *softc; 1089301771Simp union ccb *ccb; 1090301771Simp int error; 1091301771Simp 1092301771Simp CAM_PERIPH_FOREACH(periph, &ndadriver) { 1093301771Simp softc = (struct nda_softc *)periph->softc; 1094301771Simp if (SCHEDULER_STOPPED()) { 1095301771Simp /* If we paniced with the lock held, do not recurse. */ 1096301771Simp if (!cam_periph_owned(periph) && 1097301771Simp (softc->flags & NDA_FLAG_OPEN)) { 1098301771Simp ndadump(softc->disk, NULL, 0, 0, 0); 1099301771Simp } 1100301771Simp continue; 1101301771Simp } 1102301771Simp cam_periph_lock(periph); 1103301771Simp /* 1104301771Simp * We only sync the cache if the drive is still open, and 1105301771Simp * if the drive is capable of it.. 1106301771Simp */ 1107301771Simp if ((softc->flags & NDA_FLAG_OPEN) == 0) { 1108301771Simp cam_periph_unlock(periph); 1109301771Simp continue; 1110301771Simp } 1111301771Simp 1112301771Simp ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL); 1113301771Simp nda_nvme_flush(softc, &ccb->nvmeio); 1114301771Simp error = cam_periph_runccb(ccb, ndaerror, /*cam_flags*/0, 1115301771Simp /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY, 1116301771Simp softc->disk->d_devstat); 1117301771Simp if (error != 0) 1118301771Simp xpt_print(periph->path, "Synchronize cache failed\n"); 1119301771Simp xpt_release_ccb(ccb); 1120301771Simp cam_periph_unlock(periph); 1121301771Simp } 1122301771Simp} 1123301771Simp 1124301771Simpstatic void 1125301771Simpndashutdown(void *arg, int howto) 1126301771Simp{ 1127301771Simp 1128301771Simp ndaflush(); 1129301771Simp} 1130301771Simp 1131301771Simpstatic void 1132301771Simpndasuspend(void *arg) 1133301771Simp{ 1134301771Simp 1135301771Simp ndaflush(); 1136301771Simp} 1137