ahci.c revision 285789
1195534Sscottl/*- 2238805Smav * Copyright (c) 2009-2012 Alexander Motin <mav@FreeBSD.org> 3195534Sscottl * All rights reserved. 4195534Sscottl * 5195534Sscottl * Redistribution and use in source and binary forms, with or without 6195534Sscottl * modification, are permitted provided that the following conditions 7195534Sscottl * are met: 8195534Sscottl * 1. Redistributions of source code must retain the above copyright 9195534Sscottl * notice, this list of conditions and the following disclaimer, 10195534Sscottl * without modification, immediately at the beginning of the file. 11195534Sscottl * 2. Redistributions in binary form must reproduce the above copyright 12195534Sscottl * notice, this list of conditions and the following disclaimer in the 13195534Sscottl * documentation and/or other materials provided with the distribution. 14195534Sscottl * 15195534Sscottl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16195534Sscottl * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17195534Sscottl * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18195534Sscottl * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19195534Sscottl * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20195534Sscottl * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21195534Sscottl * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22195534Sscottl * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23195534Sscottl * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24195534Sscottl * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25195534Sscottl */ 26195534Sscottl 27195534Sscottl#include <sys/cdefs.h> 28195534Sscottl__FBSDID("$FreeBSD: head/sys/dev/ahci/ahci.c 285789 2015-07-22 09:46:22Z zbb $"); 29195534Sscottl 30195534Sscottl#include <sys/param.h> 31195534Sscottl#include <sys/module.h> 32195534Sscottl#include <sys/systm.h> 33195534Sscottl#include <sys/kernel.h> 34195534Sscottl#include <sys/bus.h> 35220576Smav#include <sys/conf.h> 36195534Sscottl#include <sys/endian.h> 37195534Sscottl#include <sys/malloc.h> 38195534Sscottl#include <sys/lock.h> 39195534Sscottl#include <sys/mutex.h> 40195534Sscottl#include <machine/stdarg.h> 41195534Sscottl#include <machine/resource.h> 42195534Sscottl#include <machine/bus.h> 43195534Sscottl#include <sys/rman.h> 44195534Sscottl#include "ahci.h" 45195534Sscottl 46195534Sscottl#include <cam/cam.h> 47195534Sscottl#include <cam/cam_ccb.h> 48195534Sscottl#include <cam/cam_sim.h> 49195534Sscottl#include <cam/cam_xpt_sim.h> 50195534Sscottl#include <cam/cam_debug.h> 51195534Sscottl 52195534Sscottl/* local prototypes */ 53195534Sscottlstatic void ahci_intr(void *data); 54195534Sscottlstatic void ahci_intr_one(void *data); 55256843Smavstatic void ahci_intr_one_edge(void *data); 56208375Smavstatic int ahci_ch_init(device_t dev); 57208375Smavstatic int ahci_ch_deinit(device_t dev); 58195534Sscottlstatic int ahci_ch_suspend(device_t dev); 59195534Sscottlstatic int ahci_ch_resume(device_t dev); 60196656Smavstatic void ahci_ch_pm(void *arg); 61256843Smavstatic void ahci_ch_intr(void *arg); 62256843Smavstatic void ahci_ch_intr_direct(void *arg); 63256843Smavstatic void ahci_ch_intr_main(struct ahci_channel *ch, uint32_t istatus); 64271261Smavstatic void ahci_begin_transaction(struct ahci_channel *ch, union ccb *ccb); 65195534Sscottlstatic void ahci_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error); 66195534Sscottlstatic void ahci_execute_transaction(struct ahci_slot *slot); 67195534Sscottlstatic void ahci_timeout(struct ahci_slot *slot); 68195534Sscottlstatic void ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et); 69271261Smavstatic int ahci_setup_fis(struct ahci_channel *ch, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag); 70195534Sscottlstatic void ahci_dmainit(device_t dev); 71195534Sscottlstatic void ahci_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error); 72195534Sscottlstatic void ahci_dmafini(device_t dev); 73195534Sscottlstatic void ahci_slotsalloc(device_t dev); 74195534Sscottlstatic void ahci_slotsfree(device_t dev); 75271261Smavstatic void ahci_reset(struct ahci_channel *ch); 76271261Smavstatic void ahci_start(struct ahci_channel *ch, int fbs); 77271261Smavstatic void ahci_stop(struct ahci_channel *ch); 78271261Smavstatic void ahci_clo(struct ahci_channel *ch); 79271261Smavstatic void ahci_start_fr(struct ahci_channel *ch); 80271261Smavstatic void ahci_stop_fr(struct ahci_channel *ch); 81195534Sscottl 82195534Sscottlstatic int ahci_sata_connect(struct ahci_channel *ch); 83271261Smavstatic int ahci_sata_phy_reset(struct ahci_channel *ch); 84271261Smavstatic int ahci_wait_ready(struct ahci_channel *ch, int t, int t0); 85195534Sscottl 86271261Smavstatic void ahci_issue_recovery(struct ahci_channel *ch); 87271261Smavstatic void ahci_process_read_log(struct ahci_channel *ch, union ccb *ccb); 88271261Smavstatic void ahci_process_request_sense(struct ahci_channel *ch, union ccb *ccb); 89195534Sscottl 90195534Sscottlstatic void ahciaction(struct cam_sim *sim, union ccb *ccb); 91195534Sscottlstatic void ahcipoll(struct cam_sim *sim); 92195534Sscottl 93227293Sedstatic MALLOC_DEFINE(M_AHCI, "AHCI driver", "AHCI driver data buffers"); 94195534Sscottl 95220565Smav#define recovery_type spriv_field0 96220565Smav#define RECOVERY_NONE 0 97220565Smav#define RECOVERY_READ_LOG 1 98220565Smav#define RECOVERY_REQUEST_SENSE 2 99220565Smav#define recovery_slot spriv_field1 100220565Smav 101271146Simpint 102271146Simpahci_ctlr_setup(device_t dev) 103195534Sscottl{ 104271146Simp struct ahci_controller *ctlr = device_get_softc(dev); 105271146Simp /* Clear interrupts */ 106271146Simp ATA_OUTL(ctlr->r_mem, AHCI_IS, ATA_INL(ctlr->r_mem, AHCI_IS)); 107271146Simp /* Configure CCC */ 108271146Simp if (ctlr->ccc) { 109271146Simp ATA_OUTL(ctlr->r_mem, AHCI_CCCP, ATA_INL(ctlr->r_mem, AHCI_PI)); 110271146Simp ATA_OUTL(ctlr->r_mem, AHCI_CCCC, 111271146Simp (ctlr->ccc << AHCI_CCCC_TV_SHIFT) | 112271146Simp (4 << AHCI_CCCC_CC_SHIFT) | 113271146Simp AHCI_CCCC_EN); 114271146Simp ctlr->cccv = (ATA_INL(ctlr->r_mem, AHCI_CCCC) & 115271146Simp AHCI_CCCC_INT_MASK) >> AHCI_CCCC_INT_SHIFT; 116271146Simp if (bootverbose) { 117271146Simp device_printf(dev, 118271146Simp "CCC with %dms/4cmd enabled on vector %d\n", 119271146Simp ctlr->ccc, ctlr->cccv); 120199322Smav } 121199322Smav } 122271146Simp /* Enable AHCI interrupts */ 123271146Simp ATA_OUTL(ctlr->r_mem, AHCI_GHC, 124271146Simp ATA_INL(ctlr->r_mem, AHCI_GHC) | AHCI_GHC_IE); 125271146Simp return (0); 126199322Smav} 127199322Smav 128271146Simpint 129271146Simpahci_ctlr_reset(device_t dev) 130199322Smav{ 131271146Simp struct ahci_controller *ctlr = device_get_softc(dev); 132271146Simp int timeout; 133195534Sscottl 134271146Simp /* Enable AHCI mode */ 135271146Simp ATA_OUTL(ctlr->r_mem, AHCI_GHC, AHCI_GHC_AE); 136271146Simp /* Reset AHCI controller */ 137271146Simp ATA_OUTL(ctlr->r_mem, AHCI_GHC, AHCI_GHC_AE|AHCI_GHC_HR); 138271146Simp for (timeout = 1000; timeout > 0; timeout--) { 139271146Simp DELAY(1000); 140271146Simp if ((ATA_INL(ctlr->r_mem, AHCI_GHC) & AHCI_GHC_HR) == 0) 141271146Simp break; 142199176Smav } 143271146Simp if (timeout == 0) { 144271146Simp device_printf(dev, "AHCI controller reset failure\n"); 145276019Ssmh return (ENXIO); 146271146Simp } 147271146Simp /* Reenable AHCI mode */ 148271146Simp ATA_OUTL(ctlr->r_mem, AHCI_GHC, AHCI_GHC_AE); 149280184Szbb 150280184Szbb if (ctlr->quirks & AHCI_Q_RESTORE_CAP) { 151280184Szbb /* 152280184Szbb * Restore capability field. 153280184Szbb * This is write to a read-only register to restore its state. 154280184Szbb * On fully standard-compliant hardware this is not needed and 155280184Szbb * this operation shall not take place. See ahci_pci.c for 156280184Szbb * platforms using this quirk. 157280184Szbb */ 158280184Szbb ATA_OUTL(ctlr->r_mem, AHCI_CAP, ctlr->caps); 159280184Szbb } 160280184Szbb 161271146Simp return (0); 162195534Sscottl} 163195534Sscottl 164271146Simp 165271146Simpint 166195534Sscottlahci_attach(device_t dev) 167195534Sscottl{ 168195534Sscottl struct ahci_controller *ctlr = device_get_softc(dev); 169271146Simp int error, i, u, speed, unit; 170271146Simp u_int32_t version; 171195534Sscottl device_t child; 172195534Sscottl 173195534Sscottl ctlr->dev = dev; 174271457Smav ctlr->ccc = 0; 175196656Smav resource_int_value(device_get_name(dev), 176196656Smav device_get_unit(dev), "ccc", &ctlr->ccc); 177271146Simp 178195534Sscottl /* Setup our own memory management for channels. */ 179208414Smav ctlr->sc_iomem.rm_start = rman_get_start(ctlr->r_mem); 180208414Smav ctlr->sc_iomem.rm_end = rman_get_end(ctlr->r_mem); 181195534Sscottl ctlr->sc_iomem.rm_type = RMAN_ARRAY; 182195534Sscottl ctlr->sc_iomem.rm_descr = "I/O memory addresses"; 183195534Sscottl if ((error = rman_init(&ctlr->sc_iomem)) != 0) { 184285789Szbb ahci_free_mem(dev); 185195534Sscottl return (error); 186195534Sscottl } 187195534Sscottl if ((error = rman_manage_region(&ctlr->sc_iomem, 188195534Sscottl rman_get_start(ctlr->r_mem), rman_get_end(ctlr->r_mem))) != 0) { 189285789Szbb ahci_free_mem(dev); 190195534Sscottl rman_fini(&ctlr->sc_iomem); 191195534Sscottl return (error); 192195534Sscottl } 193199322Smav /* Get the HW capabilities */ 194199322Smav version = ATA_INL(ctlr->r_mem, AHCI_VS); 195199322Smav ctlr->caps = ATA_INL(ctlr->r_mem, AHCI_CAP); 196240383Smav if (version >= 0x00010200) 197199322Smav ctlr->caps2 = ATA_INL(ctlr->r_mem, AHCI_CAP2); 198203108Smav if (ctlr->caps & AHCI_CAP_EMS) 199203108Smav ctlr->capsem = ATA_INL(ctlr->r_mem, AHCI_EM_CTL); 200280184Szbb 201280184Szbb if (ctlr->quirks & AHCI_Q_FORCE_PI) { 202280184Szbb /* 203280184Szbb * Enable ports. 204280184Szbb * The spec says that BIOS sets up bits corresponding to 205280184Szbb * available ports. On platforms where this information 206280184Szbb * is missing, the driver can define available ports on its own. 207280184Szbb */ 208280184Szbb int nports = (ctlr->caps & AHCI_CAP_NPMASK) + 1; 209280184Szbb int nmask = (1 << nports) - 1; 210280184Szbb 211280184Szbb ATA_OUTL(ctlr->r_mem, AHCI_PI, nmask); 212280184Szbb device_printf(dev, "Forcing PI to %d ports (mask = %x)\n", 213280184Szbb nports, nmask); 214280184Szbb } 215280184Szbb 216195534Sscottl ctlr->ichannels = ATA_INL(ctlr->r_mem, AHCI_PI); 217222304Smav 218222304Smav /* Identify and set separate quirks for HBA and RAID f/w Marvells. */ 219271163Smav if ((ctlr->quirks & AHCI_Q_ALTSIG) && 220222304Smav (ctlr->caps & AHCI_CAP_SPM) == 0) 221271163Smav ctlr->quirks |= AHCI_Q_NOBSYRES; 222222304Smav 223199322Smav if (ctlr->quirks & AHCI_Q_1CH) { 224199322Smav ctlr->caps &= ~AHCI_CAP_NPMASK; 225199322Smav ctlr->ichannels &= 0x01; 226199322Smav } 227199322Smav if (ctlr->quirks & AHCI_Q_2CH) { 228199322Smav ctlr->caps &= ~AHCI_CAP_NPMASK; 229199322Smav ctlr->caps |= 1; 230199322Smav ctlr->ichannels &= 0x03; 231199322Smav } 232199322Smav if (ctlr->quirks & AHCI_Q_4CH) { 233199322Smav ctlr->caps &= ~AHCI_CAP_NPMASK; 234199322Smav ctlr->caps |= 3; 235199322Smav ctlr->ichannels &= 0x0f; 236199322Smav } 237195534Sscottl ctlr->channels = MAX(flsl(ctlr->ichannels), 238199322Smav (ctlr->caps & AHCI_CAP_NPMASK) + 1); 239199322Smav if (ctlr->quirks & AHCI_Q_NOPMP) 240199322Smav ctlr->caps &= ~AHCI_CAP_SPM; 241199322Smav if (ctlr->quirks & AHCI_Q_NONCQ) 242199322Smav ctlr->caps &= ~AHCI_CAP_SNCQ; 243205422Smav if ((ctlr->caps & AHCI_CAP_CCCS) == 0) 244205422Smav ctlr->ccc = 0; 245222039Smav ctlr->emloc = ATA_INL(ctlr->r_mem, AHCI_EM_LOC); 246249346Smav 247249346Smav /* Create controller-wide DMA tag. */ 248274189Sian if (bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, 249249346Smav (ctlr->caps & AHCI_CAP_64BIT) ? BUS_SPACE_MAXADDR : 250249346Smav BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, 251249346Smav BUS_SPACE_MAXSIZE, BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, 252249346Smav 0, NULL, NULL, &ctlr->dma_tag)) { 253285789Szbb ahci_free_mem(dev); 254249346Smav rman_fini(&ctlr->sc_iomem); 255276019Ssmh return (ENXIO); 256249346Smav } 257249346Smav 258205422Smav ahci_ctlr_setup(dev); 259271146Simp 260271201Simp /* Setup interrupts. */ 261276016Ssmh if ((error = ahci_setup_interrupt(dev)) != 0) { 262271201Simp bus_dma_tag_destroy(ctlr->dma_tag); 263285789Szbb ahci_free_mem(dev); 264271201Simp rman_fini(&ctlr->sc_iomem); 265276019Ssmh return (error); 266271201Simp } 267271201Simp 268256843Smav i = 0; 269256843Smav for (u = ctlr->ichannels; u != 0; u >>= 1) 270256843Smav i += (u & 1); 271256843Smav ctlr->direct = (ctlr->msi && (ctlr->numirqs > 1 || i <= 3)); 272256843Smav resource_int_value(device_get_name(dev), device_get_unit(dev), 273256843Smav "direct", &ctlr->direct); 274195534Sscottl /* Announce HW capabilities. */ 275196656Smav speed = (ctlr->caps & AHCI_CAP_ISS) >> AHCI_CAP_ISS_SHIFT; 276195534Sscottl device_printf(dev, 277203123Smav "AHCI v%x.%02x with %d %sGbps ports, Port Multiplier %s%s\n", 278195534Sscottl ((version >> 20) & 0xf0) + ((version >> 16) & 0x0f), 279195534Sscottl ((version >> 4) & 0xf0) + (version & 0x0f), 280196656Smav (ctlr->caps & AHCI_CAP_NPMASK) + 1, 281195534Sscottl ((speed == 1) ? "1.5":((speed == 2) ? "3": 282195534Sscottl ((speed == 3) ? "6":"?"))), 283196656Smav (ctlr->caps & AHCI_CAP_SPM) ? 284203123Smav "supported" : "not supported", 285203123Smav (ctlr->caps & AHCI_CAP_FBSS) ? 286203123Smav " with FBS" : ""); 287250792Ssmh if (ctlr->quirks != 0) { 288250792Ssmh device_printf(dev, "quirks=0x%b\n", ctlr->quirks, 289250792Ssmh AHCI_Q_BIT_STRING); 290250792Ssmh } 291195534Sscottl if (bootverbose) { 292195534Sscottl device_printf(dev, "Caps:%s%s%s%s%s%s%s%s %sGbps", 293196656Smav (ctlr->caps & AHCI_CAP_64BIT) ? " 64bit":"", 294196656Smav (ctlr->caps & AHCI_CAP_SNCQ) ? " NCQ":"", 295196656Smav (ctlr->caps & AHCI_CAP_SSNTF) ? " SNTF":"", 296196656Smav (ctlr->caps & AHCI_CAP_SMPS) ? " MPS":"", 297196656Smav (ctlr->caps & AHCI_CAP_SSS) ? " SS":"", 298196656Smav (ctlr->caps & AHCI_CAP_SALP) ? " ALP":"", 299196656Smav (ctlr->caps & AHCI_CAP_SAL) ? " AL":"", 300196656Smav (ctlr->caps & AHCI_CAP_SCLO) ? " CLO":"", 301195534Sscottl ((speed == 1) ? "1.5":((speed == 2) ? "3": 302195534Sscottl ((speed == 3) ? "6":"?")))); 303195534Sscottl printf("%s%s%s%s%s%s %dcmd%s%s%s %dports\n", 304196656Smav (ctlr->caps & AHCI_CAP_SAM) ? " AM":"", 305196656Smav (ctlr->caps & AHCI_CAP_SPM) ? " PM":"", 306196656Smav (ctlr->caps & AHCI_CAP_FBSS) ? " FBS":"", 307196656Smav (ctlr->caps & AHCI_CAP_PMD) ? " PMD":"", 308196656Smav (ctlr->caps & AHCI_CAP_SSC) ? " SSC":"", 309196656Smav (ctlr->caps & AHCI_CAP_PSC) ? " PSC":"", 310196656Smav ((ctlr->caps & AHCI_CAP_NCS) >> AHCI_CAP_NCS_SHIFT) + 1, 311196656Smav (ctlr->caps & AHCI_CAP_CCCS) ? " CCC":"", 312196656Smav (ctlr->caps & AHCI_CAP_EMS) ? " EM":"", 313196656Smav (ctlr->caps & AHCI_CAP_SXS) ? " eSATA":"", 314196656Smav (ctlr->caps & AHCI_CAP_NPMASK) + 1); 315195534Sscottl } 316240383Smav if (bootverbose && version >= 0x00010200) { 317253647Smav device_printf(dev, "Caps2:%s%s%s%s%s%s\n", 318253647Smav (ctlr->caps2 & AHCI_CAP2_DESO) ? " DESO":"", 319253647Smav (ctlr->caps2 & AHCI_CAP2_SADM) ? " SADM":"", 320253647Smav (ctlr->caps2 & AHCI_CAP2_SDS) ? " SDS":"", 321196656Smav (ctlr->caps2 & AHCI_CAP2_APST) ? " APST":"", 322196656Smav (ctlr->caps2 & AHCI_CAP2_NVMP) ? " NVMP":"", 323196656Smav (ctlr->caps2 & AHCI_CAP2_BOH) ? " BOH":""); 324196656Smav } 325195534Sscottl /* Attach all channels on this controller */ 326195534Sscottl for (unit = 0; unit < ctlr->channels; unit++) { 327195534Sscottl child = device_add_child(dev, "ahcich", -1); 328227635Smav if (child == NULL) { 329195534Sscottl device_printf(dev, "failed to add channel device\n"); 330227635Smav continue; 331227635Smav } 332227635Smav device_set_ivars(child, (void *)(intptr_t)unit); 333227635Smav if ((ctlr->ichannels & (1 << unit)) == 0) 334227635Smav device_disable(child); 335195534Sscottl } 336238805Smav if (ctlr->caps & AHCI_CAP_EMS) { 337238805Smav child = device_add_child(dev, "ahciem", -1); 338238805Smav if (child == NULL) 339238805Smav device_printf(dev, "failed to add enclosure device\n"); 340238805Smav else 341238805Smav device_set_ivars(child, (void *)(intptr_t)-1); 342238805Smav } 343195534Sscottl bus_generic_attach(dev); 344276019Ssmh return (0); 345195534Sscottl} 346195534Sscottl 347271146Simpint 348195534Sscottlahci_detach(device_t dev) 349195534Sscottl{ 350195534Sscottl struct ahci_controller *ctlr = device_get_softc(dev); 351227701Shselasky int i; 352195534Sscottl 353195534Sscottl /* Detach & delete all children */ 354227849Shselasky device_delete_children(dev); 355227701Shselasky 356195534Sscottl /* Free interrupts. */ 357195534Sscottl for (i = 0; i < ctlr->numirqs; i++) { 358195534Sscottl if (ctlr->irqs[i].r_irq) { 359195534Sscottl bus_teardown_intr(dev, ctlr->irqs[i].r_irq, 360195534Sscottl ctlr->irqs[i].handle); 361195534Sscottl bus_release_resource(dev, SYS_RES_IRQ, 362195534Sscottl ctlr->irqs[i].r_irq_rid, ctlr->irqs[i].r_irq); 363195534Sscottl } 364195534Sscottl } 365249346Smav bus_dma_tag_destroy(ctlr->dma_tag); 366195534Sscottl /* Free memory. */ 367195534Sscottl rman_fini(&ctlr->sc_iomem); 368285789Szbb ahci_free_mem(dev); 369285789Szbb return (0); 370285789Szbb} 371285789Szbb 372285789Szbbvoid 373285789Szbbahci_free_mem(device_t dev) 374285789Szbb{ 375285789Szbb struct ahci_controller *ctlr = device_get_softc(dev); 376285789Szbb 377285789Szbb /* Release memory resources */ 378195534Sscottl if (ctlr->r_mem) 379195534Sscottl bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem); 380285789Szbb if (ctlr->r_msix_table) 381285789Szbb bus_release_resource(dev, SYS_RES_MEMORY, 382285789Szbb ctlr->r_msix_tab_rid, ctlr->r_msix_table); 383285789Szbb if (ctlr->r_msix_pba) 384285789Szbb bus_release_resource(dev, SYS_RES_MEMORY, 385285789Szbb ctlr->r_msix_pba_rid, ctlr->r_msix_pba); 386285789Szbb 387285789Szbb ctlr->r_msix_pba = ctlr->r_mem = ctlr->r_msix_table = NULL; 388195534Sscottl} 389195534Sscottl 390271146Simpint 391195534Sscottlahci_setup_interrupt(device_t dev) 392195534Sscottl{ 393195534Sscottl struct ahci_controller *ctlr = device_get_softc(dev); 394256843Smav int i; 395195534Sscottl 396195534Sscottl /* Check for single MSI vector fallback. */ 397195534Sscottl if (ctlr->numirqs > 1 && 398195534Sscottl (ATA_INL(ctlr->r_mem, AHCI_GHC) & AHCI_GHC_MRSM) != 0) { 399195534Sscottl device_printf(dev, "Falling back to one MSI\n"); 400195534Sscottl ctlr->numirqs = 1; 401195534Sscottl } 402276013Ssmh 403276013Ssmh /* Ensure we don't overrun irqs. */ 404276013Ssmh if (ctlr->numirqs > AHCI_MAX_IRQS) { 405276013Ssmh device_printf(dev, "Too many irqs %d > %d (clamping)\n", 406276013Ssmh ctlr->numirqs, AHCI_MAX_IRQS); 407276013Ssmh ctlr->numirqs = AHCI_MAX_IRQS; 408276013Ssmh } 409276013Ssmh 410195534Sscottl /* Allocate all IRQs. */ 411195534Sscottl for (i = 0; i < ctlr->numirqs; i++) { 412195534Sscottl ctlr->irqs[i].ctlr = ctlr; 413256843Smav ctlr->irqs[i].r_irq_rid = i + (ctlr->msi ? 1 : 0); 414272606Smav if (ctlr->channels == 1 && !ctlr->ccc && ctlr->msi) 415271207Smav ctlr->irqs[i].mode = AHCI_IRQ_MODE_ONE; 416271207Smav else if (ctlr->numirqs == 1 || i >= ctlr->channels || 417196656Smav (ctlr->ccc && i == ctlr->cccv)) 418195534Sscottl ctlr->irqs[i].mode = AHCI_IRQ_MODE_ALL; 419195534Sscottl else if (i == ctlr->numirqs - 1) 420195534Sscottl ctlr->irqs[i].mode = AHCI_IRQ_MODE_AFTER; 421195534Sscottl else 422195534Sscottl ctlr->irqs[i].mode = AHCI_IRQ_MODE_ONE; 423195534Sscottl if (!(ctlr->irqs[i].r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, 424195534Sscottl &ctlr->irqs[i].r_irq_rid, RF_SHAREABLE | RF_ACTIVE))) { 425195534Sscottl device_printf(dev, "unable to map interrupt\n"); 426276019Ssmh return (ENXIO); 427195534Sscottl } 428195534Sscottl if ((bus_setup_intr(dev, ctlr->irqs[i].r_irq, ATA_INTR_FLAGS, NULL, 429256843Smav (ctlr->irqs[i].mode != AHCI_IRQ_MODE_ONE) ? ahci_intr : 430256843Smav ((ctlr->quirks & AHCI_Q_EDGEIS) ? ahci_intr_one_edge : 431256843Smav ahci_intr_one), 432195534Sscottl &ctlr->irqs[i], &ctlr->irqs[i].handle))) { 433195534Sscottl /* SOS XXX release r_irq */ 434195534Sscottl device_printf(dev, "unable to setup interrupt\n"); 435276019Ssmh return (ENXIO); 436195534Sscottl } 437202011Smav if (ctlr->numirqs > 1) { 438202011Smav bus_describe_intr(dev, ctlr->irqs[i].r_irq, 439202011Smav ctlr->irqs[i].handle, 440202011Smav ctlr->irqs[i].mode == AHCI_IRQ_MODE_ONE ? 441202011Smav "ch%d" : "%d", i); 442202011Smav } 443195534Sscottl } 444195534Sscottl return (0); 445195534Sscottl} 446195534Sscottl 447195534Sscottl/* 448195534Sscottl * Common case interrupt handler. 449195534Sscottl */ 450195534Sscottlstatic void 451195534Sscottlahci_intr(void *data) 452195534Sscottl{ 453195534Sscottl struct ahci_controller_irq *irq = data; 454195534Sscottl struct ahci_controller *ctlr = irq->ctlr; 455205422Smav u_int32_t is, ise = 0; 456195534Sscottl void *arg; 457195534Sscottl int unit; 458195534Sscottl 459196656Smav if (irq->mode == AHCI_IRQ_MODE_ALL) { 460195534Sscottl unit = 0; 461196656Smav if (ctlr->ccc) 462196656Smav is = ctlr->ichannels; 463196656Smav else 464196656Smav is = ATA_INL(ctlr->r_mem, AHCI_IS); 465196656Smav } else { /* AHCI_IRQ_MODE_AFTER */ 466195534Sscottl unit = irq->r_irq_rid - 1; 467196656Smav is = ATA_INL(ctlr->r_mem, AHCI_IS); 468196656Smav } 469205422Smav /* CCC interrupt is edge triggered. */ 470205422Smav if (ctlr->ccc) 471205422Smav ise = 1 << ctlr->cccv; 472200814Smav /* Some controllers have edge triggered IS. */ 473200814Smav if (ctlr->quirks & AHCI_Q_EDGEIS) 474205422Smav ise |= is; 475205422Smav if (ise != 0) 476205422Smav ATA_OUTL(ctlr->r_mem, AHCI_IS, ise); 477195534Sscottl for (; unit < ctlr->channels; unit++) { 478195534Sscottl if ((is & (1 << unit)) != 0 && 479195534Sscottl (arg = ctlr->interrupt[unit].argument)) { 480199322Smav ctlr->interrupt[unit].function(arg); 481195534Sscottl } 482195534Sscottl } 483200814Smav /* AHCI declares level triggered IS. */ 484200814Smav if (!(ctlr->quirks & AHCI_Q_EDGEIS)) 485200814Smav ATA_OUTL(ctlr->r_mem, AHCI_IS, is); 486195534Sscottl} 487195534Sscottl 488195534Sscottl/* 489195534Sscottl * Simplified interrupt handler for multivector MSI mode. 490195534Sscottl */ 491195534Sscottlstatic void 492195534Sscottlahci_intr_one(void *data) 493195534Sscottl{ 494195534Sscottl struct ahci_controller_irq *irq = data; 495195534Sscottl struct ahci_controller *ctlr = irq->ctlr; 496195534Sscottl void *arg; 497195534Sscottl int unit; 498195534Sscottl 499195534Sscottl unit = irq->r_irq_rid - 1; 500195534Sscottl if ((arg = ctlr->interrupt[unit].argument)) 501195534Sscottl ctlr->interrupt[unit].function(arg); 502202011Smav /* AHCI declares level triggered IS. */ 503256843Smav ATA_OUTL(ctlr->r_mem, AHCI_IS, 1 << unit); 504195534Sscottl} 505195534Sscottl 506256843Smavstatic void 507256843Smavahci_intr_one_edge(void *data) 508256843Smav{ 509256843Smav struct ahci_controller_irq *irq = data; 510256843Smav struct ahci_controller *ctlr = irq->ctlr; 511256843Smav void *arg; 512256843Smav int unit; 513256843Smav 514256843Smav unit = irq->r_irq_rid - 1; 515256843Smav /* Some controllers have edge triggered IS. */ 516256843Smav ATA_OUTL(ctlr->r_mem, AHCI_IS, 1 << unit); 517256843Smav if ((arg = ctlr->interrupt[unit].argument)) 518256843Smav ctlr->interrupt[unit].function(arg); 519256843Smav} 520256843Smav 521271146Simpstruct resource * 522195534Sscottlahci_alloc_resource(device_t dev, device_t child, int type, int *rid, 523271146Simp u_long start, u_long end, u_long count, u_int flags) 524195534Sscottl{ 525195534Sscottl struct ahci_controller *ctlr = device_get_softc(dev); 526238805Smav struct resource *res; 527195534Sscottl long st; 528238805Smav int offset, size, unit; 529195534Sscottl 530238805Smav unit = (intptr_t)device_get_ivars(child); 531238805Smav res = NULL; 532195534Sscottl switch (type) { 533195534Sscottl case SYS_RES_MEMORY: 534238805Smav if (unit >= 0) { 535238805Smav offset = AHCI_OFFSET + (unit << 7); 536238805Smav size = 128; 537238805Smav } else if (*rid == 0) { 538238805Smav offset = AHCI_EM_CTL; 539238805Smav size = 4; 540238805Smav } else { 541238805Smav offset = (ctlr->emloc & 0xffff0000) >> 14; 542238805Smav size = (ctlr->emloc & 0x0000ffff) << 2; 543238805Smav if (*rid != 1) { 544238805Smav if (*rid == 2 && (ctlr->capsem & 545238805Smav (AHCI_EM_XMT | AHCI_EM_SMB)) == 0) 546238805Smav offset += size; 547238805Smav else 548238805Smav break; 549238805Smav } 550238805Smav } 551195534Sscottl st = rman_get_start(ctlr->r_mem); 552195534Sscottl res = rman_reserve_resource(&ctlr->sc_iomem, st + offset, 553238805Smav st + offset + size - 1, size, RF_ACTIVE, child); 554195534Sscottl if (res) { 555195534Sscottl bus_space_handle_t bsh; 556195534Sscottl bus_space_tag_t bst; 557195534Sscottl bsh = rman_get_bushandle(ctlr->r_mem); 558195534Sscottl bst = rman_get_bustag(ctlr->r_mem); 559195534Sscottl bus_space_subregion(bst, bsh, offset, 128, &bsh); 560195534Sscottl rman_set_bushandle(res, bsh); 561195534Sscottl rman_set_bustag(res, bst); 562195534Sscottl } 563195534Sscottl break; 564195534Sscottl case SYS_RES_IRQ: 565195534Sscottl if (*rid == ATA_IRQ_RID) 566195534Sscottl res = ctlr->irqs[0].r_irq; 567195534Sscottl break; 568195534Sscottl } 569195534Sscottl return (res); 570195534Sscottl} 571195534Sscottl 572271146Simpint 573195534Sscottlahci_release_resource(device_t dev, device_t child, int type, int rid, 574271146Simp struct resource *r) 575195534Sscottl{ 576195534Sscottl 577195534Sscottl switch (type) { 578195534Sscottl case SYS_RES_MEMORY: 579195534Sscottl rman_release_resource(r); 580195534Sscottl return (0); 581195534Sscottl case SYS_RES_IRQ: 582195534Sscottl if (rid != ATA_IRQ_RID) 583276019Ssmh return (ENOENT); 584195534Sscottl return (0); 585195534Sscottl } 586195534Sscottl return (EINVAL); 587195534Sscottl} 588195534Sscottl 589271146Simpint 590195534Sscottlahci_setup_intr(device_t dev, device_t child, struct resource *irq, 591271146Simp int flags, driver_filter_t *filter, driver_intr_t *function, 592271146Simp void *argument, void **cookiep) 593195534Sscottl{ 594195534Sscottl struct ahci_controller *ctlr = device_get_softc(dev); 595195534Sscottl int unit = (intptr_t)device_get_ivars(child); 596195534Sscottl 597195534Sscottl if (filter != NULL) { 598195534Sscottl printf("ahci.c: we cannot use a filter here\n"); 599195534Sscottl return (EINVAL); 600195534Sscottl } 601195534Sscottl ctlr->interrupt[unit].function = function; 602195534Sscottl ctlr->interrupt[unit].argument = argument; 603195534Sscottl return (0); 604195534Sscottl} 605195534Sscottl 606271146Simpint 607195534Sscottlahci_teardown_intr(device_t dev, device_t child, struct resource *irq, 608271146Simp void *cookie) 609195534Sscottl{ 610195534Sscottl struct ahci_controller *ctlr = device_get_softc(dev); 611195534Sscottl int unit = (intptr_t)device_get_ivars(child); 612195534Sscottl 613195534Sscottl ctlr->interrupt[unit].function = NULL; 614195534Sscottl ctlr->interrupt[unit].argument = NULL; 615195534Sscottl return (0); 616195534Sscottl} 617195534Sscottl 618271146Simpint 619195534Sscottlahci_print_child(device_t dev, device_t child) 620195534Sscottl{ 621238805Smav int retval, channel; 622195534Sscottl 623195534Sscottl retval = bus_print_child_header(dev, child); 624238805Smav channel = (int)(intptr_t)device_get_ivars(child); 625238805Smav if (channel >= 0) 626238805Smav retval += printf(" at channel %d", channel); 627195534Sscottl retval += bus_print_child_footer(dev, child); 628195534Sscottl return (retval); 629195534Sscottl} 630195534Sscottl 631271146Simpint 632208410Smavahci_child_location_str(device_t dev, device_t child, char *buf, 633208410Smav size_t buflen) 634208410Smav{ 635238805Smav int channel; 636208410Smav 637238805Smav channel = (int)(intptr_t)device_get_ivars(child); 638238805Smav if (channel >= 0) 639238805Smav snprintf(buf, buflen, "channel=%d", channel); 640208410Smav return (0); 641208410Smav} 642208410Smav 643271146Simpbus_dma_tag_t 644249346Smavahci_get_dma_tag(device_t dev, device_t child) 645249346Smav{ 646249346Smav struct ahci_controller *ctlr = device_get_softc(dev); 647249346Smav 648249346Smav return (ctlr->dma_tag); 649249346Smav} 650249346Smav 651195534Sscottlstatic int 652195534Sscottlahci_ch_probe(device_t dev) 653195534Sscottl{ 654195534Sscottl 655195534Sscottl device_set_desc_copy(dev, "AHCI channel"); 656280393Smav return (BUS_PROBE_DEFAULT); 657195534Sscottl} 658195534Sscottl 659195534Sscottlstatic int 660195534Sscottlahci_ch_attach(device_t dev) 661195534Sscottl{ 662195534Sscottl struct ahci_controller *ctlr = device_get_softc(device_get_parent(dev)); 663195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 664195534Sscottl struct cam_devq *devq; 665199821Smav int rid, error, i, sata_rev = 0; 666203123Smav u_int32_t version; 667195534Sscottl 668195534Sscottl ch->dev = dev; 669195534Sscottl ch->unit = (intptr_t)device_get_ivars(dev); 670196656Smav ch->caps = ctlr->caps; 671196656Smav ch->caps2 = ctlr->caps2; 672285090Sloos ch->start = ctlr->ch_start; 673199322Smav ch->quirks = ctlr->quirks; 674271146Simp ch->vendorid = ctlr->vendorid; 675271146Simp ch->deviceid = ctlr->deviceid; 676271146Simp ch->subvendorid = ctlr->subvendorid; 677271146Simp ch->subdeviceid = ctlr->subdeviceid; 678215725Smav ch->numslots = ((ch->caps & AHCI_CAP_NCS) >> AHCI_CAP_NCS_SHIFT) + 1; 679196656Smav mtx_init(&ch->mtx, "AHCI channel lock", NULL, MTX_DEF); 680271457Smav ch->pm_level = 0; 681195534Sscottl resource_int_value(device_get_name(dev), 682195534Sscottl device_get_unit(dev), "pm_level", &ch->pm_level); 683256843Smav STAILQ_INIT(&ch->doneq); 684196656Smav if (ch->pm_level > 3) 685196656Smav callout_init_mtx(&ch->pm_timer, &ch->mtx, 0); 686220576Smav callout_init_mtx(&ch->reset_timer, &ch->mtx, 0); 687271146Simp /* JMicron external ports (0) sometimes limited */ 688271146Simp if ((ctlr->quirks & AHCI_Q_SATA1_UNIT0) && ch->unit == 0) 689199821Smav sata_rev = 1; 690203030Smav if (ch->quirks & AHCI_Q_SATA2) 691203030Smav sata_rev = 2; 692195534Sscottl resource_int_value(device_get_name(dev), 693199821Smav device_get_unit(dev), "sata_rev", &sata_rev); 694199821Smav for (i = 0; i < 16; i++) { 695199821Smav ch->user[i].revision = sata_rev; 696199821Smav ch->user[i].mode = 0; 697199821Smav ch->user[i].bytecount = 8192; 698199821Smav ch->user[i].tags = ch->numslots; 699207499Smav ch->user[i].caps = 0; 700199821Smav ch->curr[i] = ch->user[i]; 701207499Smav if (ch->pm_level) { 702207499Smav ch->user[i].caps = CTS_SATA_CAPS_H_PMREQ | 703207499Smav CTS_SATA_CAPS_H_APST | 704207499Smav CTS_SATA_CAPS_D_PMREQ | CTS_SATA_CAPS_D_APST; 705207499Smav } 706220602Smav ch->user[i].caps |= CTS_SATA_CAPS_H_DMAAA | 707220602Smav CTS_SATA_CAPS_H_AN; 708199821Smav } 709238805Smav rid = 0; 710195534Sscottl if (!(ch->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 711195534Sscottl &rid, RF_ACTIVE))) 712195534Sscottl return (ENXIO); 713195534Sscottl ahci_dmainit(dev); 714195534Sscottl ahci_slotsalloc(dev); 715279320Smav mtx_lock(&ch->mtx); 716208375Smav ahci_ch_init(dev); 717195534Sscottl rid = ATA_IRQ_RID; 718195534Sscottl if (!(ch->r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, 719195534Sscottl &rid, RF_SHAREABLE | RF_ACTIVE))) { 720195534Sscottl device_printf(dev, "Unable to map interrupt\n"); 721208813Smav error = ENXIO; 722208813Smav goto err0; 723195534Sscottl } 724195534Sscottl if ((bus_setup_intr(dev, ch->r_irq, ATA_INTR_FLAGS, NULL, 725256843Smav ctlr->direct ? ahci_ch_intr_direct : ahci_ch_intr, 726271261Smav ch, &ch->ih))) { 727195534Sscottl device_printf(dev, "Unable to setup interrupt\n"); 728195534Sscottl error = ENXIO; 729195534Sscottl goto err1; 730195534Sscottl } 731203123Smav ch->chcaps = ATA_INL(ch->r_mem, AHCI_P_CMD); 732203123Smav version = ATA_INL(ctlr->r_mem, AHCI_VS); 733240383Smav if (version < 0x00010200 && (ctlr->caps & AHCI_CAP_FBSS)) 734203123Smav ch->chcaps |= AHCI_P_CMD_FBSCP; 735258173Smav if (ch->caps2 & AHCI_CAP2_SDS) 736258173Smav ch->chscaps = ATA_INL(ch->r_mem, AHCI_P_DEVSLP); 737203123Smav if (bootverbose) { 738258173Smav device_printf(dev, "Caps:%s%s%s%s%s%s\n", 739203123Smav (ch->chcaps & AHCI_P_CMD_HPCP) ? " HPCP":"", 740203123Smav (ch->chcaps & AHCI_P_CMD_MPSP) ? " MPSP":"", 741203123Smav (ch->chcaps & AHCI_P_CMD_CPD) ? " CPD":"", 742203123Smav (ch->chcaps & AHCI_P_CMD_ESP) ? " ESP":"", 743258173Smav (ch->chcaps & AHCI_P_CMD_FBSCP) ? " FBSCP":"", 744258173Smav (ch->chscaps & AHCI_P_DEVSLP_DSP) ? " DSP":""); 745203123Smav } 746195534Sscottl /* Create the device queue for our SIM. */ 747195534Sscottl devq = cam_simq_alloc(ch->numslots); 748195534Sscottl if (devq == NULL) { 749195534Sscottl device_printf(dev, "Unable to allocate simq\n"); 750195534Sscottl error = ENOMEM; 751195534Sscottl goto err1; 752195534Sscottl } 753195534Sscottl /* Construct SIM entry */ 754195534Sscottl ch->sim = cam_sim_alloc(ahciaction, ahcipoll, "ahcich", ch, 755271261Smav device_get_unit(dev), (struct mtx *)&ch->mtx, 756199278Smav min(2, ch->numslots), 757199278Smav (ch->caps & AHCI_CAP_SNCQ) ? ch->numslots : 0, 758199278Smav devq); 759195534Sscottl if (ch->sim == NULL) { 760208813Smav cam_simq_free(devq); 761195534Sscottl device_printf(dev, "unable to allocate sim\n"); 762195534Sscottl error = ENOMEM; 763208813Smav goto err1; 764195534Sscottl } 765195534Sscottl if (xpt_bus_register(ch->sim, dev, 0) != CAM_SUCCESS) { 766195534Sscottl device_printf(dev, "unable to register xpt bus\n"); 767195534Sscottl error = ENXIO; 768195534Sscottl goto err2; 769195534Sscottl } 770195534Sscottl if (xpt_create_path(&ch->path, /*periph*/NULL, cam_sim_path(ch->sim), 771195534Sscottl CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 772195534Sscottl device_printf(dev, "unable to create path\n"); 773195534Sscottl error = ENXIO; 774195534Sscottl goto err3; 775195534Sscottl } 776196656Smav if (ch->pm_level > 3) { 777196656Smav callout_reset(&ch->pm_timer, 778196656Smav (ch->pm_level == 4) ? hz / 1000 : hz / 8, 779271261Smav ahci_ch_pm, ch); 780196656Smav } 781195534Sscottl mtx_unlock(&ch->mtx); 782195534Sscottl return (0); 783195534Sscottl 784195534Sscottlerr3: 785195534Sscottl xpt_bus_deregister(cam_sim_path(ch->sim)); 786195534Sscottlerr2: 787195534Sscottl cam_sim_free(ch->sim, /*free_devq*/TRUE); 788195534Sscottlerr1: 789195534Sscottl bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq); 790208813Smaverr0: 791195534Sscottl bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem); 792195534Sscottl mtx_unlock(&ch->mtx); 793214325Smav mtx_destroy(&ch->mtx); 794195534Sscottl return (error); 795195534Sscottl} 796195534Sscottl 797195534Sscottlstatic int 798195534Sscottlahci_ch_detach(device_t dev) 799195534Sscottl{ 800195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 801195534Sscottl 802195534Sscottl mtx_lock(&ch->mtx); 803195534Sscottl xpt_async(AC_LOST_DEVICE, ch->path, NULL); 804220576Smav /* Forget about reset. */ 805220576Smav if (ch->resetting) { 806220576Smav ch->resetting = 0; 807220576Smav xpt_release_simq(ch->sim, TRUE); 808220576Smav } 809195534Sscottl xpt_free_path(ch->path); 810195534Sscottl xpt_bus_deregister(cam_sim_path(ch->sim)); 811195534Sscottl cam_sim_free(ch->sim, /*free_devq*/TRUE); 812195534Sscottl mtx_unlock(&ch->mtx); 813195534Sscottl 814196656Smav if (ch->pm_level > 3) 815196656Smav callout_drain(&ch->pm_timer); 816220576Smav callout_drain(&ch->reset_timer); 817195534Sscottl bus_teardown_intr(dev, ch->r_irq, ch->ih); 818195534Sscottl bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq); 819195534Sscottl 820208375Smav ahci_ch_deinit(dev); 821195534Sscottl ahci_slotsfree(dev); 822195534Sscottl ahci_dmafini(dev); 823195534Sscottl 824195534Sscottl bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem); 825195534Sscottl mtx_destroy(&ch->mtx); 826195534Sscottl return (0); 827195534Sscottl} 828195534Sscottl 829195534Sscottlstatic int 830208375Smavahci_ch_init(device_t dev) 831195534Sscottl{ 832195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 833208375Smav uint64_t work; 834195534Sscottl 835208375Smav /* Disable port interrupts */ 836208375Smav ATA_OUTL(ch->r_mem, AHCI_P_IE, 0); 837208375Smav /* Setup work areas */ 838208375Smav work = ch->dma.work_bus + AHCI_CL_OFFSET; 839208375Smav ATA_OUTL(ch->r_mem, AHCI_P_CLB, work & 0xffffffff); 840208375Smav ATA_OUTL(ch->r_mem, AHCI_P_CLBU, work >> 32); 841208375Smav work = ch->dma.rfis_bus; 842208375Smav ATA_OUTL(ch->r_mem, AHCI_P_FB, work & 0xffffffff); 843208375Smav ATA_OUTL(ch->r_mem, AHCI_P_FBU, work >> 32); 844208375Smav /* Activate the channel and power/spin up device */ 845208375Smav ATA_OUTL(ch->r_mem, AHCI_P_CMD, 846208375Smav (AHCI_P_CMD_ACTIVE | AHCI_P_CMD_POD | AHCI_P_CMD_SUD | 847208375Smav ((ch->pm_level == 2 || ch->pm_level == 3) ? AHCI_P_CMD_ALPE : 0) | 848208375Smav ((ch->pm_level > 2) ? AHCI_P_CMD_ASP : 0 ))); 849271261Smav ahci_start_fr(ch); 850271261Smav ahci_start(ch, 1); 851208375Smav return (0); 852208375Smav} 853208375Smav 854208375Smavstatic int 855208375Smavahci_ch_deinit(device_t dev) 856208375Smav{ 857208375Smav struct ahci_channel *ch = device_get_softc(dev); 858208375Smav 859195534Sscottl /* Disable port interrupts. */ 860195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_IE, 0); 861195534Sscottl /* Reset command register. */ 862271261Smav ahci_stop(ch); 863271261Smav ahci_stop_fr(ch); 864195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CMD, 0); 865195534Sscottl /* Allow everything, including partial and slumber modes. */ 866195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_SCTL, 0); 867195534Sscottl /* Request slumber mode transition and give some time to get there. */ 868195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CMD, AHCI_P_CMD_SLUMBER); 869195534Sscottl DELAY(100); 870195534Sscottl /* Disable PHY. */ 871195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_SCTL, ATA_SC_DET_DISABLE); 872195534Sscottl return (0); 873195534Sscottl} 874195534Sscottl 875195534Sscottlstatic int 876208375Smavahci_ch_suspend(device_t dev) 877208375Smav{ 878208375Smav struct ahci_channel *ch = device_get_softc(dev); 879208375Smav 880208375Smav mtx_lock(&ch->mtx); 881208375Smav xpt_freeze_simq(ch->sim, 1); 882220576Smav /* Forget about reset. */ 883220576Smav if (ch->resetting) { 884220576Smav ch->resetting = 0; 885220576Smav callout_stop(&ch->reset_timer); 886220576Smav xpt_release_simq(ch->sim, TRUE); 887220576Smav } 888208375Smav while (ch->oslots) 889208375Smav msleep(ch, &ch->mtx, PRIBIO, "ahcisusp", hz/100); 890208375Smav ahci_ch_deinit(dev); 891208375Smav mtx_unlock(&ch->mtx); 892208375Smav return (0); 893208375Smav} 894208375Smav 895208375Smavstatic int 896195534Sscottlahci_ch_resume(device_t dev) 897195534Sscottl{ 898195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 899195534Sscottl 900208375Smav mtx_lock(&ch->mtx); 901208375Smav ahci_ch_init(dev); 902271261Smav ahci_reset(ch); 903208375Smav xpt_release_simq(ch->sim, TRUE); 904208375Smav mtx_unlock(&ch->mtx); 905195534Sscottl return (0); 906195534Sscottl} 907195534Sscottl 908195534Sscottldevclass_t ahcich_devclass; 909195534Sscottlstatic device_method_t ahcich_methods[] = { 910195534Sscottl DEVMETHOD(device_probe, ahci_ch_probe), 911195534Sscottl DEVMETHOD(device_attach, ahci_ch_attach), 912195534Sscottl DEVMETHOD(device_detach, ahci_ch_detach), 913195534Sscottl DEVMETHOD(device_suspend, ahci_ch_suspend), 914195534Sscottl DEVMETHOD(device_resume, ahci_ch_resume), 915276344Smarius DEVMETHOD_END 916195534Sscottl}; 917195534Sscottlstatic driver_t ahcich_driver = { 918195534Sscottl "ahcich", 919195534Sscottl ahcich_methods, 920195534Sscottl sizeof(struct ahci_channel) 921195534Sscottl}; 922276344SmariusDRIVER_MODULE(ahcich, ahci, ahcich_driver, ahcich_devclass, NULL, NULL); 923195534Sscottl 924195534Sscottlstruct ahci_dc_cb_args { 925195534Sscottl bus_addr_t maddr; 926195534Sscottl int error; 927195534Sscottl}; 928195534Sscottl 929195534Sscottlstatic void 930195534Sscottlahci_dmainit(device_t dev) 931195534Sscottl{ 932195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 933195534Sscottl struct ahci_dc_cb_args dcba; 934203123Smav size_t rfsize; 935195534Sscottl 936195534Sscottl /* Command area. */ 937195534Sscottl if (bus_dma_tag_create(bus_get_dma_tag(dev), 1024, 0, 938249346Smav BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 939195534Sscottl NULL, NULL, AHCI_WORK_SIZE, 1, AHCI_WORK_SIZE, 940195534Sscottl 0, NULL, NULL, &ch->dma.work_tag)) 941195534Sscottl goto error; 942248687Smav if (bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work, 943248687Smav BUS_DMA_ZERO, &ch->dma.work_map)) 944195534Sscottl goto error; 945195534Sscottl if (bus_dmamap_load(ch->dma.work_tag, ch->dma.work_map, ch->dma.work, 946195534Sscottl AHCI_WORK_SIZE, ahci_dmasetupc_cb, &dcba, 0) || dcba.error) { 947195534Sscottl bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map); 948195534Sscottl goto error; 949195534Sscottl } 950195534Sscottl ch->dma.work_bus = dcba.maddr; 951195534Sscottl /* FIS receive area. */ 952203123Smav if (ch->chcaps & AHCI_P_CMD_FBSCP) 953203123Smav rfsize = 4096; 954203123Smav else 955203123Smav rfsize = 256; 956203123Smav if (bus_dma_tag_create(bus_get_dma_tag(dev), rfsize, 0, 957249346Smav BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 958203123Smav NULL, NULL, rfsize, 1, rfsize, 959195534Sscottl 0, NULL, NULL, &ch->dma.rfis_tag)) 960195534Sscottl goto error; 961195534Sscottl if (bus_dmamem_alloc(ch->dma.rfis_tag, (void **)&ch->dma.rfis, 0, 962195534Sscottl &ch->dma.rfis_map)) 963195534Sscottl goto error; 964195534Sscottl if (bus_dmamap_load(ch->dma.rfis_tag, ch->dma.rfis_map, ch->dma.rfis, 965203123Smav rfsize, ahci_dmasetupc_cb, &dcba, 0) || dcba.error) { 966195534Sscottl bus_dmamem_free(ch->dma.rfis_tag, ch->dma.rfis, ch->dma.rfis_map); 967195534Sscottl goto error; 968195534Sscottl } 969195534Sscottl ch->dma.rfis_bus = dcba.maddr; 970195534Sscottl /* Data area. */ 971195534Sscottl if (bus_dma_tag_create(bus_get_dma_tag(dev), 2, 0, 972249346Smav BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 973195534Sscottl NULL, NULL, 974195534Sscottl AHCI_SG_ENTRIES * PAGE_SIZE * ch->numslots, 975195534Sscottl AHCI_SG_ENTRIES, AHCI_PRD_MAX, 976195534Sscottl 0, busdma_lock_mutex, &ch->mtx, &ch->dma.data_tag)) { 977195534Sscottl goto error; 978195534Sscottl } 979195534Sscottl return; 980195534Sscottl 981195534Sscottlerror: 982195534Sscottl device_printf(dev, "WARNING - DMA initialization failed\n"); 983195534Sscottl ahci_dmafini(dev); 984195534Sscottl} 985195534Sscottl 986195534Sscottlstatic void 987195534Sscottlahci_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) 988195534Sscottl{ 989195534Sscottl struct ahci_dc_cb_args *dcba = (struct ahci_dc_cb_args *)xsc; 990195534Sscottl 991195534Sscottl if (!(dcba->error = error)) 992195534Sscottl dcba->maddr = segs[0].ds_addr; 993195534Sscottl} 994195534Sscottl 995195534Sscottlstatic void 996195534Sscottlahci_dmafini(device_t dev) 997195534Sscottl{ 998195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 999195534Sscottl 1000195534Sscottl if (ch->dma.data_tag) { 1001195534Sscottl bus_dma_tag_destroy(ch->dma.data_tag); 1002195534Sscottl ch->dma.data_tag = NULL; 1003195534Sscottl } 1004195534Sscottl if (ch->dma.rfis_bus) { 1005195534Sscottl bus_dmamap_unload(ch->dma.rfis_tag, ch->dma.rfis_map); 1006195534Sscottl bus_dmamem_free(ch->dma.rfis_tag, ch->dma.rfis, ch->dma.rfis_map); 1007195534Sscottl ch->dma.rfis_bus = 0; 1008195534Sscottl ch->dma.rfis = NULL; 1009195534Sscottl } 1010195534Sscottl if (ch->dma.work_bus) { 1011195534Sscottl bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map); 1012195534Sscottl bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map); 1013195534Sscottl ch->dma.work_bus = 0; 1014195534Sscottl ch->dma.work = NULL; 1015195534Sscottl } 1016195534Sscottl if (ch->dma.work_tag) { 1017195534Sscottl bus_dma_tag_destroy(ch->dma.work_tag); 1018195534Sscottl ch->dma.work_tag = NULL; 1019195534Sscottl } 1020195534Sscottl} 1021195534Sscottl 1022195534Sscottlstatic void 1023195534Sscottlahci_slotsalloc(device_t dev) 1024195534Sscottl{ 1025195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 1026195534Sscottl int i; 1027195534Sscottl 1028195534Sscottl /* Alloc and setup command/dma slots */ 1029195534Sscottl bzero(ch->slot, sizeof(ch->slot)); 1030195534Sscottl for (i = 0; i < ch->numslots; i++) { 1031195534Sscottl struct ahci_slot *slot = &ch->slot[i]; 1032195534Sscottl 1033271261Smav slot->ch = ch; 1034195534Sscottl slot->slot = i; 1035195534Sscottl slot->state = AHCI_SLOT_EMPTY; 1036195534Sscottl slot->ccb = NULL; 1037195534Sscottl callout_init_mtx(&slot->timeout, &ch->mtx, 0); 1038195534Sscottl 1039195534Sscottl if (bus_dmamap_create(ch->dma.data_tag, 0, &slot->dma.data_map)) 1040195534Sscottl device_printf(ch->dev, "FAILURE - create data_map\n"); 1041195534Sscottl } 1042195534Sscottl} 1043195534Sscottl 1044195534Sscottlstatic void 1045195534Sscottlahci_slotsfree(device_t dev) 1046195534Sscottl{ 1047195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 1048195534Sscottl int i; 1049195534Sscottl 1050195534Sscottl /* Free all dma slots */ 1051195534Sscottl for (i = 0; i < ch->numslots; i++) { 1052195534Sscottl struct ahci_slot *slot = &ch->slot[i]; 1053195534Sscottl 1054196656Smav callout_drain(&slot->timeout); 1055195534Sscottl if (slot->dma.data_map) { 1056195534Sscottl bus_dmamap_destroy(ch->dma.data_tag, slot->dma.data_map); 1057195534Sscottl slot->dma.data_map = NULL; 1058195534Sscottl } 1059195534Sscottl } 1060195534Sscottl} 1061195534Sscottl 1062220657Smavstatic int 1063271261Smavahci_phy_check_events(struct ahci_channel *ch, u_int32_t serr) 1064195534Sscottl{ 1065195534Sscottl 1066220657Smav if (((ch->pm_level == 0) && (serr & ATA_SE_PHY_CHANGED)) || 1067220657Smav ((ch->pm_level != 0 || ch->listening) && (serr & ATA_SE_EXCHANGED))) { 1068195534Sscottl u_int32_t status = ATA_INL(ch->r_mem, AHCI_P_SSTS); 1069203108Smav union ccb *ccb; 1070203108Smav 1071203165Smav if (bootverbose) { 1072220657Smav if ((status & ATA_SS_DET_MASK) != ATA_SS_DET_NO_DEVICE) 1073271261Smav device_printf(ch->dev, "CONNECT requested\n"); 1074220657Smav else 1075271261Smav device_printf(ch->dev, "DISCONNECT requested\n"); 1076195534Sscottl } 1077271261Smav ahci_reset(ch); 1078203108Smav if ((ccb = xpt_alloc_ccb_nowait()) == NULL) 1079220657Smav return (0); 1080203108Smav if (xpt_create_path(&ccb->ccb_h.path, NULL, 1081203108Smav cam_sim_path(ch->sim), 1082203108Smav CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 1083203108Smav xpt_free_ccb(ccb); 1084220657Smav return (0); 1085203108Smav } 1086203108Smav xpt_rescan(ccb); 1087220657Smav return (1); 1088195534Sscottl } 1089220657Smav return (0); 1090195534Sscottl} 1091195534Sscottl 1092195534Sscottlstatic void 1093271261Smavahci_cpd_check_events(struct ahci_channel *ch) 1094220657Smav{ 1095220657Smav u_int32_t status; 1096220657Smav union ccb *ccb; 1097271261Smav device_t dev; 1098220657Smav 1099220657Smav if (ch->pm_level == 0) 1100220657Smav return; 1101220657Smav 1102220657Smav status = ATA_INL(ch->r_mem, AHCI_P_CMD); 1103220657Smav if ((status & AHCI_P_CMD_CPD) == 0) 1104220657Smav return; 1105220657Smav 1106220657Smav if (bootverbose) { 1107271261Smav dev = ch->dev; 1108220657Smav if (status & AHCI_P_CMD_CPS) { 1109220657Smav device_printf(dev, "COLD CONNECT requested\n"); 1110220657Smav } else 1111220657Smav device_printf(dev, "COLD DISCONNECT requested\n"); 1112220657Smav } 1113271261Smav ahci_reset(ch); 1114220657Smav if ((ccb = xpt_alloc_ccb_nowait()) == NULL) 1115220657Smav return; 1116220657Smav if (xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(ch->sim), 1117220657Smav CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 1118220657Smav xpt_free_ccb(ccb); 1119220657Smav return; 1120220657Smav } 1121220657Smav xpt_rescan(ccb); 1122220657Smav} 1123220657Smav 1124220657Smavstatic void 1125271261Smavahci_notify_events(struct ahci_channel *ch, u_int32_t status) 1126196656Smav{ 1127196656Smav struct cam_path *dpath; 1128196656Smav int i; 1129196656Smav 1130200196Smav if (ch->caps & AHCI_CAP_SSNTF) 1131200196Smav ATA_OUTL(ch->r_mem, AHCI_P_SNTF, status); 1132196656Smav if (bootverbose) 1133271261Smav device_printf(ch->dev, "SNTF 0x%04x\n", status); 1134196656Smav for (i = 0; i < 16; i++) { 1135196656Smav if ((status & (1 << i)) == 0) 1136196656Smav continue; 1137196656Smav if (xpt_create_path(&dpath, NULL, 1138196656Smav xpt_path_path_id(ch->path), i, 0) == CAM_REQ_CMP) { 1139196656Smav xpt_async(AC_SCSI_AEN, dpath, NULL); 1140196656Smav xpt_free_path(dpath); 1141196656Smav } 1142196656Smav } 1143196656Smav} 1144196656Smav 1145196656Smavstatic void 1146256843Smavahci_done(struct ahci_channel *ch, union ccb *ccb) 1147195534Sscottl{ 1148256843Smav 1149256843Smav mtx_assert(&ch->mtx, MA_OWNED); 1150256843Smav if ((ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0 || 1151256843Smav ch->batch == 0) { 1152256843Smav xpt_done(ccb); 1153256843Smav return; 1154256843Smav } 1155256843Smav 1156256843Smav STAILQ_INSERT_TAIL(&ch->doneq, &ccb->ccb_h, sim_links.stqe); 1157256843Smav} 1158256843Smav 1159256843Smavstatic void 1160256843Smavahci_ch_intr(void *arg) 1161256843Smav{ 1162271261Smav struct ahci_channel *ch = (struct ahci_channel *)arg; 1163256843Smav uint32_t istatus; 1164195534Sscottl 1165256843Smav /* Read interrupt statuses. */ 1166256843Smav istatus = ATA_INL(ch->r_mem, AHCI_P_IS); 1167256843Smav if (istatus == 0) 1168256843Smav return; 1169256843Smav 1170195534Sscottl mtx_lock(&ch->mtx); 1171256843Smav ahci_ch_intr_main(ch, istatus); 1172195534Sscottl mtx_unlock(&ch->mtx); 1173195534Sscottl} 1174195534Sscottl 1175195534Sscottlstatic void 1176256843Smavahci_ch_intr_direct(void *arg) 1177256843Smav{ 1178271261Smav struct ahci_channel *ch = (struct ahci_channel *)arg; 1179256843Smav struct ccb_hdr *ccb_h; 1180256843Smav uint32_t istatus; 1181272223Ssmh STAILQ_HEAD(, ccb_hdr) tmp_doneq = STAILQ_HEAD_INITIALIZER(tmp_doneq); 1182256843Smav 1183256843Smav /* Read interrupt statuses. */ 1184256843Smav istatus = ATA_INL(ch->r_mem, AHCI_P_IS); 1185256843Smav if (istatus == 0) 1186256843Smav return; 1187256843Smav 1188256843Smav mtx_lock(&ch->mtx); 1189256843Smav ch->batch = 1; 1190256843Smav ahci_ch_intr_main(ch, istatus); 1191256843Smav ch->batch = 0; 1192272223Ssmh /* 1193272223Ssmh * Prevent the possibility of issues caused by processing the queue 1194272223Ssmh * while unlocked below by moving the contents to a local queue. 1195272223Ssmh */ 1196272223Ssmh STAILQ_CONCAT(&tmp_doneq, &ch->doneq); 1197256843Smav mtx_unlock(&ch->mtx); 1198272223Ssmh while ((ccb_h = STAILQ_FIRST(&tmp_doneq)) != NULL) { 1199272223Ssmh STAILQ_REMOVE_HEAD(&tmp_doneq, sim_links.stqe); 1200256843Smav xpt_done_direct((union ccb *)ccb_h); 1201256843Smav } 1202256843Smav} 1203256843Smav 1204256843Smavstatic void 1205196656Smavahci_ch_pm(void *arg) 1206196656Smav{ 1207271261Smav struct ahci_channel *ch = (struct ahci_channel *)arg; 1208196656Smav uint32_t work; 1209196656Smav 1210196656Smav if (ch->numrslots != 0) 1211196656Smav return; 1212196656Smav work = ATA_INL(ch->r_mem, AHCI_P_CMD); 1213196656Smav if (ch->pm_level == 4) 1214196656Smav work |= AHCI_P_CMD_PARTIAL; 1215196656Smav else 1216196656Smav work |= AHCI_P_CMD_SLUMBER; 1217196656Smav ATA_OUTL(ch->r_mem, AHCI_P_CMD, work); 1218196656Smav} 1219196656Smav 1220196656Smavstatic void 1221256843Smavahci_ch_intr_main(struct ahci_channel *ch, uint32_t istatus) 1222195534Sscottl{ 1223256843Smav uint32_t cstatus, serr = 0, sntf = 0, ok, err; 1224195534Sscottl enum ahci_err_type et; 1225220657Smav int i, ccs, port, reset = 0; 1226195534Sscottl 1227256843Smav /* Clear interrupt statuses. */ 1228195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_IS, istatus); 1229195534Sscottl /* Read command statuses. */ 1230248698Smav if (ch->numtslots != 0) 1231248698Smav cstatus = ATA_INL(ch->r_mem, AHCI_P_SACT); 1232248698Smav else 1233248698Smav cstatus = 0; 1234248698Smav if (ch->numrslots != ch->numtslots) 1235248698Smav cstatus |= ATA_INL(ch->r_mem, AHCI_P_CI); 1236248698Smav /* Read SNTF in one of possible ways. */ 1237248704Smav if ((istatus & AHCI_P_IX_SDB) && 1238248704Smav (ch->pm_present || ch->curr[0].atapi != 0)) { 1239200196Smav if (ch->caps & AHCI_CAP_SSNTF) 1240200196Smav sntf = ATA_INL(ch->r_mem, AHCI_P_SNTF); 1241203123Smav else if (ch->fbs_enabled) { 1242200196Smav u_int8_t *fis = ch->dma.rfis + 0x58; 1243200196Smav 1244203123Smav for (i = 0; i < 16; i++) { 1245203123Smav if (fis[1] & 0x80) { 1246203123Smav fis[1] &= 0x7f; 1247203123Smav sntf |= 1 << i; 1248203123Smav } 1249203123Smav fis += 256; 1250203123Smav } 1251203123Smav } else { 1252203123Smav u_int8_t *fis = ch->dma.rfis + 0x58; 1253203123Smav 1254200196Smav if (fis[1] & 0x80) 1255200196Smav sntf = (1 << (fis[1] & 0x0f)); 1256200196Smav } 1257200196Smav } 1258195534Sscottl /* Process PHY events */ 1259198319Smav if (istatus & (AHCI_P_IX_PC | AHCI_P_IX_PRC | AHCI_P_IX_OF | 1260198319Smav AHCI_P_IX_IF | AHCI_P_IX_HBD | AHCI_P_IX_HBF | AHCI_P_IX_TFE)) { 1261198319Smav serr = ATA_INL(ch->r_mem, AHCI_P_SERR); 1262198319Smav if (serr) { 1263198319Smav ATA_OUTL(ch->r_mem, AHCI_P_SERR, serr); 1264271261Smav reset = ahci_phy_check_events(ch, serr); 1265198319Smav } 1266198319Smav } 1267220657Smav /* Process cold presence detection events */ 1268220657Smav if ((istatus & AHCI_P_IX_CPD) && !reset) 1269271261Smav ahci_cpd_check_events(ch); 1270195534Sscottl /* Process command errors */ 1271198319Smav if (istatus & (AHCI_P_IX_OF | AHCI_P_IX_IF | 1272198319Smav AHCI_P_IX_HBD | AHCI_P_IX_HBF | AHCI_P_IX_TFE)) { 1273195534Sscottl ccs = (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CCS_MASK) 1274195534Sscottl >> AHCI_P_CMD_CCS_SHIFT; 1275203123Smav//device_printf(dev, "%s ERROR is %08x cs %08x ss %08x rs %08x tfd %02x serr %08x fbs %08x ccs %d\n", 1276203123Smav// __func__, istatus, cstatus, sstatus, ch->rslots, ATA_INL(ch->r_mem, AHCI_P_TFD), 1277203123Smav// serr, ATA_INL(ch->r_mem, AHCI_P_FBS), ccs); 1278203123Smav port = -1; 1279203123Smav if (ch->fbs_enabled) { 1280203123Smav uint32_t fbs = ATA_INL(ch->r_mem, AHCI_P_FBS); 1281203123Smav if (fbs & AHCI_P_FBS_SDE) { 1282203123Smav port = (fbs & AHCI_P_FBS_DWE) 1283203123Smav >> AHCI_P_FBS_DWE_SHIFT; 1284203123Smav } else { 1285203123Smav for (i = 0; i < 16; i++) { 1286203123Smav if (ch->numrslotspd[i] == 0) 1287203123Smav continue; 1288203123Smav if (port == -1) 1289203123Smav port = i; 1290203123Smav else if (port != i) { 1291203123Smav port = -2; 1292203123Smav break; 1293203123Smav } 1294203123Smav } 1295203123Smav } 1296203123Smav } 1297248698Smav err = ch->rslots & cstatus; 1298195534Sscottl } else { 1299195534Sscottl ccs = 0; 1300195534Sscottl err = 0; 1301203123Smav port = -1; 1302195534Sscottl } 1303195534Sscottl /* Complete all successfull commands. */ 1304248698Smav ok = ch->rslots & ~cstatus; 1305195534Sscottl for (i = 0; i < ch->numslots; i++) { 1306195534Sscottl if ((ok >> i) & 1) 1307195534Sscottl ahci_end_transaction(&ch->slot[i], AHCI_ERR_NONE); 1308195534Sscottl } 1309195534Sscottl /* On error, complete the rest of commands with error statuses. */ 1310195534Sscottl if (err) { 1311195534Sscottl if (ch->frozen) { 1312195534Sscottl union ccb *fccb = ch->frozen; 1313195534Sscottl ch->frozen = NULL; 1314195534Sscottl fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ; 1315198319Smav if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) { 1316198319Smav xpt_freeze_devq(fccb->ccb_h.path, 1); 1317198319Smav fccb->ccb_h.status |= CAM_DEV_QFRZN; 1318198319Smav } 1319256843Smav ahci_done(ch, fccb); 1320195534Sscottl } 1321195534Sscottl for (i = 0; i < ch->numslots; i++) { 1322195534Sscottl /* XXX: reqests in loading state. */ 1323195534Sscottl if (((err >> i) & 1) == 0) 1324195534Sscottl continue; 1325203123Smav if (port >= 0 && 1326203123Smav ch->slot[i].ccb->ccb_h.target_id != port) 1327203123Smav continue; 1328198390Smav if (istatus & AHCI_P_IX_TFE) { 1329203123Smav if (port != -2) { 1330195534Sscottl /* Task File Error */ 1331203123Smav if (ch->numtslotspd[ 1332203123Smav ch->slot[i].ccb->ccb_h.target_id] == 0) { 1333195534Sscottl /* Untagged operation. */ 1334195534Sscottl if (i == ccs) 1335195534Sscottl et = AHCI_ERR_TFE; 1336195534Sscottl else 1337195534Sscottl et = AHCI_ERR_INNOCENT; 1338195534Sscottl } else { 1339195534Sscottl /* Tagged operation. */ 1340195534Sscottl et = AHCI_ERR_NCQ; 1341195534Sscottl } 1342203123Smav } else { 1343203123Smav et = AHCI_ERR_TFE; 1344203123Smav ch->fatalerr = 1; 1345203123Smav } 1346198390Smav } else if (istatus & AHCI_P_IX_IF) { 1347203123Smav if (ch->numtslots == 0 && i != ccs && port != -2) 1348198390Smav et = AHCI_ERR_INNOCENT; 1349198390Smav else 1350198390Smav et = AHCI_ERR_SATA; 1351195534Sscottl } else 1352195534Sscottl et = AHCI_ERR_INVALID; 1353195534Sscottl ahci_end_transaction(&ch->slot[i], et); 1354195534Sscottl } 1355203123Smav /* 1356203123Smav * We can't reinit port if there are some other 1357203123Smav * commands active, use resume to complete them. 1358203123Smav */ 1359220565Smav if (ch->rslots != 0 && !ch->recoverycmd) 1360203123Smav ATA_OUTL(ch->r_mem, AHCI_P_FBS, AHCI_P_FBS_EN | AHCI_P_FBS_DEC); 1361195534Sscottl } 1362196656Smav /* Process NOTIFY events */ 1363196907Smav if (sntf) 1364271261Smav ahci_notify_events(ch, sntf); 1365195534Sscottl} 1366195534Sscottl 1367195534Sscottl/* Must be called with channel locked. */ 1368195534Sscottlstatic int 1369271261Smavahci_check_collision(struct ahci_channel *ch, union ccb *ccb) 1370195534Sscottl{ 1371203123Smav int t = ccb->ccb_h.target_id; 1372195534Sscottl 1373195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1374195534Sscottl (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 1375199747Smav /* Tagged command while we have no supported tag free. */ 1376199747Smav if (((~ch->oslots) & (0xffffffff >> (32 - 1377203123Smav ch->curr[t].tags))) == 0) 1378199747Smav return (1); 1379203123Smav /* If we have FBS */ 1380203123Smav if (ch->fbs_enabled) { 1381203123Smav /* Tagged command while untagged are active. */ 1382203123Smav if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] == 0) 1383203123Smav return (1); 1384203123Smav } else { 1385203123Smav /* Tagged command while untagged are active. */ 1386203123Smav if (ch->numrslots != 0 && ch->numtslots == 0) 1387203123Smav return (1); 1388203123Smav /* Tagged command while tagged to other target is active. */ 1389203123Smav if (ch->numtslots != 0 && 1390203123Smav ch->taggedtarget != ccb->ccb_h.target_id) 1391203123Smav return (1); 1392203123Smav } 1393195534Sscottl } else { 1394203123Smav /* If we have FBS */ 1395203123Smav if (ch->fbs_enabled) { 1396203123Smav /* Untagged command while tagged are active. */ 1397203123Smav if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] != 0) 1398203123Smav return (1); 1399203123Smav } else { 1400203123Smav /* Untagged command while tagged are active. */ 1401203123Smav if (ch->numrslots != 0 && ch->numtslots != 0) 1402203123Smav return (1); 1403203123Smav } 1404195534Sscottl } 1405195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1406195534Sscottl (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT))) { 1407195534Sscottl /* Atomic command while anything active. */ 1408195534Sscottl if (ch->numrslots != 0) 1409195534Sscottl return (1); 1410195534Sscottl } 1411195534Sscottl /* We have some atomic command running. */ 1412195534Sscottl if (ch->aslots != 0) 1413195534Sscottl return (1); 1414195534Sscottl return (0); 1415195534Sscottl} 1416195534Sscottl 1417195534Sscottl/* Must be called with channel locked. */ 1418195534Sscottlstatic void 1419271261Smavahci_begin_transaction(struct ahci_channel *ch, union ccb *ccb) 1420195534Sscottl{ 1421195534Sscottl struct ahci_slot *slot; 1422199747Smav int tag, tags; 1423195534Sscottl 1424195534Sscottl /* Choose empty slot. */ 1425199747Smav tags = ch->numslots; 1426199747Smav if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1427199747Smav (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) 1428199747Smav tags = ch->curr[ccb->ccb_h.target_id].tags; 1429271261Smav if (ch->lastslot + 1 < tags) 1430271261Smav tag = ffs(~(ch->oslots >> (ch->lastslot + 1))); 1431271261Smav else 1432271261Smav tag = 0; 1433271261Smav if (tag == 0 || tag + ch->lastslot >= tags) 1434271261Smav tag = ffs(~ch->oslots) - 1; 1435271261Smav else 1436271261Smav tag += ch->lastslot; 1437195534Sscottl ch->lastslot = tag; 1438195534Sscottl /* Occupy chosen slot. */ 1439195534Sscottl slot = &ch->slot[tag]; 1440195534Sscottl slot->ccb = ccb; 1441196656Smav /* Stop PM timer. */ 1442196656Smav if (ch->numrslots == 0 && ch->pm_level > 3) 1443196656Smav callout_stop(&ch->pm_timer); 1444195534Sscottl /* Update channel stats. */ 1445271261Smav ch->oslots |= (1 << tag); 1446195534Sscottl ch->numrslots++; 1447203123Smav ch->numrslotspd[ccb->ccb_h.target_id]++; 1448195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1449195534Sscottl (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 1450195534Sscottl ch->numtslots++; 1451203123Smav ch->numtslotspd[ccb->ccb_h.target_id]++; 1452195534Sscottl ch->taggedtarget = ccb->ccb_h.target_id; 1453195534Sscottl } 1454195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1455195534Sscottl (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT))) 1456271261Smav ch->aslots |= (1 << tag); 1457195534Sscottl if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) { 1458195534Sscottl slot->state = AHCI_SLOT_LOADING; 1459246713Skib bus_dmamap_load_ccb(ch->dma.data_tag, slot->dma.data_map, ccb, 1460246713Skib ahci_dmasetprd, slot, 0); 1461271261Smav } else { 1462271261Smav slot->dma.nsegs = 0; 1463195534Sscottl ahci_execute_transaction(slot); 1464271261Smav } 1465195534Sscottl} 1466195534Sscottl 1467195534Sscottl/* Locked by busdma engine. */ 1468195534Sscottlstatic void 1469195534Sscottlahci_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 1470195534Sscottl{ 1471195534Sscottl struct ahci_slot *slot = arg; 1472271261Smav struct ahci_channel *ch = slot->ch; 1473195534Sscottl struct ahci_cmd_tab *ctp; 1474195534Sscottl struct ahci_dma_prd *prd; 1475195534Sscottl int i; 1476195534Sscottl 1477195534Sscottl if (error) { 1478271261Smav device_printf(ch->dev, "DMA load error\n"); 1479195534Sscottl ahci_end_transaction(slot, AHCI_ERR_INVALID); 1480195534Sscottl return; 1481195534Sscottl } 1482195534Sscottl KASSERT(nsegs <= AHCI_SG_ENTRIES, ("too many DMA segment entries\n")); 1483195534Sscottl /* Get a piece of the workspace for this request */ 1484195534Sscottl ctp = (struct ahci_cmd_tab *) 1485195534Sscottl (ch->dma.work + AHCI_CT_OFFSET + (AHCI_CT_SIZE * slot->slot)); 1486195534Sscottl /* Fill S/G table */ 1487195534Sscottl prd = &ctp->prd_tab[0]; 1488195534Sscottl for (i = 0; i < nsegs; i++) { 1489195534Sscottl prd[i].dba = htole64(segs[i].ds_addr); 1490195534Sscottl prd[i].dbc = htole32((segs[i].ds_len - 1) & AHCI_PRD_MASK); 1491195534Sscottl } 1492195534Sscottl slot->dma.nsegs = nsegs; 1493195534Sscottl bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map, 1494195534Sscottl ((slot->ccb->ccb_h.flags & CAM_DIR_IN) ? 1495195534Sscottl BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE)); 1496195534Sscottl ahci_execute_transaction(slot); 1497195534Sscottl} 1498195534Sscottl 1499195534Sscottl/* Must be called with channel locked. */ 1500195534Sscottlstatic void 1501195534Sscottlahci_execute_transaction(struct ahci_slot *slot) 1502195534Sscottl{ 1503271261Smav struct ahci_channel *ch = slot->ch; 1504195534Sscottl struct ahci_cmd_tab *ctp; 1505195534Sscottl struct ahci_cmd_list *clp; 1506195534Sscottl union ccb *ccb = slot->ccb; 1507195534Sscottl int port = ccb->ccb_h.target_id & 0x0f; 1508222304Smav int fis_size, i, softreset; 1509203123Smav uint8_t *fis = ch->dma.rfis + 0x40; 1510203123Smav uint8_t val; 1511195534Sscottl 1512195534Sscottl /* Get a piece of the workspace for this request */ 1513195534Sscottl ctp = (struct ahci_cmd_tab *) 1514195534Sscottl (ch->dma.work + AHCI_CT_OFFSET + (AHCI_CT_SIZE * slot->slot)); 1515195534Sscottl /* Setup the FIS for this request */ 1516271261Smav if (!(fis_size = ahci_setup_fis(ch, ctp, ccb, slot->slot))) { 1517195534Sscottl device_printf(ch->dev, "Setting up SATA FIS failed\n"); 1518195534Sscottl ahci_end_transaction(slot, AHCI_ERR_INVALID); 1519195534Sscottl return; 1520195534Sscottl } 1521195534Sscottl /* Setup the command list entry */ 1522195534Sscottl clp = (struct ahci_cmd_list *) 1523195534Sscottl (ch->dma.work + AHCI_CL_OFFSET + (AHCI_CL_SIZE * slot->slot)); 1524214988Smav clp->cmd_flags = htole16( 1525214988Smav (ccb->ccb_h.flags & CAM_DIR_OUT ? AHCI_CMD_WRITE : 0) | 1526214988Smav (ccb->ccb_h.func_code == XPT_SCSI_IO ? 1527214988Smav (AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH) : 0) | 1528214988Smav (fis_size / sizeof(u_int32_t)) | 1529214988Smav (port << 12)); 1530214988Smav clp->prd_length = htole16(slot->dma.nsegs); 1531195534Sscottl /* Special handling for Soft Reset command. */ 1532195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1533203123Smav (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL)) { 1534203123Smav if (ccb->ataio.cmd.control & ATA_A_RESET) { 1535222304Smav softreset = 1; 1536203123Smav /* Kick controller into sane state */ 1537271261Smav ahci_stop(ch); 1538271261Smav ahci_clo(ch); 1539271261Smav ahci_start(ch, 0); 1540203123Smav clp->cmd_flags |= AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY; 1541203123Smav } else { 1542222304Smav softreset = 2; 1543203123Smav /* Prepare FIS receive area for check. */ 1544203123Smav for (i = 0; i < 20; i++) 1545203123Smav fis[i] = 0xff; 1546203123Smav } 1547222304Smav } else 1548222304Smav softreset = 0; 1549195534Sscottl clp->bytecount = 0; 1550195534Sscottl clp->cmd_table_phys = htole64(ch->dma.work_bus + AHCI_CT_OFFSET + 1551195534Sscottl (AHCI_CT_SIZE * slot->slot)); 1552195534Sscottl bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 1553214988Smav BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1554195534Sscottl bus_dmamap_sync(ch->dma.rfis_tag, ch->dma.rfis_map, 1555195534Sscottl BUS_DMASYNC_PREREAD); 1556195534Sscottl /* Set ACTIVE bit for NCQ commands. */ 1557195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1558195534Sscottl (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 1559195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_SACT, 1 << slot->slot); 1560195534Sscottl } 1561203123Smav /* If FBS is enabled, set PMP port. */ 1562203123Smav if (ch->fbs_enabled) { 1563203123Smav ATA_OUTL(ch->r_mem, AHCI_P_FBS, AHCI_P_FBS_EN | 1564203123Smav (port << AHCI_P_FBS_DEV_SHIFT)); 1565203123Smav } 1566195534Sscottl /* Issue command to the controller. */ 1567195534Sscottl slot->state = AHCI_SLOT_RUNNING; 1568195534Sscottl ch->rslots |= (1 << slot->slot); 1569195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CI, (1 << slot->slot)); 1570195534Sscottl /* Device reset commands doesn't interrupt. Poll them. */ 1571195534Sscottl if (ccb->ccb_h.func_code == XPT_ATA_IO && 1572222304Smav (ccb->ataio.cmd.command == ATA_DEVICE_RESET || softreset)) { 1573220777Smav int count, timeout = ccb->ccb_h.timeout * 100; 1574195534Sscottl enum ahci_err_type et = AHCI_ERR_NONE; 1575195534Sscottl 1576195534Sscottl for (count = 0; count < timeout; count++) { 1577220777Smav DELAY(10); 1578195534Sscottl if (!(ATA_INL(ch->r_mem, AHCI_P_CI) & (1 << slot->slot))) 1579195534Sscottl break; 1580222304Smav if ((ATA_INL(ch->r_mem, AHCI_P_TFD) & ATA_S_ERROR) && 1581222304Smav softreset != 1) { 1582222285Smav#if 0 1583195534Sscottl device_printf(ch->dev, 1584195534Sscottl "Poll error on slot %d, TFD: %04x\n", 1585195534Sscottl slot->slot, ATA_INL(ch->r_mem, AHCI_P_TFD)); 1586222285Smav#endif 1587195534Sscottl et = AHCI_ERR_TFE; 1588195534Sscottl break; 1589195534Sscottl } 1590198851Smav /* Workaround for ATI SB600/SB700 chipsets. */ 1591198851Smav if (ccb->ccb_h.target_id == 15 && 1592271146Simp (ch->quirks & AHCI_Q_ATI_PMP_BUG) && 1593198851Smav (ATA_INL(ch->r_mem, AHCI_P_IS) & AHCI_P_IX_IPM)) { 1594198851Smav et = AHCI_ERR_TIMEOUT; 1595198851Smav break; 1596198851Smav } 1597195534Sscottl } 1598222304Smav 1599271163Smav /* 1600271163Smav * Marvell HBAs with non-RAID firmware do not wait for 1601271163Smav * readiness after soft reset, so we have to wait here. 1602271196Smav * Marvell RAIDs do not have this problem, but instead 1603271196Smav * sometimes forget to update FIS receive area, breaking 1604271196Smav * this wait. 1605271163Smav */ 1606271163Smav if ((ch->quirks & AHCI_Q_NOBSYRES) == 0 && 1607271163Smav (ch->quirks & AHCI_Q_ATI_PMP_BUG) == 0 && 1608271163Smav softreset == 2 && et == AHCI_ERR_NONE) { 1609222304Smav while ((val = fis[2]) & ATA_S_BUSY) { 1610222304Smav DELAY(10); 1611222304Smav if (count++ >= timeout) 1612222304Smav break; 1613222304Smav } 1614222304Smav } 1615222304Smav 1616195534Sscottl if (timeout && (count >= timeout)) { 1617271261Smav device_printf(ch->dev, "Poll timeout on slot %d port %d\n", 1618222304Smav slot->slot, port); 1619271261Smav device_printf(ch->dev, "is %08x cs %08x ss %08x " 1620224498Smav "rs %08x tfd %02x serr %08x cmd %08x\n", 1621203108Smav ATA_INL(ch->r_mem, AHCI_P_IS), 1622203108Smav ATA_INL(ch->r_mem, AHCI_P_CI), 1623203108Smav ATA_INL(ch->r_mem, AHCI_P_SACT), ch->rslots, 1624203108Smav ATA_INL(ch->r_mem, AHCI_P_TFD), 1625224498Smav ATA_INL(ch->r_mem, AHCI_P_SERR), 1626224498Smav ATA_INL(ch->r_mem, AHCI_P_CMD)); 1627195534Sscottl et = AHCI_ERR_TIMEOUT; 1628195534Sscottl } 1629222304Smav 1630203123Smav /* Kick controller into sane state and enable FBS. */ 1631222304Smav if (softreset == 2) 1632222285Smav ch->eslots |= (1 << slot->slot); 1633222285Smav ahci_end_transaction(slot, et); 1634195534Sscottl return; 1635195534Sscottl } 1636195534Sscottl /* Start command execution timeout */ 1637274819Ssmh callout_reset_sbt(&slot->timeout, SBT_1MS * ccb->ccb_h.timeout / 2, 1638274819Ssmh 0, (timeout_t*)ahci_timeout, slot, 0); 1639195534Sscottl return; 1640195534Sscottl} 1641195534Sscottl 1642203873Smav/* Must be called with channel locked. */ 1643203873Smavstatic void 1644271261Smavahci_process_timeout(struct ahci_channel *ch) 1645203873Smav{ 1646203873Smav int i; 1647203873Smav 1648203873Smav mtx_assert(&ch->mtx, MA_OWNED); 1649203873Smav /* Handle the rest of commands. */ 1650203873Smav for (i = 0; i < ch->numslots; i++) { 1651203873Smav /* Do we have a running request on slot? */ 1652203873Smav if (ch->slot[i].state < AHCI_SLOT_RUNNING) 1653203873Smav continue; 1654203873Smav ahci_end_transaction(&ch->slot[i], AHCI_ERR_TIMEOUT); 1655203873Smav } 1656203873Smav} 1657203873Smav 1658203873Smav/* Must be called with channel locked. */ 1659203873Smavstatic void 1660271261Smavahci_rearm_timeout(struct ahci_channel *ch) 1661203873Smav{ 1662203873Smav int i; 1663203873Smav 1664203873Smav mtx_assert(&ch->mtx, MA_OWNED); 1665203873Smav for (i = 0; i < ch->numslots; i++) { 1666203873Smav struct ahci_slot *slot = &ch->slot[i]; 1667203873Smav 1668203873Smav /* Do we have a running request on slot? */ 1669203873Smav if (slot->state < AHCI_SLOT_RUNNING) 1670203873Smav continue; 1671203873Smav if ((ch->toslots & (1 << i)) == 0) 1672203873Smav continue; 1673274819Ssmh callout_reset_sbt(&slot->timeout, 1674274819Ssmh SBT_1MS * slot->ccb->ccb_h.timeout / 2, 0, 1675274819Ssmh (timeout_t*)ahci_timeout, slot, 0); 1676203873Smav } 1677203873Smav} 1678203873Smav 1679195534Sscottl/* Locked by callout mechanism. */ 1680195534Sscottlstatic void 1681195534Sscottlahci_timeout(struct ahci_slot *slot) 1682195534Sscottl{ 1683271261Smav struct ahci_channel *ch = slot->ch; 1684271261Smav device_t dev = ch->dev; 1685198319Smav uint32_t sstatus; 1686198319Smav int ccs; 1687195534Sscottl int i; 1688195534Sscottl 1689196656Smav /* Check for stale timeout. */ 1690198319Smav if (slot->state < AHCI_SLOT_RUNNING) 1691196656Smav return; 1692196656Smav 1693198319Smav /* Check if slot was not being executed last time we checked. */ 1694198319Smav if (slot->state < AHCI_SLOT_EXECUTING) { 1695198319Smav /* Check if slot started executing. */ 1696198319Smav sstatus = ATA_INL(ch->r_mem, AHCI_P_SACT); 1697198319Smav ccs = (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CCS_MASK) 1698198319Smav >> AHCI_P_CMD_CCS_SHIFT; 1699203123Smav if ((sstatus & (1 << slot->slot)) != 0 || ccs == slot->slot || 1700224498Smav ch->fbs_enabled || ch->wrongccs) 1701198319Smav slot->state = AHCI_SLOT_EXECUTING; 1702224498Smav else if ((ch->rslots & (1 << ccs)) == 0) { 1703224498Smav ch->wrongccs = 1; 1704224498Smav slot->state = AHCI_SLOT_EXECUTING; 1705224498Smav } 1706198319Smav 1707274819Ssmh callout_reset_sbt(&slot->timeout, 1708274819Ssmh SBT_1MS * slot->ccb->ccb_h.timeout / 2, 0, 1709274819Ssmh (timeout_t*)ahci_timeout, slot, 0); 1710198319Smav return; 1711198319Smav } 1712198319Smav 1713222304Smav device_printf(dev, "Timeout on slot %d port %d\n", 1714222304Smav slot->slot, slot->ccb->ccb_h.target_id & 0x0f); 1715224498Smav device_printf(dev, "is %08x cs %08x ss %08x rs %08x tfd %02x " 1716224498Smav "serr %08x cmd %08x\n", 1717198319Smav ATA_INL(ch->r_mem, AHCI_P_IS), ATA_INL(ch->r_mem, AHCI_P_CI), 1718198319Smav ATA_INL(ch->r_mem, AHCI_P_SACT), ch->rslots, 1719224498Smav ATA_INL(ch->r_mem, AHCI_P_TFD), ATA_INL(ch->r_mem, AHCI_P_SERR), 1720224498Smav ATA_INL(ch->r_mem, AHCI_P_CMD)); 1721195534Sscottl 1722197838Smav /* Handle frozen command. */ 1723195534Sscottl if (ch->frozen) { 1724195534Sscottl union ccb *fccb = ch->frozen; 1725195534Sscottl ch->frozen = NULL; 1726195534Sscottl fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ; 1727198319Smav if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) { 1728198319Smav xpt_freeze_devq(fccb->ccb_h.path, 1); 1729198319Smav fccb->ccb_h.status |= CAM_DEV_QFRZN; 1730198319Smav } 1731256843Smav ahci_done(ch, fccb); 1732195534Sscottl } 1733224498Smav if (!ch->fbs_enabled && !ch->wrongccs) { 1734203873Smav /* Without FBS we know real timeout source. */ 1735203873Smav ch->fatalerr = 1; 1736203873Smav /* Handle command with timeout. */ 1737203873Smav ahci_end_transaction(&ch->slot[slot->slot], AHCI_ERR_TIMEOUT); 1738203873Smav /* Handle the rest of commands. */ 1739203873Smav for (i = 0; i < ch->numslots; i++) { 1740203873Smav /* Do we have a running request on slot? */ 1741203873Smav if (ch->slot[i].state < AHCI_SLOT_RUNNING) 1742203873Smav continue; 1743203873Smav ahci_end_transaction(&ch->slot[i], AHCI_ERR_INNOCENT); 1744203873Smav } 1745203873Smav } else { 1746203873Smav /* With FBS we wait for other commands timeout and pray. */ 1747203873Smav if (ch->toslots == 0) 1748203873Smav xpt_freeze_simq(ch->sim, 1); 1749203873Smav ch->toslots |= (1 << slot->slot); 1750203873Smav if ((ch->rslots & ~ch->toslots) == 0) 1751271261Smav ahci_process_timeout(ch); 1752203873Smav else 1753203873Smav device_printf(dev, " ... waiting for slots %08x\n", 1754203873Smav ch->rslots & ~ch->toslots); 1755195534Sscottl } 1756195534Sscottl} 1757195534Sscottl 1758195534Sscottl/* Must be called with channel locked. */ 1759195534Sscottlstatic void 1760195534Sscottlahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et) 1761195534Sscottl{ 1762271261Smav struct ahci_channel *ch = slot->ch; 1763195534Sscottl union ccb *ccb = slot->ccb; 1764214988Smav struct ahci_cmd_list *clp; 1765212732Smav int lastto; 1766222304Smav uint32_t sig; 1767195534Sscottl 1768195534Sscottl bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 1769214988Smav BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1770214988Smav clp = (struct ahci_cmd_list *) 1771214988Smav (ch->dma.work + AHCI_CL_OFFSET + (AHCI_CL_SIZE * slot->slot)); 1772195534Sscottl /* Read result registers to the result struct 1773195534Sscottl * May be incorrect if several commands finished same time, 1774195534Sscottl * so read only when sure or have to. 1775195534Sscottl */ 1776195534Sscottl if (ccb->ccb_h.func_code == XPT_ATA_IO) { 1777195534Sscottl struct ata_res *res = &ccb->ataio.res; 1778195534Sscottl 1779195534Sscottl if ((et == AHCI_ERR_TFE) || 1780195534Sscottl (ccb->ataio.cmd.flags & CAM_ATAIO_NEEDRESULT)) { 1781195534Sscottl u_int8_t *fis = ch->dma.rfis + 0x40; 1782195534Sscottl 1783195534Sscottl bus_dmamap_sync(ch->dma.rfis_tag, ch->dma.rfis_map, 1784195534Sscottl BUS_DMASYNC_POSTREAD); 1785203123Smav if (ch->fbs_enabled) { 1786203123Smav fis += ccb->ccb_h.target_id * 256; 1787203123Smav res->status = fis[2]; 1788203123Smav res->error = fis[3]; 1789203123Smav } else { 1790203123Smav uint16_t tfd = ATA_INL(ch->r_mem, AHCI_P_TFD); 1791203123Smav 1792203123Smav res->status = tfd; 1793203123Smav res->error = tfd >> 8; 1794203123Smav } 1795195534Sscottl res->lba_low = fis[4]; 1796195534Sscottl res->lba_mid = fis[5]; 1797195534Sscottl res->lba_high = fis[6]; 1798195534Sscottl res->device = fis[7]; 1799195534Sscottl res->lba_low_exp = fis[8]; 1800195534Sscottl res->lba_mid_exp = fis[9]; 1801195534Sscottl res->lba_high_exp = fis[10]; 1802195534Sscottl res->sector_count = fis[12]; 1803195534Sscottl res->sector_count_exp = fis[13]; 1804222304Smav 1805222304Smav /* 1806222304Smav * Some weird controllers do not return signature in 1807222304Smav * FIS receive area. Read it from PxSIG register. 1808222304Smav */ 1809222304Smav if ((ch->quirks & AHCI_Q_ALTSIG) && 1810222304Smav (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) && 1811222304Smav (ccb->ataio.cmd.control & ATA_A_RESET) == 0) { 1812222304Smav sig = ATA_INL(ch->r_mem, AHCI_P_SIG); 1813222304Smav res->lba_high = sig >> 24; 1814222304Smav res->lba_mid = sig >> 16; 1815222304Smav res->lba_low = sig >> 8; 1816222304Smav res->sector_count = sig; 1817222304Smav } 1818195534Sscottl } else 1819195534Sscottl bzero(res, sizeof(*res)); 1820214988Smav if ((ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) == 0 && 1821218596Smav (ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE && 1822218596Smav (ch->quirks & AHCI_Q_NOCOUNT) == 0) { 1823214988Smav ccb->ataio.resid = 1824214988Smav ccb->ataio.dxfer_len - le32toh(clp->bytecount); 1825214988Smav } 1826214988Smav } else { 1827218596Smav if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE && 1828218596Smav (ch->quirks & AHCI_Q_NOCOUNT) == 0) { 1829214988Smav ccb->csio.resid = 1830214988Smav ccb->csio.dxfer_len - le32toh(clp->bytecount); 1831214988Smav } 1832195534Sscottl } 1833195534Sscottl if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) { 1834195534Sscottl bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map, 1835195534Sscottl (ccb->ccb_h.flags & CAM_DIR_IN) ? 1836195534Sscottl BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 1837195534Sscottl bus_dmamap_unload(ch->dma.data_tag, slot->dma.data_map); 1838195534Sscottl } 1839203123Smav if (et != AHCI_ERR_NONE) 1840203123Smav ch->eslots |= (1 << slot->slot); 1841198319Smav /* In case of error, freeze device for proper recovery. */ 1842220565Smav if ((et != AHCI_ERR_NONE) && (!ch->recoverycmd) && 1843198319Smav !(ccb->ccb_h.status & CAM_DEV_QFRZN)) { 1844198319Smav xpt_freeze_devq(ccb->ccb_h.path, 1); 1845198319Smav ccb->ccb_h.status |= CAM_DEV_QFRZN; 1846198319Smav } 1847195534Sscottl /* Set proper result status. */ 1848195534Sscottl ccb->ccb_h.status &= ~CAM_STATUS_MASK; 1849195534Sscottl switch (et) { 1850195534Sscottl case AHCI_ERR_NONE: 1851195534Sscottl ccb->ccb_h.status |= CAM_REQ_CMP; 1852195534Sscottl if (ccb->ccb_h.func_code == XPT_SCSI_IO) 1853195534Sscottl ccb->csio.scsi_status = SCSI_STATUS_OK; 1854195534Sscottl break; 1855195534Sscottl case AHCI_ERR_INVALID: 1856198851Smav ch->fatalerr = 1; 1857195534Sscottl ccb->ccb_h.status |= CAM_REQ_INVALID; 1858195534Sscottl break; 1859195534Sscottl case AHCI_ERR_INNOCENT: 1860195534Sscottl ccb->ccb_h.status |= CAM_REQUEUE_REQ; 1861195534Sscottl break; 1862195534Sscottl case AHCI_ERR_TFE: 1863198319Smav case AHCI_ERR_NCQ: 1864195534Sscottl if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 1865195534Sscottl ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR; 1866195534Sscottl ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND; 1867195534Sscottl } else { 1868195534Sscottl ccb->ccb_h.status |= CAM_ATA_STATUS_ERROR; 1869195534Sscottl } 1870195534Sscottl break; 1871195534Sscottl case AHCI_ERR_SATA: 1872198851Smav ch->fatalerr = 1; 1873220565Smav if (!ch->recoverycmd) { 1874198319Smav xpt_freeze_simq(ch->sim, 1); 1875198319Smav ccb->ccb_h.status &= ~CAM_STATUS_MASK; 1876198319Smav ccb->ccb_h.status |= CAM_RELEASE_SIMQ; 1877198319Smav } 1878198319Smav ccb->ccb_h.status |= CAM_UNCOR_PARITY; 1879195534Sscottl break; 1880195534Sscottl case AHCI_ERR_TIMEOUT: 1881220565Smav if (!ch->recoverycmd) { 1882198319Smav xpt_freeze_simq(ch->sim, 1); 1883198319Smav ccb->ccb_h.status &= ~CAM_STATUS_MASK; 1884198319Smav ccb->ccb_h.status |= CAM_RELEASE_SIMQ; 1885198319Smav } 1886195534Sscottl ccb->ccb_h.status |= CAM_CMD_TIMEOUT; 1887195534Sscottl break; 1888195534Sscottl default: 1889198851Smav ch->fatalerr = 1; 1890195534Sscottl ccb->ccb_h.status |= CAM_REQ_CMP_ERR; 1891195534Sscottl } 1892195534Sscottl /* Free slot. */ 1893199747Smav ch->oslots &= ~(1 << slot->slot); 1894195534Sscottl ch->rslots &= ~(1 << slot->slot); 1895195534Sscottl ch->aslots &= ~(1 << slot->slot); 1896195534Sscottl slot->state = AHCI_SLOT_EMPTY; 1897195534Sscottl slot->ccb = NULL; 1898195534Sscottl /* Update channel stats. */ 1899195534Sscottl ch->numrslots--; 1900203123Smav ch->numrslotspd[ccb->ccb_h.target_id]--; 1901195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1902195534Sscottl (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 1903195534Sscottl ch->numtslots--; 1904203123Smav ch->numtslotspd[ccb->ccb_h.target_id]--; 1905195534Sscottl } 1906212732Smav /* Cancel timeout state if request completed normally. */ 1907212732Smav if (et != AHCI_ERR_TIMEOUT) { 1908212732Smav lastto = (ch->toslots == (1 << slot->slot)); 1909212732Smav ch->toslots &= ~(1 << slot->slot); 1910212732Smav if (lastto) 1911212732Smav xpt_release_simq(ch->sim, TRUE); 1912212732Smav } 1913195534Sscottl /* If it was first request of reset sequence and there is no error, 1914195534Sscottl * proceed to second request. */ 1915195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1916195534Sscottl (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) && 1917195534Sscottl (ccb->ataio.cmd.control & ATA_A_RESET) && 1918195534Sscottl et == AHCI_ERR_NONE) { 1919195534Sscottl ccb->ataio.cmd.control &= ~ATA_A_RESET; 1920271261Smav ahci_begin_transaction(ch, ccb); 1921195534Sscottl return; 1922195534Sscottl } 1923198851Smav /* If it was our READ LOG command - process it. */ 1924220565Smav if (ccb->ccb_h.recovery_type == RECOVERY_READ_LOG) { 1925271261Smav ahci_process_read_log(ch, ccb); 1926220565Smav /* If it was our REQUEST SENSE command - process it. */ 1927220565Smav } else if (ccb->ccb_h.recovery_type == RECOVERY_REQUEST_SENSE) { 1928271261Smav ahci_process_request_sense(ch, ccb); 1929220565Smav /* If it was NCQ or ATAPI command error, put result on hold. */ 1930220565Smav } else if (et == AHCI_ERR_NCQ || 1931220565Smav ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_SCSI_STATUS_ERROR && 1932220565Smav (ccb->ccb_h.flags & CAM_DIS_AUTOSENSE) == 0)) { 1933195534Sscottl ch->hold[slot->slot] = ccb; 1934203123Smav ch->numhslots++; 1935198851Smav } else 1936256843Smav ahci_done(ch, ccb); 1937198851Smav /* If we have no other active commands, ... */ 1938198851Smav if (ch->rslots == 0) { 1939198851Smav /* if there was fatal error - reset port. */ 1940203873Smav if (ch->toslots != 0 || ch->fatalerr) { 1941271261Smav ahci_reset(ch); 1942203123Smav } else { 1943203123Smav /* if we have slots in error, we can reinit port. */ 1944203123Smav if (ch->eslots != 0) { 1945271261Smav ahci_stop(ch); 1946271261Smav ahci_clo(ch); 1947271261Smav ahci_start(ch, 1); 1948203123Smav } 1949203123Smav /* if there commands on hold, we can do READ LOG. */ 1950220565Smav if (!ch->recoverycmd && ch->numhslots) 1951271261Smav ahci_issue_recovery(ch); 1952198851Smav } 1953203873Smav /* If all the rest of commands are in timeout - give them chance. */ 1954203873Smav } else if ((ch->rslots & ~ch->toslots) == 0 && 1955203873Smav et != AHCI_ERR_TIMEOUT) 1956271261Smav ahci_rearm_timeout(ch); 1957222285Smav /* Unfreeze frozen command. */ 1958271261Smav if (ch->frozen && !ahci_check_collision(ch, ch->frozen)) { 1959222285Smav union ccb *fccb = ch->frozen; 1960222285Smav ch->frozen = NULL; 1961271261Smav ahci_begin_transaction(ch, fccb); 1962222285Smav xpt_release_simq(ch->sim, TRUE); 1963222285Smav } 1964196656Smav /* Start PM timer. */ 1965207499Smav if (ch->numrslots == 0 && ch->pm_level > 3 && 1966207499Smav (ch->curr[ch->pm_present ? 15 : 0].caps & CTS_SATA_CAPS_D_PMREQ)) { 1967196656Smav callout_schedule(&ch->pm_timer, 1968196656Smav (ch->pm_level == 4) ? hz / 1000 : hz / 8); 1969196656Smav } 1970195534Sscottl} 1971195534Sscottl 1972195534Sscottlstatic void 1973271261Smavahci_issue_recovery(struct ahci_channel *ch) 1974195534Sscottl{ 1975195534Sscottl union ccb *ccb; 1976195534Sscottl struct ccb_ataio *ataio; 1977220565Smav struct ccb_scsiio *csio; 1978195534Sscottl int i; 1979195534Sscottl 1980220830Smav /* Find some held command. */ 1981195534Sscottl for (i = 0; i < ch->numslots; i++) { 1982195534Sscottl if (ch->hold[i]) 1983195534Sscottl break; 1984195534Sscottl } 1985195534Sscottl ccb = xpt_alloc_ccb_nowait(); 1986195534Sscottl if (ccb == NULL) { 1987271261Smav device_printf(ch->dev, "Unable to allocate recovery command\n"); 1988220822Smavcompleteall: 1989220830Smav /* We can't do anything -- complete held commands. */ 1990220822Smav for (i = 0; i < ch->numslots; i++) { 1991220822Smav if (ch->hold[i] == NULL) 1992220822Smav continue; 1993220822Smav ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK; 1994220822Smav ch->hold[i]->ccb_h.status |= CAM_RESRC_UNAVAIL; 1995256843Smav ahci_done(ch, ch->hold[i]); 1996220822Smav ch->hold[i] = NULL; 1997220822Smav ch->numhslots--; 1998220822Smav } 1999271261Smav ahci_reset(ch); 2000220822Smav return; 2001195534Sscottl } 2002195534Sscottl ccb->ccb_h = ch->hold[i]->ccb_h; /* Reuse old header. */ 2003220565Smav if (ccb->ccb_h.func_code == XPT_ATA_IO) { 2004220565Smav /* READ LOG */ 2005220565Smav ccb->ccb_h.recovery_type = RECOVERY_READ_LOG; 2006220565Smav ccb->ccb_h.func_code = XPT_ATA_IO; 2007220565Smav ccb->ccb_h.flags = CAM_DIR_IN; 2008220565Smav ccb->ccb_h.timeout = 1000; /* 1s should be enough. */ 2009220565Smav ataio = &ccb->ataio; 2010220565Smav ataio->data_ptr = malloc(512, M_AHCI, M_NOWAIT); 2011220565Smav if (ataio->data_ptr == NULL) { 2012220565Smav xpt_free_ccb(ccb); 2013271261Smav device_printf(ch->dev, 2014220830Smav "Unable to allocate memory for READ LOG command\n"); 2015220822Smav goto completeall; 2016220565Smav } 2017220565Smav ataio->dxfer_len = 512; 2018220565Smav bzero(&ataio->cmd, sizeof(ataio->cmd)); 2019220565Smav ataio->cmd.flags = CAM_ATAIO_48BIT; 2020220565Smav ataio->cmd.command = 0x2F; /* READ LOG EXT */ 2021220565Smav ataio->cmd.sector_count = 1; 2022220565Smav ataio->cmd.sector_count_exp = 0; 2023220565Smav ataio->cmd.lba_low = 0x10; 2024220565Smav ataio->cmd.lba_mid = 0; 2025220565Smav ataio->cmd.lba_mid_exp = 0; 2026220565Smav } else { 2027220565Smav /* REQUEST SENSE */ 2028220565Smav ccb->ccb_h.recovery_type = RECOVERY_REQUEST_SENSE; 2029220565Smav ccb->ccb_h.recovery_slot = i; 2030220565Smav ccb->ccb_h.func_code = XPT_SCSI_IO; 2031220565Smav ccb->ccb_h.flags = CAM_DIR_IN; 2032220565Smav ccb->ccb_h.status = 0; 2033220565Smav ccb->ccb_h.timeout = 1000; /* 1s should be enough. */ 2034220565Smav csio = &ccb->csio; 2035220565Smav csio->data_ptr = (void *)&ch->hold[i]->csio.sense_data; 2036220565Smav csio->dxfer_len = ch->hold[i]->csio.sense_len; 2037220565Smav csio->cdb_len = 6; 2038220565Smav bzero(&csio->cdb_io, sizeof(csio->cdb_io)); 2039220565Smav csio->cdb_io.cdb_bytes[0] = 0x03; 2040220565Smav csio->cdb_io.cdb_bytes[4] = csio->dxfer_len; 2041195534Sscottl } 2042220565Smav /* Freeze SIM while doing recovery. */ 2043220822Smav ch->recoverycmd = 1; 2044198319Smav xpt_freeze_simq(ch->sim, 1); 2045271261Smav ahci_begin_transaction(ch, ccb); 2046195534Sscottl} 2047195534Sscottl 2048195534Sscottlstatic void 2049271261Smavahci_process_read_log(struct ahci_channel *ch, union ccb *ccb) 2050195534Sscottl{ 2051195534Sscottl uint8_t *data; 2052195534Sscottl struct ata_res *res; 2053195534Sscottl int i; 2054195534Sscottl 2055220565Smav ch->recoverycmd = 0; 2056195534Sscottl 2057195534Sscottl data = ccb->ataio.data_ptr; 2058195534Sscottl if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP && 2059195534Sscottl (data[0] & 0x80) == 0) { 2060195534Sscottl for (i = 0; i < ch->numslots; i++) { 2061195534Sscottl if (!ch->hold[i]) 2062195534Sscottl continue; 2063220565Smav if (ch->hold[i]->ccb_h.func_code != XPT_ATA_IO) 2064220565Smav continue; 2065195534Sscottl if ((data[0] & 0x1F) == i) { 2066195534Sscottl res = &ch->hold[i]->ataio.res; 2067195534Sscottl res->status = data[2]; 2068195534Sscottl res->error = data[3]; 2069195534Sscottl res->lba_low = data[4]; 2070195534Sscottl res->lba_mid = data[5]; 2071195534Sscottl res->lba_high = data[6]; 2072195534Sscottl res->device = data[7]; 2073195534Sscottl res->lba_low_exp = data[8]; 2074195534Sscottl res->lba_mid_exp = data[9]; 2075195534Sscottl res->lba_high_exp = data[10]; 2076195534Sscottl res->sector_count = data[12]; 2077195534Sscottl res->sector_count_exp = data[13]; 2078195534Sscottl } else { 2079195534Sscottl ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK; 2080195534Sscottl ch->hold[i]->ccb_h.status |= CAM_REQUEUE_REQ; 2081195534Sscottl } 2082256843Smav ahci_done(ch, ch->hold[i]); 2083195534Sscottl ch->hold[i] = NULL; 2084203123Smav ch->numhslots--; 2085195534Sscottl } 2086195534Sscottl } else { 2087195534Sscottl if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) 2088271261Smav device_printf(ch->dev, "Error while READ LOG EXT\n"); 2089195534Sscottl else if ((data[0] & 0x80) == 0) { 2090271261Smav device_printf(ch->dev, "Non-queued command error in READ LOG EXT\n"); 2091195534Sscottl } 2092195534Sscottl for (i = 0; i < ch->numslots; i++) { 2093195534Sscottl if (!ch->hold[i]) 2094195534Sscottl continue; 2095220565Smav if (ch->hold[i]->ccb_h.func_code != XPT_ATA_IO) 2096220565Smav continue; 2097256843Smav ahci_done(ch, ch->hold[i]); 2098195534Sscottl ch->hold[i] = NULL; 2099203123Smav ch->numhslots--; 2100195534Sscottl } 2101195534Sscottl } 2102195534Sscottl free(ccb->ataio.data_ptr, M_AHCI); 2103195534Sscottl xpt_free_ccb(ccb); 2104198319Smav xpt_release_simq(ch->sim, TRUE); 2105195534Sscottl} 2106195534Sscottl 2107195534Sscottlstatic void 2108271261Smavahci_process_request_sense(struct ahci_channel *ch, union ccb *ccb) 2109220565Smav{ 2110220565Smav int i; 2111220565Smav 2112220565Smav ch->recoverycmd = 0; 2113220565Smav 2114220565Smav i = ccb->ccb_h.recovery_slot; 2115220565Smav if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) { 2116220565Smav ch->hold[i]->ccb_h.status |= CAM_AUTOSNS_VALID; 2117220565Smav } else { 2118220565Smav ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK; 2119220565Smav ch->hold[i]->ccb_h.status |= CAM_AUTOSENSE_FAIL; 2120220565Smav } 2121256843Smav ahci_done(ch, ch->hold[i]); 2122220565Smav ch->hold[i] = NULL; 2123220565Smav ch->numhslots--; 2124220565Smav xpt_free_ccb(ccb); 2125220565Smav xpt_release_simq(ch->sim, TRUE); 2126220565Smav} 2127220565Smav 2128220565Smavstatic void 2129271261Smavahci_start(struct ahci_channel *ch, int fbs) 2130195534Sscottl{ 2131195534Sscottl u_int32_t cmd; 2132195534Sscottl 2133285090Sloos /* Run the channel start callback, if any. */ 2134285090Sloos if (ch->start) 2135285090Sloos ch->start(ch); 2136285090Sloos 2137195534Sscottl /* Clear SATA error register */ 2138195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_SERR, 0xFFFFFFFF); 2139195534Sscottl /* Clear any interrupts pending on this channel */ 2140195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_IS, 0xFFFFFFFF); 2141203123Smav /* Configure FIS-based switching if supported. */ 2142203123Smav if (ch->chcaps & AHCI_P_CMD_FBSCP) { 2143203123Smav ch->fbs_enabled = (fbs && ch->pm_present) ? 1 : 0; 2144203123Smav ATA_OUTL(ch->r_mem, AHCI_P_FBS, 2145203123Smav ch->fbs_enabled ? AHCI_P_FBS_EN : 0); 2146203123Smav } 2147195534Sscottl /* Start operations on this channel */ 2148195534Sscottl cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 2149207430Smav cmd &= ~AHCI_P_CMD_PMA; 2150195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd | AHCI_P_CMD_ST | 2151195534Sscottl (ch->pm_present ? AHCI_P_CMD_PMA : 0)); 2152195534Sscottl} 2153195534Sscottl 2154195534Sscottlstatic void 2155271261Smavahci_stop(struct ahci_channel *ch) 2156195534Sscottl{ 2157195534Sscottl u_int32_t cmd; 2158195534Sscottl int timeout; 2159195534Sscottl 2160195534Sscottl /* Kill all activity on this channel */ 2161195534Sscottl cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 2162195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd & ~AHCI_P_CMD_ST); 2163195534Sscottl /* Wait for activity stop. */ 2164195534Sscottl timeout = 0; 2165195534Sscottl do { 2166220777Smav DELAY(10); 2167220777Smav if (timeout++ > 50000) { 2168271261Smav device_printf(ch->dev, "stopping AHCI engine failed\n"); 2169195534Sscottl break; 2170195534Sscottl } 2171195534Sscottl } while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CR); 2172203123Smav ch->eslots = 0; 2173195534Sscottl} 2174195534Sscottl 2175195534Sscottlstatic void 2176271261Smavahci_clo(struct ahci_channel *ch) 2177195534Sscottl{ 2178195534Sscottl u_int32_t cmd; 2179195534Sscottl int timeout; 2180195534Sscottl 2181195534Sscottl /* Issue Command List Override if supported */ 2182195534Sscottl if (ch->caps & AHCI_CAP_SCLO) { 2183195534Sscottl cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 2184195534Sscottl cmd |= AHCI_P_CMD_CLO; 2185195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd); 2186195534Sscottl timeout = 0; 2187195534Sscottl do { 2188220777Smav DELAY(10); 2189220777Smav if (timeout++ > 50000) { 2190271261Smav device_printf(ch->dev, "executing CLO failed\n"); 2191195534Sscottl break; 2192195534Sscottl } 2193195534Sscottl } while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CLO); 2194195534Sscottl } 2195195534Sscottl} 2196195534Sscottl 2197195534Sscottlstatic void 2198271261Smavahci_stop_fr(struct ahci_channel *ch) 2199195534Sscottl{ 2200195534Sscottl u_int32_t cmd; 2201195534Sscottl int timeout; 2202195534Sscottl 2203195534Sscottl /* Kill all FIS reception on this channel */ 2204195534Sscottl cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 2205195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd & ~AHCI_P_CMD_FRE); 2206195534Sscottl /* Wait for FIS reception stop. */ 2207195534Sscottl timeout = 0; 2208195534Sscottl do { 2209220777Smav DELAY(10); 2210220777Smav if (timeout++ > 50000) { 2211271261Smav device_printf(ch->dev, "stopping AHCI FR engine failed\n"); 2212195534Sscottl break; 2213195534Sscottl } 2214195534Sscottl } while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_FR); 2215195534Sscottl} 2216195534Sscottl 2217195534Sscottlstatic void 2218271261Smavahci_start_fr(struct ahci_channel *ch) 2219195534Sscottl{ 2220195534Sscottl u_int32_t cmd; 2221195534Sscottl 2222195534Sscottl /* Start FIS reception on this channel */ 2223195534Sscottl cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 2224195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd | AHCI_P_CMD_FRE); 2225195534Sscottl} 2226195534Sscottl 2227195534Sscottlstatic int 2228271261Smavahci_wait_ready(struct ahci_channel *ch, int t, int t0) 2229195534Sscottl{ 2230195534Sscottl int timeout = 0; 2231195534Sscottl uint32_t val; 2232195534Sscottl 2233195534Sscottl while ((val = ATA_INL(ch->r_mem, AHCI_P_TFD)) & 2234195534Sscottl (ATA_S_BUSY | ATA_S_DRQ)) { 2235220576Smav if (timeout > t) { 2236220576Smav if (t != 0) { 2237271261Smav device_printf(ch->dev, 2238220576Smav "AHCI reset: device not ready after %dms " 2239220576Smav "(tfd = %08x)\n", 2240220576Smav MAX(t, 0) + t0, val); 2241220576Smav } 2242195534Sscottl return (EBUSY); 2243195534Sscottl } 2244220576Smav DELAY(1000); 2245220576Smav timeout++; 2246220576Smav } 2247195534Sscottl if (bootverbose) 2248271261Smav device_printf(ch->dev, "AHCI reset: device ready after %dms\n", 2249220576Smav timeout + t0); 2250195534Sscottl return (0); 2251195534Sscottl} 2252195534Sscottl 2253195534Sscottlstatic void 2254220576Smavahci_reset_to(void *arg) 2255220576Smav{ 2256271261Smav struct ahci_channel *ch = arg; 2257220576Smav 2258220576Smav if (ch->resetting == 0) 2259220576Smav return; 2260220576Smav ch->resetting--; 2261271261Smav if (ahci_wait_ready(ch, ch->resetting == 0 ? -1 : 0, 2262220576Smav (310 - ch->resetting) * 100) == 0) { 2263220576Smav ch->resetting = 0; 2264271261Smav ahci_start(ch, 1); 2265220576Smav xpt_release_simq(ch->sim, TRUE); 2266220576Smav return; 2267220576Smav } 2268220576Smav if (ch->resetting == 0) { 2269271261Smav ahci_clo(ch); 2270271261Smav ahci_start(ch, 1); 2271220576Smav xpt_release_simq(ch->sim, TRUE); 2272220576Smav return; 2273220576Smav } 2274220576Smav callout_schedule(&ch->reset_timer, hz / 10); 2275220576Smav} 2276220576Smav 2277220576Smavstatic void 2278271261Smavahci_reset(struct ahci_channel *ch) 2279195534Sscottl{ 2280271261Smav struct ahci_controller *ctlr = device_get_softc(device_get_parent(ch->dev)); 2281195534Sscottl int i; 2282195534Sscottl 2283203108Smav xpt_freeze_simq(ch->sim, 1); 2284195534Sscottl if (bootverbose) 2285271261Smav device_printf(ch->dev, "AHCI reset...\n"); 2286220576Smav /* Forget about previous reset. */ 2287220576Smav if (ch->resetting) { 2288220576Smav ch->resetting = 0; 2289220576Smav callout_stop(&ch->reset_timer); 2290220576Smav xpt_release_simq(ch->sim, TRUE); 2291220576Smav } 2292195534Sscottl /* Requeue freezed command. */ 2293195534Sscottl if (ch->frozen) { 2294195534Sscottl union ccb *fccb = ch->frozen; 2295195534Sscottl ch->frozen = NULL; 2296195534Sscottl fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ; 2297198319Smav if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) { 2298198319Smav xpt_freeze_devq(fccb->ccb_h.path, 1); 2299198319Smav fccb->ccb_h.status |= CAM_DEV_QFRZN; 2300198319Smav } 2301256843Smav ahci_done(ch, fccb); 2302195534Sscottl } 2303195534Sscottl /* Kill the engine and requeue all running commands. */ 2304271261Smav ahci_stop(ch); 2305195534Sscottl for (i = 0; i < ch->numslots; i++) { 2306195534Sscottl /* Do we have a running request on slot? */ 2307195534Sscottl if (ch->slot[i].state < AHCI_SLOT_RUNNING) 2308195534Sscottl continue; 2309195534Sscottl /* XXX; Commands in loading state. */ 2310195534Sscottl ahci_end_transaction(&ch->slot[i], AHCI_ERR_INNOCENT); 2311195534Sscottl } 2312198851Smav for (i = 0; i < ch->numslots; i++) { 2313198851Smav if (!ch->hold[i]) 2314198851Smav continue; 2315256843Smav ahci_done(ch, ch->hold[i]); 2316198851Smav ch->hold[i] = NULL; 2317203123Smav ch->numhslots--; 2318198851Smav } 2319203873Smav if (ch->toslots != 0) 2320203873Smav xpt_release_simq(ch->sim, TRUE); 2321203123Smav ch->eslots = 0; 2322203873Smav ch->toslots = 0; 2323224498Smav ch->wrongccs = 0; 2324198851Smav ch->fatalerr = 0; 2325198319Smav /* Tell the XPT about the event */ 2326198319Smav xpt_async(AC_BUS_RESET, ch->path, NULL); 2327195534Sscottl /* Disable port interrupts */ 2328195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_IE, 0); 2329195534Sscottl /* Reset and reconnect PHY, */ 2330271261Smav if (!ahci_sata_phy_reset(ch)) { 2331195534Sscottl if (bootverbose) 2332271261Smav device_printf(ch->dev, 2333220576Smav "AHCI reset: device not found\n"); 2334195534Sscottl ch->devices = 0; 2335195534Sscottl /* Enable wanted port interrupts */ 2336195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_IE, 2337220657Smav (((ch->pm_level != 0) ? AHCI_P_IX_CPD | AHCI_P_IX_MP : 0) | 2338220657Smav AHCI_P_IX_PRC | AHCI_P_IX_PC)); 2339203108Smav xpt_release_simq(ch->sim, TRUE); 2340195534Sscottl return; 2341195534Sscottl } 2342220576Smav if (bootverbose) 2343271261Smav device_printf(ch->dev, "AHCI reset: device found\n"); 2344195534Sscottl /* Wait for clearing busy status. */ 2345271261Smav if (ahci_wait_ready(ch, dumping ? 31000 : 0, 0)) { 2346220576Smav if (dumping) 2347271261Smav ahci_clo(ch); 2348220576Smav else 2349220576Smav ch->resetting = 310; 2350220576Smav } 2351195534Sscottl ch->devices = 1; 2352195534Sscottl /* Enable wanted port interrupts */ 2353195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_IE, 2354220657Smav (((ch->pm_level != 0) ? AHCI_P_IX_CPD | AHCI_P_IX_MP : 0) | 2355220657Smav AHCI_P_IX_TFE | AHCI_P_IX_HBF | 2356195534Sscottl AHCI_P_IX_HBD | AHCI_P_IX_IF | AHCI_P_IX_OF | 2357220657Smav ((ch->pm_level == 0) ? AHCI_P_IX_PRC : 0) | AHCI_P_IX_PC | 2358196656Smav AHCI_P_IX_DP | AHCI_P_IX_UF | (ctlr->ccc ? 0 : AHCI_P_IX_SDB) | 2359196656Smav AHCI_P_IX_DS | AHCI_P_IX_PS | (ctlr->ccc ? 0 : AHCI_P_IX_DHR))); 2360220576Smav if (ch->resetting) 2361271261Smav callout_reset(&ch->reset_timer, hz / 10, ahci_reset_to, ch); 2362220777Smav else { 2363271261Smav ahci_start(ch, 1); 2364220576Smav xpt_release_simq(ch->sim, TRUE); 2365220777Smav } 2366195534Sscottl} 2367195534Sscottl 2368195534Sscottlstatic int 2369271261Smavahci_setup_fis(struct ahci_channel *ch, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag) 2370195534Sscottl{ 2371195534Sscottl u_int8_t *fis = &ctp->cfis[0]; 2372195534Sscottl 2373270833Simp bzero(fis, 20); 2374195534Sscottl fis[0] = 0x27; /* host to device */ 2375195534Sscottl fis[1] = (ccb->ccb_h.target_id & 0x0f); 2376195534Sscottl if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 2377195534Sscottl fis[1] |= 0x80; 2378195534Sscottl fis[2] = ATA_PACKET_CMD; 2379199821Smav if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE && 2380199821Smav ch->curr[ccb->ccb_h.target_id].mode >= ATA_DMA) 2381195534Sscottl fis[3] = ATA_F_DMA; 2382195534Sscottl else { 2383195534Sscottl fis[5] = ccb->csio.dxfer_len; 2384195534Sscottl fis[6] = ccb->csio.dxfer_len >> 8; 2385195534Sscottl } 2386195534Sscottl fis[7] = ATA_D_LBA; 2387195534Sscottl fis[15] = ATA_A_4BIT; 2388195534Sscottl bcopy((ccb->ccb_h.flags & CAM_CDB_POINTER) ? 2389195534Sscottl ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes, 2390195534Sscottl ctp->acmd, ccb->csio.cdb_len); 2391248687Smav bzero(ctp->acmd + ccb->csio.cdb_len, 32 - ccb->csio.cdb_len); 2392195534Sscottl } else if ((ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) == 0) { 2393195534Sscottl fis[1] |= 0x80; 2394195534Sscottl fis[2] = ccb->ataio.cmd.command; 2395195534Sscottl fis[3] = ccb->ataio.cmd.features; 2396195534Sscottl fis[4] = ccb->ataio.cmd.lba_low; 2397195534Sscottl fis[5] = ccb->ataio.cmd.lba_mid; 2398195534Sscottl fis[6] = ccb->ataio.cmd.lba_high; 2399195534Sscottl fis[7] = ccb->ataio.cmd.device; 2400195534Sscottl fis[8] = ccb->ataio.cmd.lba_low_exp; 2401195534Sscottl fis[9] = ccb->ataio.cmd.lba_mid_exp; 2402195534Sscottl fis[10] = ccb->ataio.cmd.lba_high_exp; 2403195534Sscottl fis[11] = ccb->ataio.cmd.features_exp; 2404195534Sscottl if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) { 2405195534Sscottl fis[12] = tag << 3; 2406195534Sscottl fis[13] = 0; 2407195534Sscottl } else { 2408195534Sscottl fis[12] = ccb->ataio.cmd.sector_count; 2409195534Sscottl fis[13] = ccb->ataio.cmd.sector_count_exp; 2410195534Sscottl } 2411195534Sscottl fis[15] = ATA_A_4BIT; 2412195534Sscottl } else { 2413195534Sscottl fis[15] = ccb->ataio.cmd.control; 2414195534Sscottl } 2415195534Sscottl return (20); 2416195534Sscottl} 2417195534Sscottl 2418195534Sscottlstatic int 2419195534Sscottlahci_sata_connect(struct ahci_channel *ch) 2420195534Sscottl{ 2421195534Sscottl u_int32_t status; 2422220829Smav int timeout, found = 0; 2423195534Sscottl 2424195534Sscottl /* Wait up to 100ms for "connect well" */ 2425220777Smav for (timeout = 0; timeout < 1000 ; timeout++) { 2426195534Sscottl status = ATA_INL(ch->r_mem, AHCI_P_SSTS); 2427220829Smav if ((status & ATA_SS_DET_MASK) != ATA_SS_DET_NO_DEVICE) 2428220829Smav found = 1; 2429195534Sscottl if (((status & ATA_SS_DET_MASK) == ATA_SS_DET_PHY_ONLINE) && 2430195534Sscottl ((status & ATA_SS_SPD_MASK) != ATA_SS_SPD_NO_SPEED) && 2431195534Sscottl ((status & ATA_SS_IPM_MASK) == ATA_SS_IPM_ACTIVE)) 2432195534Sscottl break; 2433196656Smav if ((status & ATA_SS_DET_MASK) == ATA_SS_DET_PHY_OFFLINE) { 2434196656Smav if (bootverbose) { 2435196656Smav device_printf(ch->dev, "SATA offline status=%08x\n", 2436196656Smav status); 2437196656Smav } 2438196656Smav return (0); 2439196656Smav } 2440220829Smav if (found == 0 && timeout >= 100) 2441220829Smav break; 2442220777Smav DELAY(100); 2443195534Sscottl } 2444220829Smav if (timeout >= 1000 || !found) { 2445195534Sscottl if (bootverbose) { 2446220829Smav device_printf(ch->dev, 2447220829Smav "SATA connect timeout time=%dus status=%08x\n", 2448220829Smav timeout * 100, status); 2449195534Sscottl } 2450195534Sscottl return (0); 2451195534Sscottl } 2452195534Sscottl if (bootverbose) { 2453220777Smav device_printf(ch->dev, "SATA connect time=%dus status=%08x\n", 2454220777Smav timeout * 100, status); 2455195534Sscottl } 2456195534Sscottl /* Clear SATA error register */ 2457195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_SERR, 0xffffffff); 2458195534Sscottl return (1); 2459195534Sscottl} 2460195534Sscottl 2461195534Sscottlstatic int 2462271261Smavahci_sata_phy_reset(struct ahci_channel *ch) 2463195534Sscottl{ 2464199821Smav int sata_rev; 2465195534Sscottl uint32_t val; 2466195534Sscottl 2467220657Smav if (ch->listening) { 2468220657Smav val = ATA_INL(ch->r_mem, AHCI_P_CMD); 2469220657Smav val |= AHCI_P_CMD_SUD; 2470220657Smav ATA_OUTL(ch->r_mem, AHCI_P_CMD, val); 2471220657Smav ch->listening = 0; 2472220657Smav } 2473199821Smav sata_rev = ch->user[ch->pm_present ? 15 : 0].revision; 2474199821Smav if (sata_rev == 1) 2475195534Sscottl val = ATA_SC_SPD_SPEED_GEN1; 2476199821Smav else if (sata_rev == 2) 2477195534Sscottl val = ATA_SC_SPD_SPEED_GEN2; 2478199821Smav else if (sata_rev == 3) 2479195534Sscottl val = ATA_SC_SPD_SPEED_GEN3; 2480195534Sscottl else 2481195534Sscottl val = 0; 2482195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_SCTL, 2483196656Smav ATA_SC_DET_RESET | val | 2484196656Smav ATA_SC_IPM_DIS_PARTIAL | ATA_SC_IPM_DIS_SLUMBER); 2485220777Smav DELAY(1000); 2486196656Smav ATA_OUTL(ch->r_mem, AHCI_P_SCTL, 2487195534Sscottl ATA_SC_DET_IDLE | val | ((ch->pm_level > 0) ? 0 : 2488195534Sscottl (ATA_SC_IPM_DIS_PARTIAL | ATA_SC_IPM_DIS_SLUMBER))); 2489203426Smav if (!ahci_sata_connect(ch)) { 2490220657Smav if (ch->caps & AHCI_CAP_SSS) { 2491220657Smav val = ATA_INL(ch->r_mem, AHCI_P_CMD); 2492220657Smav val &= ~AHCI_P_CMD_SUD; 2493220657Smav ATA_OUTL(ch->r_mem, AHCI_P_CMD, val); 2494220657Smav ch->listening = 1; 2495220657Smav } else if (ch->pm_level > 0) 2496203426Smav ATA_OUTL(ch->r_mem, AHCI_P_SCTL, ATA_SC_DET_DISABLE); 2497203426Smav return (0); 2498203426Smav } 2499203426Smav return (1); 2500195534Sscottl} 2501195534Sscottl 2502207430Smavstatic int 2503271261Smavahci_check_ids(struct ahci_channel *ch, union ccb *ccb) 2504207430Smav{ 2505207430Smav 2506207430Smav if (ccb->ccb_h.target_id > ((ch->caps & AHCI_CAP_SPM) ? 15 : 0)) { 2507207430Smav ccb->ccb_h.status = CAM_TID_INVALID; 2508256843Smav ahci_done(ch, ccb); 2509207430Smav return (-1); 2510207430Smav } 2511207430Smav if (ccb->ccb_h.target_lun != 0) { 2512207430Smav ccb->ccb_h.status = CAM_LUN_INVALID; 2513256843Smav ahci_done(ch, ccb); 2514207430Smav return (-1); 2515207430Smav } 2516207430Smav return (0); 2517207430Smav} 2518207430Smav 2519195534Sscottlstatic void 2520195534Sscottlahciaction(struct cam_sim *sim, union ccb *ccb) 2521195534Sscottl{ 2522195534Sscottl struct ahci_channel *ch; 2523195534Sscottl 2524195534Sscottl CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("ahciaction func_code=%x\n", 2525195534Sscottl ccb->ccb_h.func_code)); 2526195534Sscottl 2527195534Sscottl ch = (struct ahci_channel *)cam_sim_softc(sim); 2528195534Sscottl switch (ccb->ccb_h.func_code) { 2529195534Sscottl /* Common cases first */ 2530195534Sscottl case XPT_ATA_IO: /* Execute the requested I/O operation */ 2531195534Sscottl case XPT_SCSI_IO: 2532271261Smav if (ahci_check_ids(ch, ccb)) 2533207430Smav return; 2534207430Smav if (ch->devices == 0 || 2535207430Smav (ch->pm_present == 0 && 2536207430Smav ccb->ccb_h.target_id > 0 && ccb->ccb_h.target_id < 15)) { 2537195534Sscottl ccb->ccb_h.status = CAM_SEL_TIMEOUT; 2538195534Sscottl break; 2539195534Sscottl } 2540220565Smav ccb->ccb_h.recovery_type = RECOVERY_NONE; 2541195534Sscottl /* Check for command collision. */ 2542271261Smav if (ahci_check_collision(ch, ccb)) { 2543195534Sscottl /* Freeze command. */ 2544195534Sscottl ch->frozen = ccb; 2545195534Sscottl /* We have only one frozen slot, so freeze simq also. */ 2546195534Sscottl xpt_freeze_simq(ch->sim, 1); 2547195534Sscottl return; 2548195534Sscottl } 2549271261Smav ahci_begin_transaction(ch, ccb); 2550207430Smav return; 2551195534Sscottl case XPT_EN_LUN: /* Enable LUN as a target */ 2552195534Sscottl case XPT_TARGET_IO: /* Execute target I/O request */ 2553195534Sscottl case XPT_ACCEPT_TARGET_IO: /* Accept Host Target Mode CDB */ 2554195534Sscottl case XPT_CONT_TARGET_IO: /* Continue Host Target I/O Connection*/ 2555195534Sscottl case XPT_ABORT: /* Abort the specified CCB */ 2556195534Sscottl /* XXX Implement */ 2557195534Sscottl ccb->ccb_h.status = CAM_REQ_INVALID; 2558195534Sscottl break; 2559195534Sscottl case XPT_SET_TRAN_SETTINGS: 2560195534Sscottl { 2561195534Sscottl struct ccb_trans_settings *cts = &ccb->cts; 2562199747Smav struct ahci_device *d; 2563195534Sscottl 2564271261Smav if (ahci_check_ids(ch, ccb)) 2565207430Smav return; 2566199747Smav if (cts->type == CTS_TYPE_CURRENT_SETTINGS) 2567199747Smav d = &ch->curr[ccb->ccb_h.target_id]; 2568199747Smav else 2569199747Smav d = &ch->user[ccb->ccb_h.target_id]; 2570199747Smav if (cts->xport_specific.sata.valid & CTS_SATA_VALID_REVISION) 2571199747Smav d->revision = cts->xport_specific.sata.revision; 2572199747Smav if (cts->xport_specific.sata.valid & CTS_SATA_VALID_MODE) 2573199747Smav d->mode = cts->xport_specific.sata.mode; 2574199747Smav if (cts->xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT) 2575199747Smav d->bytecount = min(8192, cts->xport_specific.sata.bytecount); 2576199747Smav if (cts->xport_specific.sata.valid & CTS_SATA_VALID_TAGS) 2577199747Smav d->tags = min(ch->numslots, cts->xport_specific.sata.tags); 2578199747Smav if (cts->xport_specific.sata.valid & CTS_SATA_VALID_PM) 2579195534Sscottl ch->pm_present = cts->xport_specific.sata.pm_present; 2580203376Smav if (cts->xport_specific.sata.valid & CTS_SATA_VALID_ATAPI) 2581203376Smav d->atapi = cts->xport_specific.sata.atapi; 2582207499Smav if (cts->xport_specific.sata.valid & CTS_SATA_VALID_CAPS) 2583207499Smav d->caps = cts->xport_specific.sata.caps; 2584195534Sscottl ccb->ccb_h.status = CAM_REQ_CMP; 2585195534Sscottl break; 2586195534Sscottl } 2587195534Sscottl case XPT_GET_TRAN_SETTINGS: 2588195534Sscottl /* Get default/user set transfer settings for the target */ 2589195534Sscottl { 2590195534Sscottl struct ccb_trans_settings *cts = &ccb->cts; 2591199747Smav struct ahci_device *d; 2592195534Sscottl uint32_t status; 2593195534Sscottl 2594271261Smav if (ahci_check_ids(ch, ccb)) 2595207430Smav return; 2596199747Smav if (cts->type == CTS_TYPE_CURRENT_SETTINGS) 2597199747Smav d = &ch->curr[ccb->ccb_h.target_id]; 2598199747Smav else 2599199747Smav d = &ch->user[ccb->ccb_h.target_id]; 2600236666Smav cts->protocol = PROTO_UNSPECIFIED; 2601196656Smav cts->protocol_version = PROTO_VERSION_UNSPECIFIED; 2602195534Sscottl cts->transport = XPORT_SATA; 2603196656Smav cts->transport_version = XPORT_VERSION_UNSPECIFIED; 2604195534Sscottl cts->proto_specific.valid = 0; 2605195534Sscottl cts->xport_specific.sata.valid = 0; 2606199747Smav if (cts->type == CTS_TYPE_CURRENT_SETTINGS && 2607199747Smav (ccb->ccb_h.target_id == 15 || 2608199747Smav (ccb->ccb_h.target_id == 0 && !ch->pm_present))) { 2609195534Sscottl status = ATA_INL(ch->r_mem, AHCI_P_SSTS) & ATA_SS_SPD_MASK; 2610199747Smav if (status & 0x0f0) { 2611199747Smav cts->xport_specific.sata.revision = 2612199747Smav (status & 0x0f0) >> 4; 2613199747Smav cts->xport_specific.sata.valid |= 2614199747Smav CTS_SATA_VALID_REVISION; 2615199747Smav } 2616207499Smav cts->xport_specific.sata.caps = d->caps & CTS_SATA_CAPS_D; 2617207499Smav if (ch->pm_level) { 2618207499Smav if (ch->caps & (AHCI_CAP_PSC | AHCI_CAP_SSC)) 2619207499Smav cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_PMREQ; 2620207499Smav if (ch->caps2 & AHCI_CAP2_APST) 2621207499Smav cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_APST; 2622207499Smav } 2623207499Smav if ((ch->caps & AHCI_CAP_SNCQ) && 2624207499Smav (ch->quirks & AHCI_Q_NOAA) == 0) 2625207499Smav cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_DMAAA; 2626220602Smav cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_AN; 2627207499Smav cts->xport_specific.sata.caps &= 2628207499Smav ch->user[ccb->ccb_h.target_id].caps; 2629207499Smav cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS; 2630195534Sscottl } else { 2631199747Smav cts->xport_specific.sata.revision = d->revision; 2632199747Smav cts->xport_specific.sata.valid |= CTS_SATA_VALID_REVISION; 2633207499Smav cts->xport_specific.sata.caps = d->caps; 2634207499Smav cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS; 2635195534Sscottl } 2636199747Smav cts->xport_specific.sata.mode = d->mode; 2637199747Smav cts->xport_specific.sata.valid |= CTS_SATA_VALID_MODE; 2638199747Smav cts->xport_specific.sata.bytecount = d->bytecount; 2639199747Smav cts->xport_specific.sata.valid |= CTS_SATA_VALID_BYTECOUNT; 2640199747Smav cts->xport_specific.sata.pm_present = ch->pm_present; 2641195534Sscottl cts->xport_specific.sata.valid |= CTS_SATA_VALID_PM; 2642199747Smav cts->xport_specific.sata.tags = d->tags; 2643199747Smav cts->xport_specific.sata.valid |= CTS_SATA_VALID_TAGS; 2644203376Smav cts->xport_specific.sata.atapi = d->atapi; 2645203376Smav cts->xport_specific.sata.valid |= CTS_SATA_VALID_ATAPI; 2646195534Sscottl ccb->ccb_h.status = CAM_REQ_CMP; 2647195534Sscottl break; 2648195534Sscottl } 2649195534Sscottl case XPT_RESET_BUS: /* Reset the specified SCSI bus */ 2650195534Sscottl case XPT_RESET_DEV: /* Bus Device Reset the specified SCSI device */ 2651271261Smav ahci_reset(ch); 2652195534Sscottl ccb->ccb_h.status = CAM_REQ_CMP; 2653195534Sscottl break; 2654195534Sscottl case XPT_TERM_IO: /* Terminate the I/O process */ 2655195534Sscottl /* XXX Implement */ 2656195534Sscottl ccb->ccb_h.status = CAM_REQ_INVALID; 2657195534Sscottl break; 2658195534Sscottl case XPT_PATH_INQ: /* Path routing inquiry */ 2659195534Sscottl { 2660195534Sscottl struct ccb_pathinq *cpi = &ccb->cpi; 2661195534Sscottl 2662195534Sscottl cpi->version_num = 1; /* XXX??? */ 2663199278Smav cpi->hba_inquiry = PI_SDTR_ABLE; 2664199278Smav if (ch->caps & AHCI_CAP_SNCQ) 2665199278Smav cpi->hba_inquiry |= PI_TAG_ABLE; 2666195534Sscottl if (ch->caps & AHCI_CAP_SPM) 2667195534Sscottl cpi->hba_inquiry |= PI_SATAPM; 2668195534Sscottl cpi->target_sprt = 0; 2669248522Skib cpi->hba_misc = PIM_SEQSCAN | PIM_UNMAPPED; 2670195534Sscottl cpi->hba_eng_cnt = 0; 2671195534Sscottl if (ch->caps & AHCI_CAP_SPM) 2672198322Smav cpi->max_target = 15; 2673195534Sscottl else 2674195534Sscottl cpi->max_target = 0; 2675195534Sscottl cpi->max_lun = 0; 2676195534Sscottl cpi->initiator_id = 0; 2677195534Sscottl cpi->bus_id = cam_sim_bus(sim); 2678195534Sscottl cpi->base_transfer_speed = 150000; 2679195534Sscottl strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); 2680195534Sscottl strncpy(cpi->hba_vid, "AHCI", HBA_IDLEN); 2681195534Sscottl strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); 2682195534Sscottl cpi->unit_number = cam_sim_unit(sim); 2683195534Sscottl cpi->transport = XPORT_SATA; 2684196656Smav cpi->transport_version = XPORT_VERSION_UNSPECIFIED; 2685236847Smav cpi->protocol = PROTO_ATA; 2686196656Smav cpi->protocol_version = PROTO_VERSION_UNSPECIFIED; 2687195534Sscottl cpi->maxio = MAXPHYS; 2688196777Smav /* ATI SB600 can't handle 256 sectors with FPDMA (NCQ). */ 2689271146Simp if (ch->quirks & AHCI_Q_MAXIO_64K) 2690196796Smav cpi->maxio = min(cpi->maxio, 128 * 512); 2691271146Simp cpi->hba_vendor = ch->vendorid; 2692271146Simp cpi->hba_device = ch->deviceid; 2693271146Simp cpi->hba_subvendor = ch->subvendorid; 2694271146Simp cpi->hba_subdevice = ch->subdeviceid; 2695195534Sscottl cpi->ccb_h.status = CAM_REQ_CMP; 2696195534Sscottl break; 2697195534Sscottl } 2698195534Sscottl default: 2699195534Sscottl ccb->ccb_h.status = CAM_REQ_INVALID; 2700195534Sscottl break; 2701195534Sscottl } 2702256843Smav ahci_done(ch, ccb); 2703195534Sscottl} 2704195534Sscottl 2705195534Sscottlstatic void 2706195534Sscottlahcipoll(struct cam_sim *sim) 2707195534Sscottl{ 2708195534Sscottl struct ahci_channel *ch = (struct ahci_channel *)cam_sim_softc(sim); 2709256843Smav uint32_t istatus; 2710195534Sscottl 2711256843Smav /* Read interrupt statuses and process if any. */ 2712256843Smav istatus = ATA_INL(ch->r_mem, AHCI_P_IS); 2713256843Smav if (istatus != 0) 2714256843Smav ahci_ch_intr_main(ch, istatus); 2715220789Smav if (ch->resetting != 0 && 2716220789Smav (--ch->resetpolldiv <= 0 || !callout_pending(&ch->reset_timer))) { 2717220789Smav ch->resetpolldiv = 1000; 2718271261Smav ahci_reset_to(ch); 2719220789Smav } 2720195534Sscottl} 2721271146SimpMODULE_VERSION(ahci, 1); 2722271146SimpMODULE_DEPEND(ahci, cam, 1, 1, 1); 2723