1194845Sraj/*- 2194845Sraj * Copyright (C) 2008-2009 Semihalf 3194845Sraj * All rights reserved. 4194845Sraj * 5194845Sraj * Initial version developed by Ilya Bakulin. Full functionality and bringup 6194845Sraj * by Piotr Ziecik. 7194845Sraj * 8194845Sraj * Redistribution and use in source and binary forms, with or without 9194845Sraj * modification, are permitted provided that the following conditions 10194845Sraj * are met: 11194845Sraj * 1. Redistributions of source code must retain the above copyright 12194845Sraj * notice, this list of conditions and the following disclaimer. 13194845Sraj * 2. Redistributions in binary form must reproduce the above copyright 14194845Sraj * notice, this list of conditions and the following disclaimer in the 15194845Sraj * documentation and/or other materials provided with the distribution. 16194845Sraj * 17194845Sraj * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18194845Sraj * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19194845Sraj * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20194845Sraj * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 21194845Sraj * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22194845Sraj * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23194845Sraj * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24194845Sraj * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25194845Sraj * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26194845Sraj * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27194845Sraj * SUCH DAMAGE. 28194845Sraj */ 29194845Sraj 30194845Sraj#include <sys/cdefs.h> 31194845Sraj__FBSDID("$FreeBSD$"); 32194845Sraj 33194845Sraj#include <sys/param.h> 34194845Sraj#include <sys/bus.h> 35194845Sraj#include <sys/lock.h> 36194845Sraj#include <sys/resource.h> 37194845Sraj#include <sys/systm.h> 38194845Sraj#include <sys/rman.h> 39194845Sraj#include <sys/kernel.h> 40194845Sraj#include <sys/module.h> 41194845Sraj#include <sys/mutex.h> 42194845Sraj#include <sys/endian.h> 43194845Sraj#include <sys/sema.h> 44194845Sraj#include <sys/taskqueue.h> 45194845Sraj#include <vm/uma.h> 46194845Sraj#include <machine/bus.h> 47194845Sraj#include <machine/resource.h> 48194845Sraj 49194845Sraj#include <sys/ata.h> 50194845Sraj#include <dev/ata/ata-all.h> 51209131Sraj#include <dev/ofw/ofw_bus.h> 52209131Sraj#include <dev/ofw/ofw_bus_subr.h> 53194845Sraj 54194845Sraj#include "ata_if.h" 55194845Sraj 56194845Sraj#include "mvreg.h" 57194845Sraj#include "mvvar.h" 58194845Sraj 59194845Sraj/* Useful macros */ 60194845Sraj#define EDMA_TIMEOUT 100000 /* 100 ms */ 61194845Sraj#define SATA_INL(sc, reg) ATA_INL((sc)->sc_mem_res, reg) 62194845Sraj#define SATA_OUTL(sc, reg, val) ATA_OUTL((sc)->sc_mem_res, reg, val) 63194845Sraj 64194845Sraj/* HW-related data structures */ 65194845Srajstruct sata_prdentry { 66194845Sraj uint32_t prd_addrlo; 67194845Sraj uint32_t prd_count; 68194845Sraj uint32_t prd_addrhi; 69194845Sraj uint32_t prd_reserved; 70194845Sraj}; 71194845Sraj 72194845Srajstruct sata_crqb { 73194845Sraj uint32_t crqb_prdlo; 74194845Sraj uint32_t crqb_prdhi; 75194845Sraj uint32_t crqb_flags; 76194845Sraj uint16_t crqb_count; 77194845Sraj uint16_t crqb_reserved1[2]; 78194845Sraj uint8_t crqb_ata_command; 79194845Sraj uint8_t crqb_ata_feature; 80194845Sraj uint8_t crqb_ata_lba_low; 81194845Sraj uint8_t crqb_ata_lba_mid; 82194845Sraj uint8_t crqb_ata_lba_high; 83194845Sraj uint8_t crqb_ata_device; 84194845Sraj uint8_t crqb_ata_lba_low_p; 85194845Sraj uint8_t crqb_ata_lba_mid_p; 86194845Sraj uint8_t crqb_ata_lba_high_p; 87194845Sraj uint8_t crqb_ata_feature_p; 88194845Sraj uint8_t crqb_ata_count; 89194845Sraj uint8_t crqb_ata_count_p; 90194845Sraj uint16_t crqb_reserved2; 91194845Sraj}; 92194845Sraj 93194845Srajstruct sata_crpb { 94194845Sraj uint8_t crpb_tag; 95194845Sraj uint8_t crpb_reserved; 96194845Sraj uint8_t crpb_edma_status; 97194845Sraj uint8_t crpb_dev_status; 98194845Sraj uint32_t crpb_timestamp; 99194845Sraj}; 100194845Sraj 101194845Sraj/* Identification section. */ 102194845Srajstruct sata_softc { 103194845Sraj device_t sc_dev; 104194845Sraj unsigned int sc_version; 105194845Sraj unsigned int sc_edma_qlen; 106194845Sraj uint32_t sc_edma_reqis_mask; 107194845Sraj uint32_t sc_edma_resos_mask; 108194845Sraj struct resource *sc_mem_res; 109194845Sraj bus_space_tag_t sc_mem_res_bustag; 110194845Sraj bus_space_handle_t sc_mem_res_bushdl; 111194845Sraj struct resource *sc_irq_res; 112194845Sraj void *sc_irq_cookiep; 113194845Sraj struct { 114194845Sraj void (*function)(void *); 115194845Sraj void *argument; 116194845Sraj } sc_interrupt[SATA_CHAN_NUM]; 117194845Sraj}; 118194845Sraj 119194845Sraj/* Controller functions */ 120194845Srajstatic int sata_probe(device_t dev); 121194845Srajstatic int sata_attach(device_t dev); 122194845Srajstatic int sata_detach(device_t dev); 123194845Srajstatic void sata_intr(void*); 124194845Srajstatic struct resource * sata_alloc_resource(device_t dev, device_t child, 125194845Sraj int type, int *rid, u_long start, u_long end, u_long count, u_int flags); 126194845Srajstatic int sata_release_resource(device_t dev, device_t child, int type, 127194845Sraj int rid, struct resource *r); 128194845Srajstatic int sata_setup_intr(device_t dev, device_t child, 129194845Sraj struct resource *irq, int flags, driver_filter_t *filt, 130194845Sraj driver_intr_t *function, void *argument, void **cookiep); 131194845Srajstatic int sata_teardown_intr(device_t dev, device_t child, 132194845Sraj struct resource *irq, void *cookie); 133194845Sraj 134194845Sraj/* Channel functions */ 135194845Srajstatic int sata_channel_probe(device_t dev); 136194845Srajstatic int sata_channel_attach(device_t dev); 137194845Srajstatic int sata_channel_detach(device_t dev); 138194845Srajstatic int sata_channel_begin_transaction(struct ata_request *request); 139194845Srajstatic int sata_channel_end_transaction(struct ata_request *request); 140194845Srajstatic int sata_channel_status(device_t dev); 141200171Smavstatic int sata_channel_setmode(device_t dev, int target, int mode); 142200275Smavstatic int sata_channel_getrev(device_t dev, int target); 143194845Srajstatic void sata_channel_reset(device_t dev); 144194845Srajstatic void sata_channel_dmasetprd(void *xsc, bus_dma_segment_t *segs, 145194845Sraj int nsegs, int error); 146194845Sraj 147194845Sraj/* EDMA functions */ 148194845Srajstatic int sata_edma_ctrl(device_t dev, int on); 149194845Srajstatic int sata_edma_is_running(device_t); 150194845Sraj 151194845Srajstatic device_method_t sata_methods[] = { 152194845Sraj /* Device method */ 153194845Sraj DEVMETHOD(device_probe, sata_probe), 154194845Sraj DEVMETHOD(device_attach, sata_attach), 155194845Sraj DEVMETHOD(device_detach, sata_detach), 156194845Sraj DEVMETHOD(device_shutdown, bus_generic_shutdown), 157194845Sraj DEVMETHOD(device_suspend, bus_generic_suspend), 158194845Sraj DEVMETHOD(device_resume, bus_generic_resume), 159194845Sraj 160194845Sraj /* ATA bus methods. */ 161194845Sraj DEVMETHOD(bus_alloc_resource, sata_alloc_resource), 162194845Sraj DEVMETHOD(bus_release_resource, sata_release_resource), 163194845Sraj DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), 164194845Sraj DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), 165194845Sraj DEVMETHOD(bus_setup_intr, sata_setup_intr), 166194845Sraj DEVMETHOD(bus_teardown_intr, sata_teardown_intr), 167194845Sraj { 0, 0 }, 168194845Sraj}; 169194845Sraj 170194845Srajstatic driver_t sata_driver = { 171194845Sraj "sata", 172194845Sraj sata_methods, 173194845Sraj sizeof(struct sata_softc), 174194845Sraj}; 175194845Sraj 176194845Srajdevclass_t sata_devclass; 177194845Sraj 178209131SrajDRIVER_MODULE(sata, simplebus, sata_driver, sata_devclass, 0, 0); 179194845SrajMODULE_VERSION(sata, 1); 180194845SrajMODULE_DEPEND(sata, ata, 1, 1, 1); 181194845Sraj 182194845Srajstatic int 183194845Srajsata_probe(device_t dev) 184194845Sraj{ 185194845Sraj struct sata_softc *sc; 186194845Sraj uint32_t d, r; 187194845Sraj 188209131Sraj if (!ofw_bus_is_compatible(dev, "mrvl,sata")) 189209131Sraj return (ENXIO); 190209131Sraj 191194845Sraj soc_id(&d, &r); 192194845Sraj sc = device_get_softc(dev); 193194845Sraj 194194845Sraj switch(d) { 195194845Sraj case MV_DEV_88F5182: 196194845Sraj sc->sc_version = 1; 197194845Sraj sc->sc_edma_qlen = 128; 198194845Sraj break; 199194845Sraj case MV_DEV_88F6281: 200238873Shrs case MV_DEV_88F6282: 201194845Sraj case MV_DEV_MV78100: 202194845Sraj case MV_DEV_MV78100_Z0: 203194845Sraj sc->sc_version = 2; 204194845Sraj sc->sc_edma_qlen = 32; 205194845Sraj break; 206194845Sraj default: 207194845Sraj device_printf(dev, "unsupported SoC (ID: 0x%08X)!\n", d); 208194845Sraj return (ENXIO); 209194845Sraj } 210194845Sraj 211194845Sraj sc->sc_edma_reqis_mask = (sc->sc_edma_qlen - 1) << SATA_EDMA_REQIS_OFS; 212194845Sraj sc->sc_edma_resos_mask = (sc->sc_edma_qlen - 1) << SATA_EDMA_RESOS_OFS; 213194845Sraj 214194845Sraj device_set_desc(dev, "Marvell Integrated SATA Controller"); 215194845Sraj return (0); 216194845Sraj} 217194845Sraj 218194845Srajstatic int 219194845Srajsata_attach(device_t dev) 220194845Sraj{ 221194845Sraj struct sata_softc *sc; 222194845Sraj int mem_id, irq_id, error, i; 223194845Sraj device_t ata_chan; 224194845Sraj uint32_t reg; 225194845Sraj 226194845Sraj sc = device_get_softc(dev); 227194845Sraj sc->sc_dev = dev; 228194845Sraj mem_id = 0; 229194845Sraj irq_id = 0; 230194845Sraj 231194845Sraj /* Allocate resources */ 232194845Sraj sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 233194845Sraj &mem_id, RF_ACTIVE); 234194845Sraj if (sc->sc_mem_res == NULL) { 235194845Sraj device_printf(dev, "could not allocate memory.\n"); 236194845Sraj return (ENOMEM); 237194845Sraj } 238194845Sraj 239194845Sraj sc->sc_mem_res_bustag = rman_get_bustag(sc->sc_mem_res); 240194845Sraj sc->sc_mem_res_bushdl = rman_get_bushandle(sc->sc_mem_res); 241194845Sraj KASSERT(sc->sc_mem_res_bustag && sc->sc_mem_res_bushdl, 242194845Sraj ("cannot get bus handle or tag.")); 243194845Sraj 244194845Sraj sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &irq_id, 245194845Sraj RF_ACTIVE); 246194845Sraj if (sc->sc_irq_res == NULL) { 247194845Sraj device_printf(dev, "could not allocate IRQ.\n"); 248194845Sraj error = ENOMEM; 249194845Sraj goto err; 250194845Sraj } 251194845Sraj 252194845Sraj error = bus_setup_intr(dev, sc->sc_irq_res, 253194845Sraj INTR_TYPE_BIO | INTR_MPSAFE | INTR_ENTROPY, 254194845Sraj NULL, sata_intr, sc, &sc->sc_irq_cookiep); 255194845Sraj if (error != 0) { 256194845Sraj device_printf(dev, "could not setup interrupt.\n"); 257194845Sraj goto err; 258194845Sraj } 259194845Sraj 260194845Sraj /* Attach channels */ 261194845Sraj for (i = 0; i < SATA_CHAN_NUM; i++) { 262194845Sraj ata_chan = device_add_child(dev, "ata", 263194845Sraj devclass_find_free_unit(ata_devclass, 0)); 264194845Sraj 265194845Sraj if (!ata_chan) { 266194845Sraj device_printf(dev, "cannot add channel %d.\n", i); 267194845Sraj error = ENOMEM; 268194845Sraj goto err; 269194845Sraj } 270194845Sraj } 271194845Sraj 272194845Sraj /* Disable interrupt coalescing */ 273194845Sraj reg = SATA_INL(sc, SATA_CR); 274194845Sraj for (i = 0; i < SATA_CHAN_NUM; i++) 275194845Sraj reg |= SATA_CR_COALDIS(i); 276194845Sraj 277194845Sraj /* Disable DMA byte swapping */ 278194845Sraj if (sc->sc_version == 2) 279194845Sraj reg |= SATA_CR_NODMABS | SATA_CR_NOEDMABS | 280194845Sraj SATA_CR_NOPRDPBS; 281194845Sraj 282194845Sraj SATA_OUTL(sc, SATA_CR, reg); 283194845Sraj 284194845Sraj /* Clear and mask all interrupts */ 285194845Sraj SATA_OUTL(sc, SATA_ICR, 0); 286194845Sraj SATA_OUTL(sc, SATA_MIMR, 0); 287194845Sraj 288194845Sraj return(bus_generic_attach(dev)); 289194845Sraj 290194845Srajerr: 291194845Sraj sata_detach(dev); 292194845Sraj return (error); 293194845Sraj} 294194845Sraj 295194845Srajstatic int 296194845Srajsata_detach(device_t dev) 297194845Sraj{ 298194845Sraj struct sata_softc *sc; 299194845Sraj 300194845Sraj sc = device_get_softc(dev); 301194845Sraj 302194845Sraj if (device_is_attached(dev)) 303194845Sraj bus_generic_detach(dev); 304194845Sraj 305194845Sraj if (sc->sc_mem_res != NULL) { 306194845Sraj bus_release_resource(dev, SYS_RES_MEMORY, 307194845Sraj rman_get_rid(sc->sc_mem_res), sc->sc_mem_res); 308194845Sraj sc->sc_mem_res = NULL; 309194845Sraj } 310194845Sraj 311194845Sraj if (sc->sc_irq_res != NULL) { 312194845Sraj bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_irq_cookiep); 313194845Sraj bus_release_resource(dev, SYS_RES_IRQ, 314194845Sraj rman_get_rid(sc->sc_irq_res), sc->sc_irq_res); 315194845Sraj sc->sc_irq_res = NULL; 316194845Sraj } 317194845Sraj 318194845Sraj return (0); 319194845Sraj} 320194845Sraj 321194845Srajstatic struct resource * 322194845Srajsata_alloc_resource(device_t dev, device_t child, int type, int *rid, 323194845Sraj u_long start, u_long end, u_long count, u_int flags) 324194845Sraj{ 325194845Sraj struct sata_softc *sc; 326194845Sraj 327194845Sraj sc = device_get_softc(dev); 328194845Sraj 329194845Sraj KASSERT(type == SYS_RES_IRQ && *rid == ATA_IRQ_RID, 330194845Sraj ("illegal resource request (type %u, rid %u).", 331194845Sraj type, *rid)); 332194845Sraj 333194845Sraj return (sc->sc_irq_res); 334194845Sraj} 335194845Sraj 336194845Srajstatic int 337194845Srajsata_release_resource(device_t dev, device_t child, int type, int rid, 338194845Sraj struct resource *r) 339194845Sraj{ 340194845Sraj 341194845Sraj KASSERT(type == SYS_RES_IRQ && rid == ATA_IRQ_RID, 342194845Sraj ("strange type %u and/or rid %u while releasing resource.", type, 343194845Sraj rid)); 344194845Sraj 345194845Sraj return (0); 346194845Sraj} 347194845Sraj 348194845Srajstatic int 349194845Srajsata_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, 350194845Sraj driver_filter_t *filt, driver_intr_t *function, void *argument, 351194845Sraj void **cookiep) 352194845Sraj{ 353194845Sraj struct sata_softc *sc; 354194845Sraj struct ata_channel *ch; 355194845Sraj 356194845Sraj sc = device_get_softc(dev); 357194845Sraj ch = device_get_softc(child); 358194845Sraj 359194845Sraj if (filt != NULL) { 360194845Sraj device_printf(dev, "filter interrupts are not supported.\n"); 361194845Sraj return (EINVAL); 362194845Sraj } 363194845Sraj 364194845Sraj sc->sc_interrupt[ch->unit].function = function; 365194845Sraj sc->sc_interrupt[ch->unit].argument = argument; 366194845Sraj *cookiep = sc; 367194845Sraj 368194845Sraj return (0); 369194845Sraj} 370194845Sraj 371194845Srajstatic int 372194845Srajsata_teardown_intr(device_t dev, device_t child, struct resource *irq, 373194845Sraj void *cookie) 374194845Sraj{ 375194845Sraj struct sata_softc *sc; 376194845Sraj struct ata_channel *ch; 377194845Sraj 378194845Sraj sc = device_get_softc(dev); 379194845Sraj ch = device_get_softc(child); 380194845Sraj 381194845Sraj sc->sc_interrupt[ch->unit].function = NULL; 382194845Sraj sc->sc_interrupt[ch->unit].argument = NULL; 383194845Sraj 384194845Sraj return (0); 385194845Sraj} 386194845Sraj 387194845Srajstatic void 388194845Srajsata_intr(void *xsc) 389194845Sraj{ 390194845Sraj struct sata_softc *sc; 391194845Sraj int unit; 392194845Sraj 393194845Sraj sc = xsc; 394194845Sraj 395194845Sraj /* 396194845Sraj * Behave like ata_generic_intr() for PCI controllers. 397194845Sraj * Simply invoke ISRs on all channels. 398194845Sraj */ 399194845Sraj for (unit = 0; unit < SATA_CHAN_NUM; unit++) 400194845Sraj if (sc->sc_interrupt[unit].function != NULL) 401194845Sraj sc->sc_interrupt[unit].function( 402194845Sraj sc->sc_interrupt[unit].argument); 403194845Sraj} 404194845Sraj 405194845Srajstatic int 406194845Srajsata_channel_probe(device_t dev) 407194845Sraj{ 408194845Sraj 409194845Sraj device_set_desc(dev, "Marvell Integrated SATA Channel"); 410194845Sraj return (ata_probe(dev)); 411194845Sraj} 412194845Sraj 413194845Srajstatic int 414194845Srajsata_channel_attach(device_t dev) 415194845Sraj{ 416194845Sraj struct sata_softc *sc; 417194845Sraj struct ata_channel *ch; 418194845Sraj uint64_t work; 419194845Sraj int error, i; 420194845Sraj 421194845Sraj sc = device_get_softc(device_get_parent(dev)); 422194845Sraj ch = device_get_softc(dev); 423194845Sraj 424194845Sraj if (ch->attached) 425194845Sraj return (0); 426194845Sraj 427194845Sraj ch->dev = dev; 428194845Sraj ch->unit = device_get_unit(dev); 429200275Smav ch->flags |= ATA_USE_16BIT | ATA_NO_SLAVE | ATA_SATA; 430194845Sraj 431194845Sraj /* Set legacy ATA resources. */ 432194845Sraj for (i = ATA_DATA; i <= ATA_COMMAND; i++) { 433194845Sraj ch->r_io[i].res = sc->sc_mem_res; 434194845Sraj ch->r_io[i].offset = SATA_SHADOWR_BASE(ch->unit) + (i << 2); 435194845Sraj } 436194845Sraj 437194845Sraj ch->r_io[ATA_CONTROL].res = sc->sc_mem_res; 438194845Sraj ch->r_io[ATA_CONTROL].offset = SATA_SHADOWR_CONTROL(ch->unit); 439194845Sraj 440194845Sraj ch->r_io[ATA_IDX_ADDR].res = sc->sc_mem_res; 441194845Sraj ata_default_registers(dev); 442194845Sraj 443194845Sraj /* Set SATA resources. */ 444194845Sraj ch->r_io[ATA_SSTATUS].res = sc->sc_mem_res; 445194845Sraj ch->r_io[ATA_SSTATUS].offset = SATA_SATA_SSTATUS(ch->unit); 446194845Sraj ch->r_io[ATA_SERROR].res = sc->sc_mem_res; 447194845Sraj ch->r_io[ATA_SERROR].offset = SATA_SATA_SERROR(ch->unit); 448194845Sraj ch->r_io[ATA_SCONTROL].res = sc->sc_mem_res; 449194845Sraj ch->r_io[ATA_SCONTROL].offset = SATA_SATA_SCONTROL(ch->unit); 450194845Sraj ata_generic_hw(dev); 451194845Sraj 452194845Sraj ch->hw.begin_transaction = sata_channel_begin_transaction; 453194845Sraj ch->hw.end_transaction = sata_channel_end_transaction; 454194845Sraj ch->hw.status = sata_channel_status; 455194845Sraj 456194845Sraj /* Set DMA resources */ 457194845Sraj ata_dmainit(dev); 458194845Sraj ch->dma.setprd = sata_channel_dmasetprd; 459194845Sraj 460194845Sraj /* Clear work area */ 461194845Sraj KASSERT(sc->sc_edma_qlen * (sizeof(struct sata_crqb) + 462194845Sraj sizeof(struct sata_crpb)) <= ch->dma.max_iosize, 463194845Sraj ("insufficient DMA memory for request/response queues.\n")); 464194845Sraj bzero(ch->dma.work, sc->sc_edma_qlen * (sizeof(struct sata_crqb) + 465194845Sraj sizeof(struct sata_crpb))); 466194845Sraj bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 467194845Sraj BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 468194845Sraj 469194845Sraj /* Turn off EDMA engine */ 470194845Sraj error = sata_edma_ctrl(dev, 0); 471194845Sraj if (error) { 472194845Sraj ata_dmafini(dev); 473194845Sraj return (error); 474194845Sraj } 475194845Sraj 476194845Sraj /* 477194845Sraj * Initialize EDMA engine: 478194845Sraj * - Native Command Queuing off, 479194845Sraj * - Non-Queued operation, 480194845Sraj * - Host Queue Cache enabled. 481194845Sraj */ 482194845Sraj SATA_OUTL(sc, SATA_EDMA_CFG(ch->unit), SATA_EDMA_CFG_HQCACHE | 483194845Sraj (sc->sc_version == 1) ? SATA_EDMA_CFG_QL128 : 0); 484194845Sraj 485194845Sraj /* Set request queue pointers */ 486194845Sraj work = ch->dma.work_bus; 487194845Sraj SATA_OUTL(sc, SATA_EDMA_REQBAHR(ch->unit), work >> 32); 488194845Sraj SATA_OUTL(sc, SATA_EDMA_REQIPR(ch->unit), work & 0xFFFFFFFF); 489194845Sraj SATA_OUTL(sc, SATA_EDMA_REQOPR(ch->unit), work & 0xFFFFFFFF); 490194845Sraj 491194845Sraj /* Set response queue pointers */ 492194845Sraj work += sc->sc_edma_qlen * sizeof(struct sata_crqb); 493194845Sraj SATA_OUTL(sc, SATA_EDMA_RESBAHR(ch->unit), work >> 32); 494194845Sraj SATA_OUTL(sc, SATA_EDMA_RESIPR(ch->unit), work & 0xFFFFFFFF); 495194845Sraj SATA_OUTL(sc, SATA_EDMA_RESOPR(ch->unit), work & 0xFFFFFFFF); 496194845Sraj 497194845Sraj /* Clear any outstanding interrupts */ 498194845Sraj ATA_IDX_OUTL(ch, ATA_SERROR, ATA_IDX_INL(ch, ATA_SERROR)); 499194845Sraj SATA_OUTL(sc, SATA_SATA_FISICR(ch->unit), 0); 500194845Sraj SATA_OUTL(sc, SATA_EDMA_IECR(ch->unit), 0); 501194845Sraj SATA_OUTL(sc, SATA_ICR, 502194845Sraj ~(SATA_ICR_DEV(ch->unit) | SATA_ICR_DMADONE(ch->unit))); 503194845Sraj 504194845Sraj /* Umask channel interrupts */ 505194845Sraj SATA_OUTL(sc, SATA_EDMA_IEMR(ch->unit), 0xFFFFFFFF); 506194845Sraj SATA_OUTL(sc, SATA_MIMR, SATA_INL(sc, SATA_MIMR) | 507194845Sraj SATA_MICR_DONE(ch->unit) | SATA_MICR_DMADONE(ch->unit) | 508194845Sraj SATA_MICR_ERR(ch->unit)); 509194845Sraj 510194845Sraj ch->attached = 1; 511194845Sraj 512194845Sraj return (ata_attach(dev)); 513194845Sraj} 514194845Sraj 515194845Srajstatic int 516194845Srajsata_channel_detach(device_t dev) 517194845Sraj{ 518194845Sraj struct sata_softc *sc; 519194845Sraj struct ata_channel *ch; 520194845Sraj int error; 521194845Sraj 522194845Sraj sc = device_get_softc(device_get_parent(dev)); 523194845Sraj ch = device_get_softc(dev); 524194845Sraj 525194845Sraj if (!ch->attached) 526194845Sraj return (0); 527194845Sraj 528194845Sraj /* Turn off EDMA engine */ 529194845Sraj sata_edma_ctrl(dev, 0); 530194845Sraj 531194845Sraj /* Mask chanel interrupts */ 532194845Sraj SATA_OUTL(sc, SATA_EDMA_IEMR(ch->unit), 0); 533194845Sraj SATA_OUTL(sc, SATA_MIMR, SATA_INL(sc, SATA_MIMR) & ~( 534194845Sraj SATA_MICR_DONE(ch->unit) | SATA_MICR_DMADONE(ch->unit) | 535194845Sraj SATA_MICR_ERR(ch->unit))); 536194845Sraj 537194845Sraj error = ata_detach(dev); 538194845Sraj ata_dmafini(dev); 539194845Sraj 540194845Sraj ch->attached = 0; 541194845Sraj 542194845Sraj return (error); 543194845Sraj} 544194845Sraj 545194845Srajstatic int 546194845Srajsata_channel_begin_transaction(struct ata_request *request) 547194845Sraj{ 548194845Sraj struct sata_softc *sc; 549194845Sraj struct ata_channel *ch; 550194845Sraj struct sata_crqb *crqb; 551194845Sraj uint32_t req_in; 552194845Sraj int error, slot; 553194845Sraj 554198717Smav sc = device_get_softc(device_get_parent(request->parent)); 555194845Sraj ch = device_get_softc(request->parent); 556194845Sraj 557194845Sraj mtx_assert(&ch->state_mtx, MA_OWNED); 558194845Sraj 559194845Sraj /* Only DMA R/W goes through the EDMA machine. */ 560194845Sraj if (request->u.ata.command != ATA_READ_DMA && 561198717Smav request->u.ata.command != ATA_WRITE_DMA && 562198717Smav request->u.ata.command != ATA_READ_DMA48 && 563198717Smav request->u.ata.command != ATA_WRITE_DMA48) { 564194845Sraj 565194845Sraj /* Disable EDMA before accessing legacy registers */ 566194845Sraj if (sata_edma_is_running(request->parent)) { 567194845Sraj error = sata_edma_ctrl(request->parent, 0); 568194845Sraj if (error) { 569194845Sraj request->result = error; 570194845Sraj return (ATA_OP_FINISHED); 571194845Sraj } 572194845Sraj } 573194845Sraj 574194845Sraj return (ata_begin_transaction(request)); 575194845Sraj } 576194845Sraj 577194845Sraj /* Prepare data for DMA */ 578194845Sraj if ((error = ch->dma.load(request, NULL, NULL))) { 579198717Smav device_printf(request->parent, "setting up DMA failed!\n"); 580194845Sraj request->result = error; 581194845Sraj return ATA_OP_FINISHED; 582194845Sraj } 583194845Sraj 584194845Sraj /* Get next free queue slot */ 585194845Sraj req_in = SATA_INL(sc, SATA_EDMA_REQIPR(ch->unit)); 586194845Sraj slot = (req_in & sc->sc_edma_reqis_mask) >> SATA_EDMA_REQIS_OFS; 587194845Sraj crqb = (struct sata_crqb *)(ch->dma.work + 588194845Sraj (slot << SATA_EDMA_REQIS_OFS)); 589194845Sraj 590194845Sraj /* Fill in request */ 591194845Sraj bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 592194845Sraj BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 593194845Sraj 594194845Sraj crqb->crqb_prdlo = htole32((uint64_t)request->dma->sg_bus & 0xFFFFFFFF); 595194845Sraj crqb->crqb_prdhi = htole32((uint64_t)request->dma->sg_bus >> 32); 596194845Sraj crqb->crqb_flags = htole32((request->flags & ATA_R_READ ? 0x01 : 0x00) | 597194845Sraj (request->tag << 1)); 598194845Sraj 599194845Sraj crqb->crqb_ata_command = request->u.ata.command; 600194845Sraj crqb->crqb_ata_feature = request->u.ata.feature; 601194845Sraj crqb->crqb_ata_lba_low = request->u.ata.lba; 602194845Sraj crqb->crqb_ata_lba_mid = request->u.ata.lba >> 8; 603194845Sraj crqb->crqb_ata_lba_high = request->u.ata.lba >> 16; 604194845Sraj crqb->crqb_ata_device = ((request->u.ata.lba >> 24) & 0x0F) | (1 << 6); 605206053Smav crqb->crqb_ata_lba_low_p = request->u.ata.lba >> 24; 606206053Smav crqb->crqb_ata_lba_mid_p = request->u.ata.lba >> 32; 607206053Smav crqb->crqb_ata_lba_high_p = request->u.ata.lba >> 40; 608206053Smav crqb->crqb_ata_feature_p = request->u.ata.feature >> 8; 609194845Sraj crqb->crqb_ata_count = request->u.ata.count; 610206054Smav crqb->crqb_ata_count_p = request->u.ata.count >> 8; 611194845Sraj 612194845Sraj bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 613194845Sraj BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 614194845Sraj 615194845Sraj /* Enable EDMA if disabled */ 616194845Sraj if (!sata_edma_is_running(request->parent)) { 617194845Sraj error = sata_edma_ctrl(request->parent, 1); 618194845Sraj if (error) { 619194845Sraj ch->dma.unload(request); 620194845Sraj request->result = error; 621194845Sraj return (ATA_OP_FINISHED); 622194845Sraj } 623194845Sraj } 624194845Sraj 625194845Sraj /* Tell EDMA about new request */ 626194845Sraj req_in = (req_in & ~sc->sc_edma_reqis_mask) | (((slot + 1) << 627194845Sraj SATA_EDMA_REQIS_OFS) & sc->sc_edma_reqis_mask); 628194845Sraj 629194845Sraj SATA_OUTL(sc, SATA_EDMA_REQIPR(ch->unit), req_in); 630194845Sraj 631194845Sraj return (ATA_OP_CONTINUES); 632194845Sraj} 633194845Sraj 634194845Srajstatic int 635194845Srajsata_channel_end_transaction(struct ata_request *request) 636194845Sraj{ 637194845Sraj struct sata_softc *sc; 638194845Sraj struct ata_channel *ch; 639194845Sraj struct sata_crpb *crpb; 640194845Sraj uint32_t res_in, res_out, icr; 641194845Sraj int slot; 642194845Sraj 643198717Smav sc = device_get_softc(device_get_parent(request->parent)); 644194845Sraj ch = device_get_softc(request->parent); 645194845Sraj 646194845Sraj mtx_assert(&ch->state_mtx, MA_OWNED); 647194845Sraj 648194845Sraj icr = SATA_INL(sc, SATA_ICR); 649194845Sraj if (icr & SATA_ICR_DMADONE(ch->unit)) { 650194845Sraj /* Get current response slot */ 651194845Sraj res_out = SATA_INL(sc, SATA_EDMA_RESOPR(ch->unit)); 652194845Sraj slot = (res_out & sc->sc_edma_resos_mask) >> 653194845Sraj SATA_EDMA_RESOS_OFS; 654194845Sraj crpb = (struct sata_crpb *)(ch->dma.work + 655194845Sraj (sc->sc_edma_qlen * sizeof(struct sata_crqb)) + 656194845Sraj (slot << SATA_EDMA_RESOS_OFS)); 657194845Sraj 658194845Sraj /* Record this request status */ 659194845Sraj bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 660194845Sraj BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 661194845Sraj 662194845Sraj request->status = crpb->crpb_dev_status; 663194845Sraj request->error = 0; 664194845Sraj 665194845Sraj bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 666194845Sraj BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 667194845Sraj 668194845Sraj /* Update response queue pointer */ 669194845Sraj res_out = (res_out & ~sc->sc_edma_resos_mask) | (((slot + 1) << 670194845Sraj SATA_EDMA_RESOS_OFS) & sc->sc_edma_resos_mask); 671194845Sraj 672194845Sraj SATA_OUTL(sc, SATA_EDMA_RESOPR(ch->unit), res_out); 673194845Sraj 674194845Sraj /* Ack DMA interrupt if there is nothing more to do */ 675194845Sraj res_in = SATA_INL(sc, SATA_EDMA_RESIPR(ch->unit)); 676194845Sraj res_in &= sc->sc_edma_resos_mask; 677194845Sraj res_out &= sc->sc_edma_resos_mask; 678194845Sraj 679194845Sraj if (res_in == res_out) 680194845Sraj SATA_OUTL(sc, SATA_ICR, 681194845Sraj ~SATA_ICR_DMADONE(ch->unit)); 682194845Sraj 683194845Sraj /* Update progress */ 684194845Sraj if (!(request->status & ATA_S_ERROR) && 685194845Sraj !(request->flags & ATA_R_TIMEOUT)) 686194845Sraj request->donecount = request->bytecount; 687194845Sraj 688194845Sraj /* Unload DMA data */ 689194845Sraj ch->dma.unload(request); 690194845Sraj 691194845Sraj return(ATA_OP_FINISHED); 692194845Sraj } 693194845Sraj 694194845Sraj /* Legacy ATA interrupt */ 695194845Sraj return (ata_end_transaction(request)); 696194845Sraj} 697194845Sraj 698194845Srajstatic int 699194845Srajsata_channel_status(device_t dev) 700194845Sraj{ 701194845Sraj struct sata_softc *sc; 702194845Sraj struct ata_channel *ch; 703194845Sraj uint32_t icr, iecr; 704194845Sraj 705194845Sraj sc = device_get_softc(device_get_parent(dev)); 706194845Sraj ch = device_get_softc(dev); 707194845Sraj 708194845Sraj icr = SATA_INL(sc, SATA_ICR); 709194845Sraj iecr = SATA_INL(sc, SATA_EDMA_IECR(ch->unit)); 710194845Sraj 711194845Sraj if ((icr & SATA_ICR_DEV(ch->unit)) || iecr) { 712194845Sraj /* Disable EDMA before accessing SATA registers */ 713194845Sraj sata_edma_ctrl(dev, 0); 714214016Smav ata_sata_phy_check_events(dev, -1); 715194845Sraj 716194845Sraj /* Ack device and error interrupt */ 717194845Sraj SATA_OUTL(sc, SATA_ICR, ~SATA_ICR_DEV(ch->unit)); 718194845Sraj SATA_OUTL(sc, SATA_EDMA_IECR(ch->unit), 0); 719194845Sraj } 720194845Sraj 721194845Sraj icr &= SATA_ICR_DEV(ch->unit) | SATA_ICR_DMADONE(ch->unit); 722194845Sraj return (icr); 723194845Sraj} 724194845Sraj 725194845Srajstatic void 726194845Srajsata_channel_reset(device_t dev) 727194845Sraj{ 728194845Sraj struct sata_softc *sc; 729194845Sraj struct ata_channel *ch; 730194845Sraj 731194845Sraj sc = device_get_softc(device_get_parent(dev)); 732194845Sraj ch = device_get_softc(dev); 733194845Sraj 734194845Sraj /* Disable EDMA before using legacy registers */ 735194845Sraj sata_edma_ctrl(dev, 0); 736194845Sraj 737194845Sraj /* Mask all EDMA interrups */ 738194845Sraj SATA_OUTL(sc, SATA_EDMA_IEMR(ch->unit), 0); 739194845Sraj 740194845Sraj /* Reset EDMA */ 741194845Sraj SATA_OUTL(sc, SATA_EDMA_CMD(ch->unit), SATA_EDMA_CMD_RESET); 742194845Sraj DELAY(25); 743194845Sraj SATA_OUTL(sc, SATA_EDMA_CMD(ch->unit), 0); 744194845Sraj 745194845Sraj /* Reset PHY and device */ 746194845Sraj if (ata_sata_phy_reset(dev, -1, 1)) 747194845Sraj ata_generic_reset(dev); 748194845Sraj else 749194845Sraj ch->devices = 0; 750194845Sraj 751194845Sraj /* Clear EDMA errors */ 752194845Sraj SATA_OUTL(sc, SATA_SATA_FISICR(ch->unit), 0); 753194845Sraj SATA_OUTL(sc, SATA_EDMA_IECR(ch->unit), 0); 754194845Sraj 755194845Sraj /* Unmask all EDMA interrups */ 756194845Sraj SATA_OUTL(sc, SATA_EDMA_IEMR(ch->unit), 0xFFFFFFFF); 757194845Sraj} 758194845Sraj 759200171Smavstatic int 760200171Smavsata_channel_setmode(device_t parent, int target, int mode) 761194845Sraj{ 762194845Sraj 763194845Sraj /* Disable EDMA before using legacy registers */ 764194845Sraj sata_edma_ctrl(parent, 0); 765200275Smav return (ata_sata_setmode(parent, target, mode)); 766194845Sraj} 767194845Sraj 768200275Smavstatic int 769200275Smavsata_channel_getrev(device_t parent, int target) 770200275Smav{ 771200275Smav 772200275Smav /* Disable EDMA before using legacy registers */ 773200275Smav sata_edma_ctrl(parent, 0); 774200275Smav return (ata_sata_getrev(parent, target)); 775200275Smav} 776200275Smav 777194845Srajstatic void 778194845Srajsata_channel_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, 779194845Sraj int error) 780194845Sraj{ 781194845Sraj struct ata_dmasetprd_args *args; 782194845Sraj struct sata_prdentry *prd; 783194845Sraj int i; 784194845Sraj 785194845Sraj args = xsc; 786194845Sraj prd = args->dmatab; 787194845Sraj 788194845Sraj if ((args->error = error)) 789194845Sraj return; 790194845Sraj 791194845Sraj for (i = 0; i < nsegs; i++) { 792194845Sraj prd[i].prd_addrlo = htole32(segs[i].ds_addr); 793194845Sraj prd[i].prd_addrhi = htole32((uint64_t)segs[i].ds_addr >> 32); 794194845Sraj prd[i].prd_count = htole32(segs[i].ds_len); 795194845Sraj } 796194845Sraj 797194845Sraj prd[i - 1].prd_count |= htole32(ATA_DMA_EOT); 798194845Sraj KASSERT(nsegs <= ATA_DMA_ENTRIES, ("too many DMA segment entries.\n")); 799194845Sraj args->nsegs = nsegs; 800194845Sraj} 801194845Sraj 802194845Srajstatic int 803194845Srajsata_edma_ctrl(device_t dev, int on) 804194845Sraj{ 805194845Sraj struct sata_softc *sc; 806194845Sraj struct ata_channel *ch; 807194845Sraj int bit, timeout; 808194845Sraj uint32_t reg; 809194845Sraj 810194845Sraj sc = device_get_softc(device_get_parent(dev)); 811194845Sraj ch = device_get_softc(dev); 812194845Sraj bit = on ? SATA_EDMA_CMD_ENABLE : SATA_EDMA_CMD_DISABLE; 813194845Sraj timeout = EDMA_TIMEOUT; 814194845Sraj 815194845Sraj SATA_OUTL(sc, SATA_EDMA_CMD(ch->unit), bit); 816194845Sraj 817194845Sraj while (1) { 818194845Sraj DELAY(1); 819194845Sraj 820194845Sraj reg = SATA_INL(sc, SATA_EDMA_CMD(ch->unit)); 821194845Sraj 822194845Sraj /* Enable bit will be 1 after disable command completion */ 823194845Sraj if (on && (reg & SATA_EDMA_CMD_ENABLE)) 824194845Sraj break; 825194845Sraj 826194845Sraj /* Disable bit will be 0 after disable command completion */ 827194845Sraj if (!on && !(reg & SATA_EDMA_CMD_DISABLE)) 828194845Sraj break; 829194845Sraj 830194845Sraj if (timeout-- <= 0) { 831194845Sraj device_printf(dev, "EDMA command timeout!\n"); 832194845Sraj return (ETIMEDOUT); 833194845Sraj } 834194845Sraj } 835194845Sraj 836194845Sraj return (0); 837194845Sraj} 838194845Sraj 839194845Srajstatic int 840194845Srajsata_edma_is_running(device_t dev) 841194845Sraj{ 842194845Sraj struct sata_softc *sc; 843194845Sraj struct ata_channel *ch; 844194845Sraj 845194845Sraj sc = device_get_softc(device_get_parent(dev)); 846194845Sraj ch = device_get_softc(dev); 847194845Sraj 848194845Sraj return (SATA_INL(sc, SATA_EDMA_CMD(ch->unit)) & SATA_EDMA_CMD_ENABLE); 849194845Sraj} 850194845Sraj 851194845Srajstatic device_method_t sata_channel_methods[] = { 852194845Sraj /* Device interface. */ 853194845Sraj DEVMETHOD(device_probe, sata_channel_probe), 854194845Sraj DEVMETHOD(device_attach, sata_channel_attach), 855194845Sraj DEVMETHOD(device_detach, sata_channel_detach), 856194845Sraj DEVMETHOD(device_shutdown, bus_generic_shutdown), 857194845Sraj DEVMETHOD(device_suspend, ata_suspend), 858194845Sraj DEVMETHOD(device_resume, ata_resume), 859194845Sraj 860194845Sraj /* ATA channel interface */ 861194845Sraj DEVMETHOD(ata_reset, sata_channel_reset), 862194845Sraj DEVMETHOD(ata_setmode, sata_channel_setmode), 863200275Smav DEVMETHOD(ata_getrev, sata_channel_getrev), 864194845Sraj { 0, 0 } 865194845Sraj}; 866194845Sraj 867194845Srajdriver_t sata_channel_driver = { 868194845Sraj "ata", 869194845Sraj sata_channel_methods, 870194845Sraj sizeof(struct ata_channel), 871194845Sraj}; 872194845Sraj 873194845SrajDRIVER_MODULE(ata, sata, sata_channel_driver, ata_devclass, 0, 0); 874