ata-ahci.c revision 232380
1/*- 2 * Copyright (c) 1998 - 2008 S��ren Schmidt <sos@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer, 10 * without modification, immediately at the beginning of the file. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/sys/dev/ata/chipsets/ata-ahci.c 232380 2012-03-02 08:49:07Z mav $"); 29 30#include "opt_ata.h" 31#include <sys/param.h> 32#include <sys/module.h> 33#include <sys/systm.h> 34#include <sys/kernel.h> 35#include <sys/ata.h> 36#include <sys/bus.h> 37#include <sys/endian.h> 38#include <sys/malloc.h> 39#include <sys/lock.h> 40#include <sys/mutex.h> 41#include <sys/sema.h> 42#include <sys/taskqueue.h> 43#include <vm/uma.h> 44#include <machine/stdarg.h> 45#include <machine/resource.h> 46#include <machine/bus.h> 47#include <sys/rman.h> 48#include <dev/pci/pcivar.h> 49#include <dev/pci/pcireg.h> 50#include <dev/ata/ata-all.h> 51#include <dev/ata/ata-pci.h> 52#include <ata_if.h> 53 54/* local prototypes */ 55static int ata_ahci_ch_attach(device_t dev); 56static int ata_ahci_ch_detach(device_t dev); 57static int ata_ahci_ch_suspend(device_t dev); 58static int ata_ahci_ch_resume(device_t dev); 59static int ata_ahci_ctlr_reset(device_t dev); 60static void ata_ahci_reset(device_t dev); 61static int ata_ahci_suspend(device_t dev); 62static int ata_ahci_status(device_t dev); 63static int ata_ahci_begin_transaction(struct ata_request *request); 64static int ata_ahci_end_transaction(struct ata_request *request); 65static int ata_ahci_pm_read(device_t dev, int port, int reg, u_int32_t *result); 66static int ata_ahci_pm_write(device_t dev, int port, int reg, u_int32_t result); 67static int ata_ahci_hardreset(device_t dev, int port, uint32_t *signature); 68static u_int32_t ata_ahci_softreset(device_t dev, int port); 69static void ata_ahci_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error); 70static int ata_ahci_setup_fis(struct ata_ahci_cmd_tab *ctp, struct ata_request *equest); 71static void ata_ahci_dmainit(device_t dev); 72static void ata_ahci_start(device_t dev); 73static void ata_ahci_stop(device_t dev); 74static void ata_ahci_clo(device_t dev); 75static void ata_ahci_start_fr(device_t dev); 76static void ata_ahci_stop_fr(device_t dev); 77 78/* 79 * AHCI v1.x compliant SATA chipset support functions 80 */ 81static int 82ata_ahci_probe(device_t dev) 83{ 84 struct ata_pci_controller *ctlr = device_get_softc(dev); 85 char buffer[64]; 86 87 /* is this a possible AHCI candidate ? */ 88 if (pci_get_class(dev) != PCIC_STORAGE || 89 pci_get_subclass(dev) != PCIS_STORAGE_SATA) 90 return (ENXIO); 91 92 /* is this PCI device flagged as an AHCI compliant chip ? */ 93 if (pci_get_progif(dev) != PCIP_STORAGE_SATA_AHCI_1_0) 94 return (ENXIO); 95 96 if (bootverbose) 97 sprintf(buffer, "%s (ID=%08x) AHCI controller", 98 ata_pcivendor2str(dev), pci_get_devid(dev)); 99 else 100 sprintf(buffer, "%s AHCI controller", ata_pcivendor2str(dev)); 101 device_set_desc_copy(dev, buffer); 102 ctlr->chipinit = ata_ahci_chipinit; 103 return (BUS_PROBE_GENERIC); 104} 105 106static int 107ata_ahci_ata_probe(device_t dev) 108{ 109 struct ata_pci_controller *ctlr = device_get_softc(dev); 110 111 if ((intptr_t)device_get_ivars(dev) >= 0) 112 return (ENXIO); 113 device_set_desc_copy(dev, "AHCI SATA controller"); 114 ctlr->chipinit = ata_ahci_chipinit; 115 return (BUS_PROBE_GENERIC); 116} 117 118static int 119ata_ahci_ata_attach(device_t dev) 120{ 121 struct ata_pci_controller *ctlr = device_get_softc(dev); 122 device_t child; 123 int unit; 124 125 /* do chipset specific setups only needed once */ 126 ctlr->legacy = 0; 127 ctlr->ichannels = -1; 128 ctlr->ch_attach = ata_pci_ch_attach; 129 ctlr->ch_detach = ata_pci_ch_detach; 130 ctlr->dev = dev; 131 if (ctlr->chipinit(dev)) 132 return ENXIO; 133 /* attach all channels on this controller */ 134 for (unit = 0; unit < ctlr->channels; unit++) { 135 if ((ctlr->ichannels & (1 << unit)) == 0) 136 continue; 137 child = device_add_child(dev, "ata", 138 ((unit == 0 || unit == 1) && ctlr->legacy) ? 139 unit : devclass_find_free_unit(ata_devclass, 2)); 140 if (child == NULL) 141 device_printf(dev, "failed to add ata child device\n"); 142 else 143 device_set_ivars(child, (void *)(intptr_t)unit); 144 } 145 bus_generic_attach(dev); 146 return 0; 147} 148 149int 150ata_ahci_chipinit(device_t dev) 151{ 152 struct ata_pci_controller *ctlr = device_get_softc(dev); 153 int error, speed; 154 u_int32_t caps, version; 155 156 /* if we have a memory BAR(5) we are likely on an AHCI part */ 157 ctlr->r_type2 = SYS_RES_MEMORY; 158 ctlr->r_rid2 = PCIR_BAR(5); 159 if (!(ctlr->r_res2 = bus_alloc_resource_any(dev, ctlr->r_type2, 160 &ctlr->r_rid2, RF_ACTIVE))) 161 return ENXIO; 162 163 /* setup interrupt delivery if not done allready by a vendor driver */ 164 if (!ctlr->r_irq) { 165 if (ata_setup_interrupt(dev, ata_generic_intr)) { 166 bus_release_resource(dev, ctlr->r_type2, ctlr->r_rid2, ctlr->r_res2); 167 return ENXIO; 168 } 169 } 170 else 171 device_printf(dev, "AHCI called from vendor specific driver\n"); 172 173 /* reset controller */ 174 if ((error = ata_ahci_ctlr_reset(dev)) != 0) { 175 bus_release_resource(dev, ctlr->r_type2, ctlr->r_rid2, ctlr->r_res2); 176 return (error); 177 }; 178 179 /* get the number of HW channels */ 180 ctlr->ichannels = ATA_INL(ctlr->r_res2, ATA_AHCI_PI); 181 ctlr->channels = MAX(flsl(ctlr->ichannels), 182 (ATA_INL(ctlr->r_res2, ATA_AHCI_CAP) & ATA_AHCI_CAP_NPMASK) + 1); 183 if (pci_get_devid(dev) == ATA_M88SE6111) 184 ctlr->channels = 1; 185 else if (pci_get_devid(dev) == ATA_M88SE6121) 186 ctlr->channels = 2; 187 else if (pci_get_devid(dev) == ATA_M88SE6141 || 188 pci_get_devid(dev) == ATA_M88SE6145) 189 ctlr->channels = 4; 190 191 ctlr->reset = ata_ahci_reset; 192 ctlr->ch_attach = ata_ahci_ch_attach; 193 ctlr->ch_detach = ata_ahci_ch_detach; 194 ctlr->ch_suspend = ata_ahci_ch_suspend; 195 ctlr->ch_resume = ata_ahci_ch_resume; 196 ctlr->setmode = ata_sata_setmode; 197 ctlr->getrev = ata_sata_getrev; 198 ctlr->suspend = ata_ahci_suspend; 199 ctlr->resume = ata_ahci_ctlr_reset; 200 201 /* announce we support the HW */ 202 version = ATA_INL(ctlr->r_res2, ATA_AHCI_VS); 203 caps = ATA_INL(ctlr->r_res2, ATA_AHCI_CAP); 204 speed = (caps & ATA_AHCI_CAP_ISS) >> ATA_AHCI_CAP_ISS_SHIFT; 205 device_printf(dev, 206 "AHCI v%x.%02x controller with %d %sGbps ports, PM %s\n", 207 ((version >> 20) & 0xf0) + ((version >> 16) & 0x0f), 208 ((version >> 4) & 0xf0) + (version & 0x0f), 209 (caps & ATA_AHCI_CAP_NPMASK) + 1, 210 ((speed == 1) ? "1.5":((speed == 2) ? "3": 211 ((speed == 3) ? "6":"?"))), 212 (caps & ATA_AHCI_CAP_SPM) ? 213 "supported" : "not supported"); 214 if (bootverbose) { 215 device_printf(dev, "Caps:%s%s%s%s%s%s%s%s %sGbps", 216 (caps & ATA_AHCI_CAP_64BIT) ? " 64bit":"", 217 (caps & ATA_AHCI_CAP_SNCQ) ? " NCQ":"", 218 (caps & ATA_AHCI_CAP_SSNTF) ? " SNTF":"", 219 (caps & ATA_AHCI_CAP_SMPS) ? " MPS":"", 220 (caps & ATA_AHCI_CAP_SSS) ? " SS":"", 221 (caps & ATA_AHCI_CAP_SALP) ? " ALP":"", 222 (caps & ATA_AHCI_CAP_SAL) ? " AL":"", 223 (caps & ATA_AHCI_CAP_SCLO) ? " CLO":"", 224 ((speed == 1) ? "1.5":((speed == 2) ? "3": 225 ((speed == 3) ? "6":"?")))); 226 printf("%s%s%s%s%s%s %dcmd%s%s%s %dports\n", 227 (caps & ATA_AHCI_CAP_SAM) ? " AM":"", 228 (caps & ATA_AHCI_CAP_SPM) ? " PM":"", 229 (caps & ATA_AHCI_CAP_FBSS) ? " FBS":"", 230 (caps & ATA_AHCI_CAP_PMD) ? " PMD":"", 231 (caps & ATA_AHCI_CAP_SSC) ? " SSC":"", 232 (caps & ATA_AHCI_CAP_PSC) ? " PSC":"", 233 ((caps & ATA_AHCI_CAP_NCS) >> ATA_AHCI_CAP_NCS_SHIFT) + 1, 234 (caps & ATA_AHCI_CAP_CCCS) ? " CCC":"", 235 (caps & ATA_AHCI_CAP_EMS) ? " EM":"", 236 (caps & ATA_AHCI_CAP_SXS) ? " eSATA":"", 237 (caps & ATA_AHCI_CAP_NPMASK) + 1); 238 } 239 return 0; 240} 241 242static int 243ata_ahci_ctlr_reset(device_t dev) 244{ 245 struct ata_pci_controller *ctlr = device_get_softc(dev); 246 int timeout; 247 248 /* enable AHCI mode */ 249 ATA_OUTL(ctlr->r_res2, ATA_AHCI_GHC, ATA_AHCI_GHC_AE); 250 251 /* reset AHCI controller */ 252 ATA_OUTL(ctlr->r_res2, ATA_AHCI_GHC, ATA_AHCI_GHC_AE|ATA_AHCI_GHC_HR); 253 for (timeout = 1000; timeout > 0; timeout--) { 254 DELAY(1000); 255 if ((ATA_INL(ctlr->r_res2, ATA_AHCI_GHC) & ATA_AHCI_GHC_HR) == 0) 256 break; 257 } 258 if (timeout == 0) { 259 device_printf(dev, "AHCI controller reset failure\n"); 260 return ENXIO; 261 } 262 263 /* reenable AHCI mode */ 264 ATA_OUTL(ctlr->r_res2, ATA_AHCI_GHC, ATA_AHCI_GHC_AE); 265 266 /* clear interrupts */ 267 ATA_OUTL(ctlr->r_res2, ATA_AHCI_IS, ATA_INL(ctlr->r_res2, ATA_AHCI_IS)); 268 269 /* enable AHCI interrupts */ 270 ATA_OUTL(ctlr->r_res2, ATA_AHCI_GHC, 271 ATA_INL(ctlr->r_res2, ATA_AHCI_GHC) | ATA_AHCI_GHC_IE); 272 273 return 0; 274} 275 276static int 277ata_ahci_suspend(device_t dev) 278{ 279 struct ata_pci_controller *ctlr = device_get_softc(dev); 280 281 /* disable interupts so the state change(s) doesn't trigger */ 282 ATA_OUTL(ctlr->r_res2, ATA_AHCI_GHC, 283 ATA_INL(ctlr->r_res2, ATA_AHCI_GHC) & (~ATA_AHCI_GHC_IE)); 284 return 0; 285} 286 287static int 288ata_ahci_ch_attach(device_t dev) 289{ 290 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 291 struct ata_channel *ch = device_get_softc(dev); 292 int offset = ch->unit << 7; 293 294 ata_ahci_dmainit(dev); 295 296 /* set the SATA resources */ 297 ch->r_io[ATA_SSTATUS].res = ctlr->r_res2; 298 ch->r_io[ATA_SSTATUS].offset = ATA_AHCI_P_SSTS + offset; 299 ch->r_io[ATA_SERROR].res = ctlr->r_res2; 300 ch->r_io[ATA_SERROR].offset = ATA_AHCI_P_SERR + offset; 301 ch->r_io[ATA_SCONTROL].res = ctlr->r_res2; 302 ch->r_io[ATA_SCONTROL].offset = ATA_AHCI_P_SCTL + offset; 303 ch->r_io[ATA_SACTIVE].res = ctlr->r_res2; 304 ch->r_io[ATA_SACTIVE].offset = ATA_AHCI_P_SACT + offset; 305 306 ch->hw.status = ata_ahci_status; 307 ch->hw.begin_transaction = ata_ahci_begin_transaction; 308 ch->hw.end_transaction = ata_ahci_end_transaction; 309 ch->hw.command = NULL; /* not used here */ 310 ch->hw.softreset = ata_ahci_softreset; 311 ch->hw.pm_read = ata_ahci_pm_read; 312 ch->hw.pm_write = ata_ahci_pm_write; 313 ch->flags |= ATA_NO_SLAVE; 314 ch->flags |= ATA_SATA; 315 316 ata_ahci_ch_resume(dev); 317 return 0; 318} 319 320static int 321ata_ahci_ch_detach(device_t dev) 322{ 323 struct ata_channel *ch = device_get_softc(dev); 324 325 if (ch->dma.work_tag && ch->dma.work_map) 326 bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 327 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 328 ata_ahci_ch_suspend(dev); 329 ata_dmafini(dev); 330 return (0); 331} 332 333static int 334ata_ahci_ch_suspend(device_t dev) 335{ 336 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 337 struct ata_channel *ch = device_get_softc(dev); 338 int offset = ch->unit << 7; 339 340 /* Disable port interrupts. */ 341 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IE + offset, 0); 342 /* Reset command register. */ 343 ata_ahci_stop(dev); 344 ata_ahci_stop_fr(dev); 345 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 0); 346 347 /* Allow everything including partial and slumber modes. */ 348 ATA_IDX_OUTL(ch, ATA_SCONTROL, 0); 349 /* Request slumber mode transition and give some time to get there. */ 350 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, ATA_AHCI_P_CMD_SLUMBER); 351 DELAY(100); 352 /* Disable PHY. */ 353 ATA_IDX_OUTL(ch, ATA_SCONTROL, ATA_SC_DET_DISABLE); 354 355 return (0); 356} 357 358static int 359ata_ahci_ch_resume(device_t dev) 360{ 361 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 362 struct ata_channel *ch = device_get_softc(dev); 363 uint64_t work; 364 int offset = ch->unit << 7; 365 366 /* Disable port interrupts */ 367 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IE + offset, 0); 368 369 /* setup work areas */ 370 work = ch->dma.work_bus + ATA_AHCI_CL_OFFSET; 371 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CLB + offset, work & 0xffffffff); 372 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CLBU + offset, work >> 32); 373 374 work = ch->dma.work_bus + ATA_AHCI_FB_OFFSET; 375 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_FB + offset, work & 0xffffffff); 376 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_FBU + offset, work >> 32); 377 378 /* activate the channel and power/spin up device */ 379 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 380 (ATA_AHCI_P_CMD_ACTIVE | ATA_AHCI_P_CMD_POD | ATA_AHCI_P_CMD_SUD | 381 ((ch->pm_level > 1) ? ATA_AHCI_P_CMD_ALPE : 0) | 382 ((ch->pm_level > 2) ? ATA_AHCI_P_CMD_ASP : 0 ))); 383 ata_ahci_start_fr(dev); 384 ata_ahci_start(dev); 385 386 return (0); 387} 388 389static int 390ata_ahci_status(device_t dev) 391{ 392 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 393 struct ata_channel *ch = device_get_softc(dev); 394 u_int32_t action = ATA_INL(ctlr->r_res2, ATA_AHCI_IS); 395 int offset = ch->unit << 7; 396 397#define ATA_AHCI_STATBITS \ 398 (ATA_AHCI_P_IX_IF|ATA_AHCI_P_IX_HBD|ATA_AHCI_P_IX_HBF|ATA_AHCI_P_IX_TFE) 399 400 if (action & (1 << ch->unit)) { 401 u_int32_t istatus = ATA_INL(ctlr->r_res2, ATA_AHCI_P_IS + offset); 402 u_int32_t cstatus = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CI + offset); 403 404 /* clear interrupt(s) */ 405 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IS + offset, istatus); 406 ATA_OUTL(ctlr->r_res2, ATA_AHCI_IS, 1 << ch->unit); 407 408 /* do we have any PHY events ? */ 409 if (istatus & (ATA_AHCI_P_IX_PRC | ATA_AHCI_P_IX_PC)) 410 ata_sata_phy_check_events(dev, -1); 411 412 /* do we have a potentially hanging engine to take care of? */ 413 /* XXX SOS what todo on NCQ */ 414 if ((istatus & ATA_AHCI_STATBITS) && (cstatus & 1)) { 415 416 u_int32_t cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset); 417 int timeout = 0; 418 419 /* kill off all activity on this channel */ 420 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 421 cmd & ~(ATA_AHCI_P_CMD_FRE | ATA_AHCI_P_CMD_ST)); 422 423 /* XXX SOS this is not entirely wrong */ 424 do { 425 DELAY(1000); 426 if (timeout++ > 1000) { 427 device_printf(dev, "stopping AHCI engine failed\n"); 428 break; 429 } 430 } while (ATA_INL(ctlr->r_res2, 431 ATA_AHCI_P_CMD + offset) & ATA_AHCI_P_CMD_CR); 432 433 /* start operations on this channel */ 434 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 435 cmd | (ATA_AHCI_P_CMD_FRE | ATA_AHCI_P_CMD_ST)); 436 437 return 1; 438 } 439 else 440 /* XXX SOS what todo on NCQ */ 441 return (!(cstatus & 1)); 442 } 443 return 0; 444} 445 446/* must be called with ATA channel locked and state_mtx held */ 447static int 448ata_ahci_begin_transaction(struct ata_request *request) 449{ 450 struct ata_pci_controller *ctlr=device_get_softc(device_get_parent(request->parent)); 451 struct ata_channel *ch = device_get_softc(request->parent); 452 struct ata_ahci_cmd_tab *ctp; 453 struct ata_ahci_cmd_list *clp; 454 int offset = ch->unit << 7; 455 int port = request->unit & 0x0f; 456 int entries = 0; 457 int fis_size; 458 459 /* get a piece of the workspace for this request */ 460 ctp = (struct ata_ahci_cmd_tab *) 461 (ch->dma.work + ATA_AHCI_CT_OFFSET); 462 463 /* setup the FIS for this request */ 464 if (!(fis_size = ata_ahci_setup_fis(ctp, request))) { 465 device_printf(request->parent, "setting up SATA FIS failed\n"); 466 request->result = EIO; 467 return ATA_OP_FINISHED; 468 } 469 470 /* if request moves data setup and load SG list */ 471 if (request->flags & (ATA_R_READ | ATA_R_WRITE)) { 472 if (ch->dma.load(request, ctp->prd_tab, &entries)) { 473 device_printf(request->parent, "setting up DMA failed\n"); 474 request->result = EIO; 475 return ATA_OP_FINISHED; 476 } 477 } 478 479 /* setup the command list entry */ 480 clp = (struct ata_ahci_cmd_list *) 481 (ch->dma.work + ATA_AHCI_CL_OFFSET); 482 483 clp->prd_length = entries; 484 clp->cmd_flags = (request->flags & ATA_R_WRITE ? ATA_AHCI_CMD_WRITE : 0) | 485 (request->flags & ATA_R_ATAPI ? 486 (ATA_AHCI_CMD_ATAPI | ATA_AHCI_CMD_PREFETCH) : 0) | 487 (fis_size / sizeof(u_int32_t)) | 488 (port << 12); 489 clp->bytecount = 0; 490 clp->cmd_table_phys = htole64(ch->dma.work_bus + ATA_AHCI_CT_OFFSET); 491 492 /* set command type bit */ 493 if (request->flags & ATA_R_ATAPI) 494 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 495 ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset) | 496 ATA_AHCI_P_CMD_ATAPI); 497 else 498 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 499 ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset) & 500 ~ATA_AHCI_P_CMD_ATAPI); 501 502 bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 503 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 504 505 /* issue command to controller */ 506 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CI + offset, 1); 507 508 if (!(request->flags & ATA_R_ATAPI)) { 509 /* device reset doesn't interrupt */ 510 if (request->u.ata.command == ATA_DEVICE_RESET) { 511 u_int32_t tf_data; 512 int timeout = 1000000; 513 514 do { 515 DELAY(10); 516 tf_data = ATA_INL(ctlr->r_res2, ATA_AHCI_P_TFD + (ch->unit<<7)); 517 } while ((tf_data & ATA_S_BUSY) && timeout--); 518 if (bootverbose) 519 device_printf(ch->dev, "device_reset timeout=%dus\n", 520 (1000000-timeout)*10); 521 request->status = tf_data; 522 if (request->status & ATA_S_ERROR) 523 request->error = tf_data >> 8; 524 return ATA_OP_FINISHED; 525 } 526 } 527 528 /* start the timeout */ 529 callout_reset(&request->callout, request->timeout * hz, 530 (timeout_t*)ata_timeout, request); 531 return ATA_OP_CONTINUES; 532} 533 534/* must be called with ATA channel locked and state_mtx held */ 535static int 536ata_ahci_end_transaction(struct ata_request *request) 537{ 538 struct ata_pci_controller *ctlr=device_get_softc(device_get_parent(request->parent)); 539 struct ata_channel *ch = device_get_softc(request->parent); 540 struct ata_ahci_cmd_list *clp; 541 u_int32_t tf_data; 542 int offset = ch->unit << 7; 543 544 /* kill the timeout */ 545 callout_stop(&request->callout); 546 547 bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 548 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 549 550 /* get status */ 551 tf_data = ATA_INL(ctlr->r_res2, ATA_AHCI_P_TFD + offset); 552 request->status = tf_data; 553 554 /* if error status get details */ 555 if (request->status & ATA_S_ERROR) 556 request->error = tf_data >> 8; 557 558 /* on control commands read back registers to the request struct */ 559 if (request->flags & ATA_R_CONTROL) { 560 u_int8_t *fis = ch->dma.work + ATA_AHCI_FB_OFFSET + 0x40; 561 562 request->u.ata.count = fis[12] | ((u_int16_t)fis[13] << 8); 563 request->u.ata.lba = fis[4] | ((u_int64_t)fis[5] << 8) | 564 ((u_int64_t)fis[6] << 16); 565 if (request->flags & ATA_R_48BIT) 566 request->u.ata.lba |= ((u_int64_t)fis[8] << 24) | 567 ((u_int64_t)fis[9] << 32) | 568 ((u_int64_t)fis[10] << 40); 569 else 570 request->u.ata.lba |= ((u_int64_t)(fis[7] & 0x0f) << 24); 571 } 572 573 /* record how much data we actually moved */ 574 clp = (struct ata_ahci_cmd_list *) 575 (ch->dma.work + ATA_AHCI_CL_OFFSET); 576 request->donecount = le32toh(clp->bytecount); 577 578 /* release SG list etc */ 579 ch->dma.unload(request); 580 581 return ATA_OP_FINISHED; 582} 583 584static int 585ata_ahci_issue_cmd(device_t dev, u_int16_t flags, int timeout) 586{ 587 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 588 struct ata_channel *ch = device_get_softc(dev); 589 struct ata_ahci_cmd_list *clp = 590 (struct ata_ahci_cmd_list *)(ch->dma.work + ATA_AHCI_CL_OFFSET); 591 struct ata_ahci_cmd_tab *ctp = 592 (struct ata_ahci_cmd_tab *)(ch->dma.work + ATA_AHCI_CT_OFFSET); 593 u_int32_t status = 0; 594 int offset = ch->unit << 7; 595 int port = (ctp->cfis[1] & 0x0f); 596 int count; 597 598 clp->prd_length = 0; 599 clp->cmd_flags = (20 / sizeof(u_int32_t)) | flags | (port << 12); 600 clp->bytecount = 0; 601 clp->cmd_table_phys = htole64(ch->dma.work_bus + ATA_AHCI_CT_OFFSET); 602 603 bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 604 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 605 606 /* issue command to controller */ 607 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CI + offset, 1); 608 609 /* poll for command finished */ 610 for (count = 0; count < timeout; count++) { 611 DELAY(1000); 612 if (!((status = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CI + offset)) & 1)) 613 break; 614 } 615 616 bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 617 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 618 619 /* clear interrupts */ 620 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IS + offset, 621 ATA_INL(ctlr->r_res2, ATA_AHCI_P_IS + offset)); 622 623 if (timeout && (count >= timeout)) { 624 if (bootverbose) { 625 device_printf(dev, "ahci_issue_cmd timeout: %d of %dms, status=%08x\n", 626 count, timeout, status); 627 } 628 return EIO; 629 } 630 631 return 0; 632} 633 634static int 635ata_ahci_pm_read(device_t dev, int port, int reg, u_int32_t *result) 636{ 637 struct ata_channel *ch = device_get_softc(dev); 638 struct ata_ahci_cmd_tab *ctp = 639 (struct ata_ahci_cmd_tab *)(ch->dma.work + ATA_AHCI_CT_OFFSET); 640 u_int8_t *fis = ch->dma.work + ATA_AHCI_FB_OFFSET + 0x40; 641 642 if (port < 0) { 643 *result = ATA_IDX_INL(ch, reg); 644 return (0); 645 } 646 if (port < ATA_PM) { 647 switch (reg) { 648 case ATA_SSTATUS: 649 reg = 0; 650 break; 651 case ATA_SERROR: 652 reg = 1; 653 break; 654 case ATA_SCONTROL: 655 reg = 2; 656 break; 657 default: 658 return (EINVAL); 659 } 660 } 661 bzero(ctp->cfis, 64); 662 ctp->cfis[0] = 0x27; /* host to device */ 663 ctp->cfis[1] = 0x8f; /* command FIS to PM port */ 664 ctp->cfis[2] = ATA_READ_PM; 665 ctp->cfis[3] = reg; 666 ctp->cfis[7] = port | ATA_D_LBA; 667 ctp->cfis[15] = ATA_A_4BIT; 668 669 if (ata_ahci_issue_cmd(dev, 0, 10)) { 670 device_printf(dev, "error reading PM port\n"); 671 return EIO; 672 } 673 674 *result = fis[12] | (fis[4] << 8) | (fis[5] << 16) | (fis[6] << 24); 675 return 0; 676} 677 678static int 679ata_ahci_pm_write(device_t dev, int port, int reg, u_int32_t value) 680{ 681 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 682 struct ata_channel *ch = device_get_softc(dev); 683 struct ata_ahci_cmd_tab *ctp = 684 (struct ata_ahci_cmd_tab *)(ch->dma.work + ATA_AHCI_CT_OFFSET); 685 int offset = ch->unit << 7; 686 687 if (port < 0) { 688 ATA_IDX_OUTL(ch, reg, value); 689 return (0); 690 } 691 if (port < ATA_PM) { 692 switch (reg) { 693 case ATA_SSTATUS: 694 reg = 0; 695 break; 696 case ATA_SERROR: 697 reg = 1; 698 break; 699 case ATA_SCONTROL: 700 reg = 2; 701 break; 702 default: 703 return (EINVAL); 704 } 705 } 706 bzero(ctp->cfis, 64); 707 ctp->cfis[0] = 0x27; /* host to device */ 708 ctp->cfis[1] = 0x8f; /* command FIS to PM port */ 709 ctp->cfis[2] = ATA_WRITE_PM; 710 ctp->cfis[3] = reg; 711 ctp->cfis[7] = port | ATA_D_LBA; 712 ctp->cfis[12] = value & 0xff; 713 ctp->cfis[4] = (value >> 8) & 0xff; 714 ctp->cfis[5] = (value >> 16) & 0xff; 715 ctp->cfis[6] = (value >> 24) & 0xff; 716 ctp->cfis[15] = ATA_A_4BIT; 717 718 if (ata_ahci_issue_cmd(dev, 0, 100)) { 719 device_printf(dev, "error writing PM port\n"); 720 return ATA_E_ABORT; 721 } 722 723 return (ATA_INL(ctlr->r_res2, ATA_AHCI_P_TFD + offset) >> 8) & 0xff; 724} 725 726static void 727ata_ahci_stop(device_t dev) 728{ 729 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 730 struct ata_channel *ch = device_get_softc(dev); 731 u_int32_t cmd; 732 int offset = ch->unit << 7; 733 int timeout; 734 735 /* kill off all activity on this channel */ 736 cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset); 737 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 738 cmd & ~ATA_AHCI_P_CMD_ST); 739 740 /* XXX SOS this is not entirely wrong */ 741 timeout = 0; 742 do { 743 DELAY(1000); 744 if (timeout++ > 1000) { 745 device_printf(dev, "stopping AHCI engine failed\n"); 746 break; 747 } 748 } 749 while (ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset) & ATA_AHCI_P_CMD_CR); 750} 751 752static void 753ata_ahci_clo(device_t dev) 754{ 755 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 756 struct ata_channel *ch = device_get_softc(dev); 757 u_int32_t cmd; 758 int offset = ch->unit << 7; 759 int timeout; 760 761 /* issue Command List Override if supported */ 762 if (ATA_INL(ctlr->r_res2, ATA_AHCI_CAP) & ATA_AHCI_CAP_SCLO) { 763 cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset); 764 cmd |= ATA_AHCI_P_CMD_CLO; 765 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, cmd); 766 timeout = 0; 767 do { 768 DELAY(1000); 769 if (timeout++ > 1000) { 770 device_printf(dev, "executing CLO failed\n"); 771 break; 772 } 773 } 774 while (ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD+offset)&ATA_AHCI_P_CMD_CLO); 775 } 776} 777 778static void 779ata_ahci_start(device_t dev) 780{ 781 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 782 struct ata_channel *ch = device_get_softc(dev); 783 u_int32_t cmd; 784 int offset = ch->unit << 7; 785 786 /* clear SATA error register */ 787 ATA_IDX_OUTL(ch, ATA_SERROR, ATA_IDX_INL(ch, ATA_SERROR)); 788 789 /* clear any interrupts pending on this channel */ 790 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IS + offset, 791 ATA_INL(ctlr->r_res2, ATA_AHCI_P_IS + offset)); 792 793 /* start operations on this channel */ 794 cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset); 795 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 796 cmd | ATA_AHCI_P_CMD_ST | 797 (ch->devices & ATA_PORTMULTIPLIER ? ATA_AHCI_P_CMD_PMA : 0)); 798} 799 800static void 801ata_ahci_stop_fr(device_t dev) 802{ 803 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 804 struct ata_channel *ch = device_get_softc(dev); 805 u_int32_t cmd; 806 int offset = ch->unit << 7; 807 int timeout; 808 809 /* kill off all activity on this channel */ 810 cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset); 811 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, cmd & ~ATA_AHCI_P_CMD_FRE); 812 813 timeout = 0; 814 do { 815 DELAY(1000); 816 if (timeout++ > 1000) { 817 device_printf(dev, "stopping AHCI FR engine failed\n"); 818 break; 819 } 820 } 821 while (ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset) & ATA_AHCI_P_CMD_FR); 822} 823 824static void 825ata_ahci_start_fr(device_t dev) 826{ 827 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 828 struct ata_channel *ch = device_get_softc(dev); 829 u_int32_t cmd; 830 int offset = ch->unit << 7; 831 832 /* start FIS reception on this channel */ 833 cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset); 834 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, cmd | ATA_AHCI_P_CMD_FRE); 835} 836 837static int 838ata_ahci_wait_ready(device_t dev, int t) 839{ 840 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 841 struct ata_channel *ch = device_get_softc(dev); 842 int offset = ch->unit << 7; 843 int timeout = 0; 844 uint32_t val; 845 846 while ((val = ATA_INL(ctlr->r_res2, ATA_AHCI_P_TFD + offset)) & 847 (ATA_S_BUSY | ATA_S_DRQ)) { 848 DELAY(1000); 849 if (timeout++ > t) { 850 device_printf(dev, "port is not ready (timeout %dms) tfd = %08x\n", t, val); 851 return (EBUSY); 852 } 853 } 854 if (bootverbose) 855 device_printf(dev, "ready wait time=%dms\n", timeout); 856 return (0); 857} 858 859static int 860ata_ahci_hardreset(device_t dev, int port, uint32_t *signature) 861{ 862 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 863 struct ata_channel *ch = device_get_softc(dev); 864 int offset = ch->unit << 7; 865 866 *signature = 0xffffffff; 867 ata_ahci_stop(dev); 868 /* Reset port */ 869 if (!ata_sata_phy_reset(dev, port, 0)) 870 return (ENOENT); 871 /* Wait for clearing busy status. */ 872 if (ata_ahci_wait_ready(dev, 15000)) { 873 device_printf(dev, "hardware reset timeout\n"); 874 return (EBUSY); 875 } 876 *signature = ATA_INL(ctlr->r_res2, ATA_AHCI_P_SIG + offset); 877 ata_ahci_start(dev); 878 return (0); 879} 880 881static u_int32_t 882ata_ahci_softreset(device_t dev, int port) 883{ 884 struct ata_channel *ch = device_get_softc(dev); 885 struct ata_ahci_cmd_tab *ctp = 886 (struct ata_ahci_cmd_tab *)(ch->dma.work + ATA_AHCI_CT_OFFSET); 887 u_int8_t *fis = ch->dma.work + ATA_AHCI_FB_OFFSET + 0x40; 888 889 if (bootverbose) 890 device_printf(dev, "software reset port %d...\n", port); 891 892 /* kick controller into sane state */ 893 ata_ahci_stop(dev); 894 ata_ahci_clo(dev); 895 ata_ahci_start(dev); 896 897 /* pull reset active */ 898 bzero(ctp->cfis, 64); 899 ctp->cfis[0] = 0x27; 900 ctp->cfis[1] = port & 0x0f; 901 //ctp->cfis[7] = ATA_D_LBA | ATA_D_IBM; 902 ctp->cfis[15] = (ATA_A_4BIT | ATA_A_RESET); 903 904 if (ata_ahci_issue_cmd(dev, ATA_AHCI_CMD_RESET | ATA_AHCI_CMD_CLR_BUSY,100)) { 905 device_printf(dev, "software reset set timeout\n"); 906 return (-1); 907 } 908 909 ata_udelay(50); 910 911 /* pull reset inactive -> device softreset */ 912 bzero(ctp->cfis, 64); 913 ctp->cfis[0] = 0x27; 914 ctp->cfis[1] = port & 0x0f; 915 //ctp->cfis[7] = ATA_D_LBA | ATA_D_IBM; 916 ctp->cfis[15] = ATA_A_4BIT; 917 ata_ahci_issue_cmd(dev, 0, 3000); 918 919 if (ata_ahci_wait_ready(dev, 0)) { 920 device_printf(dev, "software reset clear timeout\n"); 921 return (-1); 922 } 923 924 return (((u_int32_t)fis[6] << 24) | 925 ((u_int32_t)fis[5] << 16) | 926 ((u_int32_t)fis[4] << 8) | 927 (u_int32_t)fis[12]); 928} 929 930static void 931ata_ahci_reset(device_t dev) 932{ 933 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 934 struct ata_channel *ch = device_get_softc(dev); 935 u_int32_t signature; 936 int offset = ch->unit << 7; 937 938 if (bootverbose) 939 device_printf(dev, "AHCI reset...\n"); 940 941 /* Disable port interrupts */ 942 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IE + offset, 0); 943 944 if (ata_ahci_hardreset(dev, -1, &signature)) { 945 if (bootverbose) 946 device_printf(dev, "AHCI reset done: phy reset found no device\n"); 947 ch->devices = 0; 948 949 /* enable wanted port interrupts */ 950 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IE + offset, 951 (ATA_AHCI_P_IX_CPD | ATA_AHCI_P_IX_PRC | ATA_AHCI_P_IX_PC)); 952 return; 953 } 954 955 /* enable wanted port interrupts */ 956 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IE + offset, 957 (ATA_AHCI_P_IX_CPD | ATA_AHCI_P_IX_TFE | ATA_AHCI_P_IX_HBF | 958 ATA_AHCI_P_IX_HBD | ATA_AHCI_P_IX_IF | ATA_AHCI_P_IX_OF | 959 ((ch->pm_level == 0) ? ATA_AHCI_P_IX_PRC | ATA_AHCI_P_IX_PC : 0) | 960 ATA_AHCI_P_IX_DP | ATA_AHCI_P_IX_UF | ATA_AHCI_P_IX_SDB | 961 ATA_AHCI_P_IX_DS | ATA_AHCI_P_IX_PS | ATA_AHCI_P_IX_DHR)); 962 /* 963 * Only probe for PortMultiplier if HW has support. 964 * Ignore Marvell, which is not working, 965 */ 966 if ((ATA_INL(ctlr->r_res2, ATA_AHCI_CAP) & ATA_AHCI_CAP_SPM) && 967 pci_get_vendor(ctlr->dev) != 0x11ab) { 968 signature = ata_ahci_softreset(dev, ATA_PM); 969 /* Workaround for some ATI chips, failing to soft-reset 970 * when port multiplicator supported, but absent. 971 * XXX: We can also check PxIS.IPMS==1 here to be sure. */ 972 if (signature == 0xffffffff) 973 signature = ata_ahci_softreset(dev, 0); 974 } else { 975 signature = ata_ahci_softreset(dev, 0); 976 } 977 if (bootverbose) 978 device_printf(dev, "SIGNATURE: %08x\n", signature); 979 980 switch (signature >> 16) { 981 case 0x0000: 982 ch->devices = ATA_ATA_MASTER; 983 break; 984 case 0x9669: 985 ch->devices = ATA_PORTMULTIPLIER; 986 ata_pm_identify(dev); 987 break; 988 case 0xeb14: 989 ch->devices = ATA_ATAPI_MASTER; 990 break; 991 default: /* SOS XXX */ 992 if (bootverbose) 993 device_printf(dev, "Unknown signature, assuming disk device\n"); 994 ch->devices = ATA_ATA_MASTER; 995 } 996 if (bootverbose) 997 device_printf(dev, "AHCI reset done: devices=%08x\n", ch->devices); 998} 999 1000static void 1001ata_ahci_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) 1002{ 1003 struct ata_dmasetprd_args *args = xsc; 1004 struct ata_ahci_dma_prd *prd = args->dmatab; 1005 int i; 1006 1007 if (!(args->error = error)) { 1008 for (i = 0; i < nsegs; i++) { 1009 prd[i].dba = htole64(segs[i].ds_addr); 1010 prd[i].dbc = htole32((segs[i].ds_len - 1) & ATA_AHCI_PRD_MASK); 1011 } 1012 } 1013 1014 KASSERT(nsegs <= ATA_AHCI_DMA_ENTRIES, ("too many DMA segment entries\n")); 1015 args->nsegs = nsegs; 1016} 1017 1018static void 1019ata_ahci_dmainit(device_t dev) 1020{ 1021 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 1022 struct ata_channel *ch = device_get_softc(dev); 1023 1024 /* note start and stop are not used here */ 1025 ch->dma.setprd = ata_ahci_dmasetprd; 1026 ch->dma.max_iosize = (ATA_AHCI_DMA_ENTRIES - 1) * PAGE_SIZE; 1027 if (ATA_INL(ctlr->r_res2, ATA_AHCI_CAP) & ATA_AHCI_CAP_64BIT) 1028 ch->dma.max_address = BUS_SPACE_MAXADDR; 1029 ata_dmainit(dev); 1030} 1031 1032static int 1033ata_ahci_setup_fis(struct ata_ahci_cmd_tab *ctp, struct ata_request *request) 1034{ 1035 bzero(ctp->cfis, 64); 1036 if (request->flags & ATA_R_ATAPI) { 1037 bzero(ctp->acmd, 32); 1038 bcopy(request->u.atapi.ccb, ctp->acmd, 16); 1039 } 1040 return ata_request2fis_h2d(request, &ctp->cfis[0]); 1041} 1042 1043ATA_DECLARE_DRIVER(ata_ahci); 1044static device_method_t ata_ahci_ata_methods[] = { 1045 DEVMETHOD(device_probe, ata_ahci_ata_probe), 1046 DEVMETHOD(device_attach, ata_ahci_ata_attach), 1047 DEVMETHOD(device_detach, ata_pci_detach), 1048 DEVMETHOD(device_suspend, ata_pci_suspend), 1049 DEVMETHOD(device_resume, ata_pci_resume), 1050 DEVMETHOD(device_shutdown, bus_generic_shutdown), 1051 DEVMETHOD(bus_read_ivar, ata_pci_read_ivar), 1052 DEVMETHOD(bus_write_ivar, ata_pci_write_ivar), 1053 DEVMETHOD(bus_alloc_resource, ata_pci_alloc_resource), 1054 DEVMETHOD(bus_release_resource, ata_pci_release_resource), 1055 DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), 1056 DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), 1057 DEVMETHOD(bus_setup_intr, ata_pci_setup_intr), 1058 DEVMETHOD(bus_teardown_intr, ata_pci_teardown_intr), 1059 { 0, 0 } 1060}; 1061static driver_t ata_ahci_ata_driver = { 1062 "atapci", 1063 ata_ahci_ata_methods, 1064 sizeof(struct ata_pci_controller) 1065}; 1066DRIVER_MODULE(ata_ahci_ata, atapci, ata_ahci_ata_driver, ata_pci_devclass, 0, 0); 1067