ata-ahci.c revision 191568
1228690Sdes/*- 2228690Sdes * Copyright (c) 1998 - 2008 S�ren Schmidt <sos@FreeBSD.org> 3228690Sdes * All rights reserved. 4228690Sdes * 5228690Sdes * Redistribution and use in source and binary forms, with or without 6228690Sdes * modification, are permitted provided that the following conditions 7228690Sdes * are met: 8228690Sdes * 1. Redistributions of source code must retain the above copyright 9228690Sdes * notice, this list of conditions and the following disclaimer, 10228690Sdes * without modification, immediately at the beginning of the file. 11228690Sdes * 2. Redistributions in binary form must reproduce the above copyright 12228690Sdes * notice, this list of conditions and the following disclaimer in the 13236109Sdes * documentation and/or other materials provided with the distribution. 14236109Sdes * 15236109Sdes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16228690Sdes * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17228690Sdes * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18228690Sdes * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19228690Sdes * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20228690Sdes * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21228690Sdes * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22228690Sdes * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23228690Sdes * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24228690Sdes * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25228690Sdes */ 26228690Sdes 27228690Sdes#include <sys/cdefs.h> 28228690Sdes__FBSDID("$FreeBSD: head/sys/dev/ata/chipsets/ata-ahci.c 191568 2009-04-27 17:29:51Z jkim $"); 29271947Sdes 30228690Sdes#include "opt_ata.h" 31271947Sdes#include <sys/param.h> 32228690Sdes#include <sys/module.h> 33228690Sdes#include <sys/systm.h> 34228690Sdes#include <sys/kernel.h> 35228690Sdes#include <sys/ata.h> 36228690Sdes#include <sys/bus.h> 37271947Sdes#include <sys/endian.h> 38228690Sdes#include <sys/malloc.h> 39236109Sdes#include <sys/lock.h> 40228690Sdes#include <sys/mutex.h> 41228690Sdes#include <sys/sema.h> 42255376Sdes#include <sys/taskqueue.h> 43228690Sdes#include <vm/uma.h> 44228690Sdes#include <machine/stdarg.h> 45228690Sdes#include <machine/resource.h> 46228690Sdes#include <machine/bus.h> 47228690Sdes#include <sys/rman.h> 48228690Sdes#include <dev/pci/pcivar.h> 49228690Sdes#include <dev/pci/pcireg.h> 50228690Sdes#include <dev/ata/ata-all.h> 51228690Sdes#include <dev/ata/ata-pci.h> 52228690Sdes#include <ata_if.h> 53228690Sdes 54228690Sdes/* local prototypes */ 55228690Sdesstatic int ata_ahci_suspend(device_t dev); 56228690Sdesstatic int ata_ahci_status(device_t dev); 57228690Sdesstatic int ata_ahci_begin_transaction(struct ata_request *request); 58228690Sdesstatic int ata_ahci_end_transaction(struct ata_request *request); 59228690Sdesstatic int ata_ahci_pm_read(device_t dev, int port, int reg, u_int32_t *result); 60228690Sdesstatic int ata_ahci_pm_write(device_t dev, int port, int reg, u_int32_t result); 61228690Sdesstatic int ata_ahci_hardreset(device_t dev, int port, uint32_t *signature); 62228690Sdesstatic u_int32_t ata_ahci_softreset(device_t dev, int port); 63228690Sdesstatic void ata_ahci_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error); 64228690Sdesstatic int ata_ahci_setup_fis(struct ata_ahci_cmd_tab *ctp, struct ata_request *equest); 65228690Sdesstatic void ata_ahci_dmainit(device_t dev); 66228690Sdesstatic void ata_ahci_start(device_t dev); 67228690Sdesstatic void ata_ahci_stop(device_t dev); 68228690Sdesstatic void ata_ahci_clo(device_t dev); 69228690Sdesstatic void ata_ahci_start_fr(device_t dev); 70228690Sdesstatic void ata_ahci_stop_fr(device_t dev); 71228690Sdes 72228690Sdes/* 73228690Sdes * AHCI v1.x compliant SATA chipset support functions 74228690Sdes */ 75228690Sdesstatic int 76228690Sdesata_ahci_probe(device_t dev) 77228690Sdes{ 78228690Sdes struct ata_pci_controller *ctlr = device_get_softc(dev); 79228690Sdes char buffer[64]; 80228690Sdes 81228690Sdes /* is this a possible AHCI candidate ? */ 82228690Sdes if (pci_get_class(dev) != PCIC_STORAGE || 83228690Sdes pci_get_subclass(dev) != PCIS_STORAGE_SATA) 84228690Sdes return (ENXIO); 85228690Sdes 86228690Sdes /* is this PCI device flagged as an AHCI compliant chip ? */ 87228690Sdes if (pci_read_config(dev, PCIR_PROGIF, 1) != PCIP_STORAGE_SATA_AHCI_1_0) 88228690Sdes return ENXIO; 89228690Sdes 90228690Sdes if (bootverbose) 91228690Sdes sprintf(buffer, "%s (ID=%08x) AHCI controller", 92228690Sdes ata_pcivendor2str(dev), pci_get_devid(dev)); 93228690Sdes else 94228690Sdes sprintf(buffer, "%s AHCI controller", ata_pcivendor2str(dev)); 95228690Sdes device_set_desc_copy(dev, buffer); 96228690Sdes ctlr->chipinit = ata_ahci_chipinit; 97228690Sdes return 0; 98228690Sdes} 99228690Sdes 100228690Sdesint 101228690Sdesata_ahci_chipinit(device_t dev) 102228690Sdes{ 103228690Sdes struct ata_pci_controller *ctlr = device_get_softc(dev); 104228690Sdes int error; 105228690Sdes u_int32_t version; 106228690Sdes 107228690Sdes /* if we have a memory BAR(5) we are likely on an AHCI part */ 108228690Sdes ctlr->r_type2 = SYS_RES_MEMORY; 109228690Sdes ctlr->r_rid2 = PCIR_BAR(5); 110228690Sdes if (!(ctlr->r_res2 = bus_alloc_resource_any(dev, ctlr->r_type2, 111228690Sdes &ctlr->r_rid2, RF_ACTIVE))) 112228690Sdes return ENXIO; 113228690Sdes 114228690Sdes /* setup interrupt delivery if not done allready by a vendor driver */ 115228690Sdes if (!ctlr->r_irq) { 116228690Sdes if (ata_setup_interrupt(dev, ata_generic_intr)) { 117228690Sdes bus_release_resource(dev, ctlr->r_type2, ctlr->r_rid2, ctlr->r_res2); 118228690Sdes return ENXIO; 119228690Sdes } 120228690Sdes } 121228690Sdes else 122228690Sdes device_printf(dev, "AHCI called from vendor specific driver\n"); 123236109Sdes 124236109Sdes /* reset controller */ 125236109Sdes if ((error = ata_ahci_ctlr_reset(dev)) != 0) { 126236109Sdes bus_release_resource(dev, ctlr->r_type2, ctlr->r_rid2, ctlr->r_res2); 127236109Sdes return (error); 128228690Sdes }; 129228690Sdes 130228690Sdes /* get the number of HW channels */ 131228690Sdes ctlr->ichannels = ATA_INL(ctlr->r_res2, ATA_AHCI_PI); 132228690Sdes ctlr->channels = 133228690Sdes MAX(flsl(ctlr->ichannels), 134228690Sdes (ATA_INL(ctlr->r_res2, ATA_AHCI_CAP) & ATA_AHCI_NPMASK) + 1); 135228690Sdes 136228690Sdes ctlr->reset = ata_ahci_reset; 137228690Sdes ctlr->ch_attach = ata_ahci_ch_attach; 138228690Sdes ctlr->ch_detach = ata_ahci_ch_detach; 139228690Sdes ctlr->ch_suspend = ata_ahci_ch_suspend; 140255376Sdes ctlr->ch_resume = ata_ahci_ch_resume; 141255376Sdes ctlr->setmode = ata_sata_setmode; 142255376Sdes ctlr->suspend = ata_ahci_suspend; 143228690Sdes ctlr->resume = ata_ahci_ctlr_reset; 144228690Sdes 145228690Sdes /* announce we support the HW */ 146228690Sdes version = ATA_INL(ctlr->r_res2, ATA_AHCI_VS); 147228690Sdes device_printf(dev, 148228690Sdes "AHCI Version %x%x.%x%x controller with %d ports PM %s\n", 149228690Sdes (version >> 24) & 0xff, (version >> 16) & 0xff, 150228690Sdes (version >> 8) & 0xff, version & 0xff, 151228690Sdes (ATA_INL(ctlr->r_res2, ATA_AHCI_CAP) & ATA_AHCI_NPMASK) + 1, 152228690Sdes (ATA_INL(ctlr->r_res2, ATA_AHCI_CAP) & ATA_AHCI_CAP_SPM) ? 153228690Sdes "supported" : "not supported"); 154228690Sdes return 0; 155228690Sdes} 156228690Sdes 157228690Sdesint 158228690Sdesata_ahci_ctlr_reset(device_t dev) 159228690Sdes{ 160228690Sdes struct ata_pci_controller *ctlr = device_get_softc(dev); 161228690Sdes int timeout; 162228690Sdes 163228690Sdes /* enable AHCI mode */ 164236109Sdes ATA_OUTL(ctlr->r_res2, ATA_AHCI_GHC, ATA_AHCI_GHC_AE); 165236109Sdes 166228690Sdes /* reset AHCI controller */ 167228690Sdes ATA_OUTL(ctlr->r_res2, ATA_AHCI_GHC, ATA_AHCI_GHC_AE|ATA_AHCI_GHC_HR); 168228690Sdes for (timeout = 1000; timeout > 0; timeout--) { 169228690Sdes DELAY(1000); 170228690Sdes if ((ATA_INL(ctlr->r_res2, ATA_AHCI_GHC) & ATA_AHCI_GHC_HR) == 0) 171236109Sdes break; 172228690Sdes } 173228690Sdes if (timeout == 0) { 174228690Sdes device_printf(dev, "AHCI controller reset failure\n"); 175228690Sdes return ENXIO; 176228690Sdes } 177228690Sdes 178228690Sdes /* reenable AHCI mode */ 179228690Sdes ATA_OUTL(ctlr->r_res2, ATA_AHCI_GHC, ATA_AHCI_GHC_AE); 180228690Sdes 181228690Sdes /* clear interrupts */ 182228690Sdes ATA_OUTL(ctlr->r_res2, ATA_AHCI_IS, ATA_INL(ctlr->r_res2, ATA_AHCI_IS)); 183228690Sdes 184228690Sdes /* enable AHCI interrupts */ 185228690Sdes ATA_OUTL(ctlr->r_res2, ATA_AHCI_GHC, 186228690Sdes ATA_INL(ctlr->r_res2, ATA_AHCI_GHC) | ATA_AHCI_GHC_IE); 187228690Sdes 188228690Sdes return 0; 189228690Sdes} 190 191static int 192ata_ahci_suspend(device_t dev) 193{ 194 struct ata_pci_controller *ctlr = device_get_softc(dev); 195 196 /* disable interupts so the state change(s) doesn't trigger */ 197 ATA_OUTL(ctlr->r_res2, ATA_AHCI_GHC, 198 ATA_INL(ctlr->r_res2, ATA_AHCI_GHC) & (~ATA_AHCI_GHC_IE)); 199 return 0; 200} 201 202int 203ata_ahci_ch_attach(device_t dev) 204{ 205 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 206 struct ata_channel *ch = device_get_softc(dev); 207 int offset = ch->unit << 7; 208 209 ata_ahci_dmainit(dev); 210 211 /* set the SATA resources */ 212 ch->r_io[ATA_SSTATUS].res = ctlr->r_res2; 213 ch->r_io[ATA_SSTATUS].offset = ATA_AHCI_P_SSTS + offset; 214 ch->r_io[ATA_SERROR].res = ctlr->r_res2; 215 ch->r_io[ATA_SERROR].offset = ATA_AHCI_P_SERR + offset; 216 ch->r_io[ATA_SCONTROL].res = ctlr->r_res2; 217 ch->r_io[ATA_SCONTROL].offset = ATA_AHCI_P_SCTL + offset; 218 ch->r_io[ATA_SACTIVE].res = ctlr->r_res2; 219 ch->r_io[ATA_SACTIVE].offset = ATA_AHCI_P_SACT + offset; 220 221 ch->hw.status = ata_ahci_status; 222 ch->hw.begin_transaction = ata_ahci_begin_transaction; 223 ch->hw.end_transaction = ata_ahci_end_transaction; 224 ch->hw.command = NULL; /* not used here */ 225 ch->hw.softreset = ata_ahci_softreset; 226 ch->hw.pm_read = ata_ahci_pm_read; 227 ch->hw.pm_write = ata_ahci_pm_write; 228 229 ata_ahci_ch_resume(dev); 230 return 0; 231} 232 233int 234ata_ahci_ch_detach(device_t dev) 235{ 236 237 ata_ahci_ch_suspend(dev); 238 ata_dmafini(dev); 239 return (0); 240} 241 242int 243ata_ahci_ch_suspend(device_t dev) 244{ 245 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 246 struct ata_channel *ch = device_get_softc(dev); 247 int offset = ch->unit << 7; 248 249 /* Disable port interrupts. */ 250 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IE + offset, 0); 251 /* Reset command register. */ 252 ata_ahci_stop(dev); 253 ata_ahci_stop_fr(dev); 254 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 0); 255 256 /* Allow everything including partial and slumber modes. */ 257 ATA_IDX_OUTL(ch, ATA_SCONTROL, 0); 258 /* Request slumber mode transition and give some time to get there. */ 259 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, ATA_AHCI_P_CMD_SLUMBER); 260 DELAY(100); 261 /* Disable PHY. */ 262 ATA_IDX_OUTL(ch, ATA_SCONTROL, ATA_SC_DET_DISABLE); 263 264 return (0); 265} 266 267int 268ata_ahci_ch_resume(device_t dev) 269{ 270 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 271 struct ata_channel *ch = device_get_softc(dev); 272 uint64_t work; 273 int offset = ch->unit << 7; 274 275 /* Disable port interrupts */ 276 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IE + offset, 0); 277 278 /* setup work areas */ 279 work = ch->dma.work_bus + ATA_AHCI_CL_OFFSET; 280 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CLB + offset, work & 0xffffffff); 281 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CLBU + offset, work >> 32); 282 283 work = ch->dma.work_bus + ATA_AHCI_FB_OFFSET; 284 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_FB + offset, work & 0xffffffff); 285 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_FBU + offset, work >> 32); 286 287 /* activate the channel and power/spin up device */ 288 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 289 (ATA_AHCI_P_CMD_ACTIVE | ATA_AHCI_P_CMD_POD | ATA_AHCI_P_CMD_SUD)); 290 ata_ahci_start_fr(dev); 291 ata_ahci_start(dev); 292 293 return (0); 294} 295 296static int 297ata_ahci_status(device_t dev) 298{ 299 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 300 struct ata_channel *ch = device_get_softc(dev); 301 u_int32_t action = ATA_INL(ctlr->r_res2, ATA_AHCI_IS); 302 int offset = ch->unit << 7; 303 304#define ATA_AHCI_STATBITS \ 305 (ATA_AHCI_P_IX_IF|ATA_AHCI_P_IX_HBD|ATA_AHCI_P_IX_HBF|ATA_AHCI_P_IX_TFE) 306 307 if (action & (1 << ch->unit)) { 308 u_int32_t istatus = ATA_INL(ctlr->r_res2, ATA_AHCI_P_IS + offset); 309 u_int32_t cstatus = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CI + offset); 310 311 /* clear interrupt(s) */ 312 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IS + offset, istatus); 313 ATA_OUTL(ctlr->r_res2, ATA_AHCI_IS, 1 << ch->unit); 314 315 /* do we have any PHY events ? */ 316 if (istatus & (ATA_AHCI_P_IX_PRC | ATA_AHCI_P_IX_PC)) 317 ata_sata_phy_check_events(dev); 318 319 /* do we have a potentially hanging engine to take care of? */ 320 /* XXX SOS what todo on NCQ */ 321 if ((istatus & ATA_AHCI_STATBITS) && (cstatus & 1)) { 322 323 u_int32_t cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset); 324 int timeout = 0; 325 326 /* kill off all activity on this channel */ 327 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 328 cmd & ~(ATA_AHCI_P_CMD_FRE | ATA_AHCI_P_CMD_ST)); 329 330 /* XXX SOS this is not entirely wrong */ 331 do { 332 DELAY(1000); 333 if (timeout++ > 1000) { 334 device_printf(dev, "stopping AHCI engine failed\n"); 335 break; 336 } 337 } while (ATA_INL(ctlr->r_res2, 338 ATA_AHCI_P_CMD + offset) & ATA_AHCI_P_CMD_CR); 339 340 /* start operations on this channel */ 341 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 342 cmd | (ATA_AHCI_P_CMD_FRE | ATA_AHCI_P_CMD_ST)); 343 344 return 1; 345 } 346 else 347 /* XXX SOS what todo on NCQ */ 348 return (!(cstatus & 1)); 349 } 350 return 0; 351} 352 353/* must be called with ATA channel locked and state_mtx held */ 354static int 355ata_ahci_begin_transaction(struct ata_request *request) 356{ 357 struct ata_pci_controller *ctlr=device_get_softc(GRANDPARENT(request->dev)); 358 struct ata_channel *ch = device_get_softc(request->parent); 359 struct ata_device *atadev = device_get_softc(request->dev); 360 struct ata_ahci_cmd_tab *ctp; 361 struct ata_ahci_cmd_list *clp; 362 int offset = ch->unit << 7; 363 int port = atadev->unit & 0x0f; 364 int entries = 0; 365 int fis_size; 366 367 /* get a piece of the workspace for this request */ 368 ctp = (struct ata_ahci_cmd_tab *) 369 (ch->dma.work + ATA_AHCI_CT_OFFSET + (ATA_AHCI_CT_SIZE*request->tag)); 370 371 /* setup the FIS for this request */ 372 if (!(fis_size = ata_ahci_setup_fis(ctp, request))) { 373 device_printf(request->dev, "setting up SATA FIS failed\n"); 374 request->result = EIO; 375 return ATA_OP_FINISHED; 376 } 377 378 /* if request moves data setup and load SG list */ 379 if (request->flags & (ATA_R_READ | ATA_R_WRITE)) { 380 if (ch->dma.load(request, ctp->prd_tab, &entries)) { 381 device_printf(request->dev, "setting up DMA failed\n"); 382 request->result = EIO; 383 return ATA_OP_FINISHED; 384 } 385 } 386 387 /* setup the command list entry */ 388 clp = (struct ata_ahci_cmd_list *) 389 (ch->dma.work + ATA_AHCI_CL_OFFSET + (ATA_AHCI_CL_SIZE*request->tag)); 390 391 clp->prd_length = entries; 392 clp->cmd_flags = (request->flags & ATA_R_WRITE ? ATA_AHCI_CMD_WRITE : 0) | 393 (request->flags & ATA_R_ATAPI ? 394 (ATA_AHCI_CMD_ATAPI | ATA_AHCI_CMD_PREFETCH) : 0) | 395 (fis_size / sizeof(u_int32_t)) | 396 (port << 12); 397 clp->bytecount = 0; 398 clp->cmd_table_phys = htole64(ch->dma.work_bus + ATA_AHCI_CT_OFFSET + 399 (ATA_AHCI_CT_SIZE * request->tag)); 400 401 /* clear eventual ACTIVE bit */ 402 ATA_IDX_OUTL(ch, ATA_SACTIVE, 403 ATA_IDX_INL(ch, ATA_SACTIVE) & (1 << request->tag)); 404 405 /* set command type bit */ 406 if (request->flags & ATA_R_ATAPI) 407 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 408 ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset) | 409 ATA_AHCI_P_CMD_ATAPI); 410 else 411 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 412 ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset) & 413 ~ATA_AHCI_P_CMD_ATAPI); 414 415 /* issue command to controller */ 416 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CI + offset, (1 << request->tag)); 417 418 if (!(request->flags & ATA_R_ATAPI)) { 419 /* device reset doesn't interrupt */ 420 if (request->u.ata.command == ATA_DEVICE_RESET) { 421 u_int32_t tf_data; 422 int timeout = 1000000; 423 424 do { 425 DELAY(10); 426 tf_data = ATA_INL(ctlr->r_res2, ATA_AHCI_P_TFD + (ch->unit<<7)); 427 } while ((tf_data & ATA_S_BUSY) && timeout--); 428 if (bootverbose) 429 device_printf(ch->dev, "device_reset timeout=%dus\n", 430 (1000000-timeout)*10); 431 request->status = tf_data; 432 if (request->status & ATA_S_ERROR) 433 request->error = tf_data >> 8; 434 return ATA_OP_FINISHED; 435 } 436 } 437 438 /* start the timeout */ 439 callout_reset(&request->callout, request->timeout * hz, 440 (timeout_t*)ata_timeout, request); 441 return ATA_OP_CONTINUES; 442} 443 444/* must be called with ATA channel locked and state_mtx held */ 445static int 446ata_ahci_end_transaction(struct ata_request *request) 447{ 448 struct ata_pci_controller *ctlr=device_get_softc(GRANDPARENT(request->dev)); 449 struct ata_channel *ch = device_get_softc(request->parent); 450 struct ata_ahci_cmd_list *clp; 451 u_int32_t tf_data; 452 int offset = ch->unit << 7; 453 454 /* kill the timeout */ 455 callout_stop(&request->callout); 456 457 /* get status */ 458 tf_data = ATA_INL(ctlr->r_res2, ATA_AHCI_P_TFD + offset); 459 request->status = tf_data; 460 461 /* if error status get details */ 462 if (request->status & ATA_S_ERROR) 463 request->error = tf_data >> 8; 464 465 /* on control commands read back registers to the request struct */ 466 if (request->flags & ATA_R_CONTROL) { 467 struct ata_device *atadev = device_get_softc(request->dev); 468 u_int8_t *fis = ch->dma.work + ATA_AHCI_FB_OFFSET + 0x40; 469 470 request->u.ata.count = fis[12] | ((u_int16_t)fis[13] << 8); 471 request->u.ata.lba = fis[4] | ((u_int64_t)fis[5] << 8) | 472 ((u_int64_t)fis[6] << 16); 473 if (atadev->flags & ATA_D_48BIT_ACTIVE) 474 request->u.ata.lba |= ((u_int64_t)fis[8] << 24) | 475 ((u_int64_t)fis[9] << 32) | 476 ((u_int64_t)fis[10] << 40); 477 else 478 request->u.ata.lba |= ((u_int64_t)(fis[7] & 0x0f) << 24); 479 } 480 481 /* record how much data we actually moved */ 482 clp = (struct ata_ahci_cmd_list *) 483 (ch->dma.work + ATA_AHCI_CL_OFFSET + (ATA_AHCI_CL_SIZE*request->tag)); 484 request->donecount = clp->bytecount; 485 486 /* release SG list etc */ 487 ch->dma.unload(request); 488 489 return ATA_OP_FINISHED; 490} 491 492static int 493ata_ahci_issue_cmd(device_t dev, u_int16_t flags, int timeout) 494{ 495 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 496 struct ata_channel *ch = device_get_softc(dev); 497 struct ata_ahci_cmd_list *clp = 498 (struct ata_ahci_cmd_list *)(ch->dma.work + ATA_AHCI_CL_OFFSET); 499 struct ata_ahci_cmd_tab *ctp = 500 (struct ata_ahci_cmd_tab *)(ch->dma.work + ATA_AHCI_CT_OFFSET); 501 u_int32_t status = 0; 502 int offset = ch->unit << 7; 503 int port = (ctp->cfis[1] & 0x0f); 504 int count; 505 506 clp->prd_length = 0; 507 clp->cmd_flags = (20 / sizeof(u_int32_t)) | flags | (port << 12); 508 clp->bytecount = 0; 509 clp->cmd_table_phys = htole64(ch->dma.work_bus + ATA_AHCI_CT_OFFSET); 510 511 /* issue command to controller */ 512 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CI + offset, 1); 513 514 /* poll for command finished */ 515 for (count = 0; count < timeout; count++) { 516 DELAY(1000); 517 if (!((status = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CI + offset)) & 1)) 518 break; 519 } 520 521 /* clear interrupts */ 522 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IS + offset, 523 ATA_INL(ctlr->r_res2, ATA_AHCI_P_IS + offset)); 524 525 if (timeout && (count >= timeout)) { 526 if (bootverbose) { 527 device_printf(dev, "ahci_issue_cmd timeout: %d of %dms, status=%08x\n", 528 count, timeout, status); 529 } 530 return EIO; 531 } 532 533 return 0; 534} 535 536static int 537ata_ahci_pm_read(device_t dev, int port, int reg, u_int32_t *result) 538{ 539 struct ata_channel *ch = device_get_softc(dev); 540 struct ata_ahci_cmd_tab *ctp = 541 (struct ata_ahci_cmd_tab *)(ch->dma.work + ATA_AHCI_CT_OFFSET); 542 u_int8_t *fis = ch->dma.work + ATA_AHCI_FB_OFFSET + 0x40; 543 544 bzero(ctp->cfis, 64); 545 ctp->cfis[0] = 0x27; /* host to device */ 546 ctp->cfis[1] = 0x8f; /* command FIS to PM port */ 547 ctp->cfis[2] = ATA_READ_PM; 548 ctp->cfis[3] = reg; 549 ctp->cfis[7] = port | ATA_D_LBA; 550 ctp->cfis[15] = ATA_A_4BIT; 551 552 if (ata_ahci_issue_cmd(dev, 0, 10)) { 553 device_printf(dev, "error reading PM port\n"); 554 return EIO; 555 } 556 557 *result = fis[12] | (fis[4] << 8) | (fis[5] << 16) | (fis[6] << 24); 558 return 0; 559} 560 561static int 562ata_ahci_pm_write(device_t dev, int port, int reg, u_int32_t value) 563{ 564 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 565 struct ata_channel *ch = device_get_softc(dev); 566 struct ata_ahci_cmd_tab *ctp = 567 (struct ata_ahci_cmd_tab *)(ch->dma.work + ATA_AHCI_CT_OFFSET); 568 int offset = ch->unit << 7; 569 570 bzero(ctp->cfis, 64); 571 ctp->cfis[0] = 0x27; /* host to device */ 572 ctp->cfis[1] = 0x8f; /* command FIS to PM port */ 573 ctp->cfis[2] = ATA_WRITE_PM; 574 ctp->cfis[3] = reg; 575 ctp->cfis[7] = port | ATA_D_LBA; 576 ctp->cfis[12] = value & 0xff; 577 ctp->cfis[4] = (value >> 8) & 0xff;; 578 ctp->cfis[5] = (value >> 16) & 0xff;; 579 ctp->cfis[6] = (value >> 24) & 0xff;; 580 ctp->cfis[15] = ATA_A_4BIT; 581 582 if (ata_ahci_issue_cmd(dev, 0, 100)) { 583 device_printf(dev, "error writing PM port\n"); 584 return ATA_E_ABORT; 585 } 586 587 return (ATA_INL(ctlr->r_res2, ATA_AHCI_P_TFD + offset) >> 8) & 0xff; 588} 589 590static void 591ata_ahci_stop(device_t dev) 592{ 593 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 594 struct ata_channel *ch = device_get_softc(dev); 595 u_int32_t cmd; 596 int offset = ch->unit << 7; 597 int timeout; 598 599 /* kill off all activity on this channel */ 600 cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset); 601 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 602 cmd & ~ATA_AHCI_P_CMD_ST); 603 604 /* XXX SOS this is not entirely wrong */ 605 timeout = 0; 606 do { 607 DELAY(1000); 608 if (timeout++ > 1000) { 609 device_printf(dev, "stopping AHCI engine failed\n"); 610 break; 611 } 612 } 613 while (ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset) & ATA_AHCI_P_CMD_CR); 614} 615 616static void 617ata_ahci_clo(device_t dev) 618{ 619 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 620 struct ata_channel *ch = device_get_softc(dev); 621 u_int32_t cmd; 622 int offset = ch->unit << 7; 623 int timeout; 624 625 /* issue Command List Override if supported */ 626 if (ATA_INL(ctlr->r_res2, ATA_AHCI_CAP) & ATA_AHCI_CAP_CLO) { 627 cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset); 628 cmd |= ATA_AHCI_P_CMD_CLO; 629 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, cmd); 630 timeout = 0; 631 do { 632 DELAY(1000); 633 if (timeout++ > 1000) { 634 device_printf(dev, "executing CLO failed\n"); 635 break; 636 } 637 } 638 while (ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD+offset)&ATA_AHCI_P_CMD_CLO); 639 } 640} 641 642static void 643ata_ahci_start(device_t dev) 644{ 645 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 646 struct ata_channel *ch = device_get_softc(dev); 647 u_int32_t cmd; 648 int offset = ch->unit << 7; 649 650 /* clear SATA error register */ 651 ATA_IDX_OUTL(ch, ATA_SERROR, ATA_IDX_INL(ch, ATA_SERROR)); 652 653 /* clear any interrupts pending on this channel */ 654 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IS + offset, 655 ATA_INL(ctlr->r_res2, ATA_AHCI_P_IS + offset)); 656 657 /* start operations on this channel */ 658 cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset); 659 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 660 cmd | ATA_AHCI_P_CMD_ST | 661 (ch->devices & ATA_PORTMULTIPLIER ? ATA_AHCI_P_CMD_PMA : 0)); 662} 663 664static void 665ata_ahci_stop_fr(device_t dev) 666{ 667 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 668 struct ata_channel *ch = device_get_softc(dev); 669 u_int32_t cmd; 670 int offset = ch->unit << 7; 671 int timeout; 672 673 /* kill off all activity on this channel */ 674 cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset); 675 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, cmd & ~ATA_AHCI_P_CMD_FRE); 676 677 timeout = 0; 678 do { 679 DELAY(1000); 680 if (timeout++ > 1000) { 681 device_printf(dev, "stopping AHCI FR engine failed\n"); 682 break; 683 } 684 } 685 while (ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset) & ATA_AHCI_P_CMD_FR); 686} 687 688static void 689ata_ahci_start_fr(device_t dev) 690{ 691 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 692 struct ata_channel *ch = device_get_softc(dev); 693 u_int32_t cmd; 694 int offset = ch->unit << 7; 695 696 /* start FIS reception on this channel */ 697 cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset); 698 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, cmd | ATA_AHCI_P_CMD_FRE); 699} 700 701static int 702ata_ahci_wait_ready(device_t dev, int t) 703{ 704 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 705 struct ata_channel *ch = device_get_softc(dev); 706 int offset = ch->unit << 7; 707 int timeout = 0; 708 uint32_t val; 709 710 while ((val = ATA_INL(ctlr->r_res2, ATA_AHCI_P_TFD + offset)) & 711 (ATA_S_BUSY | ATA_S_DRQ)) { 712 DELAY(1000); 713 if (timeout++ > t) { 714 device_printf(dev, "port is not ready (timeout %dms) tfd = %08x\n", t, val); 715 return (EBUSY); 716 } 717 } 718 if (bootverbose) 719 device_printf(dev, "ready wait time=%dms\n", timeout); 720 return (0); 721} 722 723static int 724ata_ahci_hardreset(device_t dev, int port, uint32_t *signature) 725{ 726 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 727 struct ata_channel *ch = device_get_softc(dev); 728 int offset = ch->unit << 7; 729 730 *signature = 0xffffffff; 731 ata_ahci_stop(dev); 732 /* Reset port */ 733 if (!ata_sata_phy_reset(dev, port, 0)) 734 return (ENOENT); 735 /* Wait for clearing busy status. */ 736 if (ata_ahci_wait_ready(dev, 10000)) { 737 device_printf(dev, "hardware reset timeout\n"); 738 return (EBUSY); 739 } 740 *signature = ATA_INL(ctlr->r_res2, ATA_AHCI_P_SIG + offset); 741 ata_ahci_start(dev); 742 return (0); 743} 744 745static u_int32_t 746ata_ahci_softreset(device_t dev, int port) 747{ 748 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 749 struct ata_channel *ch = device_get_softc(dev); 750 int offset = ch->unit << 7; 751 struct ata_ahci_cmd_tab *ctp = 752 (struct ata_ahci_cmd_tab *)(ch->dma.work + ATA_AHCI_CT_OFFSET); 753 754 if (bootverbose) 755 device_printf(dev, "software reset port %d...\n", port); 756 757 /* kick controller into sane state */ 758 ata_ahci_stop(dev); 759 ata_ahci_clo(dev); 760 ata_ahci_start(dev); 761 762 /* pull reset active */ 763 bzero(ctp->cfis, 64); 764 ctp->cfis[0] = 0x27; 765 ctp->cfis[1] = port & 0x0f; 766 //ctp->cfis[7] = ATA_D_LBA | ATA_D_IBM; 767 ctp->cfis[15] = (ATA_A_4BIT | ATA_A_RESET); 768 769 if (ata_ahci_issue_cmd(dev, ATA_AHCI_CMD_RESET | ATA_AHCI_CMD_CLR_BUSY,100)) { 770 device_printf(dev, "software reset set timeout\n"); 771 return (-1); 772 } 773 774 ata_udelay(50); 775 776 /* pull reset inactive -> device softreset */ 777 bzero(ctp->cfis, 64); 778 ctp->cfis[0] = 0x27; 779 ctp->cfis[1] = port & 0x0f; 780 //ctp->cfis[7] = ATA_D_LBA | ATA_D_IBM; 781 ctp->cfis[15] = ATA_A_4BIT; 782 ata_ahci_issue_cmd(dev, 0, 3000); 783 784 if (ata_ahci_wait_ready(dev, 0)) { 785 device_printf(dev, "software reset clear timeout\n"); 786 return (-1); 787 } 788 789 return ATA_INL(ctlr->r_res2, ATA_AHCI_P_SIG + offset); 790} 791 792void 793ata_ahci_reset(device_t dev) 794{ 795 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 796 struct ata_channel *ch = device_get_softc(dev); 797 u_int32_t signature; 798 int offset = ch->unit << 7; 799 800 if (bootverbose) 801 device_printf(dev, "AHCI reset...\n"); 802 803 /* Disable port interrupts */ 804 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IE + offset, 0); 805 806 if (ata_ahci_hardreset(dev, -1, &signature)) { 807 if (bootverbose) 808 device_printf(dev, "AHCI reset done: phy reset found no device\n"); 809 ch->devices = 0; 810 811 /* enable wanted port interrupts */ 812 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IE + offset, 813 (ATA_AHCI_P_IX_CPD | ATA_AHCI_P_IX_PRC | ATA_AHCI_P_IX_PC)); 814 return; 815 } 816 817 /* enable wanted port interrupts */ 818 ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IE + offset, 819 (ATA_AHCI_P_IX_CPD | ATA_AHCI_P_IX_TFE | ATA_AHCI_P_IX_HBF | 820 ATA_AHCI_P_IX_HBD | ATA_AHCI_P_IX_IF | ATA_AHCI_P_IX_OF | 821 ATA_AHCI_P_IX_PRC | ATA_AHCI_P_IX_PC | ATA_AHCI_P_IX_DP | 822 ATA_AHCI_P_IX_UF | ATA_AHCI_P_IX_SDB | ATA_AHCI_P_IX_DS | 823 ATA_AHCI_P_IX_PS | ATA_AHCI_P_IX_DHR)); 824 825 /* only probe for PortMultiplier if HW has support */ 826 if (ATA_INL(ctlr->r_res2, ATA_AHCI_CAP) & ATA_AHCI_CAP_SPM) { 827 signature = ata_ahci_softreset(dev, ATA_PM); 828 /* Workaround for some ATI chips, failing to soft-reset 829 * when port multiplicator supported, but absent. 830 * XXX: We can also check PxIS.IPMS==1 here to be sure. */ 831 if (signature == 0xffffffff) 832 signature = ata_ahci_softreset(dev, 0); 833 } else { 834 signature = ata_ahci_softreset(dev, 0); 835 } 836 if (bootverbose) 837 device_printf(dev, "SIGNATURE: %08x\n", signature); 838 839 switch (signature >> 16) { 840 case 0x0000: 841 ch->devices = ATA_ATA_MASTER; 842 break; 843 case 0x9669: 844 ch->devices = ATA_PORTMULTIPLIER; 845 ata_pm_identify(dev); 846 break; 847 case 0xeb14: 848 ch->devices = ATA_ATAPI_MASTER; 849 break; 850 default: /* SOS XXX */ 851 if (bootverbose) 852 device_printf(dev, "Unknown signature, assuming disk device\n"); 853 ch->devices = ATA_ATA_MASTER; 854 } 855 if (bootverbose) 856 device_printf(dev, "AHCI reset done: devices=%08x\n", ch->devices); 857} 858 859static void 860ata_ahci_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) 861{ 862 struct ata_dmasetprd_args *args = xsc; 863 struct ata_ahci_dma_prd *prd = args->dmatab; 864 int i; 865 866 if (!(args->error = error)) { 867 for (i = 0; i < nsegs; i++) { 868 prd[i].dba = htole64(segs[i].ds_addr); 869 prd[i].dbc = htole32((segs[i].ds_len - 1) & ATA_AHCI_PRD_MASK); 870 } 871 } 872 873 KASSERT(nsegs <= ATA_AHCI_DMA_ENTRIES, ("too many DMA segment entries\n")); 874 args->nsegs = nsegs; 875} 876 877static void 878ata_ahci_dmainit(device_t dev) 879{ 880 struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev)); 881 struct ata_channel *ch = device_get_softc(dev); 882 883 ata_dmainit(dev); 884 /* note start and stop are not used here */ 885 ch->dma.setprd = ata_ahci_dmasetprd; 886 ch->dma.max_iosize = 8192 * DEV_BSIZE; 887 if (ATA_INL(ctlr->r_res2, ATA_AHCI_CAP) & ATA_AHCI_CAP_64BIT) 888 ch->dma.max_address = BUS_SPACE_MAXADDR; 889} 890 891static int 892ata_ahci_setup_fis(struct ata_ahci_cmd_tab *ctp, struct ata_request *request) 893{ 894 bzero(ctp->cfis, 64); 895 if (request->flags & ATA_R_ATAPI) { 896 bzero(ctp->acmd, 32); 897 bcopy(request->u.atapi.ccb, ctp->acmd, 16); 898 } 899 return ata_request2fis_h2d(request, &ctp->cfis[0]); 900} 901 902ATA_DECLARE_DRIVER(ata_ahci); 903