1// Copyright 2016 The Fuchsia Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include <ddk/binding.h> 6#include <ddk/debug.h> 7#include <ddk/device.h> 8#include <ddk/driver.h> 9#include <ddk/io-buffer.h> 10#include <ddk/phys-iter.h> 11#include <ddk/protocol/pci.h> 12 13#include <assert.h> 14#include <zircon/listnode.h> 15#include <zircon/syscalls.h> 16#include <zircon/types.h> 17#include <zircon/assert.h> 18#include <pretty/hexdump.h> 19#include <lib/sync/completion.h> 20#include <inttypes.h> 21#include <stdio.h> 22#include <stdlib.h> 23#include <string.h> 24#include <sys/param.h> 25#include <threads.h> 26#include <unistd.h> 27 28#include "ahci.h" 29#include "sata.h" 30 31#define ahci_read(reg) pcie_read32(reg) 32#define ahci_write(reg, val) pcie_write32(reg, val) 33 34#define HI32(val) (((val) >> 32) & 0xffffffff) 35#define LO32(val) ((val) & 0xffffffff) 36 37#define PAGE_MASK (PAGE_SIZE - 1ull) 38 39// port is implemented by the controller 40#define AHCI_PORT_FLAG_IMPLEMENTED (1 << 0) 41// a device is present on port 42#define AHCI_PORT_FLAG_PRESENT (1 << 1) 43// port is paused (no queued transactions will be processed) 44// until pending transactions are done 45#define AHCI_PORT_FLAG_SYNC_PAUSED (1 << 2) 46 47//clang-format on 48 49typedef struct ahci_port { 50 int nr; // 0-based 51 int flags; 52 53 sata_devinfo_t devinfo; 54 55 ahci_port_reg_t* regs; 56 ahci_cl_t* cl; 57 ahci_fis_t* fis; 58 ahci_ct_t* ct[AHCI_MAX_COMMANDS]; 59 60 mtx_t lock; 61 62 list_node_t txn_list; 63 io_buffer_t buffer; 64 65 uint32_t running; // bitmask of running commands 66 uint32_t completed; // bitmask of completed commands 67 sata_txn_t* commands[AHCI_MAX_COMMANDS]; // commands in flight 68 sata_txn_t* sync; // FLUSH command in flight 69} ahci_port_t; 70 71struct ahci_device { 72 zx_device_t* zxdev; 73 74 ahci_hba_t* regs; 75 uint64_t regs_size; 76 zx_handle_t regs_handle; 77 78 pci_protocol_t pci; 79 80 zx_handle_t irq_handle; 81 thrd_t irq_thread; 82 83 zx_handle_t bti_handle; 84 85 thrd_t worker_thread; 86 sync_completion_t worker_completion; 87 88 thrd_t watchdog_thread; 89 sync_completion_t watchdog_completion; 90 91 uint32_t cap; 92 93 // TODO(ZX-1641): lazily allocate these 94 ahci_port_t ports[AHCI_MAX_PORTS]; 95}; 96 97static inline zx_status_t ahci_wait_for_clear(const volatile uint32_t* reg, uint32_t mask, 98 zx_time_t timeout) { 99 int i = 0; 100 zx_time_t start_time = zx_clock_get_monotonic(); 101 do { 102 if (!(ahci_read(reg) & mask)) return ZX_OK; 103 usleep(10 * 1000); 104 i++; 105 } while (zx_clock_get_monotonic() - start_time < timeout); 106 return ZX_ERR_TIMED_OUT; 107} 108 109static inline zx_status_t ahci_wait_for_set(const volatile uint32_t* reg, uint32_t mask, 110 zx_time_t timeout) { 111 int i = 0; 112 zx_time_t start_time = zx_clock_get_monotonic(); 113 do { 114 if (ahci_read(reg) & mask) return ZX_OK; 115 usleep(10 * 1000); 116 i++; 117 } while (zx_clock_get_monotonic() - start_time < timeout); 118 return ZX_ERR_TIMED_OUT; 119} 120 121static bool ahci_port_valid(ahci_device_t* dev, int portnr) { 122 if (portnr >= AHCI_MAX_PORTS) { 123 return false; 124 } 125 ahci_port_t* port = &dev->ports[portnr]; 126 int flags = AHCI_PORT_FLAG_IMPLEMENTED | AHCI_PORT_FLAG_PRESENT; 127 return (port->flags & flags) == flags; 128} 129 130static void ahci_port_disable(ahci_port_t* port) { 131 uint32_t cmd = ahci_read(&port->regs->cmd); 132 if (!(cmd & AHCI_PORT_CMD_ST)) return; 133 cmd &= ~AHCI_PORT_CMD_ST; 134 ahci_write(&port->regs->cmd, cmd); 135 zx_status_t status = ahci_wait_for_clear(&port->regs->cmd, AHCI_PORT_CMD_CR, 500 * 1000 * 1000); 136 if (status) { 137 zxlogf(ERROR, "ahci.%d: port disable timed out\n", port->nr); 138 } 139} 140 141static void ahci_port_enable(ahci_port_t* port) { 142 uint32_t cmd = ahci_read(&port->regs->cmd); 143 if (cmd & AHCI_PORT_CMD_ST) return; 144 if (!(cmd & AHCI_PORT_CMD_FRE)) { 145 zxlogf(ERROR, "ahci.%d: cannot enable port without FRE enabled\n", port->nr); 146 return; 147 } 148 zx_status_t status = ahci_wait_for_clear(&port->regs->cmd, AHCI_PORT_CMD_CR, 500 * 1000 * 1000); 149 if (status) { 150 zxlogf(ERROR, "ahci.%d: dma engine still running when enabling port\n", port->nr); 151 } 152 cmd |= AHCI_PORT_CMD_ST; 153 ahci_write(&port->regs->cmd, cmd); 154} 155 156static void ahci_port_reset(ahci_port_t* port) { 157 // disable port 158 ahci_port_disable(port); 159 160 // clear error 161 ahci_write(&port->regs->serr, ahci_read(&port->regs->serr)); 162 163 // wait for device idle 164 zx_status_t status = ahci_wait_for_clear(&port->regs->tfd, AHCI_PORT_TFD_BUSY | AHCI_PORT_TFD_DATA_REQUEST, 1000 * 1000 * 1000); 165 if (status < 0) { 166 // if busy is not cleared, do a full comreset 167 zxlogf(SPEW, "ahci.%d: timed out waiting for port idle, resetting\n", port->nr); 168 // v1.3.1, 10.4.2 port reset 169 uint32_t sctl = AHCI_PORT_SCTL_IPM_ACTIVE | AHCI_PORT_SCTL_IPM_PARTIAL | AHCI_PORT_SCTL_DET_INIT; 170 ahci_write(&port->regs->sctl, sctl); 171 usleep(1000); 172 sctl = ahci_read(&port->regs->sctl); 173 sctl &= ~AHCI_PORT_SCTL_DET_MASK; 174 ahci_write(&port->regs->sctl, sctl); 175 } 176 177 // enable port 178 ahci_port_enable(port); 179 180 // wait for device detect 181 status = ahci_wait_for_set(&port->regs->ssts, AHCI_PORT_SSTS_DET_PRESENT, 1llu * 1000 * 1000 * 1000); 182 if ((driver_get_log_flags() & DDK_LOG_SPEW) && (status < 0)) { 183 zxlogf(SPEW, "ahci.%d: no device detected\n", port->nr); 184 } 185 186 // clear error 187 ahci_write(&port->regs->serr, ahci_read(&port->regs->serr)); 188} 189 190static bool ahci_port_cmd_busy(ahci_port_t* port, int slot) { 191 // a command slot is busy if a transaction is in flight or pending to be completed 192 return ((ahci_read(&port->regs->sact) | ahci_read(&port->regs->ci)) & (1 << slot)) || 193 (port->commands[slot] != NULL) || 194 (port->running & (1 << slot)) || (port->completed & (1 << slot)); 195} 196 197static bool cmd_is_read(uint8_t cmd) { 198 if (cmd == SATA_CMD_READ_DMA || 199 cmd == SATA_CMD_READ_DMA_EXT || 200 cmd == SATA_CMD_READ_FPDMA_QUEUED) { 201 return true; 202 } else { 203 return false; 204 } 205} 206 207static bool cmd_is_write(uint8_t cmd) { 208 if (cmd == SATA_CMD_WRITE_DMA || 209 cmd == SATA_CMD_WRITE_DMA_EXT || 210 cmd == SATA_CMD_WRITE_FPDMA_QUEUED) { 211 return true; 212 } else { 213 return false; 214 } 215} 216 217static bool cmd_is_queued(uint8_t cmd) { 218 return (cmd == SATA_CMD_READ_FPDMA_QUEUED) || (cmd == SATA_CMD_WRITE_FPDMA_QUEUED); 219} 220 221static void ahci_port_complete_txn(ahci_device_t* dev, ahci_port_t* port, zx_status_t status) { 222 mtx_lock(&port->lock); 223 uint32_t sact = ahci_read(&port->regs->sact); 224 uint32_t running = port->running; 225 uint32_t done = sact ^ running; 226 // assert if a command slot without an outstanding transaction is active 227 ZX_DEBUG_ASSERT(!(done & sact)); 228 port->completed |= done; 229 mtx_unlock(&port->lock); 230 // hit the worker thread to complete commands 231 sync_completion_signal(&dev->worker_completion); 232} 233 234static zx_status_t ahci_do_txn(ahci_device_t* dev, ahci_port_t* port, int slot, sata_txn_t* txn) { 235 assert(slot < AHCI_MAX_COMMANDS); 236 assert(!ahci_port_cmd_busy(port, slot)); 237 238 uint64_t offset_vmo = txn->bop.rw.offset_vmo * port->devinfo.block_size; 239 uint64_t bytes = txn->bop.rw.length * port->devinfo.block_size; 240 size_t pagecount = ((offset_vmo & (PAGE_SIZE - 1)) + bytes + (PAGE_SIZE - 1)) / 241 PAGE_SIZE; 242 zx_paddr_t pages[AHCI_MAX_PAGES]; 243 if (pagecount > AHCI_MAX_PAGES) { 244 zxlogf(SPEW, "ahci.%d: txn %p too many pages (%zd)\n", port->nr, txn, pagecount); 245 return ZX_ERR_INVALID_ARGS; 246 } 247 248 zx_handle_t vmo = txn->bop.rw.vmo; 249 bool is_write = cmd_is_write(txn->cmd); 250 uint32_t options = is_write ? ZX_BTI_PERM_READ : ZX_BTI_PERM_WRITE; 251 zx_handle_t pmt; 252 zx_status_t st = zx_bti_pin(dev->bti_handle, options, vmo, offset_vmo & ~PAGE_MASK, 253 pagecount * PAGE_SIZE, pages, pagecount, &pmt); 254 if (st != ZX_OK) { 255 zxlogf(SPEW, "ahci.%d: failed to pin pages, err = %d\n", port->nr, st); 256 return st; 257 } 258 txn->pmt = pmt; 259 260 phys_iter_buffer_t physbuf = { 261 .phys = pages, 262 .phys_count = pagecount, 263 .length = bytes, 264 .vmo_offset = offset_vmo, 265 }; 266 phys_iter_t iter; 267 phys_iter_init(&iter, &physbuf, AHCI_PRD_MAX_SIZE); 268 269 uint8_t cmd = txn->cmd; 270 uint8_t device = txn->device; 271 uint64_t lba = txn->bop.rw.offset_dev; 272 uint64_t count = txn->bop.rw.length; 273 274 // use queued command if available 275 if (dev->cap & AHCI_CAP_NCQ) { 276 if (cmd == SATA_CMD_READ_DMA_EXT) { 277 cmd = SATA_CMD_READ_FPDMA_QUEUED; 278 } else if (cmd == SATA_CMD_WRITE_DMA_EXT) { 279 cmd = SATA_CMD_WRITE_FPDMA_QUEUED; 280 } 281 } 282 283 // build the command 284 ahci_cl_t* cl = port->cl + slot; 285 // don't clear the cl since we set up ctba/ctbau at init 286 cl->prdtl_flags_cfl = 0; 287 cl->cfl = 5; // 20 bytes 288 cl->w = is_write ? 1 : 0; 289 cl->prdbc = 0; 290 memset(port->ct[slot], 0, sizeof(ahci_ct_t)); 291 292 uint8_t* cfis = port->ct[slot]->cfis; 293 cfis[0] = 0x27; // host-to-device 294 cfis[1] = 0x80; // command 295 cfis[2] = cmd; 296 cfis[7] = device; 297 298 // some commands have lba/count fields 299 if (cmd == SATA_CMD_READ_DMA_EXT || 300 cmd == SATA_CMD_WRITE_DMA_EXT) { 301 cfis[4] = lba & 0xff; 302 cfis[5] = (lba >> 8) & 0xff; 303 cfis[6] = (lba >> 16) & 0xff; 304 cfis[8] = (lba >> 24) & 0xff; 305 cfis[9] = (lba >> 32) & 0xff; 306 cfis[10] = (lba >> 40) & 0xff; 307 cfis[12] = count & 0xff; 308 cfis[13] = (count >> 8) & 0xff; 309 } else if (cmd_is_queued(cmd)) { 310 cfis[4] = lba & 0xff; 311 cfis[5] = (lba >> 8) & 0xff; 312 cfis[6] = (lba >> 16) & 0xff; 313 cfis[8] = (lba >> 24) & 0xff; 314 cfis[9] = (lba >> 32) & 0xff; 315 cfis[10] = (lba >> 40) & 0xff; 316 cfis[3] = count & 0xff; 317 cfis[11] = (count >> 8) & 0xff; 318 cfis[12] = (slot << 3) & 0xff; // tag 319 cfis[13] = 0; // normal priority 320 } 321 322 cl->prdtl = 0; 323 ahci_prd_t* prd = (ahci_prd_t*)((void*)port->ct[slot] + sizeof(ahci_ct_t)); 324 size_t length; 325 zx_paddr_t paddr; 326 for (;;) { 327 length = phys_iter_next(&iter, &paddr); 328 if (length == 0) { 329 break; 330 } else if (length > AHCI_PRD_MAX_SIZE) { 331 zxlogf(ERROR, "ahci.%d: chunk size > %zu is unsupported\n", port->nr, length); 332 return ZX_ERR_NOT_SUPPORTED;; 333 } else if (cl->prdtl == AHCI_MAX_PRDS) { 334 zxlogf(ERROR, "ahci.%d: txn with more than %d chunks is unsupported\n", 335 port->nr, cl->prdtl); 336 return ZX_ERR_NOT_SUPPORTED; 337 } 338 339 prd->dba = LO32(paddr); 340 prd->dbau = HI32(paddr); 341 prd->dbc = ((length - 1) & (AHCI_PRD_MAX_SIZE - 1)); // 0-based byte count 342 cl->prdtl += 1; 343 prd += 1; 344 } 345 346 port->running |= (1 << slot); 347 port->commands[slot] = txn; 348 349 zxlogf(SPEW, "ahci.%d: do_txn txn %p (%c) offset 0x%" PRIx64 " length 0x%" PRIx64 350 " slot %d prdtl %u\n", 351 port->nr, txn, cl->w ? 'w' : 'r', lba, count, slot, cl->prdtl); 352 prd = (ahci_prd_t*)((void*)port->ct[slot] + sizeof(ahci_ct_t)); 353 if (driver_get_log_flags() & DDK_LOG_SPEW) { 354 for (uint i = 0; i < cl->prdtl; i++) { 355 zxlogf(SPEW, "%04u: dbau=0x%08x dba=0x%08x dbc=0x%x\n", 356 i, prd->dbau, prd->dba, prd->dbc); 357 prd += 1; 358 } 359 } 360 361 // start command 362 if (cmd_is_queued(cmd)) { 363 ahci_write(&port->regs->sact, (1 << slot)); 364 } 365 ahci_write(&port->regs->ci, (1 << slot)); 366 367 // set the watchdog 368 // TODO: general timeout mechanism 369 txn->timeout = zx_clock_get_monotonic() + ZX_SEC(1); 370 sync_completion_signal(&dev->watchdog_completion); 371 return ZX_OK; 372} 373 374static zx_status_t ahci_port_initialize(ahci_device_t* dev, ahci_port_t* port) { 375 uint32_t cmd = ahci_read(&port->regs->cmd); 376 if (cmd & (AHCI_PORT_CMD_ST | AHCI_PORT_CMD_FRE | AHCI_PORT_CMD_CR | AHCI_PORT_CMD_FR)) { 377 zxlogf(ERROR, "ahci.%d: port busy\n", port->nr); 378 return ZX_ERR_UNAVAILABLE; 379 } 380 381 // allocate memory for the command list, FIS receive area, command table and PRDT 382 size_t ct_prd_sz = sizeof(ahci_ct_t) + sizeof(ahci_prd_t) * AHCI_MAX_PRDS; 383 size_t ct_prd_padding = 0x80 - (ct_prd_sz & (0x80 - 1)); // 128-byte aligned 384 size_t mem_sz = sizeof(ahci_fis_t) + sizeof(ahci_cl_t) * AHCI_MAX_COMMANDS 385 + (ct_prd_sz + ct_prd_padding) * AHCI_MAX_COMMANDS; 386 zx_status_t status = io_buffer_init(&port->buffer, dev->bti_handle, 387 mem_sz, IO_BUFFER_RW | IO_BUFFER_CONTIG); 388 if (status < 0) { 389 zxlogf(ERROR, "ahci.%d: error %d allocating dma memory\n", port->nr, status); 390 return status; 391 } 392 zx_paddr_t mem_phys = io_buffer_phys(&port->buffer); 393 void* mem = io_buffer_virt(&port->buffer); 394 395 // clear memory area 396 // order is command list (1024-byte aligned) 397 // FIS receive area (256-byte aligned) 398 // command table + PRDT (127-byte aligned) 399 memset(mem, 0, mem_sz); 400 401 // command list 402 ahci_write(&port->regs->clb, LO32(mem_phys)); 403 ahci_write(&port->regs->clbu, HI32(mem_phys)); 404 mem_phys += sizeof(ahci_cl_t) * AHCI_MAX_COMMANDS; 405 port->cl = mem; 406 mem += sizeof(ahci_cl_t) * AHCI_MAX_COMMANDS; 407 408 // FIS receive area 409 ahci_write(&port->regs->fb, LO32(mem_phys)); 410 ahci_write(&port->regs->fbu, HI32(mem_phys)); 411 mem_phys += sizeof(ahci_fis_t); 412 port->fis = mem; 413 mem += sizeof(ahci_fis_t); 414 415 // command table, followed by PRDT 416 for (int i = 0; i < AHCI_MAX_COMMANDS; i++) { 417 port->cl[i].ctba = LO32(mem_phys); 418 port->cl[i].ctbau = HI32(mem_phys); 419 mem_phys += ct_prd_sz + ct_prd_padding; 420 port->ct[i] = mem; 421 mem += ct_prd_sz + ct_prd_padding; 422 } 423 424 // clear port interrupts 425 ahci_write(&port->regs->is, ahci_read(&port->regs->is)); 426 427 // clear error 428 ahci_write(&port->regs->serr, ahci_read(&port->regs->serr)); 429 430 // spin up 431 cmd |= AHCI_PORT_CMD_SUD; 432 ahci_write(&port->regs->cmd, cmd); 433 434 // activate link 435 cmd &= ~AHCI_PORT_CMD_ICC_MASK; 436 cmd |= AHCI_PORT_CMD_ICC_ACTIVE; 437 ahci_write(&port->regs->cmd, cmd); 438 439 // enable FIS receive 440 cmd |= AHCI_PORT_CMD_FRE; 441 ahci_write(&port->regs->cmd, cmd); 442 443 return ZX_OK; 444} 445 446static void ahci_enable_ahci(ahci_device_t* dev) { 447 uint32_t ghc = ahci_read(&dev->regs->ghc); 448 if (ghc & AHCI_GHC_AE) return; 449 for (int i = 0; i < 5; i++) { 450 ghc |= AHCI_GHC_AE; 451 ahci_write(&dev->regs->ghc, ghc); 452 ghc = ahci_read(&dev->regs->ghc); 453 if (ghc & AHCI_GHC_AE) return; 454 usleep(10 * 1000); 455 } 456} 457 458static void ahci_hba_reset(ahci_device_t* dev) { 459 // AHCI 1.3: Software may perform an HBA reset prior to initializing the controller 460 uint32_t ghc = ahci_read(&dev->regs->ghc); 461 ghc |= AHCI_GHC_AE; 462 ahci_write(&dev->regs->ghc, ghc); 463 ghc |= AHCI_GHC_HR; 464 ahci_write(&dev->regs->ghc, ghc); 465 // reset should complete within 1 second 466 zx_status_t status = ahci_wait_for_clear(&dev->regs->ghc, AHCI_GHC_HR, 1000 * 1000 * 1000); 467 if (status) { 468 zxlogf(ERROR, "ahci: hba reset timed out\n"); 469 } 470} 471 472void ahci_set_devinfo(ahci_device_t* device, int portnr, sata_devinfo_t* devinfo) { 473 ZX_DEBUG_ASSERT(ahci_port_valid(device, portnr)); 474 ahci_port_t* port = &device->ports[portnr]; 475 memcpy(&port->devinfo, devinfo, sizeof(port->devinfo)); 476} 477 478void ahci_queue(ahci_device_t* device, int portnr, sata_txn_t* txn) { 479 ZX_DEBUG_ASSERT(ahci_port_valid(device, portnr)); 480 481 ahci_port_t* port = &device->ports[portnr]; 482 483 zxlogf(SPEW, "ahci.%d: queue_txn txn %p offset_dev 0x%" PRIx64 " length 0x%x\n", 484 port->nr, txn, txn->bop.rw.offset_dev, txn->bop.rw.length); 485 486 // reset the physical address 487 txn->pmt = ZX_HANDLE_INVALID; 488 489 // put the cmd on the queue 490 mtx_lock(&port->lock); 491 list_add_tail(&port->txn_list, &txn->node); 492 493 // hit the worker thread 494 sync_completion_signal(&device->worker_completion); 495 mtx_unlock(&port->lock); 496} 497 498static void ahci_release(void* ctx) { 499 // FIXME - join threads created by this driver 500 ahci_device_t* device = ctx; 501 zx_handle_close(device->irq_handle); 502 zx_handle_close(device->bti_handle); 503 free(device); 504} 505 506// worker thread 507 508static int ahci_worker_thread(void* arg) { 509 ahci_device_t* dev = (ahci_device_t*)arg; 510 ahci_port_t* port; 511 sata_txn_t* txn; 512 for (;;) { 513 // iterate all the ports and run or complete commands 514 for (int i = 0; i < AHCI_MAX_PORTS; i++) { 515 port = &dev->ports[i]; 516 mtx_lock(&port->lock); 517 if (!ahci_port_valid(dev, i)) { 518 goto next; 519 } 520 521 // complete commands first 522 while (port->completed) { 523 unsigned slot = 32 - __builtin_clz(port->completed) - 1; 524 txn = port->commands[slot]; 525 if (txn == NULL) { 526 zxlogf(ERROR, "ahci.%d: illegal state, completing slot %d but txn == NULL\n", 527 port->nr, slot); 528 } else { 529 mtx_unlock(&port->lock); 530 if (txn->pmt != ZX_HANDLE_INVALID) { 531 zx_pmt_unpin(txn->pmt); 532 } 533 zxlogf(SPEW, "ahci.%d: complete txn %p\n", port->nr, txn); 534 block_complete(&txn->bop, ZX_OK); 535 mtx_lock(&port->lock); 536 } 537 port->completed &= ~(1 << slot); 538 port->running &= ~(1 << slot); 539 port->commands[slot] = NULL; 540 // resume the port if paused for sync and no outstanding transactions 541 if ((port->flags & AHCI_PORT_FLAG_SYNC_PAUSED) && !port->running) { 542 port->flags &= ~AHCI_PORT_FLAG_SYNC_PAUSED; 543 if (port->sync) { 544 block_op_t* sop = &port->sync->bop; 545 port->sync = NULL; 546 mtx_unlock(&port->lock); 547 block_complete(sop, ZX_OK); 548 mtx_lock(&port->lock); 549 } 550 } 551 } 552 553 if (port->flags & AHCI_PORT_FLAG_SYNC_PAUSED) { 554 goto next; 555 } 556 557 // process queued txns 558 for (;;) { 559 txn = list_peek_head_type(&port->txn_list, sata_txn_t, node); 560 if (!txn) { 561 break; 562 } 563 564 // find a free command tag 565 int max = MIN(port->devinfo.max_cmd, (int)((dev->cap >> 8) & 0x1f)); 566 int i = 0; 567 for (i = 0; i <= max; i++) { 568 if (!ahci_port_cmd_busy(port, i)) break; 569 } 570 if (i > max) { 571 break; 572 } 573 574 list_delete(&txn->node); 575 576 if (BLOCK_OP(txn->bop.command) == BLOCK_OP_FLUSH) { 577 if (port->running) { 578 ZX_DEBUG_ASSERT(port->sync == NULL); 579 // pause the port if FLUSH command 580 port->flags |= AHCI_PORT_FLAG_SYNC_PAUSED; 581 port->sync = txn; 582 } else { 583 // complete immediately if nothing in flight 584 mtx_unlock(&port->lock); 585 block_complete(&txn->bop, ZX_OK); 586 mtx_lock(&port->lock); 587 } 588 } else { 589 // run the transaction 590 zx_status_t st = ahci_do_txn(dev, port, i, txn); 591 // complete the transaction with if it failed during processing 592 if (st != ZX_OK) { 593 mtx_unlock(&port->lock); 594 block_complete(&txn->bop, st); 595 mtx_lock(&port->lock); 596 continue; 597 } 598 } 599 } 600next: 601 mtx_unlock(&port->lock); 602 } 603 // wait here until more commands are queued, or a port becomes idle 604 sync_completion_wait(&dev->worker_completion, ZX_TIME_INFINITE); 605 sync_completion_reset(&dev->worker_completion); 606 } 607 return 0; 608} 609 610static int ahci_watchdog_thread(void* arg) { 611 ahci_device_t* dev = (ahci_device_t*)arg; 612 for (;;) { 613 bool idle = true; 614 zx_time_t now = zx_clock_get_monotonic(); 615 for (int i = 0; i < AHCI_MAX_PORTS; i++) { 616 ahci_port_t* port = &dev->ports[i]; 617 if (!ahci_port_valid(dev, i)) { 618 continue; 619 } 620 621 mtx_lock(&port->lock); 622 uint32_t pending = port->running & ~port->completed; 623 while (pending) { 624 idle = false; 625 unsigned slot = 32 - __builtin_clz(pending) - 1; 626 sata_txn_t* txn = port->commands[slot]; 627 if (!txn) { 628 zxlogf(ERROR, "ahci: command %u pending but txn is NULL\n", slot); 629 } else { 630 if (txn->timeout < now) { 631 // time out 632 zxlogf(ERROR, "ahci: txn time out on port %d txn %p\n", port->nr, txn); 633 port->running &= ~(1 << slot); 634 port->commands[slot] = NULL; 635 mtx_unlock(&port->lock); 636 block_complete(&txn->bop, ZX_ERR_TIMED_OUT); 637 mtx_lock(&port->lock); 638 } 639 } 640 pending &= ~(1 << slot); 641 } 642 mtx_unlock(&port->lock); 643 } 644 645 // no need to run the watchdog if there are no active xfers 646 sync_completion_wait(&dev->watchdog_completion, idle ? ZX_TIME_INFINITE : 5ULL * 1000 * 1000 * 1000); 647 sync_completion_reset(&dev->watchdog_completion); 648 } 649 return 0; 650} 651 652// irq handler: 653 654static void ahci_port_irq(ahci_device_t* dev, int nr) { 655 ahci_port_t* port = &dev->ports[nr]; 656 // clear interrupt 657 uint32_t is = ahci_read(&port->regs->is); 658 ahci_write(&port->regs->is, is); 659 660 if (is & AHCI_PORT_INT_PRC) { // PhyRdy change 661 uint32_t serr = ahci_read(&port->regs->serr); 662 ahci_write(&port->regs->serr, serr & ~0x1); 663 } 664 if (is & AHCI_PORT_INT_ERROR) { // error 665 zxlogf(ERROR, "ahci.%d: error is=0x%08x\n", nr, is); 666 ahci_port_complete_txn(dev, port, ZX_ERR_INTERNAL); 667 } else if (is) { 668 ahci_port_complete_txn(dev, port, ZX_OK); 669 } 670} 671 672static int ahci_irq_thread(void* arg) { 673 ahci_device_t* dev = (ahci_device_t*)arg; 674 zx_status_t status; 675 for (;;) { 676 status = zx_interrupt_wait(dev->irq_handle, NULL); 677 if (status) { 678 zxlogf(ERROR, "ahci: error %d waiting for interrupt\n", status); 679 continue; 680 } 681 // mask hba interrupts while interrupts are being handled 682 uint32_t ghc = ahci_read(&dev->regs->ghc); 683 ahci_write(&dev->regs->ghc, ghc & ~AHCI_GHC_IE); 684 685 // handle interrupt for each port 686 uint32_t is = ahci_read(&dev->regs->is); 687 ahci_write(&dev->regs->is, is); 688 for (int i = 0; is && i < AHCI_MAX_PORTS; i++) { 689 if (is & 0x1) { 690 ahci_port_irq(dev, i); 691 } 692 is >>= 1; 693 } 694 695 // unmask hba interrupts 696 ghc = ahci_read(&dev->regs->ghc); 697 ahci_write(&dev->regs->ghc, ghc | AHCI_GHC_IE); 698 } 699 return 0; 700} 701 702// implement device protocol: 703 704static zx_protocol_device_t ahci_device_proto = { 705 .version = DEVICE_OPS_VERSION, 706 .release = ahci_release, 707}; 708 709extern zx_protocol_device_t ahci_port_device_proto; 710 711static int ahci_init_thread(void* arg) { 712 ahci_device_t* dev = (ahci_device_t*)arg; 713 714 // reset 715 ahci_hba_reset(dev); 716 717 // enable ahci mode 718 ahci_enable_ahci(dev); 719 720 dev->cap = ahci_read(&dev->regs->cap); 721 722 // count number of ports 723 uint32_t port_map = ahci_read(&dev->regs->pi); 724 725 // initialize ports 726 zx_status_t status; 727 ahci_port_t* port; 728 for (int i = 0; i < AHCI_MAX_PORTS; i++) { 729 port = &dev->ports[i]; 730 port->nr = i; 731 732 if (!(port_map & (1 << i))) continue; // port not implemented 733 734 port->flags = AHCI_PORT_FLAG_IMPLEMENTED; 735 port->regs = &dev->regs->ports[i]; 736 list_initialize(&port->txn_list); 737 738 status = ahci_port_initialize(dev, port); 739 if (status) goto fail; 740 } 741 742 // clear hba interrupts 743 ahci_write(&dev->regs->is, ahci_read(&dev->regs->is)); 744 745 // enable hba interrupts 746 uint32_t ghc = ahci_read(&dev->regs->ghc); 747 ghc |= AHCI_GHC_IE; 748 ahci_write(&dev->regs->ghc, ghc); 749 750 // this part of port init happens after enabling interrupts in ghc 751 for (int i = 0; i < AHCI_MAX_PORTS; i++) { 752 port = &dev->ports[i]; 753 if (!(port->flags & AHCI_PORT_FLAG_IMPLEMENTED)) continue; 754 755 // enable port 756 ahci_port_enable(port); 757 758 // enable interrupts 759 ahci_write(&port->regs->ie, AHCI_PORT_INT_MASK); 760 761 // reset port 762 ahci_port_reset(port); 763 764 // FIXME proper layering? 765 if (ahci_read(&port->regs->ssts) & AHCI_PORT_SSTS_DET_PRESENT) { 766 port->flags |= AHCI_PORT_FLAG_PRESENT; 767 if (ahci_read(&port->regs->sig) == AHCI_PORT_SIG_SATA) { 768 sata_bind(dev, dev->zxdev, port->nr); 769 } 770 } 771 } 772 773 return ZX_OK; 774fail: 775 free(dev->ports); 776 return status; 777} 778 779// implement driver object: 780 781static zx_status_t ahci_bind(void* ctx, zx_device_t* dev) { 782 // map resources and initalize the device 783 ahci_device_t* device = calloc(1, sizeof(ahci_device_t)); 784 if (!device) { 785 zxlogf(ERROR, "ahci: out of memory\n"); 786 return ZX_ERR_NO_MEMORY; 787 } 788 789 if (device_get_protocol(dev, ZX_PROTOCOL_PCI, &device->pci)) { 790 free(device); 791 return ZX_ERR_NOT_SUPPORTED; 792 } 793 794 // map register window 795 zx_status_t status = pci_map_bar(&device->pci, 796 5u, 797 ZX_CACHE_POLICY_UNCACHED_DEVICE, 798 (void**)&device->regs, 799 &device->regs_size, 800 &device->regs_handle); 801 if (status != ZX_OK) { 802 zxlogf(ERROR, "ahci: error %d mapping register window\n", status); 803 goto fail; 804 } 805 806 zx_pcie_device_info_t config; 807 status = pci_get_device_info(&device->pci, &config); 808 if (status != ZX_OK) { 809 zxlogf(ERROR, "ahci: error getting config information\n"); 810 goto fail; 811 } 812 813 if (config.sub_class != 0x06 && config.base_class == 0x01) { // SATA 814 status = ZX_ERR_NOT_SUPPORTED; 815 zxlogf(ERROR, "ahci: device class 0x%x unsupported!\n", config.sub_class); 816 goto fail; 817 } 818 819 // FIXME intel devices need to set SATA port enable at config + 0x92 820 // ahci controller is bus master 821 status = pci_enable_bus_master(&device->pci, true); 822 if (status < 0) { 823 zxlogf(ERROR, "ahci: error %d in enable bus master\n", status); 824 goto fail; 825 } 826 827 // Query and configure IRQ modes by trying MSI first and falling back to 828 // legacy if necessary. 829 uint32_t irq_cnt; 830 zx_pci_irq_mode_t irq_mode = ZX_PCIE_IRQ_MODE_MSI; 831 status = pci_query_irq_mode(&device->pci, ZX_PCIE_IRQ_MODE_MSI, &irq_cnt); 832 if (status == ZX_ERR_NOT_SUPPORTED) { 833 status = pci_query_irq_mode(&device->pci, ZX_PCIE_IRQ_MODE_LEGACY, &irq_cnt); 834 if (status != ZX_OK) { 835 zxlogf(ERROR, "ahci: neither MSI nor legacy interrupts are supported\n"); 836 goto fail; 837 } else { 838 irq_mode = ZX_PCIE_IRQ_MODE_LEGACY; 839 } 840 } 841 842 if (irq_cnt == 0) { 843 zxlogf(ERROR, "ahci: no interrupts available\n"); 844 status = ZX_ERR_NO_RESOURCES; 845 goto fail; 846 } 847 848 zxlogf(INFO, "ahci: using %s interrupt\n", (irq_mode == ZX_PCIE_IRQ_MODE_MSI) ? "MSI" : "legacy"); 849 status = pci_set_irq_mode(&device->pci, irq_mode, 1); 850 if (status != ZX_OK) { 851 zxlogf(ERROR, "ahci: error %d setting irq mode\n", status); 852 goto fail; 853 } 854 855 // get bti handle 856 status = pci_get_bti(&device->pci, 0, &device->bti_handle); 857 if (status != ZX_OK) { 858 zxlogf(ERROR, "ahci: error %d getting bti handle\n", status); 859 goto fail; 860 } 861 862 // get irq handle 863 status = pci_map_interrupt(&device->pci, 0, &device->irq_handle); 864 if (status != ZX_OK) { 865 zxlogf(ERROR, "ahci: error %d getting irq handle\n", status); 866 goto fail; 867 } 868 869 // start irq thread 870 int ret = thrd_create_with_name(&device->irq_thread, ahci_irq_thread, device, "ahci-irq"); 871 if (ret != thrd_success) { 872 zxlogf(ERROR, "ahci: error %d in irq thread create\n", ret); 873 goto fail; 874 } 875 876 // start watchdog thread 877 device->watchdog_completion = SYNC_COMPLETION_INIT; 878 thrd_create_with_name(&device->watchdog_thread, ahci_watchdog_thread, device, "ahci-watchdog"); 879 880 // start worker thread (for iotxn queue) 881 device->worker_completion = SYNC_COMPLETION_INIT; 882 ret = thrd_create_with_name(&device->worker_thread, ahci_worker_thread, device, "ahci-worker"); 883 if (ret != thrd_success) { 884 zxlogf(ERROR, "ahci: error %d in worker thread create\n", ret); 885 goto fail; 886 } 887 888 // add the device for the controller 889 device_add_args_t args = { 890 .version = DEVICE_ADD_ARGS_VERSION, 891 .name = "ahci", 892 .ctx = device, 893 .ops = &ahci_device_proto, 894 .flags = DEVICE_ADD_NON_BINDABLE, 895 }; 896 897 status = device_add(dev, &args, &device->zxdev); 898 if (status != ZX_OK) { 899 zxlogf(ERROR, "ahci: error %d in device_add\n", status); 900 goto fail; 901 } 902 903 // initialize controller and detect devices 904 thrd_t t; 905 ret = thrd_create_with_name(&t, ahci_init_thread, device, "ahci-init"); 906 if (ret != thrd_success) { 907 zxlogf(ERROR, "ahci: error %d in init thread create\n", status); 908 goto fail; 909 } 910 911 return ZX_OK; 912fail: 913 // FIXME unmap, and join any threads created above 914 free(device); 915 return status; 916} 917 918static zx_driver_ops_t ahci_driver_ops = { 919 .version = DRIVER_OPS_VERSION, 920 .bind = ahci_bind, 921}; 922 923// clang-format off 924ZIRCON_DRIVER_BEGIN(ahci, ahci_driver_ops, "zircon", "0.1", 4) 925 BI_ABORT_IF(NE, BIND_PROTOCOL, ZX_PROTOCOL_PCI), 926 BI_ABORT_IF(NE, BIND_PCI_CLASS, 0x01), 927 BI_ABORT_IF(NE, BIND_PCI_SUBCLASS, 0x06), 928 BI_MATCH_IF(EQ, BIND_PCI_INTERFACE, 0x01), 929ZIRCON_DRIVER_END(ahci) 930