1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 1998 - 2008 S��ren Schmidt <sos@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer, 12 * without modification, immediately at the beginning of the file. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29#include <sys/param.h> 30#include <sys/systm.h> 31#include <sys/kernel.h> 32#include <sys/endian.h> 33#include <sys/ata.h> 34#include <sys/bio.h> 35#include <sys/conf.h> 36#include <sys/ctype.h> 37#include <sys/bus.h> 38#include <sys/sema.h> 39#include <sys/taskqueue.h> 40#include <vm/uma.h> 41#include <machine/bus.h> 42#include <sys/rman.h> 43#include <dev/ata/ata-all.h> 44#include <dev/ata/ata-pci.h> 45#include <ata_if.h> 46 47#include <vm/vm.h> 48#include <vm/pmap.h> 49 50#include <cam/cam.h> 51#include <cam/cam_ccb.h> 52 53/* prototypes */ 54static int ata_generic_status(device_t dev); 55static int ata_wait(struct ata_channel *ch, int unit, u_int8_t); 56static void ata_pio_read(struct ata_request *, int); 57static void ata_pio_write(struct ata_request *, int); 58static void ata_tf_read(struct ata_request *); 59static void ata_tf_write(struct ata_request *); 60 61/* 62 * low level ATA functions 63 */ 64void 65ata_generic_hw(device_t dev) 66{ 67 struct ata_channel *ch = device_get_softc(dev); 68 69 ch->hw.begin_transaction = ata_begin_transaction; 70 ch->hw.end_transaction = ata_end_transaction; 71 ch->hw.status = ata_generic_status; 72 ch->hw.softreset = NULL; 73 ch->hw.command = ata_generic_command; 74 ch->hw.tf_read = ata_tf_read; 75 ch->hw.tf_write = ata_tf_write; 76 ch->hw.pm_read = NULL; 77 ch->hw.pm_write = NULL; 78} 79 80/* must be called with ATA channel locked and state_mtx held */ 81int 82ata_begin_transaction(struct ata_request *request) 83{ 84 struct ata_channel *ch = device_get_softc(request->parent); 85 int dummy, error; 86 87 ATA_DEBUG_RQ(request, "begin transaction"); 88 89 /* disable ATAPI DMA writes if HW doesn't support it */ 90 if ((ch->flags & ATA_NO_ATAPI_DMA) && 91 (request->flags & ATA_R_ATAPI) == ATA_R_ATAPI) 92 request->flags &= ~ATA_R_DMA; 93 if ((ch->flags & ATA_ATAPI_DMA_RO) && 94 ((request->flags & (ATA_R_ATAPI | ATA_R_DMA | ATA_R_WRITE)) == 95 (ATA_R_ATAPI | ATA_R_DMA | ATA_R_WRITE))) 96 request->flags &= ~ATA_R_DMA; 97 98 switch (request->flags & (ATA_R_ATAPI | ATA_R_DMA)) { 99 /* ATA PIO data transfer and control commands */ 100 default: 101 { 102 /* record command direction here as our request might be gone later */ 103 int write = (request->flags & ATA_R_WRITE); 104 105 /* issue command */ 106 if (ch->hw.command(request)) { 107 device_printf(request->parent, "error issuing %s command\n", 108 ata_cmd2str(request)); 109 request->result = EIO; 110 goto begin_finished; 111 } 112 113 /* device reset doesn't interrupt */ 114 if (request->u.ata.command == ATA_DEVICE_RESET) { 115 int timeout = 1000000; 116 do { 117 DELAY(10); 118 request->status = ATA_IDX_INB(ch, ATA_STATUS); 119 } while (request->status & ATA_S_BUSY && timeout--); 120 if (request->status & ATA_S_ERROR) 121 request->error = ATA_IDX_INB(ch, ATA_ERROR); 122 ch->hw.tf_read(request); 123 goto begin_finished; 124 } 125 126 /* if write command output the data */ 127 if (write) { 128 if (ata_wait(ch, request->unit, (ATA_S_READY | ATA_S_DRQ)) < 0) { 129 device_printf(request->parent, 130 "timeout waiting for write DRQ\n"); 131 request->result = EIO; 132 goto begin_finished; 133 } 134 ata_pio_write(request, request->transfersize); 135 } 136 } 137 goto begin_continue; 138 139 /* ATA DMA data transfer commands */ 140 case ATA_R_DMA: 141 /* check sanity, setup SG list and DMA engine */ 142 if ((error = ch->dma.load(request, NULL, &dummy))) { 143 device_printf(request->parent, "setting up DMA failed\n"); 144 request->result = error; 145 goto begin_finished; 146 } 147 148 /* start DMA engine if necessary */ 149 if ((ch->flags & ATA_DMA_BEFORE_CMD) && 150 ch->dma.start && ch->dma.start(request)) { 151 device_printf(request->parent, "error starting DMA\n"); 152 request->result = EIO; 153 goto begin_finished; 154 } 155 156 /* issue command */ 157 if (ch->hw.command(request)) { 158 device_printf(request->parent, "error issuing %s command\n", 159 ata_cmd2str(request)); 160 request->result = EIO; 161 goto begin_finished; 162 } 163 164 /* start DMA engine */ 165 if (!(ch->flags & ATA_DMA_BEFORE_CMD) && 166 ch->dma.start && ch->dma.start(request)) { 167 device_printf(request->parent, "error starting DMA\n"); 168 request->result = EIO; 169 goto begin_finished; 170 } 171 goto begin_continue; 172 173 /* ATAPI PIO commands */ 174 case ATA_R_ATAPI: 175 /* is this just a POLL DSC command ? */ 176 if (request->u.atapi.ccb[0] == ATAPI_POLL_DSC) { 177 ATA_IDX_OUTB(ch, ATA_DRIVE, ATA_D_IBM | ATA_DEV(request->unit)); 178 DELAY(10); 179 if (!(ATA_IDX_INB(ch, ATA_ALTSTAT) & ATA_S_DSC)) 180 request->result = EBUSY; 181 goto begin_finished; 182 } 183 184 /* start ATAPI operation */ 185 if (ch->hw.command(request)) { 186 device_printf(request->parent, "error issuing ATA PACKET command\n"); 187 request->result = EIO; 188 goto begin_finished; 189 } 190 goto begin_continue; 191 192 /* ATAPI DMA commands */ 193 case ATA_R_ATAPI|ATA_R_DMA: 194 /* is this just a POLL DSC command ? */ 195 if (request->u.atapi.ccb[0] == ATAPI_POLL_DSC) { 196 ATA_IDX_OUTB(ch, ATA_DRIVE, ATA_D_IBM | ATA_DEV(request->unit)); 197 DELAY(10); 198 if (!(ATA_IDX_INB(ch, ATA_ALTSTAT) & ATA_S_DSC)) 199 request->result = EBUSY; 200 goto begin_finished; 201 } 202 203 /* check sanity, setup SG list and DMA engine */ 204 if ((error = ch->dma.load(request, NULL, &dummy))) { 205 device_printf(request->parent, "setting up DMA failed\n"); 206 request->result = error; 207 goto begin_finished; 208 } 209 210 /* start ATAPI operation */ 211 if (ch->hw.command(request)) { 212 device_printf(request->parent, "error issuing ATA PACKET command\n"); 213 request->result = EIO; 214 goto begin_finished; 215 } 216 217 /* start DMA engine */ 218 if (ch->dma.start && ch->dma.start(request)) { 219 request->result = EIO; 220 goto begin_finished; 221 } 222 goto begin_continue; 223 } 224 /* NOT REACHED */ 225 printf("ata_begin_transaction OOPS!!!\n"); 226 227begin_finished: 228 if (ch->dma.unload) { 229 ch->dma.unload(request); 230 } 231 return ATA_OP_FINISHED; 232 233begin_continue: 234 callout_reset(&request->callout, request->timeout * hz, 235 ata_timeout, request); 236 return ATA_OP_CONTINUES; 237} 238 239/* must be called with ATA channel locked and state_mtx held */ 240int 241ata_end_transaction(struct ata_request *request) 242{ 243 struct ata_channel *ch = device_get_softc(request->parent); 244 int length; 245 246 ATA_DEBUG_RQ(request, "end transaction"); 247 248 /* clear interrupt and get status */ 249 request->status = ATA_IDX_INB(ch, ATA_STATUS); 250 251 switch (request->flags & (ATA_R_ATAPI | ATA_R_DMA | ATA_R_CONTROL)) { 252 /* ATA PIO data transfer and control commands */ 253 default: 254 255 /* on timeouts we have no data or anything so just return */ 256 if (request->flags & ATA_R_TIMEOUT) 257 goto end_finished; 258 259 /* Read back registers to the request struct. */ 260 if ((request->status & ATA_S_ERROR) || 261 (request->flags & (ATA_R_CONTROL | ATA_R_NEEDRESULT))) { 262 ch->hw.tf_read(request); 263 } 264 265 /* if we got an error we are done with the HW */ 266 if (request->status & ATA_S_ERROR) { 267 request->error = ATA_IDX_INB(ch, ATA_ERROR); 268 goto end_finished; 269 } 270 271 /* are we moving data ? */ 272 if (request->flags & (ATA_R_READ | ATA_R_WRITE)) { 273 /* if read data get it */ 274 if (request->flags & ATA_R_READ) { 275 int flags = ATA_S_DRQ; 276 277 if (request->u.ata.command != ATA_ATAPI_IDENTIFY) 278 flags |= ATA_S_READY; 279 if (ata_wait(ch, request->unit, flags) < 0) { 280 device_printf(request->parent, 281 "timeout waiting for read DRQ\n"); 282 request->result = EIO; 283 goto end_finished; 284 } 285 ata_pio_read(request, request->transfersize); 286 } 287 288 /* update how far we've gotten */ 289 request->donecount += request->transfersize; 290 291 /* do we need a scoop more ? */ 292 if (request->bytecount > request->donecount) { 293 /* set this transfer size according to HW capabilities */ 294 request->transfersize = 295 min((request->bytecount - request->donecount), 296 request->transfersize); 297 298 /* if data write command, output the data */ 299 if (request->flags & ATA_R_WRITE) { 300 /* if we get an error here we are done with the HW */ 301 if (ata_wait(ch, request->unit, (ATA_S_READY | ATA_S_DRQ)) < 0) { 302 device_printf(request->parent, 303 "timeout waiting for write DRQ\n"); 304 request->status = ATA_IDX_INB(ch, ATA_STATUS); 305 goto end_finished; 306 } 307 308 /* output data and return waiting for new interrupt */ 309 ata_pio_write(request, request->transfersize); 310 goto end_continue; 311 } 312 313 /* if data read command, return & wait for interrupt */ 314 if (request->flags & ATA_R_READ) 315 goto end_continue; 316 } 317 } 318 /* done with HW */ 319 goto end_finished; 320 321 /* ATA DMA data transfer commands */ 322 case ATA_R_DMA: 323 324 /* stop DMA engine and get status */ 325 if (ch->dma.stop) 326 request->dma->status = ch->dma.stop(request); 327 328 /* did we get error or data */ 329 if (request->status & ATA_S_ERROR) 330 request->error = ATA_IDX_INB(ch, ATA_ERROR); 331 else if (request->dma->status & ATA_BMSTAT_ERROR) 332 request->status |= ATA_S_ERROR; 333 else if (!(request->flags & ATA_R_TIMEOUT)) 334 request->donecount = request->bytecount; 335 336 /* Read back registers to the request struct. */ 337 if ((request->status & ATA_S_ERROR) || 338 (request->flags & (ATA_R_CONTROL | ATA_R_NEEDRESULT))) { 339 ch->hw.tf_read(request); 340 } 341 342 /* release SG list etc */ 343 ch->dma.unload(request); 344 345 /* done with HW */ 346 goto end_finished; 347 348 /* ATAPI PIO commands */ 349 case ATA_R_ATAPI: 350 length = ATA_IDX_INB(ch, ATA_CYL_LSB)|(ATA_IDX_INB(ch, ATA_CYL_MSB)<<8); 351 352 /* on timeouts we have no data or anything so just return */ 353 if (request->flags & ATA_R_TIMEOUT) 354 goto end_finished; 355 356 switch ((ATA_IDX_INB(ch, ATA_IREASON) & (ATA_I_CMD | ATA_I_IN)) | 357 (request->status & ATA_S_DRQ)) { 358 case ATAPI_P_CMDOUT: 359 /* this seems to be needed for some (slow) devices */ 360 DELAY(10); 361 362 if (!(request->status & ATA_S_DRQ)) { 363 device_printf(request->parent, "command interrupt without DRQ\n"); 364 request->status = ATA_S_ERROR; 365 goto end_finished; 366 } 367 ATA_IDX_OUTSW_STRM(ch, ATA_DATA, (int16_t *)request->u.atapi.ccb, 368 (request->flags & ATA_R_ATAPI16) ? 8 : 6); 369 /* return wait for interrupt */ 370 goto end_continue; 371 372 case ATAPI_P_WRITE: 373 if (request->flags & ATA_R_READ) { 374 request->status = ATA_S_ERROR; 375 device_printf(request->parent, 376 "%s trying to write on read buffer\n", 377 ata_cmd2str(request)); 378 goto end_finished; 379 } 380 ata_pio_write(request, length); 381 request->donecount += length; 382 383 /* set next transfer size according to HW capabilities */ 384 request->transfersize = min((request->bytecount-request->donecount), 385 request->transfersize); 386 /* return wait for interrupt */ 387 goto end_continue; 388 389 case ATAPI_P_READ: 390 if (request->flags & ATA_R_WRITE) { 391 request->status = ATA_S_ERROR; 392 device_printf(request->parent, 393 "%s trying to read on write buffer\n", 394 ata_cmd2str(request)); 395 goto end_finished; 396 } 397 ata_pio_read(request, length); 398 request->donecount += length; 399 400 /* set next transfer size according to HW capabilities */ 401 request->transfersize = min((request->bytecount-request->donecount), 402 request->transfersize); 403 /* return wait for interrupt */ 404 goto end_continue; 405 406 case ATAPI_P_DONEDRQ: 407 device_printf(request->parent, 408 "WARNING - %s DONEDRQ non conformant device\n", 409 ata_cmd2str(request)); 410 if (request->flags & ATA_R_READ) { 411 ata_pio_read(request, length); 412 request->donecount += length; 413 } 414 else if (request->flags & ATA_R_WRITE) { 415 ata_pio_write(request, length); 416 request->donecount += length; 417 } 418 else 419 request->status = ATA_S_ERROR; 420 /* FALLTHROUGH */ 421 422 case ATAPI_P_ABORT: 423 case ATAPI_P_DONE: 424 if (request->status & (ATA_S_ERROR | ATA_S_DWF)) 425 request->error = ATA_IDX_INB(ch, ATA_ERROR); 426 goto end_finished; 427 428 default: 429 device_printf(request->parent, "unknown transfer phase\n"); 430 request->status = ATA_S_ERROR; 431 } 432 433 /* done with HW */ 434 goto end_finished; 435 436 /* ATAPI DMA commands */ 437 case ATA_R_ATAPI|ATA_R_DMA: 438 439 /* stop DMA engine and get status */ 440 if (ch->dma.stop) 441 request->dma->status = ch->dma.stop(request); 442 443 /* did we get error or data */ 444 if (request->status & (ATA_S_ERROR | ATA_S_DWF)) 445 request->error = ATA_IDX_INB(ch, ATA_ERROR); 446 else if (request->dma->status & ATA_BMSTAT_ERROR) 447 request->status |= ATA_S_ERROR; 448 else if (!(request->flags & ATA_R_TIMEOUT)) 449 request->donecount = request->bytecount; 450 451 /* release SG list etc */ 452 ch->dma.unload(request); 453 454 /* done with HW */ 455 goto end_finished; 456 } 457 /* NOT REACHED */ 458 printf("ata_end_transaction OOPS!!\n"); 459 460end_finished: 461 callout_stop(&request->callout); 462 return ATA_OP_FINISHED; 463 464end_continue: 465 return ATA_OP_CONTINUES; 466} 467 468/* must be called with ATA channel locked and state_mtx held */ 469void 470ata_generic_reset(device_t dev) 471{ 472 struct ata_channel *ch = device_get_softc(dev); 473 474 u_int8_t ostat0 = 0, stat0 = 0, ostat1 = 0, stat1 = 0; 475 u_int8_t err = 0, lsb = 0, msb = 0; 476 int mask = 0, timeout; 477 478 /* do we have any signs of ATA/ATAPI HW being present ? */ 479 ATA_IDX_OUTB(ch, ATA_DRIVE, ATA_D_IBM | ATA_D_LBA | ATA_DEV(ATA_MASTER)); 480 DELAY(10); 481 ostat0 = ATA_IDX_INB(ch, ATA_STATUS); 482 if (((ostat0 & 0xf8) != 0xf8 || (ch->flags & ATA_KNOWN_PRESENCE)) && 483 ostat0 != 0xa5) { 484 stat0 = ATA_S_BUSY; 485 mask |= 0x01; 486 } 487 488 /* in some setups we dont want to test for a slave */ 489 if (!(ch->flags & ATA_NO_SLAVE)) { 490 ATA_IDX_OUTB(ch, ATA_DRIVE, ATA_D_IBM | ATA_D_LBA | ATA_DEV(ATA_SLAVE)); 491 DELAY(10); 492 ostat1 = ATA_IDX_INB(ch, ATA_STATUS); 493 if (((ostat1 & 0xf8) != 0xf8 || (ch->flags & ATA_KNOWN_PRESENCE)) && 494 ostat1 != 0xa5) { 495 stat1 = ATA_S_BUSY; 496 mask |= 0x02; 497 } 498 } 499 500 if (bootverbose) 501 device_printf(dev, "reset tp1 mask=%02x ostat0=%02x ostat1=%02x\n", 502 mask, ostat0, ostat1); 503 504 /* if nothing showed up there is no need to get any further */ 505 /* XXX SOS is that too strong?, we just might lose devices here */ 506 ch->devices = 0; 507 if (!mask) 508 return; 509 510 /* reset (both) devices on this channel */ 511 ATA_IDX_OUTB(ch, ATA_DRIVE, ATA_D_IBM | ATA_D_LBA | ATA_DEV(ATA_MASTER)); 512 DELAY(10); 513 ATA_IDX_OUTB(ch, ATA_CONTROL, ATA_A_IDS | ATA_A_RESET); 514 ata_udelay(10000); 515 ATA_IDX_OUTB(ch, ATA_CONTROL, ATA_A_IDS); 516 ata_udelay(100000); 517 ATA_IDX_INB(ch, ATA_ERROR); 518 519 /* wait for BUSY to go inactive */ 520 for (timeout = 0; timeout < 310; timeout++) { 521 if ((mask & 0x01) && (stat0 & ATA_S_BUSY)) { 522 ATA_IDX_OUTB(ch, ATA_DRIVE, ATA_D_IBM | ATA_DEV(ATA_MASTER)); 523 DELAY(10); 524 if (ch->flags & ATA_STATUS_IS_LONG) 525 stat0 = ATA_IDX_INL(ch, ATA_STATUS) & 0xff; 526 else 527 stat0 = ATA_IDX_INB(ch, ATA_STATUS); 528 err = ATA_IDX_INB(ch, ATA_ERROR); 529 lsb = ATA_IDX_INB(ch, ATA_CYL_LSB); 530 msb = ATA_IDX_INB(ch, ATA_CYL_MSB); 531 if (bootverbose) 532 device_printf(dev, 533 "stat0=0x%02x err=0x%02x lsb=0x%02x msb=0x%02x\n", 534 stat0, err, lsb, msb); 535 if (stat0 == err && lsb == err && msb == err && 536 timeout > (stat0 & ATA_S_BUSY ? 100 : 10)) 537 mask &= ~0x01; 538 if (!(stat0 & ATA_S_BUSY)) { 539 if ((err & 0x7f) == ATA_E_ILI) { 540 if (lsb == ATAPI_MAGIC_LSB && msb == ATAPI_MAGIC_MSB) { 541 ch->devices |= ATA_ATAPI_MASTER; 542 } 543 else if (lsb == 0 && msb == 0 && (stat0 & ATA_S_READY)) { 544 ch->devices |= ATA_ATA_MASTER; 545 } 546 } 547 else if ((stat0 & 0x0f) && err == lsb && err == msb) { 548 stat0 |= ATA_S_BUSY; 549 } 550 } 551 } 552 553 if ((mask & 0x02) && (stat1 & ATA_S_BUSY) && 554 !((mask & 0x01) && (stat0 & ATA_S_BUSY))) { 555 ATA_IDX_OUTB(ch, ATA_DRIVE, ATA_D_IBM | ATA_DEV(ATA_SLAVE)); 556 DELAY(10); 557 if (ch->flags & ATA_STATUS_IS_LONG) 558 stat1 = ATA_IDX_INL(ch, ATA_STATUS) & 0xff; 559 else 560 stat1 = ATA_IDX_INB(ch, ATA_STATUS); 561 err = ATA_IDX_INB(ch, ATA_ERROR); 562 lsb = ATA_IDX_INB(ch, ATA_CYL_LSB); 563 msb = ATA_IDX_INB(ch, ATA_CYL_MSB); 564 if (bootverbose) 565 device_printf(dev, 566 "stat1=0x%02x err=0x%02x lsb=0x%02x msb=0x%02x\n", 567 stat1, err, lsb, msb); 568 if (stat1 == err && lsb == err && msb == err && 569 timeout > (stat1 & ATA_S_BUSY ? 100 : 10)) 570 mask &= ~0x02; 571 if (!(stat1 & ATA_S_BUSY)) { 572 if ((err & 0x7f) == ATA_E_ILI) { 573 if (lsb == ATAPI_MAGIC_LSB && msb == ATAPI_MAGIC_MSB) { 574 ch->devices |= ATA_ATAPI_SLAVE; 575 } 576 else if (lsb == 0 && msb == 0 && (stat1 & ATA_S_READY)) { 577 ch->devices |= ATA_ATA_SLAVE; 578 } 579 } 580 else if ((stat1 & 0x0f) && err == lsb && err == msb) { 581 stat1 |= ATA_S_BUSY; 582 } 583 } 584 } 585 586 if ((ch->flags & ATA_KNOWN_PRESENCE) == 0 && 587 timeout > ((mask == 0x03) ? 20 : 10)) { 588 if ((mask & 0x01) && stat0 == 0xff) 589 mask &= ~0x01; 590 if ((mask & 0x02) && stat1 == 0xff) 591 mask &= ~0x02; 592 } 593 if (((mask & 0x01) == 0 || !(stat0 & ATA_S_BUSY)) && 594 ((mask & 0x02) == 0 || !(stat1 & ATA_S_BUSY))) 595 break; 596 ata_udelay(100000); 597 } 598 599 if (bootverbose) 600 device_printf(dev, "reset tp2 stat0=%02x stat1=%02x devices=0x%x\n", 601 stat0, stat1, ch->devices); 602} 603 604/* must be called with ATA channel locked and state_mtx held */ 605static int 606ata_generic_status(device_t dev) 607{ 608 struct ata_channel *ch = device_get_softc(dev); 609 610 if (ATA_IDX_INB(ch, ATA_ALTSTAT) & ATA_S_BUSY) { 611 DELAY(100); 612 if (ATA_IDX_INB(ch, ATA_ALTSTAT) & ATA_S_BUSY) 613 return 0; 614 } 615 return 1; 616} 617 618static int 619ata_wait(struct ata_channel *ch, int unit, u_int8_t mask) 620{ 621 u_int8_t status; 622 int timeout = 0; 623 624 DELAY(1); 625 626 /* wait at max 1 second for device to get !BUSY */ 627 while (timeout < 1000000) { 628 status = ATA_IDX_INB(ch, ATA_ALTSTAT); 629 630 /* if drive fails status, reselect the drive and try again */ 631 if (status == 0xff) { 632 ATA_IDX_OUTB(ch, ATA_DRIVE, ATA_D_IBM | ATA_DEV(unit)); 633 timeout += 1000; 634 DELAY(1000); 635 continue; 636 } 637 638 /* are we done ? */ 639 if (!(status & ATA_S_BUSY)) 640 break; 641 642 if (timeout > 1000) { 643 timeout += 1000; 644 DELAY(1000); 645 } 646 else { 647 timeout += 10; 648 DELAY(10); 649 } 650 } 651 if (timeout >= 1000000) 652 return -2; 653 if (!mask) 654 return (status & ATA_S_ERROR); 655 656 DELAY(1); 657 658 /* wait 50 msec for bits wanted */ 659 timeout = 5000; 660 while (timeout--) { 661 status = ATA_IDX_INB(ch, ATA_ALTSTAT); 662 if ((status & mask) == mask) 663 return (status & ATA_S_ERROR); 664 DELAY(10); 665 } 666 return -3; 667} 668 669int 670ata_generic_command(struct ata_request *request) 671{ 672 struct ata_channel *ch = device_get_softc(request->parent); 673 674 /* select device */ 675 ATA_IDX_OUTB(ch, ATA_DRIVE, ATA_D_IBM | ATA_D_LBA | ATA_DEV(request->unit)); 676 677 /* ready to issue command ? */ 678 if (ata_wait(ch, request->unit, 0) < 0) { 679 device_printf(request->parent, "timeout waiting to issue command\n"); 680 request->flags |= ATA_R_TIMEOUT; 681 return (-1); 682 } 683 684 /* enable interrupt */ 685 ATA_IDX_OUTB(ch, ATA_CONTROL, ATA_A_4BIT); 686 687 if (request->flags & ATA_R_ATAPI) { 688 int timeout = 5000; 689 int res; 690 691 /* issue packet command to controller */ 692 if (request->flags & ATA_R_DMA) { 693 ATA_IDX_OUTB(ch, ATA_FEATURE, ATA_F_DMA); 694 ATA_IDX_OUTB(ch, ATA_CYL_LSB, 0); 695 ATA_IDX_OUTB(ch, ATA_CYL_MSB, 0); 696 } 697 else { 698 ATA_IDX_OUTB(ch, ATA_FEATURE, 0); 699 ATA_IDX_OUTB(ch, ATA_CYL_LSB, request->transfersize); 700 ATA_IDX_OUTB(ch, ATA_CYL_MSB, request->transfersize >> 8); 701 } 702 ATA_IDX_OUTB(ch, ATA_COMMAND, ATA_PACKET_CMD); 703 704 /* command interrupt device ? just return and wait for interrupt */ 705 if (request->flags & ATA_R_ATAPI_INTR) 706 return (0); 707 708 /* command processed ? */ 709 res = ata_wait(ch, request->unit, 0); 710 if (res != 0) { 711 if (res < 0) { 712 device_printf(request->parent, 713 "timeout waiting for PACKET command\n"); 714 request->flags |= ATA_R_TIMEOUT; 715 } 716 return (-1); 717 } 718 /* wait for ready to write ATAPI command block */ 719 while (timeout--) { 720 int reason = ATA_IDX_INB(ch, ATA_IREASON); 721 int status = ATA_IDX_INB(ch, ATA_STATUS); 722 723 if (((reason & (ATA_I_CMD | ATA_I_IN)) | 724 (status & (ATA_S_DRQ | ATA_S_BUSY))) == ATAPI_P_CMDOUT) 725 break; 726 DELAY(20); 727 } 728 if (timeout <= 0) { 729 device_printf(request->parent, 730 "timeout waiting for ATAPI ready\n"); 731 request->flags |= ATA_R_TIMEOUT; 732 return (-1); 733 } 734 735 /* this seems to be needed for some (slow) devices */ 736 DELAY(10); 737 738 /* output command block */ 739 ATA_IDX_OUTSW_STRM(ch, ATA_DATA, (int16_t *)request->u.atapi.ccb, 740 (request->flags & ATA_R_ATAPI16) ? 8 : 6); 741 } 742 else { 743 ch->hw.tf_write(request); 744 745 /* issue command to controller */ 746 ATA_IDX_OUTB(ch, ATA_COMMAND, request->u.ata.command); 747 } 748 return (0); 749} 750 751static void 752ata_tf_read(struct ata_request *request) 753{ 754 struct ata_channel *ch = device_get_softc(request->parent); 755 756 if (request->flags & ATA_R_48BIT) { 757 ATA_IDX_OUTB(ch, ATA_CONTROL, ATA_A_4BIT | ATA_A_HOB); 758 request->u.ata.count = (ATA_IDX_INB(ch, ATA_COUNT) << 8); 759 request->u.ata.lba = 760 ((u_int64_t)(ATA_IDX_INB(ch, ATA_SECTOR)) << 24) | 761 ((u_int64_t)(ATA_IDX_INB(ch, ATA_CYL_LSB)) << 32) | 762 ((u_int64_t)(ATA_IDX_INB(ch, ATA_CYL_MSB)) << 40); 763 764 ATA_IDX_OUTB(ch, ATA_CONTROL, ATA_A_4BIT); 765 request->u.ata.count |= ATA_IDX_INB(ch, ATA_COUNT); 766 request->u.ata.lba |= 767 (ATA_IDX_INB(ch, ATA_SECTOR) | 768 (ATA_IDX_INB(ch, ATA_CYL_LSB) << 8) | 769 (ATA_IDX_INB(ch, ATA_CYL_MSB) << 16)); 770 } 771 else { 772 request->u.ata.count = ATA_IDX_INB(ch, ATA_COUNT); 773 request->u.ata.lba = ATA_IDX_INB(ch, ATA_SECTOR) | 774 (ATA_IDX_INB(ch, ATA_CYL_LSB) << 8) | 775 (ATA_IDX_INB(ch, ATA_CYL_MSB) << 16) | 776 ((ATA_IDX_INB(ch, ATA_DRIVE) & 0xf) << 24); 777 } 778} 779 780static void 781ata_tf_write(struct ata_request *request) 782{ 783 struct ata_channel *ch = device_get_softc(request->parent); 784 785 if (request->flags & ATA_R_48BIT) { 786 ATA_IDX_OUTB(ch, ATA_FEATURE, request->u.ata.feature >> 8); 787 ATA_IDX_OUTB(ch, ATA_FEATURE, request->u.ata.feature); 788 ATA_IDX_OUTB(ch, ATA_COUNT, request->u.ata.count >> 8); 789 ATA_IDX_OUTB(ch, ATA_COUNT, request->u.ata.count); 790 ATA_IDX_OUTB(ch, ATA_SECTOR, request->u.ata.lba >> 24); 791 ATA_IDX_OUTB(ch, ATA_SECTOR, request->u.ata.lba); 792 ATA_IDX_OUTB(ch, ATA_CYL_LSB, request->u.ata.lba >> 32); 793 ATA_IDX_OUTB(ch, ATA_CYL_LSB, request->u.ata.lba >> 8); 794 ATA_IDX_OUTB(ch, ATA_CYL_MSB, request->u.ata.lba >> 40); 795 ATA_IDX_OUTB(ch, ATA_CYL_MSB, request->u.ata.lba >> 16); 796 ATA_IDX_OUTB(ch, ATA_DRIVE, ATA_D_LBA | ATA_DEV(request->unit)); 797 } 798 else { 799 ATA_IDX_OUTB(ch, ATA_FEATURE, request->u.ata.feature); 800 ATA_IDX_OUTB(ch, ATA_COUNT, request->u.ata.count); 801 ATA_IDX_OUTB(ch, ATA_SECTOR, request->u.ata.lba); 802 ATA_IDX_OUTB(ch, ATA_CYL_LSB, request->u.ata.lba >> 8); 803 ATA_IDX_OUTB(ch, ATA_CYL_MSB, request->u.ata.lba >> 16); 804 ATA_IDX_OUTB(ch, ATA_DRIVE, 805 ATA_D_IBM | ATA_D_LBA | ATA_DEV(request->unit) | 806 ((request->u.ata.lba >> 24) & 0x0f)); 807 } 808} 809 810static void 811ata_pio_read(struct ata_request *request, int length) 812{ 813 struct ata_channel *ch = device_get_softc(request->parent); 814 struct bio *bio; 815 uint8_t *addr; 816 vm_offset_t page; 817 int todo, done, off, moff, resid, size, i; 818 uint8_t buf[2] __aligned(2); 819 820 todo = min(request->transfersize, length); 821 page = done = resid = 0; 822 while (done < todo) { 823 size = todo - done; 824 825 /* Prepare data address and limit size (if not sequential). */ 826 off = request->donecount + done; 827 if ((request->flags & ATA_R_DATA_IN_CCB) == 0 || 828 (request->ccb->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) { 829 addr = (uint8_t *)request->data + off; 830 } else if ((request->ccb->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_BIO) { 831 bio = (struct bio *)request->data; 832 if ((bio->bio_flags & BIO_UNMAPPED) == 0) { 833 addr = (uint8_t *)bio->bio_data + off; 834 } else { 835 moff = bio->bio_ma_offset + off; 836 page = pmap_quick_enter_page( 837 bio->bio_ma[moff / PAGE_SIZE]); 838 moff %= PAGE_SIZE; 839 size = min(size, PAGE_SIZE - moff); 840 addr = (void *)(page + moff); 841 } 842 } else 843 panic("ata_pio_read: Unsupported CAM data type %x\n", 844 (request->ccb->ccb_h.flags & CAM_DATA_MASK)); 845 846 /* We may have extra byte already read but not stored. */ 847 if (resid) { 848 addr[0] = buf[1]; 849 addr++; 850 done++; 851 size--; 852 } 853 854 /* Process main part of data. */ 855 resid = size % 2; 856 if (__predict_false((ch->flags & ATA_USE_16BIT) || 857 (size % 4) != 0 || ((uintptr_t)addr % 4) != 0)) { 858#ifndef __NO_STRICT_ALIGNMENT 859 if (__predict_false((uintptr_t)addr % 2)) { 860 for (i = 0; i + 1 < size; i += 2) { 861 *(uint16_t *)&buf = 862 ATA_IDX_INW_STRM(ch, ATA_DATA); 863 addr[i] = buf[0]; 864 addr[i + 1] = buf[1]; 865 } 866 } else 867#endif 868 ATA_IDX_INSW_STRM(ch, ATA_DATA, (void*)addr, 869 size / 2); 870 871 /* If we have extra byte of data, leave it for later. */ 872 if (resid) { 873 *(uint16_t *)&buf = 874 ATA_IDX_INW_STRM(ch, ATA_DATA); 875 addr[size - 1] = buf[0]; 876 } 877 } else 878 ATA_IDX_INSL_STRM(ch, ATA_DATA, (void*)addr, size / 4); 879 880 if (page) { 881 pmap_quick_remove_page(page); 882 page = 0; 883 } 884 done += size; 885 } 886 887 if (length > done) { 888 device_printf(request->parent, 889 "WARNING - %s read data overrun %d > %d\n", 890 ata_cmd2str(request), length, done); 891 for (i = done + resid; i < length; i += 2) 892 ATA_IDX_INW(ch, ATA_DATA); 893 } 894} 895 896static void 897ata_pio_write(struct ata_request *request, int length) 898{ 899 struct ata_channel *ch = device_get_softc(request->parent); 900 struct bio *bio; 901 uint8_t *addr; 902 vm_offset_t page; 903 int todo, done, off, moff, resid, size, i; 904 uint8_t buf[2] __aligned(2); 905 906 todo = min(request->transfersize, length); 907 page = done = resid = 0; 908 while (done < todo) { 909 size = todo - done; 910 911 /* Prepare data address and limit size (if not sequential). */ 912 off = request->donecount + done; 913 if ((request->flags & ATA_R_DATA_IN_CCB) == 0 || 914 (request->ccb->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) { 915 addr = (uint8_t *)request->data + off; 916 } else if ((request->ccb->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_BIO) { 917 bio = (struct bio *)request->data; 918 if ((bio->bio_flags & BIO_UNMAPPED) == 0) { 919 addr = (uint8_t *)bio->bio_data + off; 920 } else { 921 moff = bio->bio_ma_offset + off; 922 page = pmap_quick_enter_page( 923 bio->bio_ma[moff / PAGE_SIZE]); 924 moff %= PAGE_SIZE; 925 size = min(size, PAGE_SIZE - moff); 926 addr = (void *)(page + moff); 927 } 928 } else 929 panic("ata_pio_write: Unsupported CAM data type %x\n", 930 (request->ccb->ccb_h.flags & CAM_DATA_MASK)); 931 932 /* We may have extra byte to be written first. */ 933 if (resid) { 934 buf[1] = addr[0]; 935 ATA_IDX_OUTW_STRM(ch, ATA_DATA, *(uint16_t *)&buf); 936 addr++; 937 done++; 938 size--; 939 } 940 941 /* Process main part of data. */ 942 resid = size % 2; 943 if (__predict_false((ch->flags & ATA_USE_16BIT) || 944 (size % 4) != 0 || ((uintptr_t)addr % 4) != 0)) { 945#ifndef __NO_STRICT_ALIGNMENT 946 if (__predict_false((uintptr_t)addr % 2)) { 947 for (i = 0; i + 1 < size; i += 2) { 948 buf[0] = addr[i]; 949 buf[1] = addr[i + 1]; 950 ATA_IDX_OUTW_STRM(ch, ATA_DATA, 951 *(uint16_t *)&buf); 952 } 953 } else 954#endif 955 ATA_IDX_OUTSW_STRM(ch, ATA_DATA, (void*)addr, 956 size / 2); 957 958 /* If we have extra byte of data, save it for later. */ 959 if (resid) 960 buf[0] = addr[size - 1]; 961 } else 962 ATA_IDX_OUTSL_STRM(ch, ATA_DATA, 963 (void*)addr, size / sizeof(int32_t)); 964 965 if (page) { 966 pmap_quick_remove_page(page); 967 page = 0; 968 } 969 done += size; 970 } 971 972 /* We may have extra byte of data to be written. Pad it with zero. */ 973 if (resid) { 974 buf[1] = 0; 975 ATA_IDX_OUTW_STRM(ch, ATA_DATA, *(uint16_t *)&buf); 976 } 977 978 if (length > done) { 979 device_printf(request->parent, 980 "WARNING - %s write data underrun %d > %d\n", 981 ata_cmd2str(request), length, done); 982 for (i = done + resid; i < length; i += 2) 983 ATA_IDX_OUTW(ch, ATA_DATA, 0); 984 } 985} 986