1/*- 2 * Inline routines shareable across OS platforms. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 * 6 * Copyright (c) 1994-2001 Justin T. Gibbs. 7 * Copyright (c) 2000-2003 Adaptec Inc. 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions, and the following disclaimer, 15 * without modification. 16 * 2. Redistributions in binary form must reproduce at minimum a disclaimer 17 * substantially similar to the "NO WARRANTY" disclaimer below 18 * ("Disclaimer") and any redistribution must be conditioned upon 19 * including a substantially similar Disclaimer requirement for further 20 * binary redistribution. 21 * 3. Neither the names of the above-listed copyright holders nor the names 22 * of any contributors may be used to endorse or promote products derived 23 * from this software without specific prior written permission. 24 * 25 * Alternatively, this software may be distributed under the terms of the 26 * GNU General Public License ("GPL") version 2 as published by the Free 27 * Software Foundation. 28 * 29 * NO WARRANTY 30 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 31 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 32 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR 33 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 34 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 35 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 36 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 37 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 38 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 39 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGES. 41 * 42 * $Id: //depot/aic7xxx/aic7xxx/aic79xx_inline.h#57 $ 43 * 44 * $FreeBSD$ 45 */ 46 47#ifndef _AIC79XX_INLINE_H_ 48#define _AIC79XX_INLINE_H_ 49 50/******************************** Debugging ***********************************/ 51static __inline char *ahd_name(struct ahd_softc *ahd); 52 53static __inline char * 54ahd_name(struct ahd_softc *ahd) 55{ 56 return (ahd->name); 57} 58 59/************************ Sequencer Execution Control *************************/ 60static __inline void ahd_known_modes(struct ahd_softc *ahd, 61 ahd_mode src, ahd_mode dst); 62static __inline ahd_mode_state ahd_build_mode_state(struct ahd_softc *ahd, 63 ahd_mode src, 64 ahd_mode dst); 65static __inline void ahd_extract_mode_state(struct ahd_softc *ahd, 66 ahd_mode_state state, 67 ahd_mode *src, ahd_mode *dst); 68static __inline void ahd_set_modes(struct ahd_softc *ahd, ahd_mode src, 69 ahd_mode dst); 70static __inline void ahd_update_modes(struct ahd_softc *ahd); 71static __inline void ahd_assert_modes(struct ahd_softc *ahd, ahd_mode srcmode, 72 ahd_mode dstmode, const char *file, 73 int line); 74static __inline ahd_mode_state ahd_save_modes(struct ahd_softc *ahd); 75static __inline void ahd_restore_modes(struct ahd_softc *ahd, 76 ahd_mode_state state); 77static __inline int ahd_is_paused(struct ahd_softc *ahd); 78static __inline void ahd_pause(struct ahd_softc *ahd); 79static __inline void ahd_unpause(struct ahd_softc *ahd); 80 81static __inline void 82ahd_known_modes(struct ahd_softc *ahd, ahd_mode src, ahd_mode dst) 83{ 84 ahd->src_mode = src; 85 ahd->dst_mode = dst; 86 ahd->saved_src_mode = src; 87 ahd->saved_dst_mode = dst; 88} 89 90static __inline ahd_mode_state 91ahd_build_mode_state(struct ahd_softc *ahd, ahd_mode src, ahd_mode dst) 92{ 93 return ((src << SRC_MODE_SHIFT) | (dst << DST_MODE_SHIFT)); 94} 95 96static __inline void 97ahd_extract_mode_state(struct ahd_softc *ahd, ahd_mode_state state, 98 ahd_mode *src, ahd_mode *dst) 99{ 100 *src = (state & SRC_MODE) >> SRC_MODE_SHIFT; 101 *dst = (state & DST_MODE) >> DST_MODE_SHIFT; 102} 103 104static __inline void 105ahd_set_modes(struct ahd_softc *ahd, ahd_mode src, ahd_mode dst) 106{ 107 if (ahd->src_mode == src && ahd->dst_mode == dst) 108 return; 109#ifdef AHD_DEBUG 110 if (ahd->src_mode == AHD_MODE_UNKNOWN 111 || ahd->dst_mode == AHD_MODE_UNKNOWN) 112 panic("Setting mode prior to saving it.\n"); 113 if ((ahd_debug & AHD_SHOW_MODEPTR) != 0) 114 printf("%s: Setting mode 0x%x\n", ahd_name(ahd), 115 ahd_build_mode_state(ahd, src, dst)); 116#endif 117 ahd_outb(ahd, MODE_PTR, ahd_build_mode_state(ahd, src, dst)); 118 ahd->src_mode = src; 119 ahd->dst_mode = dst; 120} 121 122static __inline void 123ahd_update_modes(struct ahd_softc *ahd) 124{ 125 ahd_mode_state mode_ptr; 126 ahd_mode src; 127 ahd_mode dst; 128 129 mode_ptr = ahd_inb(ahd, MODE_PTR); 130#ifdef AHD_DEBUG 131 if ((ahd_debug & AHD_SHOW_MODEPTR) != 0) 132 printf("Reading mode 0x%x\n", mode_ptr); 133#endif 134 ahd_extract_mode_state(ahd, mode_ptr, &src, &dst); 135 ahd_known_modes(ahd, src, dst); 136} 137 138static __inline void 139ahd_assert_modes(struct ahd_softc *ahd, ahd_mode srcmode, 140 ahd_mode dstmode, const char *file, int line) 141{ 142#ifdef AHD_DEBUG 143 if ((srcmode & AHD_MK_MSK(ahd->src_mode)) == 0 144 || (dstmode & AHD_MK_MSK(ahd->dst_mode)) == 0) { 145 panic("%s:%s:%d: Mode assertion failed.\n", 146 ahd_name(ahd), file, line); 147 } 148#endif 149} 150 151static __inline ahd_mode_state 152ahd_save_modes(struct ahd_softc *ahd) 153{ 154 if (ahd->src_mode == AHD_MODE_UNKNOWN 155 || ahd->dst_mode == AHD_MODE_UNKNOWN) 156 ahd_update_modes(ahd); 157 158 return (ahd_build_mode_state(ahd, ahd->src_mode, ahd->dst_mode)); 159} 160 161static __inline void 162ahd_restore_modes(struct ahd_softc *ahd, ahd_mode_state state) 163{ 164 ahd_mode src; 165 ahd_mode dst; 166 167 ahd_extract_mode_state(ahd, state, &src, &dst); 168 ahd_set_modes(ahd, src, dst); 169} 170 171#define AHD_ASSERT_MODES(ahd, source, dest) \ 172 ahd_assert_modes(ahd, source, dest, __FILE__, __LINE__); 173 174/* 175 * Determine whether the sequencer has halted code execution. 176 * Returns non-zero status if the sequencer is stopped. 177 */ 178static __inline int 179ahd_is_paused(struct ahd_softc *ahd) 180{ 181 return ((ahd_inb(ahd, HCNTRL) & PAUSE) != 0); 182} 183 184/* 185 * Request that the sequencer stop and wait, indefinitely, for it 186 * to stop. The sequencer will only acknowledge that it is paused 187 * once it has reached an instruction boundary and PAUSEDIS is 188 * cleared in the SEQCTL register. The sequencer may use PAUSEDIS 189 * for critical sections. 190 */ 191static __inline void 192ahd_pause(struct ahd_softc *ahd) 193{ 194 ahd_outb(ahd, HCNTRL, ahd->pause); 195 196 /* 197 * Since the sequencer can disable pausing in a critical section, we 198 * must loop until it actually stops. 199 */ 200 while (ahd_is_paused(ahd) == 0) 201 ; 202} 203 204/* 205 * Allow the sequencer to continue program execution. 206 * We check here to ensure that no additional interrupt 207 * sources that would cause the sequencer to halt have been 208 * asserted. If, for example, a SCSI bus reset is detected 209 * while we are fielding a different, pausing, interrupt type, 210 * we don't want to release the sequencer before going back 211 * into our interrupt handler and dealing with this new 212 * condition. 213 */ 214static __inline void 215ahd_unpause(struct ahd_softc *ahd) 216{ 217 /* 218 * Automatically restore our modes to those saved 219 * prior to the first change of the mode. 220 */ 221 if (ahd->saved_src_mode != AHD_MODE_UNKNOWN 222 && ahd->saved_dst_mode != AHD_MODE_UNKNOWN) { 223 if ((ahd->flags & AHD_UPDATE_PEND_CMDS) != 0) 224 ahd_reset_cmds_pending(ahd); 225 ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode); 226 } 227 228 if ((ahd_inb(ahd, INTSTAT) & ~CMDCMPLT) == 0) 229 ahd_outb(ahd, HCNTRL, ahd->unpause); 230 231 ahd_known_modes(ahd, AHD_MODE_UNKNOWN, AHD_MODE_UNKNOWN); 232} 233 234/*********************** Scatter Gather List Handling *************************/ 235static __inline void *ahd_sg_setup(struct ahd_softc *ahd, struct scb *scb, 236 void *sgptr, bus_addr_t addr, 237 bus_size_t len, int last); 238static __inline void ahd_setup_scb_common(struct ahd_softc *ahd, 239 struct scb *scb); 240static __inline void ahd_setup_data_scb(struct ahd_softc *ahd, 241 struct scb *scb); 242static __inline void ahd_setup_noxfer_scb(struct ahd_softc *ahd, 243 struct scb *scb); 244 245static __inline void * 246ahd_sg_setup(struct ahd_softc *ahd, struct scb *scb, 247 void *sgptr, bus_addr_t addr, bus_size_t len, int last) 248{ 249 scb->sg_count++; 250 if (sizeof(bus_addr_t) > 4 251 && (ahd->flags & AHD_64BIT_ADDRESSING) != 0) { 252 struct ahd_dma64_seg *sg; 253 254 sg = (struct ahd_dma64_seg *)sgptr; 255 sg->addr = aic_htole64(addr); 256 sg->len = aic_htole32(len | (last ? AHD_DMA_LAST_SEG : 0)); 257 return (sg + 1); 258 } else { 259 struct ahd_dma_seg *sg; 260 261 sg = (struct ahd_dma_seg *)sgptr; 262 sg->addr = aic_htole32(addr & 0xFFFFFFFF); 263 sg->len = aic_htole32(len | ((addr >> 8) & 0x7F000000) 264 | (last ? AHD_DMA_LAST_SEG : 0)); 265 return (sg + 1); 266 } 267} 268 269static __inline void 270ahd_setup_scb_common(struct ahd_softc *ahd, struct scb *scb) 271{ 272 /* XXX Handle target mode SCBs. */ 273 scb->crc_retry_count = 0; 274 if ((scb->flags & SCB_PACKETIZED) != 0) { 275 /* XXX what about ACA?? It is type 4, but TAG_TYPE == 0x3. */ 276 scb->hscb->task_attribute = scb->hscb->control & SCB_TAG_TYPE; 277 } else { 278 if (aic_get_transfer_length(scb) & 0x01) 279 scb->hscb->task_attribute = SCB_XFERLEN_ODD; 280 else 281 scb->hscb->task_attribute = 0; 282 } 283 284 if (scb->hscb->cdb_len <= MAX_CDB_LEN_WITH_SENSE_ADDR 285 || (scb->hscb->cdb_len & SCB_CDB_LEN_PTR) != 0) 286 scb->hscb->shared_data.idata.cdb_plus_saddr.sense_addr = 287 aic_htole32(scb->sense_busaddr); 288} 289 290static __inline void 291ahd_setup_data_scb(struct ahd_softc *ahd, struct scb *scb) 292{ 293 /* 294 * Copy the first SG into the "current" data ponter area. 295 */ 296 if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) { 297 struct ahd_dma64_seg *sg; 298 299 sg = (struct ahd_dma64_seg *)scb->sg_list; 300 scb->hscb->dataptr = sg->addr; 301 scb->hscb->datacnt = sg->len; 302 } else { 303 struct ahd_dma_seg *sg; 304 uint32_t *dataptr_words; 305 306 sg = (struct ahd_dma_seg *)scb->sg_list; 307 dataptr_words = (uint32_t*)&scb->hscb->dataptr; 308 dataptr_words[0] = sg->addr; 309 dataptr_words[1] = 0; 310 if ((ahd->flags & AHD_39BIT_ADDRESSING) != 0) { 311 uint64_t high_addr; 312 313 high_addr = aic_le32toh(sg->len) & 0x7F000000; 314 scb->hscb->dataptr |= aic_htole64(high_addr << 8); 315 } 316 scb->hscb->datacnt = sg->len; 317 } 318 /* 319 * Note where to find the SG entries in bus space. 320 * We also set the full residual flag which the 321 * sequencer will clear as soon as a data transfer 322 * occurs. 323 */ 324 scb->hscb->sgptr = aic_htole32(scb->sg_list_busaddr|SG_FULL_RESID); 325} 326 327static __inline void 328ahd_setup_noxfer_scb(struct ahd_softc *ahd, struct scb *scb) 329{ 330 scb->hscb->sgptr = aic_htole32(SG_LIST_NULL); 331 scb->hscb->dataptr = 0; 332 scb->hscb->datacnt = 0; 333} 334 335/************************** Memory mapping routines ***************************/ 336static __inline size_t ahd_sg_size(struct ahd_softc *ahd); 337static __inline void * 338 ahd_sg_bus_to_virt(struct ahd_softc *ahd, 339 struct scb *scb, 340 uint32_t sg_busaddr); 341static __inline uint32_t 342 ahd_sg_virt_to_bus(struct ahd_softc *ahd, 343 struct scb *scb, 344 void *sg); 345static __inline void ahd_sync_scb(struct ahd_softc *ahd, 346 struct scb *scb, int op); 347static __inline void ahd_sync_sglist(struct ahd_softc *ahd, 348 struct scb *scb, int op); 349static __inline void ahd_sync_sense(struct ahd_softc *ahd, 350 struct scb *scb, int op); 351static __inline uint32_t 352 ahd_targetcmd_offset(struct ahd_softc *ahd, 353 u_int index); 354 355static __inline size_t 356ahd_sg_size(struct ahd_softc *ahd) 357{ 358 if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) 359 return (sizeof(struct ahd_dma64_seg)); 360 return (sizeof(struct ahd_dma_seg)); 361} 362 363static __inline void * 364ahd_sg_bus_to_virt(struct ahd_softc *ahd, struct scb *scb, uint32_t sg_busaddr) 365{ 366 bus_addr_t sg_offset; 367 368 /* sg_list_phys points to entry 1, not 0 */ 369 sg_offset = sg_busaddr - (scb->sg_list_busaddr - ahd_sg_size(ahd)); 370 return ((uint8_t *)scb->sg_list + sg_offset); 371} 372 373static __inline uint32_t 374ahd_sg_virt_to_bus(struct ahd_softc *ahd, struct scb *scb, void *sg) 375{ 376 bus_addr_t sg_offset; 377 378 /* sg_list_phys points to entry 1, not 0 */ 379 sg_offset = ((uint8_t *)sg - (uint8_t *)scb->sg_list) 380 - ahd_sg_size(ahd); 381 382 return (scb->sg_list_busaddr + sg_offset); 383} 384 385static __inline void 386ahd_sync_scb(struct ahd_softc *ahd, struct scb *scb, int op) 387{ 388 aic_dmamap_sync(ahd, ahd->scb_data.hscb_dmat, 389 scb->hscb_map->dmamap, 390 /*offset*/(uint8_t*)scb->hscb - scb->hscb_map->vaddr, 391 /*len*/sizeof(*scb->hscb), op); 392} 393 394static __inline void 395ahd_sync_sglist(struct ahd_softc *ahd, struct scb *scb, int op) 396{ 397 if (scb->sg_count == 0) 398 return; 399 400 aic_dmamap_sync(ahd, ahd->scb_data.sg_dmat, 401 scb->sg_map->dmamap, 402 /*offset*/scb->sg_list_busaddr - ahd_sg_size(ahd), 403 /*len*/ahd_sg_size(ahd) * scb->sg_count, op); 404} 405 406static __inline void 407ahd_sync_sense(struct ahd_softc *ahd, struct scb *scb, int op) 408{ 409 aic_dmamap_sync(ahd, ahd->scb_data.sense_dmat, 410 scb->sense_map->dmamap, 411 /*offset*/scb->sense_busaddr, 412 /*len*/AHD_SENSE_BUFSIZE, op); 413} 414 415static __inline uint32_t 416ahd_targetcmd_offset(struct ahd_softc *ahd, u_int index) 417{ 418 return (((uint8_t *)&ahd->targetcmds[index]) 419 - (uint8_t *)ahd->qoutfifo); 420} 421 422/********************** Miscellaneous Support Functions ***********************/ 423static __inline void ahd_complete_scb(struct ahd_softc *ahd, 424 struct scb *scb); 425static __inline void ahd_update_residual(struct ahd_softc *ahd, 426 struct scb *scb); 427static __inline struct ahd_initiator_tinfo * 428 ahd_fetch_transinfo(struct ahd_softc *ahd, 429 char channel, u_int our_id, 430 u_int remote_id, 431 struct ahd_tmode_tstate **tstate); 432static __inline uint16_t 433 ahd_inw(struct ahd_softc *ahd, u_int port); 434static __inline void ahd_outw(struct ahd_softc *ahd, u_int port, 435 u_int value); 436static __inline uint32_t 437 ahd_inl(struct ahd_softc *ahd, u_int port); 438static __inline void ahd_outl(struct ahd_softc *ahd, u_int port, 439 uint32_t value); 440static __inline uint64_t 441 ahd_inq(struct ahd_softc *ahd, u_int port); 442static __inline void ahd_outq(struct ahd_softc *ahd, u_int port, 443 uint64_t value); 444static __inline u_int ahd_get_scbptr(struct ahd_softc *ahd); 445static __inline void ahd_set_scbptr(struct ahd_softc *ahd, u_int scbptr); 446static __inline u_int ahd_get_hnscb_qoff(struct ahd_softc *ahd); 447static __inline void ahd_set_hnscb_qoff(struct ahd_softc *ahd, u_int value); 448static __inline u_int ahd_get_hescb_qoff(struct ahd_softc *ahd); 449static __inline void ahd_set_hescb_qoff(struct ahd_softc *ahd, u_int value); 450static __inline u_int ahd_get_snscb_qoff(struct ahd_softc *ahd); 451static __inline void ahd_set_snscb_qoff(struct ahd_softc *ahd, u_int value); 452static __inline u_int ahd_get_sescb_qoff(struct ahd_softc *ahd); 453static __inline void ahd_set_sescb_qoff(struct ahd_softc *ahd, u_int value); 454static __inline u_int ahd_get_sdscb_qoff(struct ahd_softc *ahd); 455static __inline void ahd_set_sdscb_qoff(struct ahd_softc *ahd, u_int value); 456static __inline u_int ahd_inb_scbram(struct ahd_softc *ahd, u_int offset); 457static __inline u_int ahd_inw_scbram(struct ahd_softc *ahd, u_int offset); 458static __inline uint32_t 459 ahd_inl_scbram(struct ahd_softc *ahd, u_int offset); 460static __inline uint64_t 461 ahd_inq_scbram(struct ahd_softc *ahd, u_int offset); 462static __inline void ahd_swap_with_next_hscb(struct ahd_softc *ahd, 463 struct scb *scb); 464static __inline void ahd_queue_scb(struct ahd_softc *ahd, struct scb *scb); 465static __inline uint8_t * 466 ahd_get_sense_buf(struct ahd_softc *ahd, 467 struct scb *scb); 468static __inline uint32_t 469 ahd_get_sense_bufaddr(struct ahd_softc *ahd, 470 struct scb *scb); 471 472static __inline void 473ahd_complete_scb(struct ahd_softc *ahd, struct scb *scb) 474{ 475 uint32_t sgptr; 476 477 sgptr = aic_le32toh(scb->hscb->sgptr); 478 if ((sgptr & SG_STATUS_VALID) != 0) 479 ahd_handle_scb_status(ahd, scb); 480 else 481 ahd_done(ahd, scb); 482} 483 484/* 485 * Determine whether the sequencer reported a residual 486 * for this SCB/transaction. 487 */ 488static __inline void 489ahd_update_residual(struct ahd_softc *ahd, struct scb *scb) 490{ 491 uint32_t sgptr; 492 493 sgptr = aic_le32toh(scb->hscb->sgptr); 494 if ((sgptr & SG_STATUS_VALID) != 0) 495 ahd_calc_residual(ahd, scb); 496} 497 498/* 499 * Return pointers to the transfer negotiation information 500 * for the specified our_id/remote_id pair. 501 */ 502static __inline struct ahd_initiator_tinfo * 503ahd_fetch_transinfo(struct ahd_softc *ahd, char channel, u_int our_id, 504 u_int remote_id, struct ahd_tmode_tstate **tstate) 505{ 506 /* 507 * Transfer data structures are stored from the perspective 508 * of the target role. Since the parameters for a connection 509 * in the initiator role to a given target are the same as 510 * when the roles are reversed, we pretend we are the target. 511 */ 512 if (channel == 'B') 513 our_id += 8; 514 *tstate = ahd->enabled_targets[our_id]; 515 return (&(*tstate)->transinfo[remote_id]); 516} 517 518#define AHD_COPY_COL_IDX(dst, src) \ 519do { \ 520 dst->hscb->scsiid = src->hscb->scsiid; \ 521 dst->hscb->lun = src->hscb->lun; \ 522} while (0) 523 524static __inline uint16_t 525ahd_inw(struct ahd_softc *ahd, u_int port) 526{ 527 /* 528 * Read high byte first as some registers increment 529 * or have other side effects when the low byte is 530 * read. 531 */ 532 return ((ahd_inb(ahd, port+1) << 8) | ahd_inb(ahd, port)); 533} 534 535static __inline void 536ahd_outw(struct ahd_softc *ahd, u_int port, u_int value) 537{ 538 /* 539 * Write low byte first to accommodate registers 540 * such as PRGMCNT where the order maters. 541 */ 542 ahd_outb(ahd, port, value & 0xFF); 543 ahd_outb(ahd, port+1, (value >> 8) & 0xFF); 544} 545 546static __inline uint32_t 547ahd_inl(struct ahd_softc *ahd, u_int port) 548{ 549 return ((ahd_inb(ahd, port)) 550 | (ahd_inb(ahd, port+1) << 8) 551 | (ahd_inb(ahd, port+2) << 16) 552 | (ahd_inb(ahd, port+3) << 24)); 553} 554 555static __inline void 556ahd_outl(struct ahd_softc *ahd, u_int port, uint32_t value) 557{ 558 ahd_outb(ahd, port, (value) & 0xFF); 559 ahd_outb(ahd, port+1, ((value) >> 8) & 0xFF); 560 ahd_outb(ahd, port+2, ((value) >> 16) & 0xFF); 561 ahd_outb(ahd, port+3, ((value) >> 24) & 0xFF); 562} 563 564static __inline uint64_t 565ahd_inq(struct ahd_softc *ahd, u_int port) 566{ 567 return ((ahd_inb(ahd, port)) 568 | (ahd_inb(ahd, port+1) << 8) 569 | (ahd_inb(ahd, port+2) << 16) 570 | (((uint64_t)ahd_inb(ahd, port+3)) << 24) 571 | (((uint64_t)ahd_inb(ahd, port+4)) << 32) 572 | (((uint64_t)ahd_inb(ahd, port+5)) << 40) 573 | (((uint64_t)ahd_inb(ahd, port+6)) << 48) 574 | (((uint64_t)ahd_inb(ahd, port+7)) << 56)); 575} 576 577static __inline void 578ahd_outq(struct ahd_softc *ahd, u_int port, uint64_t value) 579{ 580 ahd_outb(ahd, port, value & 0xFF); 581 ahd_outb(ahd, port+1, (value >> 8) & 0xFF); 582 ahd_outb(ahd, port+2, (value >> 16) & 0xFF); 583 ahd_outb(ahd, port+3, (value >> 24) & 0xFF); 584 ahd_outb(ahd, port+4, (value >> 32) & 0xFF); 585 ahd_outb(ahd, port+5, (value >> 40) & 0xFF); 586 ahd_outb(ahd, port+6, (value >> 48) & 0xFF); 587 ahd_outb(ahd, port+7, (value >> 56) & 0xFF); 588} 589 590static __inline u_int 591ahd_get_scbptr(struct ahd_softc *ahd) 592{ 593 AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK), 594 ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK)); 595 return (ahd_inb(ahd, SCBPTR) | (ahd_inb(ahd, SCBPTR + 1) << 8)); 596} 597 598static __inline void 599ahd_set_scbptr(struct ahd_softc *ahd, u_int scbptr) 600{ 601 AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK), 602 ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK)); 603 ahd_outb(ahd, SCBPTR, scbptr & 0xFF); 604 ahd_outb(ahd, SCBPTR+1, (scbptr >> 8) & 0xFF); 605} 606 607static __inline u_int 608ahd_get_hnscb_qoff(struct ahd_softc *ahd) 609{ 610 return (ahd_inw_atomic(ahd, HNSCB_QOFF)); 611} 612 613static __inline void 614ahd_set_hnscb_qoff(struct ahd_softc *ahd, u_int value) 615{ 616 ahd_outw_atomic(ahd, HNSCB_QOFF, value); 617} 618 619static __inline u_int 620ahd_get_hescb_qoff(struct ahd_softc *ahd) 621{ 622 return (ahd_inb(ahd, HESCB_QOFF)); 623} 624 625static __inline void 626ahd_set_hescb_qoff(struct ahd_softc *ahd, u_int value) 627{ 628 ahd_outb(ahd, HESCB_QOFF, value); 629} 630 631static __inline u_int 632ahd_get_snscb_qoff(struct ahd_softc *ahd) 633{ 634 u_int oldvalue; 635 636 AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK); 637 oldvalue = ahd_inw(ahd, SNSCB_QOFF); 638 ahd_outw(ahd, SNSCB_QOFF, oldvalue); 639 return (oldvalue); 640} 641 642static __inline void 643ahd_set_snscb_qoff(struct ahd_softc *ahd, u_int value) 644{ 645 AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK); 646 ahd_outw(ahd, SNSCB_QOFF, value); 647} 648 649static __inline u_int 650ahd_get_sescb_qoff(struct ahd_softc *ahd) 651{ 652 AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK); 653 return (ahd_inb(ahd, SESCB_QOFF)); 654} 655 656static __inline void 657ahd_set_sescb_qoff(struct ahd_softc *ahd, u_int value) 658{ 659 AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK); 660 ahd_outb(ahd, SESCB_QOFF, value); 661} 662 663static __inline u_int 664ahd_get_sdscb_qoff(struct ahd_softc *ahd) 665{ 666 AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK); 667 return (ahd_inb(ahd, SDSCB_QOFF) | (ahd_inb(ahd, SDSCB_QOFF + 1) << 8)); 668} 669 670static __inline void 671ahd_set_sdscb_qoff(struct ahd_softc *ahd, u_int value) 672{ 673 AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK); 674 ahd_outb(ahd, SDSCB_QOFF, value & 0xFF); 675 ahd_outb(ahd, SDSCB_QOFF+1, (value >> 8) & 0xFF); 676} 677 678static __inline u_int 679ahd_inb_scbram(struct ahd_softc *ahd, u_int offset) 680{ 681 u_int value; 682 683 /* 684 * Workaround PCI-X Rev A. hardware bug. 685 * After a host read of SCB memory, the chip 686 * may become confused into thinking prefetch 687 * was required. This starts the discard timer 688 * running and can cause an unexpected discard 689 * timer interrupt. The work around is to read 690 * a normal register prior to the exhaustion of 691 * the discard timer. The mode pointer register 692 * has no side effects and so serves well for 693 * this purpose. 694 * 695 * Razor #528 696 */ 697 value = ahd_inb(ahd, offset); 698 if ((ahd->bugs & AHD_PCIX_SCBRAM_RD_BUG) != 0) 699 ahd_inb(ahd, MODE_PTR); 700 return (value); 701} 702 703static __inline u_int 704ahd_inw_scbram(struct ahd_softc *ahd, u_int offset) 705{ 706 return (ahd_inb_scbram(ahd, offset) 707 | (ahd_inb_scbram(ahd, offset+1) << 8)); 708} 709 710static __inline uint32_t 711ahd_inl_scbram(struct ahd_softc *ahd, u_int offset) 712{ 713 return (ahd_inw_scbram(ahd, offset) 714 | (ahd_inw_scbram(ahd, offset+2) << 16)); 715} 716 717static __inline uint64_t 718ahd_inq_scbram(struct ahd_softc *ahd, u_int offset) 719{ 720 return (ahd_inl_scbram(ahd, offset) 721 | ((uint64_t)ahd_inl_scbram(ahd, offset+4)) << 32); 722} 723 724static __inline struct scb * 725ahd_lookup_scb(struct ahd_softc *ahd, u_int tag) 726{ 727 struct scb* scb; 728 729 if (tag >= AHD_SCB_MAX) 730 return (NULL); 731 scb = ahd->scb_data.scbindex[tag]; 732 if (scb != NULL) 733 ahd_sync_scb(ahd, scb, 734 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 735 return (scb); 736} 737 738static __inline void 739ahd_swap_with_next_hscb(struct ahd_softc *ahd, struct scb *scb) 740{ 741 struct hardware_scb *q_hscb; 742 struct map_node *q_hscb_map; 743 uint32_t saved_hscb_busaddr; 744 745 /* 746 * Our queuing method is a bit tricky. The card 747 * knows in advance which HSCB (by address) to download, 748 * and we can't disappoint it. To achieve this, the next 749 * HSCB to download is saved off in ahd->next_queued_hscb. 750 * When we are called to queue "an arbitrary scb", 751 * we copy the contents of the incoming HSCB to the one 752 * the sequencer knows about, swap HSCB pointers and 753 * finally assign the SCB to the tag indexed location 754 * in the scb_array. This makes sure that we can still 755 * locate the correct SCB by SCB_TAG. 756 */ 757 q_hscb = ahd->next_queued_hscb; 758 q_hscb_map = ahd->next_queued_hscb_map; 759 saved_hscb_busaddr = q_hscb->hscb_busaddr; 760 memcpy(q_hscb, scb->hscb, sizeof(*scb->hscb)); 761 q_hscb->hscb_busaddr = saved_hscb_busaddr; 762 q_hscb->next_hscb_busaddr = scb->hscb->hscb_busaddr; 763 764 /* Now swap HSCB pointers. */ 765 ahd->next_queued_hscb = scb->hscb; 766 ahd->next_queued_hscb_map = scb->hscb_map; 767 scb->hscb = q_hscb; 768 scb->hscb_map = q_hscb_map; 769 770 /* Now define the mapping from tag to SCB in the scbindex */ 771 ahd->scb_data.scbindex[SCB_GET_TAG(scb)] = scb; 772} 773 774/* 775 * Tell the sequencer about a new transaction to execute. 776 */ 777static __inline void 778ahd_queue_scb(struct ahd_softc *ahd, struct scb *scb) 779{ 780 ahd_swap_with_next_hscb(ahd, scb); 781 782 if (SCBID_IS_NULL(SCB_GET_TAG(scb))) 783 panic("Attempt to queue invalid SCB tag %x\n", 784 SCB_GET_TAG(scb)); 785 786 /* 787 * Keep a history of SCBs we've downloaded in the qinfifo. 788 */ 789 ahd->qinfifo[AHD_QIN_WRAP(ahd->qinfifonext)] = SCB_GET_TAG(scb); 790 ahd->qinfifonext++; 791 792 if (scb->sg_count != 0) 793 ahd_setup_data_scb(ahd, scb); 794 else 795 ahd_setup_noxfer_scb(ahd, scb); 796 ahd_setup_scb_common(ahd, scb); 797 798 /* 799 * Make sure our data is consistent from the 800 * perspective of the adapter. 801 */ 802 ahd_sync_scb(ahd, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 803 804#ifdef AHD_DEBUG 805 if ((ahd_debug & AHD_SHOW_QUEUE) != 0) { 806 uint64_t host_dataptr; 807 808 host_dataptr = aic_le64toh(scb->hscb->dataptr); 809 printf("%s: Queueing SCB %d:0x%x bus addr 0x%x - 0x%x%x/0x%x\n", 810 ahd_name(ahd), 811 SCB_GET_TAG(scb), scb->hscb->scsiid, 812 aic_le32toh(scb->hscb->hscb_busaddr), 813 (u_int)((host_dataptr >> 32) & 0xFFFFFFFF), 814 (u_int)(host_dataptr & 0xFFFFFFFF), 815 aic_le32toh(scb->hscb->datacnt)); 816 } 817#endif 818 /* Tell the adapter about the newly queued SCB */ 819 ahd_set_hnscb_qoff(ahd, ahd->qinfifonext); 820} 821 822static __inline uint8_t * 823ahd_get_sense_buf(struct ahd_softc *ahd, struct scb *scb) 824{ 825 return (scb->sense_data); 826} 827 828static __inline uint32_t 829ahd_get_sense_bufaddr(struct ahd_softc *ahd, struct scb *scb) 830{ 831 return (scb->sense_busaddr); 832} 833 834/************************** Interrupt Processing ******************************/ 835static __inline void ahd_sync_qoutfifo(struct ahd_softc *ahd, int op); 836static __inline void ahd_sync_tqinfifo(struct ahd_softc *ahd, int op); 837static __inline u_int ahd_check_cmdcmpltqueues(struct ahd_softc *ahd); 838static __inline int ahd_intr(struct ahd_softc *ahd); 839 840static __inline void 841ahd_sync_qoutfifo(struct ahd_softc *ahd, int op) 842{ 843 aic_dmamap_sync(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap, 844 /*offset*/0, 845 /*len*/AHD_SCB_MAX * sizeof(struct ahd_completion), op); 846} 847 848static __inline void 849ahd_sync_tqinfifo(struct ahd_softc *ahd, int op) 850{ 851#ifdef AHD_TARGET_MODE 852 if ((ahd->flags & AHD_TARGETROLE) != 0) { 853 aic_dmamap_sync(ahd, ahd->shared_data_dmat, 854 ahd->shared_data_map.dmamap, 855 ahd_targetcmd_offset(ahd, 0), 856 sizeof(struct target_cmd) * AHD_TMODE_CMDS, 857 op); 858 } 859#endif 860} 861 862/* 863 * See if the firmware has posted any completed commands 864 * into our in-core command complete fifos. 865 */ 866#define AHD_RUN_QOUTFIFO 0x1 867#define AHD_RUN_TQINFIFO 0x2 868static __inline u_int 869ahd_check_cmdcmpltqueues(struct ahd_softc *ahd) 870{ 871 u_int retval; 872 873 retval = 0; 874 aic_dmamap_sync(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap, 875 /*offset*/ahd->qoutfifonext * sizeof(*ahd->qoutfifo), 876 /*len*/sizeof(*ahd->qoutfifo), BUS_DMASYNC_POSTREAD); 877 if (ahd->qoutfifo[ahd->qoutfifonext].valid_tag 878 == ahd->qoutfifonext_valid_tag) 879 retval |= AHD_RUN_QOUTFIFO; 880#ifdef AHD_TARGET_MODE 881 if ((ahd->flags & AHD_TARGETROLE) != 0 882 && (ahd->flags & AHD_TQINFIFO_BLOCKED) == 0) { 883 aic_dmamap_sync(ahd, ahd->shared_data_dmat, 884 ahd->shared_data_map.dmamap, 885 ahd_targetcmd_offset(ahd, ahd->tqinfifofnext), 886 /*len*/sizeof(struct target_cmd), 887 BUS_DMASYNC_POSTREAD); 888 if (ahd->targetcmds[ahd->tqinfifonext].cmd_valid != 0) 889 retval |= AHD_RUN_TQINFIFO; 890 } 891#endif 892 return (retval); 893} 894 895/* 896 * Catch an interrupt from the adapter 897 */ 898static __inline int 899ahd_intr(struct ahd_softc *ahd) 900{ 901 u_int intstat; 902 903 if ((ahd->pause & INTEN) == 0) { 904 /* 905 * Our interrupt is not enabled on the chip 906 * and may be disabled for re-entrancy reasons, 907 * so just return. This is likely just a shared 908 * interrupt. 909 */ 910 return (0); 911 } 912 913 /* 914 * Instead of directly reading the interrupt status register, 915 * infer the cause of the interrupt by checking our in-core 916 * completion queues. This avoids a costly PCI bus read in 917 * most cases. 918 */ 919 if ((ahd->flags & AHD_ALL_INTERRUPTS) == 0 920 && (ahd_check_cmdcmpltqueues(ahd) != 0)) 921 intstat = CMDCMPLT; 922 else 923 intstat = ahd_inb(ahd, INTSTAT); 924 925 if ((intstat & INT_PEND) == 0) 926 return (0); 927 928 if (intstat & CMDCMPLT) { 929 ahd_outb(ahd, CLRINT, CLRCMDINT); 930 931 /* 932 * Ensure that the chip sees that we've cleared 933 * this interrupt before we walk the output fifo. 934 * Otherwise, we may, due to posted bus writes, 935 * clear the interrupt after we finish the scan, 936 * and after the sequencer has added new entries 937 * and asserted the interrupt again. 938 */ 939 if ((ahd->bugs & AHD_INTCOLLISION_BUG) != 0) { 940 if (ahd_is_paused(ahd)) { 941 /* 942 * Potentially lost SEQINT. 943 * If SEQINTCODE is non-zero, 944 * simulate the SEQINT. 945 */ 946 if (ahd_inb(ahd, SEQINTCODE) != NO_SEQINT) 947 intstat |= SEQINT; 948 } 949 } else { 950 ahd_flush_device_writes(ahd); 951 } 952 ahd_run_qoutfifo(ahd); 953 ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket]++; 954 ahd->cmdcmplt_total++; 955#ifdef AHD_TARGET_MODE 956 if ((ahd->flags & AHD_TARGETROLE) != 0) 957 ahd_run_tqinfifo(ahd, /*paused*/FALSE); 958#endif 959 } 960 961 /* 962 * Handle statuses that may invalidate our cached 963 * copy of INTSTAT separately. 964 */ 965 if (intstat == 0xFF && (ahd->features & AHD_REMOVABLE) != 0) { 966 /* Hot eject. Do nothing */ 967 } else if (intstat & HWERRINT) { 968 ahd_handle_hwerrint(ahd); 969 } else if ((intstat & (PCIINT|SPLTINT)) != 0) { 970 ahd->bus_intr(ahd); 971 } else { 972 if ((intstat & SEQINT) != 0) 973 ahd_handle_seqint(ahd, intstat); 974 975 if ((intstat & SCSIINT) != 0) 976 ahd_handle_scsiint(ahd, intstat); 977 } 978 return (1); 979} 980 981#endif /* _AIC79XX_INLINE_H_ */ 982