1/*- 2 * SPDX-License-Identifier: BSD-3-Clause AND BSD-2-Clause-NetBSDE 3 * 4 * Copyright (c) KATO Takenori, 1999. 5 * 6 * All rights reserved. Unpublished rights reserved under the copyright 7 * laws of Japan. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer as 15 * the first lines of this file unmodified. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. The name of the author may not be used to endorse or promote products 20 * derived from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 27 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 31 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 * 33 * $FreeBSD$ 34 */ 35 36/* $NetBSD: bus.h,v 1.12 1997/10/01 08:25:15 fvdl Exp $ */ 37 38/*- 39 * Copyright (c) 1996, 1997 The NetBSD Foundation, Inc. 40 * All rights reserved. 41 * 42 * This code is derived from software contributed to The NetBSD Foundation 43 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 44 * NASA Ames Research Center. 45 * 46 * Redistribution and use in source and binary forms, with or without 47 * modification, are permitted provided that the following conditions 48 * are met: 49 * 1. Redistributions of source code must retain the above copyright 50 * notice, this list of conditions and the following disclaimer. 51 * 2. Redistributions in binary form must reproduce the above copyright 52 * notice, this list of conditions and the following disclaimer in the 53 * documentation and/or other materials provided with the distribution. 54 * 55 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 56 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 57 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 58 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 59 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 60 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 61 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 62 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 63 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 64 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 65 * POSSIBILITY OF SUCH DAMAGE. 66 */ 67 68/*- 69 * Copyright (c) 1996 Charles M. Hannum. All rights reserved. 70 * Copyright (c) 1996 Christopher G. Demetriou. All rights reserved. 71 * 72 * Redistribution and use in source and binary forms, with or without 73 * modification, are permitted provided that the following conditions 74 * are met: 75 * 1. Redistributions of source code must retain the above copyright 76 * notice, this list of conditions and the following disclaimer. 77 * 2. Redistributions in binary form must reproduce the above copyright 78 * notice, this list of conditions and the following disclaimer in the 79 * documentation and/or other materials provided with the distribution. 80 * 3. All advertising materials mentioning features or use of this software 81 * must display the following acknowledgement: 82 * This product includes software developed by Christopher G. Demetriou 83 * for the NetBSD Project. 84 * 4. The name of the author may not be used to endorse or promote products 85 * derived from this software without specific prior written permission 86 * 87 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 88 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 89 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 90 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 91 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 92 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 93 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 94 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 95 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 96 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 97 */ 98 99#ifndef _X86_BUS_H_ 100#define _X86_BUS_H_ 101 102#include <machine/_bus.h> 103#include <machine/cpufunc.h> 104#include <machine/bus_dma.h> 105 106#ifndef __GNUCLIKE_ASM 107#error "no assembler code for your compiler" 108#endif 109 110/* 111 * Values for the x86 bus space tag, not to be used directly by MI code. 112 */ 113#define X86_BUS_SPACE_IO 0 /* space is i/o space */ 114#define X86_BUS_SPACE_MEM 1 /* space is mem space */ 115 116#define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFF 117#define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF 118#if defined(__amd64__) 119#define BUS_SPACE_MAXSIZE 0xFFFFFFFFFFFFFFFFULL 120#else 121#define BUS_SPACE_MAXSIZE 0xFFFFFFFF 122#endif 123#define BUS_SPACE_MAXADDR_24BIT 0xFFFFFF 124#define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF 125#if defined(__amd64__) || defined(PAE) 126#define BUS_SPACE_MAXADDR_48BIT 0xFFFFFFFFFFFFULL 127#define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFULL 128#else 129#define BUS_SPACE_MAXADDR 0xFFFFFFFF 130#endif 131 132#define BUS_SPACE_INVALID_DATA (~0) 133#define BUS_SPACE_UNRESTRICTED (~0) 134 135#define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */ 136#define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */ 137 138#if defined(KCSAN) && !defined(KCSAN_RUNTIME) 139#include <sys/bus_san.h> 140#else 141 142/* 143 * Map a region of device bus space into CPU virtual address space. 144 */ 145 146int bus_space_map(bus_space_tag_t tag, bus_addr_t addr, bus_size_t size, 147 int flags, bus_space_handle_t *bshp); 148 149/* 150 * Unmap a region of device bus space. 151 */ 152 153void bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t bsh, 154 bus_size_t size); 155 156/* 157 * Get a new handle for a subregion of an already-mapped area of bus space. 158 */ 159 160static __inline int bus_space_subregion(bus_space_tag_t t, 161 bus_space_handle_t bsh, 162 bus_size_t offset, bus_size_t size, 163 bus_space_handle_t *nbshp); 164 165static __inline int 166bus_space_subregion(bus_space_tag_t t __unused, bus_space_handle_t bsh, 167 bus_size_t offset, bus_size_t size __unused, 168 bus_space_handle_t *nbshp) 169{ 170 171 *nbshp = bsh + offset; 172 return (0); 173} 174 175/* 176 * Allocate a region of memory that is accessible to devices in bus space. 177 */ 178 179int bus_space_alloc(bus_space_tag_t t, bus_addr_t rstart, 180 bus_addr_t rend, bus_size_t size, bus_size_t align, 181 bus_size_t boundary, int flags, bus_addr_t *addrp, 182 bus_space_handle_t *bshp); 183 184/* 185 * Free a region of bus space accessible memory. 186 */ 187 188static __inline void bus_space_free(bus_space_tag_t t, bus_space_handle_t bsh, 189 bus_size_t size); 190 191static __inline void 192bus_space_free(bus_space_tag_t t __unused, bus_space_handle_t bsh __unused, 193 bus_size_t size __unused) 194{ 195} 196 197/* 198 * Read a 1, 2, 4, or 8 byte quantity from bus space 199 * described by tag/handle/offset. 200 */ 201static __inline u_int8_t bus_space_read_1(bus_space_tag_t tag, 202 bus_space_handle_t handle, 203 bus_size_t offset); 204 205static __inline u_int16_t bus_space_read_2(bus_space_tag_t tag, 206 bus_space_handle_t handle, 207 bus_size_t offset); 208 209static __inline u_int32_t bus_space_read_4(bus_space_tag_t tag, 210 bus_space_handle_t handle, 211 bus_size_t offset); 212 213#ifdef __amd64__ 214static __inline uint64_t bus_space_read_8(bus_space_tag_t tag, 215 bus_space_handle_t handle, 216 bus_size_t offset); 217#endif 218 219static __inline u_int8_t 220bus_space_read_1(bus_space_tag_t tag, bus_space_handle_t handle, 221 bus_size_t offset) 222{ 223 224 if (tag == X86_BUS_SPACE_IO) 225 return (inb(handle + offset)); 226 return (*(volatile u_int8_t *)(handle + offset)); 227} 228 229static __inline u_int16_t 230bus_space_read_2(bus_space_tag_t tag, bus_space_handle_t handle, 231 bus_size_t offset) 232{ 233 234 if (tag == X86_BUS_SPACE_IO) 235 return (inw(handle + offset)); 236 return (*(volatile u_int16_t *)(handle + offset)); 237} 238 239static __inline u_int32_t 240bus_space_read_4(bus_space_tag_t tag, bus_space_handle_t handle, 241 bus_size_t offset) 242{ 243 244 if (tag == X86_BUS_SPACE_IO) 245 return (inl(handle + offset)); 246 return (*(volatile u_int32_t *)(handle + offset)); 247} 248 249#ifdef __amd64__ 250static __inline uint64_t 251bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 252 bus_size_t offset) 253{ 254 255 if (tag == X86_BUS_SPACE_IO) /* No 8 byte IO space access on x86 */ 256 return (BUS_SPACE_INVALID_DATA); 257 return (*(volatile uint64_t *)(handle + offset)); 258} 259#endif 260 261/* 262 * Read `count' 1, 2, 4, or 8 byte quantities from bus space 263 * described by tag/handle/offset and copy into buffer provided. 264 */ 265static __inline void bus_space_read_multi_1(bus_space_tag_t tag, 266 bus_space_handle_t bsh, 267 bus_size_t offset, u_int8_t *addr, 268 size_t count); 269 270static __inline void bus_space_read_multi_2(bus_space_tag_t tag, 271 bus_space_handle_t bsh, 272 bus_size_t offset, u_int16_t *addr, 273 size_t count); 274 275static __inline void bus_space_read_multi_4(bus_space_tag_t tag, 276 bus_space_handle_t bsh, 277 bus_size_t offset, u_int32_t *addr, 278 size_t count); 279 280static __inline void 281bus_space_read_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, 282 bus_size_t offset, u_int8_t *addr, size_t count) 283{ 284 285 if (tag == X86_BUS_SPACE_IO) 286 insb(bsh + offset, addr, count); 287 else { 288#ifdef __GNUCLIKE_ASM 289 __asm __volatile(" \n\ 290 1: movb (%2),%%al \n\ 291 stosb \n\ 292 loop 1b" : 293 "=D" (addr), "=c" (count) : 294 "r" (bsh + offset), "0" (addr), "1" (count) : 295 "%eax", "memory"); 296#endif 297 } 298} 299 300static __inline void 301bus_space_read_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, 302 bus_size_t offset, u_int16_t *addr, size_t count) 303{ 304 305 if (tag == X86_BUS_SPACE_IO) 306 insw(bsh + offset, addr, count); 307 else { 308#ifdef __GNUCLIKE_ASM 309 __asm __volatile(" \n\ 310 1: movw (%2),%%ax \n\ 311 stosw \n\ 312 loop 1b" : 313 "=D" (addr), "=c" (count) : 314 "r" (bsh + offset), "0" (addr), "1" (count) : 315 "%eax", "memory"); 316#endif 317 } 318} 319 320static __inline void 321bus_space_read_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, 322 bus_size_t offset, u_int32_t *addr, size_t count) 323{ 324 325 if (tag == X86_BUS_SPACE_IO) 326 insl(bsh + offset, addr, count); 327 else { 328#ifdef __GNUCLIKE_ASM 329 __asm __volatile(" \n\ 330 1: movl (%2),%%eax \n\ 331 stosl \n\ 332 loop 1b" : 333 "=D" (addr), "=c" (count) : 334 "r" (bsh + offset), "0" (addr), "1" (count) : 335 "%eax", "memory"); 336#endif 337 } 338} 339 340#if 0 /* Cause a link error for bus_space_read_multi_8 */ 341#define bus_space_read_multi_8 !!! bus_space_read_multi_8 unimplemented !!! 342#endif 343 344/* 345 * Read `count' 1, 2, 4, or 8 byte quantities from bus space 346 * described by tag/handle and starting at `offset' and copy into 347 * buffer provided. 348 */ 349static __inline void bus_space_read_region_1(bus_space_tag_t tag, 350 bus_space_handle_t bsh, 351 bus_size_t offset, u_int8_t *addr, 352 size_t count); 353 354static __inline void bus_space_read_region_2(bus_space_tag_t tag, 355 bus_space_handle_t bsh, 356 bus_size_t offset, u_int16_t *addr, 357 size_t count); 358 359static __inline void bus_space_read_region_4(bus_space_tag_t tag, 360 bus_space_handle_t bsh, 361 bus_size_t offset, u_int32_t *addr, 362 size_t count); 363 364static __inline void 365bus_space_read_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, 366 bus_size_t offset, u_int8_t *addr, size_t count) 367{ 368 369 if (tag == X86_BUS_SPACE_IO) { 370 int _port_ = bsh + offset; 371#ifdef __GNUCLIKE_ASM 372 __asm __volatile(" \n\ 373 1: inb %w2,%%al \n\ 374 stosb \n\ 375 incl %2 \n\ 376 loop 1b" : 377 "=D" (addr), "=c" (count), "=d" (_port_) : 378 "0" (addr), "1" (count), "2" (_port_) : 379 "%eax", "memory", "cc"); 380#endif 381 } else { 382 bus_space_handle_t _port_ = bsh + offset; 383#ifdef __GNUCLIKE_ASM 384 __asm __volatile(" \n\ 385 repne \n\ 386 movsb" : 387 "=D" (addr), "=c" (count), "=S" (_port_) : 388 "0" (addr), "1" (count), "2" (_port_) : 389 "memory", "cc"); 390#endif 391 } 392} 393 394static __inline void 395bus_space_read_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, 396 bus_size_t offset, u_int16_t *addr, size_t count) 397{ 398 399 if (tag == X86_BUS_SPACE_IO) { 400 int _port_ = bsh + offset; 401#ifdef __GNUCLIKE_ASM 402 __asm __volatile(" \n\ 403 1: inw %w2,%%ax \n\ 404 stosw \n\ 405 addl $2,%2 \n\ 406 loop 1b" : 407 "=D" (addr), "=c" (count), "=d" (_port_) : 408 "0" (addr), "1" (count), "2" (_port_) : 409 "%eax", "memory", "cc"); 410#endif 411 } else { 412 bus_space_handle_t _port_ = bsh + offset; 413#ifdef __GNUCLIKE_ASM 414 __asm __volatile(" \n\ 415 repne \n\ 416 movsw" : 417 "=D" (addr), "=c" (count), "=S" (_port_) : 418 "0" (addr), "1" (count), "2" (_port_) : 419 "memory", "cc"); 420#endif 421 } 422} 423 424static __inline void 425bus_space_read_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, 426 bus_size_t offset, u_int32_t *addr, size_t count) 427{ 428 429 if (tag == X86_BUS_SPACE_IO) { 430 int _port_ = bsh + offset; 431#ifdef __GNUCLIKE_ASM 432 __asm __volatile(" \n\ 433 1: inl %w2,%%eax \n\ 434 stosl \n\ 435 addl $4,%2 \n\ 436 loop 1b" : 437 "=D" (addr), "=c" (count), "=d" (_port_) : 438 "0" (addr), "1" (count), "2" (_port_) : 439 "%eax", "memory", "cc"); 440#endif 441 } else { 442 bus_space_handle_t _port_ = bsh + offset; 443#ifdef __GNUCLIKE_ASM 444 __asm __volatile(" \n\ 445 repne \n\ 446 movsl" : 447 "=D" (addr), "=c" (count), "=S" (_port_) : 448 "0" (addr), "1" (count), "2" (_port_) : 449 "memory", "cc"); 450#endif 451 } 452} 453 454#if 0 /* Cause a link error for bus_space_read_region_8 */ 455#define bus_space_read_region_8 !!! bus_space_read_region_8 unimplemented !!! 456#endif 457 458/* 459 * Write the 1, 2, 4, or 8 byte value `value' to bus space 460 * described by tag/handle/offset. 461 */ 462 463static __inline void bus_space_write_1(bus_space_tag_t tag, 464 bus_space_handle_t bsh, 465 bus_size_t offset, u_int8_t value); 466 467static __inline void bus_space_write_2(bus_space_tag_t tag, 468 bus_space_handle_t bsh, 469 bus_size_t offset, u_int16_t value); 470 471static __inline void bus_space_write_4(bus_space_tag_t tag, 472 bus_space_handle_t bsh, 473 bus_size_t offset, u_int32_t value); 474 475#ifdef __amd64__ 476static __inline void bus_space_write_8(bus_space_tag_t tag, 477 bus_space_handle_t bsh, 478 bus_size_t offset, uint64_t value); 479#endif 480 481static __inline void 482bus_space_write_1(bus_space_tag_t tag, bus_space_handle_t bsh, 483 bus_size_t offset, u_int8_t value) 484{ 485 486 if (tag == X86_BUS_SPACE_IO) 487 outb(bsh + offset, value); 488 else 489 *(volatile u_int8_t *)(bsh + offset) = value; 490} 491 492static __inline void 493bus_space_write_2(bus_space_tag_t tag, bus_space_handle_t bsh, 494 bus_size_t offset, u_int16_t value) 495{ 496 497 if (tag == X86_BUS_SPACE_IO) 498 outw(bsh + offset, value); 499 else 500 *(volatile u_int16_t *)(bsh + offset) = value; 501} 502 503static __inline void 504bus_space_write_4(bus_space_tag_t tag, bus_space_handle_t bsh, 505 bus_size_t offset, u_int32_t value) 506{ 507 508 if (tag == X86_BUS_SPACE_IO) 509 outl(bsh + offset, value); 510 else 511 *(volatile u_int32_t *)(bsh + offset) = value; 512} 513 514#ifdef __amd64__ 515static __inline void 516bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, 517 bus_size_t offset, uint64_t value) 518{ 519 520 if (tag == X86_BUS_SPACE_IO) /* No 8 byte IO space access on x86 */ 521 return; 522 else 523 *(volatile uint64_t *)(bsh + offset) = value; 524} 525#endif 526 527/* 528 * Write `count' 1, 2, 4, or 8 byte quantities from the buffer 529 * provided to bus space described by tag/handle/offset. 530 */ 531 532static __inline void bus_space_write_multi_1(bus_space_tag_t tag, 533 bus_space_handle_t bsh, 534 bus_size_t offset, 535 const u_int8_t *addr, 536 size_t count); 537static __inline void bus_space_write_multi_2(bus_space_tag_t tag, 538 bus_space_handle_t bsh, 539 bus_size_t offset, 540 const u_int16_t *addr, 541 size_t count); 542 543static __inline void bus_space_write_multi_4(bus_space_tag_t tag, 544 bus_space_handle_t bsh, 545 bus_size_t offset, 546 const u_int32_t *addr, 547 size_t count); 548 549static __inline void 550bus_space_write_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, 551 bus_size_t offset, const u_int8_t *addr, size_t count) 552{ 553 554 if (tag == X86_BUS_SPACE_IO) 555 outsb(bsh + offset, addr, count); 556 else { 557#ifdef __GNUCLIKE_ASM 558 __asm __volatile(" \n\ 559 1: lodsb \n\ 560 movb %%al,(%2) \n\ 561 loop 1b" : 562 "=S" (addr), "=c" (count) : 563 "r" (bsh + offset), "0" (addr), "1" (count) : 564 "%eax", "memory", "cc"); 565#endif 566 } 567} 568 569static __inline void 570bus_space_write_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, 571 bus_size_t offset, const u_int16_t *addr, size_t count) 572{ 573 574 if (tag == X86_BUS_SPACE_IO) 575 outsw(bsh + offset, addr, count); 576 else { 577#ifdef __GNUCLIKE_ASM 578 __asm __volatile(" \n\ 579 1: lodsw \n\ 580 movw %%ax,(%2) \n\ 581 loop 1b" : 582 "=S" (addr), "=c" (count) : 583 "r" (bsh + offset), "0" (addr), "1" (count) : 584 "%eax", "memory", "cc"); 585#endif 586 } 587} 588 589static __inline void 590bus_space_write_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, 591 bus_size_t offset, const u_int32_t *addr, size_t count) 592{ 593 594 if (tag == X86_BUS_SPACE_IO) 595 outsl(bsh + offset, addr, count); 596 else { 597#ifdef __GNUCLIKE_ASM 598 __asm __volatile(" \n\ 599 1: lodsl \n\ 600 movl %%eax,(%2) \n\ 601 loop 1b" : 602 "=S" (addr), "=c" (count) : 603 "r" (bsh + offset), "0" (addr), "1" (count) : 604 "%eax", "memory", "cc"); 605#endif 606 } 607} 608 609#if 0 /* Cause a link error for bus_space_write_multi_8 */ 610#define bus_space_write_multi_8(t, h, o, a, c) \ 611 !!! bus_space_write_multi_8 unimplemented !!! 612#endif 613 614/* 615 * Write `count' 1, 2, 4, or 8 byte quantities from the buffer provided 616 * to bus space described by tag/handle starting at `offset'. 617 */ 618 619static __inline void bus_space_write_region_1(bus_space_tag_t tag, 620 bus_space_handle_t bsh, 621 bus_size_t offset, 622 const u_int8_t *addr, 623 size_t count); 624static __inline void bus_space_write_region_2(bus_space_tag_t tag, 625 bus_space_handle_t bsh, 626 bus_size_t offset, 627 const u_int16_t *addr, 628 size_t count); 629static __inline void bus_space_write_region_4(bus_space_tag_t tag, 630 bus_space_handle_t bsh, 631 bus_size_t offset, 632 const u_int32_t *addr, 633 size_t count); 634 635static __inline void 636bus_space_write_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, 637 bus_size_t offset, const u_int8_t *addr, size_t count) 638{ 639 640 if (tag == X86_BUS_SPACE_IO) { 641 int _port_ = bsh + offset; 642#ifdef __GNUCLIKE_ASM 643 __asm __volatile(" \n\ 644 1: lodsb \n\ 645 outb %%al,%w0 \n\ 646 incl %0 \n\ 647 loop 1b" : 648 "=d" (_port_), "=S" (addr), "=c" (count) : 649 "0" (_port_), "1" (addr), "2" (count) : 650 "%eax", "memory", "cc"); 651#endif 652 } else { 653 bus_space_handle_t _port_ = bsh + offset; 654#ifdef __GNUCLIKE_ASM 655 __asm __volatile(" \n\ 656 repne \n\ 657 movsb" : 658 "=D" (_port_), "=S" (addr), "=c" (count) : 659 "0" (_port_), "1" (addr), "2" (count) : 660 "memory", "cc"); 661#endif 662 } 663} 664 665static __inline void 666bus_space_write_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, 667 bus_size_t offset, const u_int16_t *addr, size_t count) 668{ 669 670 if (tag == X86_BUS_SPACE_IO) { 671 int _port_ = bsh + offset; 672#ifdef __GNUCLIKE_ASM 673 __asm __volatile(" \n\ 674 1: lodsw \n\ 675 outw %%ax,%w0 \n\ 676 addl $2,%0 \n\ 677 loop 1b" : 678 "=d" (_port_), "=S" (addr), "=c" (count) : 679 "0" (_port_), "1" (addr), "2" (count) : 680 "%eax", "memory", "cc"); 681#endif 682 } else { 683 bus_space_handle_t _port_ = bsh + offset; 684#ifdef __GNUCLIKE_ASM 685 __asm __volatile(" \n\ 686 repne \n\ 687 movsw" : 688 "=D" (_port_), "=S" (addr), "=c" (count) : 689 "0" (_port_), "1" (addr), "2" (count) : 690 "memory", "cc"); 691#endif 692 } 693} 694 695static __inline void 696bus_space_write_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, 697 bus_size_t offset, const u_int32_t *addr, size_t count) 698{ 699 700 if (tag == X86_BUS_SPACE_IO) { 701 int _port_ = bsh + offset; 702#ifdef __GNUCLIKE_ASM 703 __asm __volatile(" \n\ 704 1: lodsl \n\ 705 outl %%eax,%w0 \n\ 706 addl $4,%0 \n\ 707 loop 1b" : 708 "=d" (_port_), "=S" (addr), "=c" (count) : 709 "0" (_port_), "1" (addr), "2" (count) : 710 "%eax", "memory", "cc"); 711#endif 712 } else { 713 bus_space_handle_t _port_ = bsh + offset; 714#ifdef __GNUCLIKE_ASM 715 __asm __volatile(" \n\ 716 repne \n\ 717 movsl" : 718 "=D" (_port_), "=S" (addr), "=c" (count) : 719 "0" (_port_), "1" (addr), "2" (count) : 720 "memory", "cc"); 721#endif 722 } 723} 724 725#if 0 /* Cause a link error for bus_space_write_region_8 */ 726#define bus_space_write_region_8 \ 727 !!! bus_space_write_region_8 unimplemented !!! 728#endif 729 730/* 731 * Write the 1, 2, 4, or 8 byte value `val' to bus space described 732 * by tag/handle/offset `count' times. 733 */ 734 735static __inline void bus_space_set_multi_1(bus_space_tag_t tag, 736 bus_space_handle_t bsh, 737 bus_size_t offset, 738 u_int8_t value, size_t count); 739static __inline void bus_space_set_multi_2(bus_space_tag_t tag, 740 bus_space_handle_t bsh, 741 bus_size_t offset, 742 u_int16_t value, size_t count); 743static __inline void bus_space_set_multi_4(bus_space_tag_t tag, 744 bus_space_handle_t bsh, 745 bus_size_t offset, 746 u_int32_t value, size_t count); 747 748static __inline void 749bus_space_set_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, 750 bus_size_t offset, u_int8_t value, size_t count) 751{ 752 bus_space_handle_t addr = bsh + offset; 753 754 if (tag == X86_BUS_SPACE_IO) 755 while (count--) 756 outb(addr, value); 757 else 758 while (count--) 759 *(volatile u_int8_t *)(addr) = value; 760} 761 762static __inline void 763bus_space_set_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, 764 bus_size_t offset, u_int16_t value, size_t count) 765{ 766 bus_space_handle_t addr = bsh + offset; 767 768 if (tag == X86_BUS_SPACE_IO) 769 while (count--) 770 outw(addr, value); 771 else 772 while (count--) 773 *(volatile u_int16_t *)(addr) = value; 774} 775 776static __inline void 777bus_space_set_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, 778 bus_size_t offset, u_int32_t value, size_t count) 779{ 780 bus_space_handle_t addr = bsh + offset; 781 782 if (tag == X86_BUS_SPACE_IO) 783 while (count--) 784 outl(addr, value); 785 else 786 while (count--) 787 *(volatile u_int32_t *)(addr) = value; 788} 789 790#if 0 /* Cause a link error for bus_space_set_multi_8 */ 791#define bus_space_set_multi_8 !!! bus_space_set_multi_8 unimplemented !!! 792#endif 793 794/* 795 * Write `count' 1, 2, 4, or 8 byte value `val' to bus space described 796 * by tag/handle starting at `offset'. 797 */ 798 799static __inline void bus_space_set_region_1(bus_space_tag_t tag, 800 bus_space_handle_t bsh, 801 bus_size_t offset, u_int8_t value, 802 size_t count); 803static __inline void bus_space_set_region_2(bus_space_tag_t tag, 804 bus_space_handle_t bsh, 805 bus_size_t offset, u_int16_t value, 806 size_t count); 807static __inline void bus_space_set_region_4(bus_space_tag_t tag, 808 bus_space_handle_t bsh, 809 bus_size_t offset, u_int32_t value, 810 size_t count); 811 812static __inline void 813bus_space_set_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, 814 bus_size_t offset, u_int8_t value, size_t count) 815{ 816 bus_space_handle_t addr = bsh + offset; 817 818 if (tag == X86_BUS_SPACE_IO) 819 for (; count != 0; count--, addr++) 820 outb(addr, value); 821 else 822 for (; count != 0; count--, addr++) 823 *(volatile u_int8_t *)(addr) = value; 824} 825 826static __inline void 827bus_space_set_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, 828 bus_size_t offset, u_int16_t value, size_t count) 829{ 830 bus_space_handle_t addr = bsh + offset; 831 832 if (tag == X86_BUS_SPACE_IO) 833 for (; count != 0; count--, addr += 2) 834 outw(addr, value); 835 else 836 for (; count != 0; count--, addr += 2) 837 *(volatile u_int16_t *)(addr) = value; 838} 839 840static __inline void 841bus_space_set_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, 842 bus_size_t offset, u_int32_t value, size_t count) 843{ 844 bus_space_handle_t addr = bsh + offset; 845 846 if (tag == X86_BUS_SPACE_IO) 847 for (; count != 0; count--, addr += 4) 848 outl(addr, value); 849 else 850 for (; count != 0; count--, addr += 4) 851 *(volatile u_int32_t *)(addr) = value; 852} 853 854#if 0 /* Cause a link error for bus_space_set_region_8 */ 855#define bus_space_set_region_8 !!! bus_space_set_region_8 unimplemented !!! 856#endif 857 858/* 859 * Copy `count' 1, 2, 4, or 8 byte values from bus space starting 860 * at tag/bsh1/off1 to bus space starting at tag/bsh2/off2. 861 */ 862 863static __inline void bus_space_copy_region_1(bus_space_tag_t tag, 864 bus_space_handle_t bsh1, 865 bus_size_t off1, 866 bus_space_handle_t bsh2, 867 bus_size_t off2, size_t count); 868 869static __inline void bus_space_copy_region_2(bus_space_tag_t tag, 870 bus_space_handle_t bsh1, 871 bus_size_t off1, 872 bus_space_handle_t bsh2, 873 bus_size_t off2, size_t count); 874 875static __inline void bus_space_copy_region_4(bus_space_tag_t tag, 876 bus_space_handle_t bsh1, 877 bus_size_t off1, 878 bus_space_handle_t bsh2, 879 bus_size_t off2, size_t count); 880 881static __inline void 882bus_space_copy_region_1(bus_space_tag_t tag, bus_space_handle_t bsh1, 883 bus_size_t off1, bus_space_handle_t bsh2, 884 bus_size_t off2, size_t count) 885{ 886 bus_space_handle_t addr1 = bsh1 + off1; 887 bus_space_handle_t addr2 = bsh2 + off2; 888 889 if (tag == X86_BUS_SPACE_IO) { 890 if (addr1 >= addr2) { 891 /* src after dest: copy forward */ 892 for (; count != 0; count--, addr1++, addr2++) 893 outb(addr2, inb(addr1)); 894 } else { 895 /* dest after src: copy backwards */ 896 for (addr1 += (count - 1), addr2 += (count - 1); 897 count != 0; count--, addr1--, addr2--) 898 outb(addr2, inb(addr1)); 899 } 900 } else { 901 if (addr1 >= addr2) { 902 /* src after dest: copy forward */ 903 for (; count != 0; count--, addr1++, addr2++) 904 *(volatile u_int8_t *)(addr2) = 905 *(volatile u_int8_t *)(addr1); 906 } else { 907 /* dest after src: copy backwards */ 908 for (addr1 += (count - 1), addr2 += (count - 1); 909 count != 0; count--, addr1--, addr2--) 910 *(volatile u_int8_t *)(addr2) = 911 *(volatile u_int8_t *)(addr1); 912 } 913 } 914} 915 916static __inline void 917bus_space_copy_region_2(bus_space_tag_t tag, bus_space_handle_t bsh1, 918 bus_size_t off1, bus_space_handle_t bsh2, 919 bus_size_t off2, size_t count) 920{ 921 bus_space_handle_t addr1 = bsh1 + off1; 922 bus_space_handle_t addr2 = bsh2 + off2; 923 924 if (tag == X86_BUS_SPACE_IO) { 925 if (addr1 >= addr2) { 926 /* src after dest: copy forward */ 927 for (; count != 0; count--, addr1 += 2, addr2 += 2) 928 outw(addr2, inw(addr1)); 929 } else { 930 /* dest after src: copy backwards */ 931 for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1); 932 count != 0; count--, addr1 -= 2, addr2 -= 2) 933 outw(addr2, inw(addr1)); 934 } 935 } else { 936 if (addr1 >= addr2) { 937 /* src after dest: copy forward */ 938 for (; count != 0; count--, addr1 += 2, addr2 += 2) 939 *(volatile u_int16_t *)(addr2) = 940 *(volatile u_int16_t *)(addr1); 941 } else { 942 /* dest after src: copy backwards */ 943 for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1); 944 count != 0; count--, addr1 -= 2, addr2 -= 2) 945 *(volatile u_int16_t *)(addr2) = 946 *(volatile u_int16_t *)(addr1); 947 } 948 } 949} 950 951static __inline void 952bus_space_copy_region_4(bus_space_tag_t tag, bus_space_handle_t bsh1, 953 bus_size_t off1, bus_space_handle_t bsh2, 954 bus_size_t off2, size_t count) 955{ 956 bus_space_handle_t addr1 = bsh1 + off1; 957 bus_space_handle_t addr2 = bsh2 + off2; 958 959 if (tag == X86_BUS_SPACE_IO) { 960 if (addr1 >= addr2) { 961 /* src after dest: copy forward */ 962 for (; count != 0; count--, addr1 += 4, addr2 += 4) 963 outl(addr2, inl(addr1)); 964 } else { 965 /* dest after src: copy backwards */ 966 for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1); 967 count != 0; count--, addr1 -= 4, addr2 -= 4) 968 outl(addr2, inl(addr1)); 969 } 970 } else { 971 if (addr1 >= addr2) { 972 /* src after dest: copy forward */ 973 for (; count != 0; count--, addr1 += 4, addr2 += 4) 974 *(volatile u_int32_t *)(addr2) = 975 *(volatile u_int32_t *)(addr1); 976 } else { 977 /* dest after src: copy backwards */ 978 for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1); 979 count != 0; count--, addr1 -= 4, addr2 -= 4) 980 *(volatile u_int32_t *)(addr2) = 981 *(volatile u_int32_t *)(addr1); 982 } 983 } 984} 985 986#if 0 /* Cause a link error for bus_space_copy_8 */ 987#define bus_space_copy_region_8 !!! bus_space_copy_region_8 unimplemented !!! 988#endif 989 990/* 991 * Bus read/write barrier methods. 992 * 993 * void bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t bsh, 994 * bus_size_t offset, bus_size_t len, int flags); 995 * 996 * 997 * Note that BUS_SPACE_BARRIER_WRITE doesn't do anything other than 998 * prevent reordering by the compiler; all Intel x86 processors currently 999 * retire operations outside the CPU in program order. 1000 */ 1001static __inline void 1002bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused, 1003 bus_size_t offset __unused, bus_size_t len __unused, int flags) 1004{ 1005#ifdef __GNUCLIKE_ASM 1006 if (flags & BUS_SPACE_BARRIER_READ) 1007#ifdef __amd64__ 1008 __asm __volatile("lock; addl $0,0(%%rsp)" : : : "memory"); 1009#else 1010 __asm __volatile("lock; addl $0,0(%%esp)" : : : "memory"); 1011#endif 1012 else 1013 __compiler_membar(); 1014#endif 1015} 1016 1017#ifdef BUS_SPACE_NO_LEGACY 1018#undef inb 1019#undef outb 1020#define inb(a) compiler_error 1021#define inw(a) compiler_error 1022#define inl(a) compiler_error 1023#define outb(a, b) compiler_error 1024#define outw(a, b) compiler_error 1025#define outl(a, b) compiler_error 1026#endif 1027 1028/* 1029 * Stream accesses are the same as normal accesses on x86; there are no 1030 * supported bus systems with an endianess different from the host one. 1031 */ 1032#define bus_space_read_stream_1(t, h, o) bus_space_read_1((t), (h), (o)) 1033#define bus_space_read_stream_2(t, h, o) bus_space_read_2((t), (h), (o)) 1034#define bus_space_read_stream_4(t, h, o) bus_space_read_4((t), (h), (o)) 1035 1036#define bus_space_read_multi_stream_1(t, h, o, a, c) \ 1037 bus_space_read_multi_1((t), (h), (o), (a), (c)) 1038#define bus_space_read_multi_stream_2(t, h, o, a, c) \ 1039 bus_space_read_multi_2((t), (h), (o), (a), (c)) 1040#define bus_space_read_multi_stream_4(t, h, o, a, c) \ 1041 bus_space_read_multi_4((t), (h), (o), (a), (c)) 1042 1043#define bus_space_write_stream_1(t, h, o, v) \ 1044 bus_space_write_1((t), (h), (o), (v)) 1045#define bus_space_write_stream_2(t, h, o, v) \ 1046 bus_space_write_2((t), (h), (o), (v)) 1047#define bus_space_write_stream_4(t, h, o, v) \ 1048 bus_space_write_4((t), (h), (o), (v)) 1049 1050#define bus_space_write_multi_stream_1(t, h, o, a, c) \ 1051 bus_space_write_multi_1((t), (h), (o), (a), (c)) 1052#define bus_space_write_multi_stream_2(t, h, o, a, c) \ 1053 bus_space_write_multi_2((t), (h), (o), (a), (c)) 1054#define bus_space_write_multi_stream_4(t, h, o, a, c) \ 1055 bus_space_write_multi_4((t), (h), (o), (a), (c)) 1056 1057#define bus_space_set_multi_stream_1(t, h, o, v, c) \ 1058 bus_space_set_multi_1((t), (h), (o), (v), (c)) 1059#define bus_space_set_multi_stream_2(t, h, o, v, c) \ 1060 bus_space_set_multi_2((t), (h), (o), (v), (c)) 1061#define bus_space_set_multi_stream_4(t, h, o, v, c) \ 1062 bus_space_set_multi_4((t), (h), (o), (v), (c)) 1063 1064#define bus_space_read_region_stream_1(t, h, o, a, c) \ 1065 bus_space_read_region_1((t), (h), (o), (a), (c)) 1066#define bus_space_read_region_stream_2(t, h, o, a, c) \ 1067 bus_space_read_region_2((t), (h), (o), (a), (c)) 1068#define bus_space_read_region_stream_4(t, h, o, a, c) \ 1069 bus_space_read_region_4((t), (h), (o), (a), (c)) 1070 1071#define bus_space_write_region_stream_1(t, h, o, a, c) \ 1072 bus_space_write_region_1((t), (h), (o), (a), (c)) 1073#define bus_space_write_region_stream_2(t, h, o, a, c) \ 1074 bus_space_write_region_2((t), (h), (o), (a), (c)) 1075#define bus_space_write_region_stream_4(t, h, o, a, c) \ 1076 bus_space_write_region_4((t), (h), (o), (a), (c)) 1077 1078#define bus_space_set_region_stream_1(t, h, o, v, c) \ 1079 bus_space_set_region_1((t), (h), (o), (v), (c)) 1080#define bus_space_set_region_stream_2(t, h, o, v, c) \ 1081 bus_space_set_region_2((t), (h), (o), (v), (c)) 1082#define bus_space_set_region_stream_4(t, h, o, v, c) \ 1083 bus_space_set_region_4((t), (h), (o), (v), (c)) 1084 1085#define bus_space_copy_region_stream_1(t, h1, o1, h2, o2, c) \ 1086 bus_space_copy_region_1((t), (h1), (o1), (h2), (o2), (c)) 1087#define bus_space_copy_region_stream_2(t, h1, o1, h2, o2, c) \ 1088 bus_space_copy_region_2((t), (h1), (o1), (h2), (o2), (c)) 1089#define bus_space_copy_region_stream_4(t, h1, o1, h2, o2, c) \ 1090 bus_space_copy_region_4((t), (h1), (o1), (h2), (o2), (c)) 1091 1092#define BUS_PEEK_FUNC(width, type) \ 1093 static inline int \ 1094 bus_space_peek_##width(bus_space_tag_t tag, \ 1095 bus_space_handle_t hnd, bus_size_t offset, type *value) \ 1096 { \ 1097 type tmp; \ 1098 tmp = bus_space_read_##width(tag, hnd, offset); \ 1099 *value = (type)tmp; \ 1100 return (0); \ 1101 } 1102BUS_PEEK_FUNC(1, uint8_t) 1103BUS_PEEK_FUNC(2, uint16_t) 1104BUS_PEEK_FUNC(4, uint32_t) 1105#ifdef __amd64__ 1106BUS_PEEK_FUNC(8, uint64_t) 1107#endif 1108 1109#define BUS_POKE_FUNC(width, type) \ 1110 static inline int \ 1111 bus_space_poke_##width(bus_space_tag_t tag, \ 1112 bus_space_handle_t hnd, bus_size_t offset, type value) \ 1113 { \ 1114 bus_space_write_##width(tag, hnd, offset, value); \ 1115 return (0); \ 1116 } 1117BUS_POKE_FUNC(1, uint8_t) 1118BUS_POKE_FUNC(2, uint16_t) 1119BUS_POKE_FUNC(4, uint32_t) 1120#ifdef __amd64__ 1121BUS_POKE_FUNC(8, uint64_t) 1122#endif 1123 1124#endif /* KCSAN && !KCSAN_RUNTIME */ 1125 1126#endif /* _X86_BUS_H_ */ 1127