1/*- 2 * SPDX-License-Identifier: BSD-3-Clause AND BSD-2-ClauseE 3 * 4 * Copyright (c) KATO Takenori, 1999. 5 * 6 * All rights reserved. Unpublished rights reserved under the copyright 7 * laws of Japan. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer as 15 * the first lines of this file unmodified. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. The name of the author may not be used to endorse or promote products 20 * derived from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 27 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 31 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34/* $NetBSD: bus.h,v 1.12 1997/10/01 08:25:15 fvdl Exp $ */ 35 36/*- 37 * Copyright (c) 1996, 1997 The NetBSD Foundation, Inc. 38 * All rights reserved. 39 * 40 * This code is derived from software contributed to The NetBSD Foundation 41 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 42 * NASA Ames Research Center. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 55 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 56 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 57 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 58 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 59 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 60 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 61 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 62 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 63 * POSSIBILITY OF SUCH DAMAGE. 64 */ 65 66/*- 67 * Copyright (c) 1996 Charles M. Hannum. All rights reserved. 68 * Copyright (c) 1996 Christopher G. Demetriou. All rights reserved. 69 * 70 * Redistribution and use in source and binary forms, with or without 71 * modification, are permitted provided that the following conditions 72 * are met: 73 * 1. Redistributions of source code must retain the above copyright 74 * notice, this list of conditions and the following disclaimer. 75 * 2. Redistributions in binary form must reproduce the above copyright 76 * notice, this list of conditions and the following disclaimer in the 77 * documentation and/or other materials provided with the distribution. 78 * 3. All advertising materials mentioning features or use of this software 79 * must display the following acknowledgement: 80 * This product includes software developed by Christopher G. Demetriou 81 * for the NetBSD Project. 82 * 4. The name of the author may not be used to endorse or promote products 83 * derived from this software without specific prior written permission 84 * 85 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 86 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 87 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 88 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 89 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 90 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 91 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 92 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 93 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 94 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 95 */ 96 97#ifndef _MACHINE_BUS_H_ 98#define _MACHINE_BUS_H_ 99 100#include <machine/_bus.h> 101#include <machine/cpufunc.h> 102#include <machine/bus_dma.h> 103 104/* 105 * Values for the x86 bus space tag, not to be used directly by MI code. 106 */ 107#define X86_BUS_SPACE_IO 0 /* space is i/o space */ 108#define X86_BUS_SPACE_MEM 1 /* space is mem space */ 109 110#define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFF 111#define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF 112#if defined(__amd64__) 113#define BUS_SPACE_MAXSIZE 0xFFFFFFFFFFFFFFFFULL 114#else 115#define BUS_SPACE_MAXSIZE 0xFFFFFFFF 116#endif 117#define BUS_SPACE_MAXADDR_24BIT 0xFFFFFF 118#define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF 119#if defined(__amd64__) || defined(PAE) 120#define BUS_SPACE_MAXADDR_48BIT 0xFFFFFFFFFFFFULL 121#define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFULL 122#else 123#define BUS_SPACE_MAXADDR 0xFFFFFFFF 124#endif 125 126#define BUS_SPACE_INVALID_DATA (~0) 127#define BUS_SPACE_UNRESTRICTED (~0) 128 129#define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */ 130#define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */ 131 132#if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME) 133#include <sys/bus_san.h> 134#else 135 136/* 137 * Map a region of device bus space into CPU virtual address space. 138 */ 139 140int bus_space_map(bus_space_tag_t tag, bus_addr_t addr, bus_size_t size, 141 int flags, bus_space_handle_t *bshp); 142 143/* 144 * Unmap a region of device bus space. 145 */ 146 147void bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t bsh, 148 bus_size_t size); 149 150/* 151 * Get a new handle for a subregion of an already-mapped area of bus space. 152 */ 153 154static __inline int bus_space_subregion(bus_space_tag_t t, 155 bus_space_handle_t bsh, 156 bus_size_t offset, bus_size_t size, 157 bus_space_handle_t *nbshp); 158 159static __inline int 160bus_space_subregion(bus_space_tag_t t __unused, bus_space_handle_t bsh, 161 bus_size_t offset, bus_size_t size __unused, 162 bus_space_handle_t *nbshp) 163{ 164 165 *nbshp = bsh + offset; 166 return (0); 167} 168 169/* 170 * Allocate a region of memory that is accessible to devices in bus space. 171 */ 172 173int bus_space_alloc(bus_space_tag_t t, bus_addr_t rstart, 174 bus_addr_t rend, bus_size_t size, bus_size_t align, 175 bus_size_t boundary, int flags, bus_addr_t *addrp, 176 bus_space_handle_t *bshp); 177 178/* 179 * Free a region of bus space accessible memory. 180 */ 181 182static __inline void bus_space_free(bus_space_tag_t t, bus_space_handle_t bsh, 183 bus_size_t size); 184 185static __inline void 186bus_space_free(bus_space_tag_t t __unused, bus_space_handle_t bsh __unused, 187 bus_size_t size __unused) 188{ 189} 190 191/* 192 * Read a 1, 2, 4, or 8 byte quantity from bus space 193 * described by tag/handle/offset. 194 */ 195static __inline u_int8_t bus_space_read_1(bus_space_tag_t tag, 196 bus_space_handle_t handle, 197 bus_size_t offset); 198 199static __inline u_int16_t bus_space_read_2(bus_space_tag_t tag, 200 bus_space_handle_t handle, 201 bus_size_t offset); 202 203static __inline u_int32_t bus_space_read_4(bus_space_tag_t tag, 204 bus_space_handle_t handle, 205 bus_size_t offset); 206 207#ifdef __amd64__ 208static __inline uint64_t bus_space_read_8(bus_space_tag_t tag, 209 bus_space_handle_t handle, 210 bus_size_t offset); 211#endif 212 213static __inline u_int8_t 214bus_space_read_1(bus_space_tag_t tag, bus_space_handle_t handle, 215 bus_size_t offset) 216{ 217 218 if (tag == X86_BUS_SPACE_IO) 219 return (inb(handle + offset)); 220 return (*(volatile u_int8_t *)(handle + offset)); 221} 222 223static __inline u_int16_t 224bus_space_read_2(bus_space_tag_t tag, bus_space_handle_t handle, 225 bus_size_t offset) 226{ 227 228 if (tag == X86_BUS_SPACE_IO) 229 return (inw(handle + offset)); 230 return (*(volatile u_int16_t *)(handle + offset)); 231} 232 233static __inline u_int32_t 234bus_space_read_4(bus_space_tag_t tag, bus_space_handle_t handle, 235 bus_size_t offset) 236{ 237 238 if (tag == X86_BUS_SPACE_IO) 239 return (inl(handle + offset)); 240 return (*(volatile u_int32_t *)(handle + offset)); 241} 242 243#ifdef __amd64__ 244static __inline uint64_t 245bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 246 bus_size_t offset) 247{ 248 249 if (tag == X86_BUS_SPACE_IO) /* No 8 byte IO space access on x86 */ 250 return (BUS_SPACE_INVALID_DATA); 251 return (*(volatile uint64_t *)(handle + offset)); 252} 253#endif 254 255/* 256 * Read `count' 1, 2, 4, or 8 byte quantities from bus space 257 * described by tag/handle/offset and copy into buffer provided. 258 */ 259static __inline void bus_space_read_multi_1(bus_space_tag_t tag, 260 bus_space_handle_t bsh, 261 bus_size_t offset, u_int8_t *addr, 262 size_t count); 263 264static __inline void bus_space_read_multi_2(bus_space_tag_t tag, 265 bus_space_handle_t bsh, 266 bus_size_t offset, u_int16_t *addr, 267 size_t count); 268 269static __inline void bus_space_read_multi_4(bus_space_tag_t tag, 270 bus_space_handle_t bsh, 271 bus_size_t offset, u_int32_t *addr, 272 size_t count); 273 274static __inline void 275bus_space_read_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, 276 bus_size_t offset, u_int8_t *addr, size_t count) 277{ 278 279 if (tag == X86_BUS_SPACE_IO) 280 insb(bsh + offset, addr, count); 281 else { 282 __asm __volatile(" \n\ 283 1: movb (%2),%%al \n\ 284 stosb \n\ 285 loop 1b" : 286 "=D" (addr), "=c" (count) : 287 "r" (bsh + offset), "0" (addr), "1" (count) : 288 "%eax", "memory"); 289 } 290} 291 292static __inline void 293bus_space_read_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, 294 bus_size_t offset, u_int16_t *addr, size_t count) 295{ 296 297 if (tag == X86_BUS_SPACE_IO) 298 insw(bsh + offset, addr, count); 299 else { 300 __asm __volatile(" \n\ 301 1: movw (%2),%%ax \n\ 302 stosw \n\ 303 loop 1b" : 304 "=D" (addr), "=c" (count) : 305 "r" (bsh + offset), "0" (addr), "1" (count) : 306 "%eax", "memory"); 307 } 308} 309 310static __inline void 311bus_space_read_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, 312 bus_size_t offset, u_int32_t *addr, size_t count) 313{ 314 315 if (tag == X86_BUS_SPACE_IO) 316 insl(bsh + offset, addr, count); 317 else { 318 __asm __volatile(" \n\ 319 1: movl (%2),%%eax \n\ 320 stosl \n\ 321 loop 1b" : 322 "=D" (addr), "=c" (count) : 323 "r" (bsh + offset), "0" (addr), "1" (count) : 324 "%eax", "memory"); 325 } 326} 327 328#if 0 /* Cause a link error for bus_space_read_multi_8 */ 329#define bus_space_read_multi_8 !!! bus_space_read_multi_8 unimplemented !!! 330#endif 331 332/* 333 * Read `count' 1, 2, 4, or 8 byte quantities from bus space 334 * described by tag/handle and starting at `offset' and copy into 335 * buffer provided. 336 */ 337static __inline void bus_space_read_region_1(bus_space_tag_t tag, 338 bus_space_handle_t bsh, 339 bus_size_t offset, u_int8_t *addr, 340 size_t count); 341 342static __inline void bus_space_read_region_2(bus_space_tag_t tag, 343 bus_space_handle_t bsh, 344 bus_size_t offset, u_int16_t *addr, 345 size_t count); 346 347static __inline void bus_space_read_region_4(bus_space_tag_t tag, 348 bus_space_handle_t bsh, 349 bus_size_t offset, u_int32_t *addr, 350 size_t count); 351 352static __inline void 353bus_space_read_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, 354 bus_size_t offset, u_int8_t *addr, size_t count) 355{ 356 357 if (tag == X86_BUS_SPACE_IO) { 358 int _port_ = bsh + offset; 359 __asm __volatile(" \n\ 360 1: inb %w2,%%al \n\ 361 stosb \n\ 362 incl %2 \n\ 363 loop 1b" : 364 "=D" (addr), "=c" (count), "=d" (_port_) : 365 "0" (addr), "1" (count), "2" (_port_) : 366 "%eax", "memory", "cc"); 367 } else { 368 bus_space_handle_t _port_ = bsh + offset; 369 __asm __volatile(" \n\ 370 repne \n\ 371 movsb" : 372 "=D" (addr), "=c" (count), "=S" (_port_) : 373 "0" (addr), "1" (count), "2" (_port_) : 374 "memory", "cc"); 375 } 376} 377 378static __inline void 379bus_space_read_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, 380 bus_size_t offset, u_int16_t *addr, size_t count) 381{ 382 383 if (tag == X86_BUS_SPACE_IO) { 384 int _port_ = bsh + offset; 385 __asm __volatile(" \n\ 386 1: inw %w2,%%ax \n\ 387 stosw \n\ 388 addl $2,%2 \n\ 389 loop 1b" : 390 "=D" (addr), "=c" (count), "=d" (_port_) : 391 "0" (addr), "1" (count), "2" (_port_) : 392 "%eax", "memory", "cc"); 393 } else { 394 bus_space_handle_t _port_ = bsh + offset; 395 __asm __volatile(" \n\ 396 repne \n\ 397 movsw" : 398 "=D" (addr), "=c" (count), "=S" (_port_) : 399 "0" (addr), "1" (count), "2" (_port_) : 400 "memory", "cc"); 401 } 402} 403 404static __inline void 405bus_space_read_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, 406 bus_size_t offset, u_int32_t *addr, size_t count) 407{ 408 409 if (tag == X86_BUS_SPACE_IO) { 410 int _port_ = bsh + offset; 411 __asm __volatile(" \n\ 412 1: inl %w2,%%eax \n\ 413 stosl \n\ 414 addl $4,%2 \n\ 415 loop 1b" : 416 "=D" (addr), "=c" (count), "=d" (_port_) : 417 "0" (addr), "1" (count), "2" (_port_) : 418 "%eax", "memory", "cc"); 419 } else { 420 bus_space_handle_t _port_ = bsh + offset; 421 __asm __volatile(" \n\ 422 repne \n\ 423 movsl" : 424 "=D" (addr), "=c" (count), "=S" (_port_) : 425 "0" (addr), "1" (count), "2" (_port_) : 426 "memory", "cc"); 427 } 428} 429 430#if 0 /* Cause a link error for bus_space_read_region_8 */ 431#define bus_space_read_region_8 !!! bus_space_read_region_8 unimplemented !!! 432#endif 433 434/* 435 * Write the 1, 2, 4, or 8 byte value `value' to bus space 436 * described by tag/handle/offset. 437 */ 438 439static __inline void bus_space_write_1(bus_space_tag_t tag, 440 bus_space_handle_t bsh, 441 bus_size_t offset, u_int8_t value); 442 443static __inline void bus_space_write_2(bus_space_tag_t tag, 444 bus_space_handle_t bsh, 445 bus_size_t offset, u_int16_t value); 446 447static __inline void bus_space_write_4(bus_space_tag_t tag, 448 bus_space_handle_t bsh, 449 bus_size_t offset, u_int32_t value); 450 451#ifdef __amd64__ 452static __inline void bus_space_write_8(bus_space_tag_t tag, 453 bus_space_handle_t bsh, 454 bus_size_t offset, uint64_t value); 455#endif 456 457static __inline void 458bus_space_write_1(bus_space_tag_t tag, bus_space_handle_t bsh, 459 bus_size_t offset, u_int8_t value) 460{ 461 462 if (tag == X86_BUS_SPACE_IO) 463 outb(bsh + offset, value); 464 else 465 *(volatile u_int8_t *)(bsh + offset) = value; 466} 467 468static __inline void 469bus_space_write_2(bus_space_tag_t tag, bus_space_handle_t bsh, 470 bus_size_t offset, u_int16_t value) 471{ 472 473 if (tag == X86_BUS_SPACE_IO) 474 outw(bsh + offset, value); 475 else 476 *(volatile u_int16_t *)(bsh + offset) = value; 477} 478 479static __inline void 480bus_space_write_4(bus_space_tag_t tag, bus_space_handle_t bsh, 481 bus_size_t offset, u_int32_t value) 482{ 483 484 if (tag == X86_BUS_SPACE_IO) 485 outl(bsh + offset, value); 486 else 487 *(volatile u_int32_t *)(bsh + offset) = value; 488} 489 490#ifdef __amd64__ 491static __inline void 492bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, 493 bus_size_t offset, uint64_t value) 494{ 495 496 if (tag == X86_BUS_SPACE_IO) /* No 8 byte IO space access on x86 */ 497 return; 498 else 499 *(volatile uint64_t *)(bsh + offset) = value; 500} 501#endif 502 503/* 504 * Write `count' 1, 2, 4, or 8 byte quantities from the buffer 505 * provided to bus space described by tag/handle/offset. 506 */ 507 508static __inline void bus_space_write_multi_1(bus_space_tag_t tag, 509 bus_space_handle_t bsh, 510 bus_size_t offset, 511 const u_int8_t *addr, 512 size_t count); 513static __inline void bus_space_write_multi_2(bus_space_tag_t tag, 514 bus_space_handle_t bsh, 515 bus_size_t offset, 516 const u_int16_t *addr, 517 size_t count); 518 519static __inline void bus_space_write_multi_4(bus_space_tag_t tag, 520 bus_space_handle_t bsh, 521 bus_size_t offset, 522 const u_int32_t *addr, 523 size_t count); 524 525static __inline void 526bus_space_write_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, 527 bus_size_t offset, const u_int8_t *addr, size_t count) 528{ 529 530 if (tag == X86_BUS_SPACE_IO) 531 outsb(bsh + offset, addr, count); 532 else { 533 __asm __volatile(" \n\ 534 1: lodsb \n\ 535 movb %%al,(%2) \n\ 536 loop 1b" : 537 "=S" (addr), "=c" (count) : 538 "r" (bsh + offset), "0" (addr), "1" (count) : 539 "%eax", "memory", "cc"); 540 } 541} 542 543static __inline void 544bus_space_write_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, 545 bus_size_t offset, const u_int16_t *addr, size_t count) 546{ 547 548 if (tag == X86_BUS_SPACE_IO) 549 outsw(bsh + offset, addr, count); 550 else { 551 __asm __volatile(" \n\ 552 1: lodsw \n\ 553 movw %%ax,(%2) \n\ 554 loop 1b" : 555 "=S" (addr), "=c" (count) : 556 "r" (bsh + offset), "0" (addr), "1" (count) : 557 "%eax", "memory", "cc"); 558 } 559} 560 561static __inline void 562bus_space_write_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, 563 bus_size_t offset, const u_int32_t *addr, size_t count) 564{ 565 566 if (tag == X86_BUS_SPACE_IO) 567 outsl(bsh + offset, addr, count); 568 else { 569 __asm __volatile(" \n\ 570 1: lodsl \n\ 571 movl %%eax,(%2) \n\ 572 loop 1b" : 573 "=S" (addr), "=c" (count) : 574 "r" (bsh + offset), "0" (addr), "1" (count) : 575 "%eax", "memory", "cc"); 576 } 577} 578 579#if 0 /* Cause a link error for bus_space_write_multi_8 */ 580#define bus_space_write_multi_8(t, h, o, a, c) \ 581 !!! bus_space_write_multi_8 unimplemented !!! 582#endif 583 584/* 585 * Write `count' 1, 2, 4, or 8 byte quantities from the buffer provided 586 * to bus space described by tag/handle starting at `offset'. 587 */ 588 589static __inline void bus_space_write_region_1(bus_space_tag_t tag, 590 bus_space_handle_t bsh, 591 bus_size_t offset, 592 const u_int8_t *addr, 593 size_t count); 594static __inline void bus_space_write_region_2(bus_space_tag_t tag, 595 bus_space_handle_t bsh, 596 bus_size_t offset, 597 const u_int16_t *addr, 598 size_t count); 599static __inline void bus_space_write_region_4(bus_space_tag_t tag, 600 bus_space_handle_t bsh, 601 bus_size_t offset, 602 const u_int32_t *addr, 603 size_t count); 604 605static __inline void 606bus_space_write_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, 607 bus_size_t offset, const u_int8_t *addr, size_t count) 608{ 609 610 if (tag == X86_BUS_SPACE_IO) { 611 int _port_ = bsh + offset; 612 __asm __volatile(" \n\ 613 1: lodsb \n\ 614 outb %%al,%w0 \n\ 615 incl %0 \n\ 616 loop 1b" : 617 "=d" (_port_), "=S" (addr), "=c" (count) : 618 "0" (_port_), "1" (addr), "2" (count) : 619 "%eax", "memory", "cc"); 620 } else { 621 bus_space_handle_t _port_ = bsh + offset; 622 __asm __volatile(" \n\ 623 repne \n\ 624 movsb" : 625 "=D" (_port_), "=S" (addr), "=c" (count) : 626 "0" (_port_), "1" (addr), "2" (count) : 627 "memory", "cc"); 628 } 629} 630 631static __inline void 632bus_space_write_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, 633 bus_size_t offset, const u_int16_t *addr, size_t count) 634{ 635 636 if (tag == X86_BUS_SPACE_IO) { 637 int _port_ = bsh + offset; 638 __asm __volatile(" \n\ 639 1: lodsw \n\ 640 outw %%ax,%w0 \n\ 641 addl $2,%0 \n\ 642 loop 1b" : 643 "=d" (_port_), "=S" (addr), "=c" (count) : 644 "0" (_port_), "1" (addr), "2" (count) : 645 "%eax", "memory", "cc"); 646 } else { 647 bus_space_handle_t _port_ = bsh + offset; 648 __asm __volatile(" \n\ 649 repne \n\ 650 movsw" : 651 "=D" (_port_), "=S" (addr), "=c" (count) : 652 "0" (_port_), "1" (addr), "2" (count) : 653 "memory", "cc"); 654 } 655} 656 657static __inline void 658bus_space_write_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, 659 bus_size_t offset, const u_int32_t *addr, size_t count) 660{ 661 662 if (tag == X86_BUS_SPACE_IO) { 663 int _port_ = bsh + offset; 664 __asm __volatile(" \n\ 665 1: lodsl \n\ 666 outl %%eax,%w0 \n\ 667 addl $4,%0 \n\ 668 loop 1b" : 669 "=d" (_port_), "=S" (addr), "=c" (count) : 670 "0" (_port_), "1" (addr), "2" (count) : 671 "%eax", "memory", "cc"); 672 } else { 673 bus_space_handle_t _port_ = bsh + offset; 674 __asm __volatile(" \n\ 675 repne \n\ 676 movsl" : 677 "=D" (_port_), "=S" (addr), "=c" (count) : 678 "0" (_port_), "1" (addr), "2" (count) : 679 "memory", "cc"); 680 } 681} 682 683#if 0 /* Cause a link error for bus_space_write_region_8 */ 684#define bus_space_write_region_8 \ 685 !!! bus_space_write_region_8 unimplemented !!! 686#endif 687 688/* 689 * Write the 1, 2, 4, or 8 byte value `val' to bus space described 690 * by tag/handle/offset `count' times. 691 */ 692 693static __inline void bus_space_set_multi_1(bus_space_tag_t tag, 694 bus_space_handle_t bsh, 695 bus_size_t offset, 696 u_int8_t value, size_t count); 697static __inline void bus_space_set_multi_2(bus_space_tag_t tag, 698 bus_space_handle_t bsh, 699 bus_size_t offset, 700 u_int16_t value, size_t count); 701static __inline void bus_space_set_multi_4(bus_space_tag_t tag, 702 bus_space_handle_t bsh, 703 bus_size_t offset, 704 u_int32_t value, size_t count); 705 706static __inline void 707bus_space_set_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, 708 bus_size_t offset, u_int8_t value, size_t count) 709{ 710 bus_space_handle_t addr = bsh + offset; 711 712 if (tag == X86_BUS_SPACE_IO) 713 while (count--) 714 outb(addr, value); 715 else 716 while (count--) 717 *(volatile u_int8_t *)(addr) = value; 718} 719 720static __inline void 721bus_space_set_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, 722 bus_size_t offset, u_int16_t value, size_t count) 723{ 724 bus_space_handle_t addr = bsh + offset; 725 726 if (tag == X86_BUS_SPACE_IO) 727 while (count--) 728 outw(addr, value); 729 else 730 while (count--) 731 *(volatile u_int16_t *)(addr) = value; 732} 733 734static __inline void 735bus_space_set_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, 736 bus_size_t offset, u_int32_t value, size_t count) 737{ 738 bus_space_handle_t addr = bsh + offset; 739 740 if (tag == X86_BUS_SPACE_IO) 741 while (count--) 742 outl(addr, value); 743 else 744 while (count--) 745 *(volatile u_int32_t *)(addr) = value; 746} 747 748#if 0 /* Cause a link error for bus_space_set_multi_8 */ 749#define bus_space_set_multi_8 !!! bus_space_set_multi_8 unimplemented !!! 750#endif 751 752/* 753 * Write `count' 1, 2, 4, or 8 byte value `val' to bus space described 754 * by tag/handle starting at `offset'. 755 */ 756 757static __inline void bus_space_set_region_1(bus_space_tag_t tag, 758 bus_space_handle_t bsh, 759 bus_size_t offset, u_int8_t value, 760 size_t count); 761static __inline void bus_space_set_region_2(bus_space_tag_t tag, 762 bus_space_handle_t bsh, 763 bus_size_t offset, u_int16_t value, 764 size_t count); 765static __inline void bus_space_set_region_4(bus_space_tag_t tag, 766 bus_space_handle_t bsh, 767 bus_size_t offset, u_int32_t value, 768 size_t count); 769 770static __inline void 771bus_space_set_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, 772 bus_size_t offset, u_int8_t value, size_t count) 773{ 774 bus_space_handle_t addr = bsh + offset; 775 776 if (tag == X86_BUS_SPACE_IO) 777 for (; count != 0; count--, addr++) 778 outb(addr, value); 779 else 780 for (; count != 0; count--, addr++) 781 *(volatile u_int8_t *)(addr) = value; 782} 783 784static __inline void 785bus_space_set_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, 786 bus_size_t offset, u_int16_t value, size_t count) 787{ 788 bus_space_handle_t addr = bsh + offset; 789 790 if (tag == X86_BUS_SPACE_IO) 791 for (; count != 0; count--, addr += 2) 792 outw(addr, value); 793 else 794 for (; count != 0; count--, addr += 2) 795 *(volatile u_int16_t *)(addr) = value; 796} 797 798static __inline void 799bus_space_set_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, 800 bus_size_t offset, u_int32_t value, size_t count) 801{ 802 bus_space_handle_t addr = bsh + offset; 803 804 if (tag == X86_BUS_SPACE_IO) 805 for (; count != 0; count--, addr += 4) 806 outl(addr, value); 807 else 808 for (; count != 0; count--, addr += 4) 809 *(volatile u_int32_t *)(addr) = value; 810} 811 812#if 0 /* Cause a link error for bus_space_set_region_8 */ 813#define bus_space_set_region_8 !!! bus_space_set_region_8 unimplemented !!! 814#endif 815 816/* 817 * Copy `count' 1, 2, 4, or 8 byte values from bus space starting 818 * at tag/bsh1/off1 to bus space starting at tag/bsh2/off2. 819 */ 820 821static __inline void bus_space_copy_region_1(bus_space_tag_t tag, 822 bus_space_handle_t bsh1, 823 bus_size_t off1, 824 bus_space_handle_t bsh2, 825 bus_size_t off2, size_t count); 826 827static __inline void bus_space_copy_region_2(bus_space_tag_t tag, 828 bus_space_handle_t bsh1, 829 bus_size_t off1, 830 bus_space_handle_t bsh2, 831 bus_size_t off2, size_t count); 832 833static __inline void bus_space_copy_region_4(bus_space_tag_t tag, 834 bus_space_handle_t bsh1, 835 bus_size_t off1, 836 bus_space_handle_t bsh2, 837 bus_size_t off2, size_t count); 838 839static __inline void 840bus_space_copy_region_1(bus_space_tag_t tag, bus_space_handle_t bsh1, 841 bus_size_t off1, bus_space_handle_t bsh2, 842 bus_size_t off2, size_t count) 843{ 844 bus_space_handle_t addr1 = bsh1 + off1; 845 bus_space_handle_t addr2 = bsh2 + off2; 846 847 if (tag == X86_BUS_SPACE_IO) { 848 if (addr1 >= addr2) { 849 /* src after dest: copy forward */ 850 for (; count != 0; count--, addr1++, addr2++) 851 outb(addr2, inb(addr1)); 852 } else { 853 /* dest after src: copy backwards */ 854 for (addr1 += (count - 1), addr2 += (count - 1); 855 count != 0; count--, addr1--, addr2--) 856 outb(addr2, inb(addr1)); 857 } 858 } else { 859 if (addr1 >= addr2) { 860 /* src after dest: copy forward */ 861 for (; count != 0; count--, addr1++, addr2++) 862 *(volatile u_int8_t *)(addr2) = 863 *(volatile u_int8_t *)(addr1); 864 } else { 865 /* dest after src: copy backwards */ 866 for (addr1 += (count - 1), addr2 += (count - 1); 867 count != 0; count--, addr1--, addr2--) 868 *(volatile u_int8_t *)(addr2) = 869 *(volatile u_int8_t *)(addr1); 870 } 871 } 872} 873 874static __inline void 875bus_space_copy_region_2(bus_space_tag_t tag, bus_space_handle_t bsh1, 876 bus_size_t off1, bus_space_handle_t bsh2, 877 bus_size_t off2, size_t count) 878{ 879 bus_space_handle_t addr1 = bsh1 + off1; 880 bus_space_handle_t addr2 = bsh2 + off2; 881 882 if (tag == X86_BUS_SPACE_IO) { 883 if (addr1 >= addr2) { 884 /* src after dest: copy forward */ 885 for (; count != 0; count--, addr1 += 2, addr2 += 2) 886 outw(addr2, inw(addr1)); 887 } else { 888 /* dest after src: copy backwards */ 889 for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1); 890 count != 0; count--, addr1 -= 2, addr2 -= 2) 891 outw(addr2, inw(addr1)); 892 } 893 } else { 894 if (addr1 >= addr2) { 895 /* src after dest: copy forward */ 896 for (; count != 0; count--, addr1 += 2, addr2 += 2) 897 *(volatile u_int16_t *)(addr2) = 898 *(volatile u_int16_t *)(addr1); 899 } else { 900 /* dest after src: copy backwards */ 901 for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1); 902 count != 0; count--, addr1 -= 2, addr2 -= 2) 903 *(volatile u_int16_t *)(addr2) = 904 *(volatile u_int16_t *)(addr1); 905 } 906 } 907} 908 909static __inline void 910bus_space_copy_region_4(bus_space_tag_t tag, bus_space_handle_t bsh1, 911 bus_size_t off1, bus_space_handle_t bsh2, 912 bus_size_t off2, size_t count) 913{ 914 bus_space_handle_t addr1 = bsh1 + off1; 915 bus_space_handle_t addr2 = bsh2 + off2; 916 917 if (tag == X86_BUS_SPACE_IO) { 918 if (addr1 >= addr2) { 919 /* src after dest: copy forward */ 920 for (; count != 0; count--, addr1 += 4, addr2 += 4) 921 outl(addr2, inl(addr1)); 922 } else { 923 /* dest after src: copy backwards */ 924 for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1); 925 count != 0; count--, addr1 -= 4, addr2 -= 4) 926 outl(addr2, inl(addr1)); 927 } 928 } else { 929 if (addr1 >= addr2) { 930 /* src after dest: copy forward */ 931 for (; count != 0; count--, addr1 += 4, addr2 += 4) 932 *(volatile u_int32_t *)(addr2) = 933 *(volatile u_int32_t *)(addr1); 934 } else { 935 /* dest after src: copy backwards */ 936 for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1); 937 count != 0; count--, addr1 -= 4, addr2 -= 4) 938 *(volatile u_int32_t *)(addr2) = 939 *(volatile u_int32_t *)(addr1); 940 } 941 } 942} 943 944#if 0 /* Cause a link error for bus_space_copy_8 */ 945#define bus_space_copy_region_8 !!! bus_space_copy_region_8 unimplemented !!! 946#endif 947 948/* 949 * Bus read/write barrier methods. 950 * 951 * void bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t bsh, 952 * bus_size_t offset, bus_size_t len, int flags); 953 * 954 * 955 * Note that BUS_SPACE_BARRIER_WRITE doesn't do anything other than 956 * prevent reordering by the compiler; all Intel x86 processors currently 957 * retire operations outside the CPU in program order. 958 */ 959static __inline void 960bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused, 961 bus_size_t offset __unused, bus_size_t len __unused, int flags) 962{ 963 if (flags & BUS_SPACE_BARRIER_READ) 964#ifdef __amd64__ 965 __asm __volatile("lock; addl $0,0(%%rsp)" : : : "memory"); 966#else 967 __asm __volatile("lock; addl $0,0(%%esp)" : : : "memory"); 968#endif 969 else 970 __compiler_membar(); 971} 972 973#ifdef BUS_SPACE_NO_LEGACY 974#undef inb 975#undef outb 976#define inb(a) compiler_error 977#define inw(a) compiler_error 978#define inl(a) compiler_error 979#define outb(a, b) compiler_error 980#define outw(a, b) compiler_error 981#define outl(a, b) compiler_error 982#endif 983 984/* 985 * Stream accesses are the same as normal accesses on x86; there are no 986 * supported bus systems with an endianess different from the host one. 987 */ 988#define bus_space_read_stream_1(t, h, o) bus_space_read_1((t), (h), (o)) 989#define bus_space_read_stream_2(t, h, o) bus_space_read_2((t), (h), (o)) 990#define bus_space_read_stream_4(t, h, o) bus_space_read_4((t), (h), (o)) 991 992#define bus_space_read_multi_stream_1(t, h, o, a, c) \ 993 bus_space_read_multi_1((t), (h), (o), (a), (c)) 994#define bus_space_read_multi_stream_2(t, h, o, a, c) \ 995 bus_space_read_multi_2((t), (h), (o), (a), (c)) 996#define bus_space_read_multi_stream_4(t, h, o, a, c) \ 997 bus_space_read_multi_4((t), (h), (o), (a), (c)) 998 999#define bus_space_write_stream_1(t, h, o, v) \ 1000 bus_space_write_1((t), (h), (o), (v)) 1001#define bus_space_write_stream_2(t, h, o, v) \ 1002 bus_space_write_2((t), (h), (o), (v)) 1003#define bus_space_write_stream_4(t, h, o, v) \ 1004 bus_space_write_4((t), (h), (o), (v)) 1005 1006#define bus_space_write_multi_stream_1(t, h, o, a, c) \ 1007 bus_space_write_multi_1((t), (h), (o), (a), (c)) 1008#define bus_space_write_multi_stream_2(t, h, o, a, c) \ 1009 bus_space_write_multi_2((t), (h), (o), (a), (c)) 1010#define bus_space_write_multi_stream_4(t, h, o, a, c) \ 1011 bus_space_write_multi_4((t), (h), (o), (a), (c)) 1012 1013#define bus_space_set_multi_stream_1(t, h, o, v, c) \ 1014 bus_space_set_multi_1((t), (h), (o), (v), (c)) 1015#define bus_space_set_multi_stream_2(t, h, o, v, c) \ 1016 bus_space_set_multi_2((t), (h), (o), (v), (c)) 1017#define bus_space_set_multi_stream_4(t, h, o, v, c) \ 1018 bus_space_set_multi_4((t), (h), (o), (v), (c)) 1019 1020#define bus_space_read_region_stream_1(t, h, o, a, c) \ 1021 bus_space_read_region_1((t), (h), (o), (a), (c)) 1022#define bus_space_read_region_stream_2(t, h, o, a, c) \ 1023 bus_space_read_region_2((t), (h), (o), (a), (c)) 1024#define bus_space_read_region_stream_4(t, h, o, a, c) \ 1025 bus_space_read_region_4((t), (h), (o), (a), (c)) 1026 1027#define bus_space_write_region_stream_1(t, h, o, a, c) \ 1028 bus_space_write_region_1((t), (h), (o), (a), (c)) 1029#define bus_space_write_region_stream_2(t, h, o, a, c) \ 1030 bus_space_write_region_2((t), (h), (o), (a), (c)) 1031#define bus_space_write_region_stream_4(t, h, o, a, c) \ 1032 bus_space_write_region_4((t), (h), (o), (a), (c)) 1033 1034#define bus_space_set_region_stream_1(t, h, o, v, c) \ 1035 bus_space_set_region_1((t), (h), (o), (v), (c)) 1036#define bus_space_set_region_stream_2(t, h, o, v, c) \ 1037 bus_space_set_region_2((t), (h), (o), (v), (c)) 1038#define bus_space_set_region_stream_4(t, h, o, v, c) \ 1039 bus_space_set_region_4((t), (h), (o), (v), (c)) 1040 1041#define bus_space_copy_region_stream_1(t, h1, o1, h2, o2, c) \ 1042 bus_space_copy_region_1((t), (h1), (o1), (h2), (o2), (c)) 1043#define bus_space_copy_region_stream_2(t, h1, o1, h2, o2, c) \ 1044 bus_space_copy_region_2((t), (h1), (o1), (h2), (o2), (c)) 1045#define bus_space_copy_region_stream_4(t, h1, o1, h2, o2, c) \ 1046 bus_space_copy_region_4((t), (h1), (o1), (h2), (o2), (c)) 1047 1048#define BUS_PEEK_FUNC(width, type) \ 1049 static inline int \ 1050 bus_space_peek_##width(bus_space_tag_t tag, \ 1051 bus_space_handle_t hnd, bus_size_t offset, type *value) \ 1052 { \ 1053 type tmp; \ 1054 tmp = bus_space_read_##width(tag, hnd, offset); \ 1055 *value = (type)tmp; \ 1056 return (0); \ 1057 } 1058BUS_PEEK_FUNC(1, uint8_t) 1059BUS_PEEK_FUNC(2, uint16_t) 1060BUS_PEEK_FUNC(4, uint32_t) 1061#ifdef __amd64__ 1062BUS_PEEK_FUNC(8, uint64_t) 1063#endif 1064 1065#define BUS_POKE_FUNC(width, type) \ 1066 static inline int \ 1067 bus_space_poke_##width(bus_space_tag_t tag, \ 1068 bus_space_handle_t hnd, bus_size_t offset, type value) \ 1069 { \ 1070 bus_space_write_##width(tag, hnd, offset, value); \ 1071 return (0); \ 1072 } 1073BUS_POKE_FUNC(1, uint8_t) 1074BUS_POKE_FUNC(2, uint16_t) 1075BUS_POKE_FUNC(4, uint32_t) 1076#ifdef __amd64__ 1077BUS_POKE_FUNC(8, uint64_t) 1078#endif 1079 1080#endif /* !SAN_NEEDS_INTERCEPTORS && SAN_RUNTIME */ 1081 1082#endif /* !_MACHINE_BUS_H_ */ 1083