1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Implement the default iomap interfaces 4 * 5 * (C) Copyright 2004 Linus Torvalds 6 */ 7#include <linux/pci.h> 8#include <linux/io.h> 9#include <linux/kmsan-checks.h> 10 11#include <linux/export.h> 12 13/* 14 * Read/write from/to an (offsettable) iomem cookie. It might be a PIO 15 * access or a MMIO access, these functions don't care. The info is 16 * encoded in the hardware mapping set up by the mapping functions 17 * (or the cookie itself, depending on implementation and hw). 18 * 19 * The generic routines don't assume any hardware mappings, and just 20 * encode the PIO/MMIO as part of the cookie. They coldly assume that 21 * the MMIO IO mappings are not in the low address range. 22 * 23 * Architectures for which this is not true can't use this generic 24 * implementation and should do their own copy. 25 */ 26 27#ifndef HAVE_ARCH_PIO_SIZE 28/* 29 * We encode the physical PIO addresses (0-0xffff) into the 30 * pointer by offsetting them with a constant (0x10000) and 31 * assuming that all the low addresses are always PIO. That means 32 * we can do some sanity checks on the low bits, and don't 33 * need to just take things for granted. 34 */ 35#define PIO_OFFSET 0x10000UL 36#define PIO_MASK 0x0ffffUL 37#define PIO_RESERVED 0x40000UL 38#endif 39 40static void bad_io_access(unsigned long port, const char *access) 41{ 42 static int count = 10; 43 if (count) { 44 count--; 45 WARN(1, KERN_ERR "Bad IO access at port %#lx (%s)\n", port, access); 46 } 47} 48 49/* 50 * Ugly macros are a way of life. 51 */ 52#define IO_COND(addr, is_pio, is_mmio) do { \ 53 unsigned long port = (unsigned long __force)addr; \ 54 if (port >= PIO_RESERVED) { \ 55 is_mmio; \ 56 } else if (port > PIO_OFFSET) { \ 57 port &= PIO_MASK; \ 58 is_pio; \ 59 } else \ 60 bad_io_access(port, #is_pio ); \ 61} while (0) 62 63#ifndef pio_read16be 64#define pio_read16be(port) swab16(inw(port)) 65#define pio_read32be(port) swab32(inl(port)) 66#endif 67 68#ifndef mmio_read16be 69#define mmio_read16be(addr) swab16(readw(addr)) 70#define mmio_read32be(addr) swab32(readl(addr)) 71#define mmio_read64be(addr) swab64(readq(addr)) 72#endif 73 74/* 75 * Here and below, we apply __no_kmsan_checks to functions reading data from 76 * hardware, to ensure that KMSAN marks their return values as initialized. 77 */ 78__no_kmsan_checks 79unsigned int ioread8(const void __iomem *addr) 80{ 81 IO_COND(addr, return inb(port), return readb(addr)); 82 return 0xff; 83} 84__no_kmsan_checks 85unsigned int ioread16(const void __iomem *addr) 86{ 87 IO_COND(addr, return inw(port), return readw(addr)); 88 return 0xffff; 89} 90__no_kmsan_checks 91unsigned int ioread16be(const void __iomem *addr) 92{ 93 IO_COND(addr, return pio_read16be(port), return mmio_read16be(addr)); 94 return 0xffff; 95} 96__no_kmsan_checks 97unsigned int ioread32(const void __iomem *addr) 98{ 99 IO_COND(addr, return inl(port), return readl(addr)); 100 return 0xffffffff; 101} 102__no_kmsan_checks 103unsigned int ioread32be(const void __iomem *addr) 104{ 105 IO_COND(addr, return pio_read32be(port), return mmio_read32be(addr)); 106 return 0xffffffff; 107} 108EXPORT_SYMBOL(ioread8); 109EXPORT_SYMBOL(ioread16); 110EXPORT_SYMBOL(ioread16be); 111EXPORT_SYMBOL(ioread32); 112EXPORT_SYMBOL(ioread32be); 113 114#ifdef readq 115static u64 pio_read64_lo_hi(unsigned long port) 116{ 117 u64 lo, hi; 118 119 lo = inl(port); 120 hi = inl(port + sizeof(u32)); 121 122 return lo | (hi << 32); 123} 124 125static u64 pio_read64_hi_lo(unsigned long port) 126{ 127 u64 lo, hi; 128 129 hi = inl(port + sizeof(u32)); 130 lo = inl(port); 131 132 return lo | (hi << 32); 133} 134 135static u64 pio_read64be_lo_hi(unsigned long port) 136{ 137 u64 lo, hi; 138 139 lo = pio_read32be(port + sizeof(u32)); 140 hi = pio_read32be(port); 141 142 return lo | (hi << 32); 143} 144 145static u64 pio_read64be_hi_lo(unsigned long port) 146{ 147 u64 lo, hi; 148 149 hi = pio_read32be(port); 150 lo = pio_read32be(port + sizeof(u32)); 151 152 return lo | (hi << 32); 153} 154 155__no_kmsan_checks 156u64 ioread64_lo_hi(const void __iomem *addr) 157{ 158 IO_COND(addr, return pio_read64_lo_hi(port), return readq(addr)); 159 return 0xffffffffffffffffULL; 160} 161 162__no_kmsan_checks 163u64 ioread64_hi_lo(const void __iomem *addr) 164{ 165 IO_COND(addr, return pio_read64_hi_lo(port), return readq(addr)); 166 return 0xffffffffffffffffULL; 167} 168 169__no_kmsan_checks 170u64 ioread64be_lo_hi(const void __iomem *addr) 171{ 172 IO_COND(addr, return pio_read64be_lo_hi(port), 173 return mmio_read64be(addr)); 174 return 0xffffffffffffffffULL; 175} 176 177__no_kmsan_checks 178u64 ioread64be_hi_lo(const void __iomem *addr) 179{ 180 IO_COND(addr, return pio_read64be_hi_lo(port), 181 return mmio_read64be(addr)); 182 return 0xffffffffffffffffULL; 183} 184 185EXPORT_SYMBOL(ioread64_lo_hi); 186EXPORT_SYMBOL(ioread64_hi_lo); 187EXPORT_SYMBOL(ioread64be_lo_hi); 188EXPORT_SYMBOL(ioread64be_hi_lo); 189 190#endif /* readq */ 191 192#ifndef pio_write16be 193#define pio_write16be(val,port) outw(swab16(val),port) 194#define pio_write32be(val,port) outl(swab32(val),port) 195#endif 196 197#ifndef mmio_write16be 198#define mmio_write16be(val,port) writew(swab16(val),port) 199#define mmio_write32be(val,port) writel(swab32(val),port) 200#define mmio_write64be(val,port) writeq(swab64(val),port) 201#endif 202 203void iowrite8(u8 val, void __iomem *addr) 204{ 205 /* Make sure uninitialized memory isn't copied to devices. */ 206 kmsan_check_memory(&val, sizeof(val)); 207 IO_COND(addr, outb(val,port), writeb(val, addr)); 208} 209void iowrite16(u16 val, void __iomem *addr) 210{ 211 /* Make sure uninitialized memory isn't copied to devices. */ 212 kmsan_check_memory(&val, sizeof(val)); 213 IO_COND(addr, outw(val,port), writew(val, addr)); 214} 215void iowrite16be(u16 val, void __iomem *addr) 216{ 217 /* Make sure uninitialized memory isn't copied to devices. */ 218 kmsan_check_memory(&val, sizeof(val)); 219 IO_COND(addr, pio_write16be(val,port), mmio_write16be(val, addr)); 220} 221void iowrite32(u32 val, void __iomem *addr) 222{ 223 /* Make sure uninitialized memory isn't copied to devices. */ 224 kmsan_check_memory(&val, sizeof(val)); 225 IO_COND(addr, outl(val,port), writel(val, addr)); 226} 227void iowrite32be(u32 val, void __iomem *addr) 228{ 229 /* Make sure uninitialized memory isn't copied to devices. */ 230 kmsan_check_memory(&val, sizeof(val)); 231 IO_COND(addr, pio_write32be(val,port), mmio_write32be(val, addr)); 232} 233EXPORT_SYMBOL(iowrite8); 234EXPORT_SYMBOL(iowrite16); 235EXPORT_SYMBOL(iowrite16be); 236EXPORT_SYMBOL(iowrite32); 237EXPORT_SYMBOL(iowrite32be); 238 239#ifdef writeq 240static void pio_write64_lo_hi(u64 val, unsigned long port) 241{ 242 outl(val, port); 243 outl(val >> 32, port + sizeof(u32)); 244} 245 246static void pio_write64_hi_lo(u64 val, unsigned long port) 247{ 248 outl(val >> 32, port + sizeof(u32)); 249 outl(val, port); 250} 251 252static void pio_write64be_lo_hi(u64 val, unsigned long port) 253{ 254 pio_write32be(val, port + sizeof(u32)); 255 pio_write32be(val >> 32, port); 256} 257 258static void pio_write64be_hi_lo(u64 val, unsigned long port) 259{ 260 pio_write32be(val >> 32, port); 261 pio_write32be(val, port + sizeof(u32)); 262} 263 264void iowrite64_lo_hi(u64 val, void __iomem *addr) 265{ 266 /* Make sure uninitialized memory isn't copied to devices. */ 267 kmsan_check_memory(&val, sizeof(val)); 268 IO_COND(addr, pio_write64_lo_hi(val, port), 269 writeq(val, addr)); 270} 271 272void iowrite64_hi_lo(u64 val, void __iomem *addr) 273{ 274 /* Make sure uninitialized memory isn't copied to devices. */ 275 kmsan_check_memory(&val, sizeof(val)); 276 IO_COND(addr, pio_write64_hi_lo(val, port), 277 writeq(val, addr)); 278} 279 280void iowrite64be_lo_hi(u64 val, void __iomem *addr) 281{ 282 /* Make sure uninitialized memory isn't copied to devices. */ 283 kmsan_check_memory(&val, sizeof(val)); 284 IO_COND(addr, pio_write64be_lo_hi(val, port), 285 mmio_write64be(val, addr)); 286} 287 288void iowrite64be_hi_lo(u64 val, void __iomem *addr) 289{ 290 /* Make sure uninitialized memory isn't copied to devices. */ 291 kmsan_check_memory(&val, sizeof(val)); 292 IO_COND(addr, pio_write64be_hi_lo(val, port), 293 mmio_write64be(val, addr)); 294} 295 296EXPORT_SYMBOL(iowrite64_lo_hi); 297EXPORT_SYMBOL(iowrite64_hi_lo); 298EXPORT_SYMBOL(iowrite64be_lo_hi); 299EXPORT_SYMBOL(iowrite64be_hi_lo); 300 301#endif /* readq */ 302 303/* 304 * These are the "repeat MMIO read/write" functions. 305 * Note the "__raw" accesses, since we don't want to 306 * convert to CPU byte order. We write in "IO byte 307 * order" (we also don't have IO barriers). 308 */ 309#ifndef mmio_insb 310static inline void mmio_insb(const void __iomem *addr, u8 *dst, int count) 311{ 312 while (--count >= 0) { 313 u8 data = __raw_readb(addr); 314 *dst = data; 315 dst++; 316 } 317} 318static inline void mmio_insw(const void __iomem *addr, u16 *dst, int count) 319{ 320 while (--count >= 0) { 321 u16 data = __raw_readw(addr); 322 *dst = data; 323 dst++; 324 } 325} 326static inline void mmio_insl(const void __iomem *addr, u32 *dst, int count) 327{ 328 while (--count >= 0) { 329 u32 data = __raw_readl(addr); 330 *dst = data; 331 dst++; 332 } 333} 334#endif 335 336#ifndef mmio_outsb 337static inline void mmio_outsb(void __iomem *addr, const u8 *src, int count) 338{ 339 while (--count >= 0) { 340 __raw_writeb(*src, addr); 341 src++; 342 } 343} 344static inline void mmio_outsw(void __iomem *addr, const u16 *src, int count) 345{ 346 while (--count >= 0) { 347 __raw_writew(*src, addr); 348 src++; 349 } 350} 351static inline void mmio_outsl(void __iomem *addr, const u32 *src, int count) 352{ 353 while (--count >= 0) { 354 __raw_writel(*src, addr); 355 src++; 356 } 357} 358#endif 359 360void ioread8_rep(const void __iomem *addr, void *dst, unsigned long count) 361{ 362 IO_COND(addr, insb(port,dst,count), mmio_insb(addr, dst, count)); 363 /* KMSAN must treat values read from devices as initialized. */ 364 kmsan_unpoison_memory(dst, count); 365} 366void ioread16_rep(const void __iomem *addr, void *dst, unsigned long count) 367{ 368 IO_COND(addr, insw(port,dst,count), mmio_insw(addr, dst, count)); 369 /* KMSAN must treat values read from devices as initialized. */ 370 kmsan_unpoison_memory(dst, count * 2); 371} 372void ioread32_rep(const void __iomem *addr, void *dst, unsigned long count) 373{ 374 IO_COND(addr, insl(port,dst,count), mmio_insl(addr, dst, count)); 375 /* KMSAN must treat values read from devices as initialized. */ 376 kmsan_unpoison_memory(dst, count * 4); 377} 378EXPORT_SYMBOL(ioread8_rep); 379EXPORT_SYMBOL(ioread16_rep); 380EXPORT_SYMBOL(ioread32_rep); 381 382void iowrite8_rep(void __iomem *addr, const void *src, unsigned long count) 383{ 384 /* Make sure uninitialized memory isn't copied to devices. */ 385 kmsan_check_memory(src, count); 386 IO_COND(addr, outsb(port, src, count), mmio_outsb(addr, src, count)); 387} 388void iowrite16_rep(void __iomem *addr, const void *src, unsigned long count) 389{ 390 /* Make sure uninitialized memory isn't copied to devices. */ 391 kmsan_check_memory(src, count * 2); 392 IO_COND(addr, outsw(port, src, count), mmio_outsw(addr, src, count)); 393} 394void iowrite32_rep(void __iomem *addr, const void *src, unsigned long count) 395{ 396 /* Make sure uninitialized memory isn't copied to devices. */ 397 kmsan_check_memory(src, count * 4); 398 IO_COND(addr, outsl(port, src,count), mmio_outsl(addr, src, count)); 399} 400EXPORT_SYMBOL(iowrite8_rep); 401EXPORT_SYMBOL(iowrite16_rep); 402EXPORT_SYMBOL(iowrite32_rep); 403 404#ifdef CONFIG_HAS_IOPORT_MAP 405/* Create a virtual mapping cookie for an IO port range */ 406void __iomem *ioport_map(unsigned long port, unsigned int nr) 407{ 408 if (port > PIO_MASK) 409 return NULL; 410 return (void __iomem *) (unsigned long) (port + PIO_OFFSET); 411} 412 413void ioport_unmap(void __iomem *addr) 414{ 415 /* Nothing to do */ 416} 417EXPORT_SYMBOL(ioport_map); 418EXPORT_SYMBOL(ioport_unmap); 419#endif /* CONFIG_HAS_IOPORT_MAP */ 420 421#ifdef CONFIG_PCI 422/* Hide the details if this is a MMIO or PIO address space and just do what 423 * you expect in the correct way. */ 424void pci_iounmap(struct pci_dev *dev, void __iomem * addr) 425{ 426 IO_COND(addr, /* nothing */, iounmap(addr)); 427} 428EXPORT_SYMBOL(pci_iounmap); 429#endif /* CONFIG_PCI */ 430