1/* $NetBSD: subr_csan.c,v 1.14 2022/07/30 14:13:27 riastradh Exp $ */ 2 3/* 4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the KCSAN subsystem of the NetBSD kernel. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31#include <sys/cdefs.h> 32__KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.14 2022/07/30 14:13:27 riastradh Exp $"); 33 34#include <sys/param.h> 35#include <sys/device.h> 36#include <sys/kernel.h> 37#include <sys/param.h> 38#include <sys/conf.h> 39#include <sys/systm.h> 40#include <sys/types.h> 41#include <sys/csan.h> 42#include <sys/cpu.h> 43#include <sys/pserialize.h> 44 45#ifdef KCSAN_PANIC 46#define REPORT panic 47#else 48#define REPORT printf 49#endif 50 51typedef struct { 52 uintptr_t addr; 53 uint32_t size; 54 bool write:1; 55 bool atomic:1; 56 uintptr_t pc; 57} csan_cell_t; 58 59typedef struct { 60 bool inited; 61 uint32_t cnt; 62 csan_cell_t cell; 63} csan_cpu_t; 64 65static csan_cpu_t kcsan_cpus[MAXCPUS]; 66static bool kcsan_enabled __read_mostly; 67 68#define __RET_ADDR (uintptr_t)__builtin_return_address(0) 69 70#define KCSAN_NACCESSES 1024 71#define KCSAN_DELAY 10 /* 10 microseconds */ 72 73/* -------------------------------------------------------------------------- */ 74 75/* The MD code. */ 76#include <machine/csan.h> 77 78/* -------------------------------------------------------------------------- */ 79 80void 81kcsan_init(void) 82{ 83 kcsan_enabled = true; 84} 85 86void 87kcsan_cpu_init(struct cpu_info *ci) 88{ 89 kcsan_cpus[cpu_index(ci)].inited = true; 90} 91 92/* -------------------------------------------------------------------------- */ 93 94static inline void 95kcsan_report(csan_cell_t *new, cpuid_t newcpu, csan_cell_t *old, cpuid_t oldcpu) 96{ 97 const char *newsym, *oldsym; 98 int s; 99 100 s = pserialize_read_enter(); 101 if (ksyms_getname(NULL, &newsym, (vaddr_t)new->pc, KSYMS_PROC) != 0) { 102 newsym = "Unknown"; 103 } 104 if (ksyms_getname(NULL, &oldsym, (vaddr_t)old->pc, KSYMS_PROC) != 0) { 105 oldsym = "Unknown"; 106 } 107 REPORT("CSan: Racy Access " 108 "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>] " 109 "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>]\n", 110 newcpu, 111 (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), 112 (void *)new->addr, new->size, (void *)new->pc, newsym, 113 oldcpu, 114 (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"), 115 (void *)old->addr, old->size, (void *)old->pc, oldsym); 116 pserialize_read_exit(s); 117 kcsan_md_unwind(); 118} 119 120static inline bool 121kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old) 122{ 123 if (new->write && !new->atomic) 124 return false; 125 if (old->write && !old->atomic) 126 return false; 127 return true; 128} 129 130static inline void 131kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) 132{ 133 csan_cell_t old, new; 134 csan_cpu_t *cpu; 135 uint64_t intr; 136 size_t i; 137 138 if (__predict_false(!kcsan_enabled)) 139 return; 140 if (__predict_false(kcsan_md_unsupported((vaddr_t)addr))) 141 return; 142 143 new.addr = addr; 144 new.size = size; 145 new.write = write; 146 new.atomic = atomic; 147 new.pc = pc; 148 149 for (i = 0; i < ncpu; i++) { 150 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); 151 152 if (old.addr + old.size <= new.addr) 153 continue; 154 if (new.addr + new.size <= old.addr) 155 continue; 156 if (__predict_true(!old.write && !new.write)) 157 continue; 158 if (__predict_true(kcsan_access_is_atomic(&new, &old))) 159 continue; 160 161 kcsan_report(&new, cpu_number(), &old, i); 162 break; 163 } 164 165 if (__predict_false(!kcsan_md_is_avail())) 166 return; 167 168 kcsan_md_disable_intrs(&intr); 169 170 cpu = &kcsan_cpus[cpu_number()]; 171 if (__predict_false(!cpu->inited)) 172 goto out; 173 cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; 174 if (__predict_true(cpu->cnt != 0)) 175 goto out; 176 177 __builtin_memcpy(&cpu->cell, &new, sizeof(new)); 178 kcsan_md_delay(KCSAN_DELAY); 179 __builtin_memset(&cpu->cell, 0, sizeof(new)); 180 181out: 182 kcsan_md_enable_intrs(&intr); 183} 184 185#define CSAN_READ(size) \ 186 void __tsan_read##size(uintptr_t); \ 187 void __tsan_read##size(uintptr_t addr) \ 188 { \ 189 kcsan_access(addr, size, false, false, __RET_ADDR); \ 190 } 191 192CSAN_READ(1) 193CSAN_READ(2) 194CSAN_READ(4) 195CSAN_READ(8) 196CSAN_READ(16) 197 198#define CSAN_WRITE(size) \ 199 void __tsan_write##size(uintptr_t); \ 200 void __tsan_write##size(uintptr_t addr) \ 201 { \ 202 kcsan_access(addr, size, true, false, __RET_ADDR); \ 203 } 204 205CSAN_WRITE(1) 206CSAN_WRITE(2) 207CSAN_WRITE(4) 208CSAN_WRITE(8) 209CSAN_WRITE(16) 210 211void __tsan_read_range(uintptr_t, size_t); 212void __tsan_write_range(uintptr_t, size_t); 213 214void 215__tsan_read_range(uintptr_t addr, size_t size) 216{ 217 kcsan_access(addr, size, false, false, __RET_ADDR); 218} 219 220void 221__tsan_write_range(uintptr_t addr, size_t size) 222{ 223 kcsan_access(addr, size, true, false, __RET_ADDR); 224} 225 226void __tsan_init(void); 227void __tsan_func_entry(void *); 228void __tsan_func_exit(void); 229 230void 231__tsan_init(void) 232{ 233} 234 235void 236__tsan_func_entry(void *call_pc) 237{ 238} 239 240void 241__tsan_func_exit(void) 242{ 243} 244 245/* -------------------------------------------------------------------------- */ 246 247void * 248kcsan_memcpy(void *dst, const void *src, size_t len) 249{ 250 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 251 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 252 return __builtin_memcpy(dst, src, len); 253} 254 255int 256kcsan_memcmp(const void *b1, const void *b2, size_t len) 257{ 258 kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR); 259 kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR); 260 return __builtin_memcmp(b1, b2, len); 261} 262 263void * 264kcsan_memset(void *b, int c, size_t len) 265{ 266 kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR); 267 return __builtin_memset(b, c, len); 268} 269 270void * 271kcsan_memmove(void *dst, const void *src, size_t len) 272{ 273 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 274 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 275 return __builtin_memmove(dst, src, len); 276} 277 278char * 279kcsan_strcpy(char *dst, const char *src) 280{ 281 char *save = dst; 282 283 while (1) { 284 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR); 285 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR); 286 *dst = *src; 287 if (*src == '\0') 288 break; 289 src++, dst++; 290 } 291 292 return save; 293} 294 295int 296kcsan_strcmp(const char *s1, const char *s2) 297{ 298 while (1) { 299 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR); 300 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR); 301 if (*s1 != *s2) 302 break; 303 if (*s1 == '\0') 304 return 0; 305 s1++, s2++; 306 } 307 308 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 309} 310 311size_t 312kcsan_strlen(const char *str) 313{ 314 const char *s; 315 316 s = str; 317 while (1) { 318 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR); 319 if (*s == '\0') 320 break; 321 s++; 322 } 323 324 return (s - str); 325} 326 327#undef kcopy 328#undef copyinstr 329#undef copyoutstr 330#undef copyin 331#undef copyout 332 333int kcsan_kcopy(const void *, void *, size_t); 334int kcsan_copyinstr(const void *, void *, size_t, size_t *); 335int kcsan_copyoutstr(const void *, void *, size_t, size_t *); 336int kcsan_copyin(const void *, void *, size_t); 337int kcsan_copyout(const void *, void *, size_t); 338int kcopy(const void *, void *, size_t); 339int copyinstr(const void *, void *, size_t, size_t *); 340int copyoutstr(const void *, void *, size_t, size_t *); 341int copyin(const void *, void *, size_t); 342int copyout(const void *, void *, size_t); 343 344int 345kcsan_kcopy(const void *src, void *dst, size_t len) 346{ 347 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 348 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 349 return kcopy(src, dst, len); 350} 351 352int 353kcsan_copyin(const void *uaddr, void *kaddr, size_t len) 354{ 355 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 356 return copyin(uaddr, kaddr, len); 357} 358 359int 360kcsan_copyout(const void *kaddr, void *uaddr, size_t len) 361{ 362 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); 363 return copyout(kaddr, uaddr, len); 364} 365 366int 367kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 368{ 369 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 370 return copyinstr(uaddr, kaddr, len, done); 371} 372 373int 374kcsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 375{ 376 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); 377 return copyoutstr(kaddr, uaddr, len, done); 378} 379 380/* -------------------------------------------------------------------------- */ 381 382#undef atomic_add_32 383#undef atomic_add_int 384#undef atomic_add_long 385#undef atomic_add_ptr 386#undef atomic_add_64 387#undef atomic_add_32_nv 388#undef atomic_add_int_nv 389#undef atomic_add_long_nv 390#undef atomic_add_ptr_nv 391#undef atomic_add_64_nv 392#undef atomic_and_32 393#undef atomic_and_uint 394#undef atomic_and_ulong 395#undef atomic_and_64 396#undef atomic_and_32_nv 397#undef atomic_and_uint_nv 398#undef atomic_and_ulong_nv 399#undef atomic_and_64_nv 400#undef atomic_or_32 401#undef atomic_or_uint 402#undef atomic_or_ulong 403#undef atomic_or_64 404#undef atomic_or_32_nv 405#undef atomic_or_uint_nv 406#undef atomic_or_ulong_nv 407#undef atomic_or_64_nv 408#undef atomic_cas_32 409#undef atomic_cas_uint 410#undef atomic_cas_ulong 411#undef atomic_cas_ptr 412#undef atomic_cas_64 413#undef atomic_cas_32_ni 414#undef atomic_cas_uint_ni 415#undef atomic_cas_ulong_ni 416#undef atomic_cas_ptr_ni 417#undef atomic_cas_64_ni 418#undef atomic_swap_32 419#undef atomic_swap_uint 420#undef atomic_swap_ulong 421#undef atomic_swap_ptr 422#undef atomic_swap_64 423#undef atomic_dec_32 424#undef atomic_dec_uint 425#undef atomic_dec_ulong 426#undef atomic_dec_ptr 427#undef atomic_dec_64 428#undef atomic_dec_32_nv 429#undef atomic_dec_uint_nv 430#undef atomic_dec_ulong_nv 431#undef atomic_dec_ptr_nv 432#undef atomic_dec_64_nv 433#undef atomic_inc_32 434#undef atomic_inc_uint 435#undef atomic_inc_ulong 436#undef atomic_inc_ptr 437#undef atomic_inc_64 438#undef atomic_inc_32_nv 439#undef atomic_inc_uint_nv 440#undef atomic_inc_ulong_nv 441#undef atomic_inc_ptr_nv 442#undef atomic_inc_64_nv 443 444#define CSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 445 void atomic_add_##name(volatile targ1 *, targ2); \ 446 void kcsan_atomic_add_##name(volatile targ1 *, targ2); \ 447 void kcsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 448 { \ 449 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 450 __RET_ADDR); \ 451 atomic_add_##name(ptr, val); \ 452 } \ 453 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 454 tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 455 tret kcsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 456 { \ 457 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 458 __RET_ADDR); \ 459 return atomic_add_##name##_nv(ptr, val); \ 460 } 461 462#define CSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 463 void atomic_and_##name(volatile targ1 *, targ2); \ 464 void kcsan_atomic_and_##name(volatile targ1 *, targ2); \ 465 void kcsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 466 { \ 467 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 468 __RET_ADDR); \ 469 atomic_and_##name(ptr, val); \ 470 } \ 471 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 472 tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 473 tret kcsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 474 { \ 475 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 476 __RET_ADDR); \ 477 return atomic_and_##name##_nv(ptr, val); \ 478 } 479 480#define CSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 481 void atomic_or_##name(volatile targ1 *, targ2); \ 482 void kcsan_atomic_or_##name(volatile targ1 *, targ2); \ 483 void kcsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 484 { \ 485 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 486 __RET_ADDR); \ 487 atomic_or_##name(ptr, val); \ 488 } \ 489 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 490 tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 491 tret kcsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 492 { \ 493 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 494 __RET_ADDR); \ 495 return atomic_or_##name##_nv(ptr, val); \ 496 } 497 498#define CSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 499 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 500 tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 501 tret kcsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 502 { \ 503 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 504 __RET_ADDR); \ 505 return atomic_cas_##name(ptr, exp, new); \ 506 } \ 507 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 508 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 509 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 510 { \ 511 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 512 __RET_ADDR); \ 513 return atomic_cas_##name##_ni(ptr, exp, new); \ 514 } 515 516#define CSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 517 tret atomic_swap_##name(volatile targ1 *, targ2); \ 518 tret kcsan_atomic_swap_##name(volatile targ1 *, targ2); \ 519 tret kcsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 520 { \ 521 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 522 __RET_ADDR); \ 523 return atomic_swap_##name(ptr, val); \ 524 } 525 526#define CSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 527 void atomic_dec_##name(volatile targ1 *); \ 528 void kcsan_atomic_dec_##name(volatile targ1 *); \ 529 void kcsan_atomic_dec_##name(volatile targ1 *ptr) \ 530 { \ 531 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 532 __RET_ADDR); \ 533 atomic_dec_##name(ptr); \ 534 } \ 535 tret atomic_dec_##name##_nv(volatile targ1 *); \ 536 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *); \ 537 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 538 { \ 539 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 540 __RET_ADDR); \ 541 return atomic_dec_##name##_nv(ptr); \ 542 } 543 544#define CSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 545 void atomic_inc_##name(volatile targ1 *); \ 546 void kcsan_atomic_inc_##name(volatile targ1 *); \ 547 void kcsan_atomic_inc_##name(volatile targ1 *ptr) \ 548 { \ 549 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 550 __RET_ADDR); \ 551 atomic_inc_##name(ptr); \ 552 } \ 553 tret atomic_inc_##name##_nv(volatile targ1 *); \ 554 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *); \ 555 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 556 { \ 557 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 558 __RET_ADDR); \ 559 return atomic_inc_##name##_nv(ptr); \ 560 } 561 562CSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 563CSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 564CSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 565CSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 566CSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 567 568CSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 569CSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 570CSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 571CSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 572 573CSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 574CSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 575CSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 576CSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 577 578CSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 579CSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 580CSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 581CSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 582CSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 583 584CSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 585CSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 586CSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 587CSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 588CSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 589 590CSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 591CSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 592CSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 593CSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 594CSAN_ATOMIC_FUNC_DEC(ptr, void *, void); 595 596CSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 597CSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 598CSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 599CSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 600CSAN_ATOMIC_FUNC_INC(ptr, void *, void); 601 602void 603kcsan_atomic_load(const volatile void *p, void *v, int size) 604{ 605 kcsan_access((uintptr_t)p, size, false, true, __RET_ADDR); 606 switch (size) { 607 case 1: *(uint8_t *)v = *(const volatile uint8_t *)p; break; 608 case 2: *(uint16_t *)v = *(const volatile uint16_t *)p; break; 609 case 4: *(uint32_t *)v = *(const volatile uint32_t *)p; break; 610 case 8: *(uint64_t *)v = *(const volatile uint64_t *)p; break; 611 } 612} 613 614void 615kcsan_atomic_store(volatile void *p, const void *v, int size) 616{ 617 kcsan_access((uintptr_t)p, size, true, true, __RET_ADDR); 618#ifdef __HAVE_HASHLOCKED_ATOMICS 619 __do_atomic_store(p, v, size); 620#else 621 switch (size) { 622 case 1: *(volatile uint8_t *)p = *(const uint8_t *)v; break; 623 case 2: *(volatile uint16_t *)p = *(const uint16_t *)v; break; 624 case 4: *(volatile uint32_t *)p = *(const uint32_t *)v; break; 625 case 8: *(volatile uint64_t *)p = *(const uint64_t *)v; break; 626 } 627#endif 628} 629 630/* -------------------------------------------------------------------------- */ 631 632#include <sys/bus.h> 633 634#undef bus_space_read_multi_1 635#undef bus_space_read_multi_2 636#undef bus_space_read_multi_4 637#undef bus_space_read_multi_8 638#undef bus_space_read_multi_stream_1 639#undef bus_space_read_multi_stream_2 640#undef bus_space_read_multi_stream_4 641#undef bus_space_read_multi_stream_8 642#undef bus_space_read_region_1 643#undef bus_space_read_region_2 644#undef bus_space_read_region_4 645#undef bus_space_read_region_8 646#undef bus_space_read_region_stream_1 647#undef bus_space_read_region_stream_2 648#undef bus_space_read_region_stream_4 649#undef bus_space_read_region_stream_8 650#undef bus_space_write_multi_1 651#undef bus_space_write_multi_2 652#undef bus_space_write_multi_4 653#undef bus_space_write_multi_8 654#undef bus_space_write_multi_stream_1 655#undef bus_space_write_multi_stream_2 656#undef bus_space_write_multi_stream_4 657#undef bus_space_write_multi_stream_8 658#undef bus_space_write_region_1 659#undef bus_space_write_region_2 660#undef bus_space_write_region_4 661#undef bus_space_write_region_8 662#undef bus_space_write_region_stream_1 663#undef bus_space_write_region_stream_2 664#undef bus_space_write_region_stream_4 665#undef bus_space_write_region_stream_8 666 667#define CSAN_BUS_READ_FUNC(bytes, bits) \ 668 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 669 bus_size_t, uint##bits##_t *, bus_size_t); \ 670 void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 671 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 672 void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 673 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 674 bus_size_t count) \ 675 { \ 676 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 677 false, false, __RET_ADDR); \ 678 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 679 } \ 680 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 681 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 682 void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 683 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 684 void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 685 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 686 bus_size_t count) \ 687 { \ 688 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 689 false, false, __RET_ADDR); \ 690 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 691 } \ 692 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 693 bus_size_t, uint##bits##_t *, bus_size_t); \ 694 void kcsan_bus_space_read_region_##bytes(bus_space_tag_t, \ 695 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 696 void kcsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 697 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 698 bus_size_t count) \ 699 { \ 700 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 701 false, false, __RET_ADDR); \ 702 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 703 } \ 704 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 705 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 706 void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 707 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 708 void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 709 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 710 bus_size_t count) \ 711 { \ 712 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 713 false, false, __RET_ADDR); \ 714 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 715 } 716 717#define CSAN_BUS_WRITE_FUNC(bytes, bits) \ 718 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 719 bus_size_t, const uint##bits##_t *, bus_size_t); \ 720 void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 721 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 722 void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 723 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 724 bus_size_t count) \ 725 { \ 726 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 727 true, false, __RET_ADDR); \ 728 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 729 } \ 730 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 731 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 732 void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 733 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 734 void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 735 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 736 bus_size_t count) \ 737 { \ 738 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 739 true, false, __RET_ADDR); \ 740 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 741 } \ 742 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 743 bus_size_t, const uint##bits##_t *, bus_size_t); \ 744 void kcsan_bus_space_write_region_##bytes(bus_space_tag_t, \ 745 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 746 void kcsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 747 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 748 bus_size_t count) \ 749 { \ 750 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 751 true, false, __RET_ADDR); \ 752 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 753 } \ 754 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 755 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 756 void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 757 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 758 void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 759 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 760 bus_size_t count) \ 761 { \ 762 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 763 true, false, __RET_ADDR); \ 764 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 765 } 766 767CSAN_BUS_READ_FUNC(1, 8) 768CSAN_BUS_READ_FUNC(2, 16) 769CSAN_BUS_READ_FUNC(4, 32) 770CSAN_BUS_READ_FUNC(8, 64) 771 772CSAN_BUS_WRITE_FUNC(1, 8) 773CSAN_BUS_WRITE_FUNC(2, 16) 774CSAN_BUS_WRITE_FUNC(4, 32) 775CSAN_BUS_WRITE_FUNC(8, 64) 776