1/* $OpenBSD: cache_mips64r2.c,v 1.4 2022/08/29 02:08:13 jsg Exp $ */ 2 3/* 4 * Copyright (c) 2014 Miodrag Vallat. 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19/* 20 * Cache handling code for mips64r2 compatible processors 21 */ 22 23#include <sys/param.h> 24#include <sys/systm.h> 25 26#include <mips64/cache.h> 27#include <mips64/mips_cpu.h> 28#include <machine/cpu.h> 29 30#include <uvm/uvm_extern.h> 31 32#define IndexInvalidate_I 0x00 33#define IndexWBInvalidate_D 0x01 34#define IndexWBInvalidate_T 0x02 35#define IndexWBInvalidate_S 0x03 36 37#define HitInvalidate_D 0x11 38#define HitInvalidate_T 0x12 39#define HitInvalidate_S 0x13 40 41#define HitWBInvalidate_D 0x15 42#define HitWBInvalidate_T 0x16 43#define HitWBInvalidate_S 0x17 44 45#define cache(op,addr) \ 46 __asm__ volatile \ 47 ("cache %0, 0(%1)" :: "i"(op), "r"(addr) : "memory") 48 49static __inline__ void mips64r2_hitinv_primary(vaddr_t, vsize_t, vsize_t); 50static __inline__ void mips64r2_hitinv_secondary(vaddr_t, vsize_t, vsize_t); 51static __inline__ void mips64r2_hitinv_ternary(vaddr_t, vsize_t, vsize_t); 52static __inline__ void mips64r2_hitwbinv_primary(vaddr_t, vsize_t, vsize_t); 53static __inline__ void mips64r2_hitwbinv_secondary(vaddr_t, vsize_t, vsize_t); 54static __inline__ void mips64r2_hitwbinv_ternary(vaddr_t, vsize_t, vsize_t); 55 56void 57mips64r2_ConfigCache(struct cpu_info *ci) 58{ 59 uint32_t cfg, valias_mask; 60 uint32_t s, l, a; 61 62 cfg = cp0_get_config(); 63 if ((cfg & 0x80000000) == 0) 64 panic("no M bit in cfg0.0"); 65 66 cfg = cp0_get_config_1(); 67 68 a = 1 + ((cfg & CONFIG1_DA) >> CONFIG1_DA_SHIFT); 69 l = (cfg & CONFIG1_DL) >> CONFIG1_DL_SHIFT; 70 s = (cfg & CONFIG1_DS) >> CONFIG1_DS_SHIFT; 71 ci->ci_l1data.linesize = 2 << l; 72 ci->ci_l1data.setsize = (64 << s) * ci->ci_l1data.linesize; 73 ci->ci_l1data.sets = a; 74 ci->ci_l1data.size = ci->ci_l1data.sets * ci->ci_l1data.setsize; 75 76 a = 1 + ((cfg & CONFIG1_IA) >> CONFIG1_IA_SHIFT); 77 l = (cfg & CONFIG1_IL) >> CONFIG1_IL_SHIFT; 78 s = (cfg & CONFIG1_IS) >> CONFIG1_IS_SHIFT; 79 ci->ci_l1inst.linesize = 2 << l; 80 ci->ci_l1inst.setsize = (64 << s) * ci->ci_l1inst.linesize; 81 ci->ci_l1inst.sets = a; 82 ci->ci_l1inst.size = ci->ci_l1inst.sets * ci->ci_l1inst.setsize; 83 84 memset(&ci->ci_l2, 0, sizeof(struct cache_info)); 85 memset(&ci->ci_l3, 0, sizeof(struct cache_info)); 86 87 if ((cfg & 0x80000000) != 0) { 88 cfg = cp0_get_config_2(); 89 90 a = 1 + ((cfg >> 0) & 0x0f); 91 l = (cfg >> 4) & 0x0f; 92 s = (cfg >> 8) & 0x0f; 93 if (l != 0) { 94 ci->ci_l2.linesize = 2 << l; 95 ci->ci_l2.setsize = (64 << s) * ci->ci_l2.linesize; 96 ci->ci_l2.sets = a; 97 ci->ci_l2.size = ci->ci_l2.sets * ci->ci_l2.setsize; 98 } 99 100 a = 1 + ((cfg >> 16) & 0x0f); 101 l = (cfg >> 20) & 0x0f; 102 s = (cfg >> 24) & 0x0f; 103 if (l != 0) { 104 ci->ci_l3.linesize = 2 << l; 105 ci->ci_l3.setsize = (64 << s) * ci->ci_l3.linesize; 106 ci->ci_l3.sets = a; 107 ci->ci_l3.size = ci->ci_l3.sets * ci->ci_l3.setsize; 108 } 109 } 110 111 valias_mask = (max(ci->ci_l1inst.setsize, ci->ci_l1data.setsize) - 1) & 112 ~PAGE_MASK; 113 114 if (valias_mask != 0) { 115 valias_mask |= PAGE_MASK; 116#ifdef MULTIPROCESSOR 117 if (valias_mask > cache_valias_mask) { 118#endif 119 cache_valias_mask = valias_mask; 120 pmap_prefer_mask = valias_mask; 121#ifdef MULTIPROCESSOR 122 } 123#endif 124 } 125 126 ci->ci_SyncCache = mips64r2_SyncCache; 127 ci->ci_InvalidateICache = mips64r2_InvalidateICache; 128 ci->ci_InvalidateICachePage = mips64r2_InvalidateICachePage; 129 ci->ci_SyncICache = mips64r2_SyncICache; 130 ci->ci_SyncDCachePage = mips64r2_SyncDCachePage; 131 ci->ci_HitSyncDCachePage = mips64r2_HitSyncDCachePage; 132 ci->ci_HitSyncDCache = mips64r2_HitSyncDCache; 133 ci->ci_HitInvalidateDCache = mips64r2_HitInvalidateDCache; 134 ci->ci_IOSyncDCache = mips64r2_IOSyncDCache; 135} 136 137static __inline__ void 138mips64r2_hitwbinv_primary(vaddr_t va, vsize_t sz, vsize_t line) 139{ 140 vaddr_t eva; 141 142 eva = va + sz; 143 while (va != eva) { 144 cache(HitWBInvalidate_D, va); 145 va += line; 146 } 147} 148 149static __inline__ void 150mips64r2_hitwbinv_secondary(vaddr_t va, vsize_t sz, vsize_t line) 151{ 152 vaddr_t eva; 153 154 eva = va + sz; 155 while (va != eva) { 156 cache(HitWBInvalidate_S, va); 157 va += line; 158 } 159} 160 161static __inline__ void 162mips64r2_hitwbinv_ternary(vaddr_t va, vsize_t sz, vsize_t line) 163{ 164 vaddr_t eva; 165 166 eva = va + sz; 167 while (va != eva) { 168 cache(HitWBInvalidate_T, va); 169 va += line; 170 } 171} 172 173static __inline__ void 174mips64r2_hitinv_primary(vaddr_t va, vsize_t sz, vsize_t line) 175{ 176 vaddr_t eva; 177 178 eva = va + sz; 179 while (va != eva) { 180 cache(HitInvalidate_D, va); 181 va += line; 182 } 183} 184 185static __inline__ void 186mips64r2_hitinv_secondary(vaddr_t va, vsize_t sz, vsize_t line) 187{ 188 vaddr_t eva; 189 190 eva = va + sz; 191 while (va != eva) { 192 cache(HitInvalidate_S, va); 193 va += line; 194 } 195} 196 197static __inline__ void 198mips64r2_hitinv_ternary(vaddr_t va, vsize_t sz, vsize_t line) 199{ 200 vaddr_t eva; 201 202 eva = va + sz; 203 while (va != eva) { 204 cache(HitInvalidate_T, va); 205 va += line; 206 } 207} 208 209/* 210 * Writeback and invalidate all caches. 211 */ 212void 213mips64r2_SyncCache(struct cpu_info *ci) 214{ 215 vaddr_t sva, eva; 216 217 sva = PHYS_TO_XKPHYS(0, CCA_CACHED); 218 eva = sva + ci->ci_l1inst.linesize; 219 while (sva != eva) { 220 cache(IndexInvalidate_I, sva); 221 sva += ci->ci_l1inst.linesize; 222 } 223 224 sva = PHYS_TO_XKPHYS(0, CCA_CACHED); 225 eva = sva + ci->ci_l1data.linesize; 226 while (sva != eva) { 227 cache(IndexWBInvalidate_D, sva); 228 sva += ci->ci_l1data.linesize; 229 } 230 231 if (ci->ci_l2.size != 0) { 232 sva = PHYS_TO_XKPHYS(0, CCA_CACHED); 233 eva = sva + ci->ci_l2.size; 234 while (sva != eva) { 235 cache(IndexWBInvalidate_S, sva); 236 sva += ci->ci_l2.linesize; 237 } 238 } 239 240 if (ci->ci_l3.size != 0) { 241 sva = PHYS_TO_XKPHYS(0, CCA_CACHED); 242 eva = sva + ci->ci_l3.size; 243 while (sva != eva) { 244 cache(IndexWBInvalidate_T, sva); 245 sva += ci->ci_l3.linesize; 246 } 247 } 248} 249 250/* 251 * Invalidate I$ for the given range. 252 */ 253void 254mips64r2_InvalidateICache(struct cpu_info *ci, vaddr_t _va, size_t _sz) 255{ 256 vaddr_t va, sva, eva, iva; 257 vsize_t sz, offs; 258 uint set, nsets; 259 260 /* extend the range to integral cache lines */ 261 va = _va & ~(ci->ci_l1inst.linesize - 1); 262 sz = ((_va + _sz + ci->ci_l1inst.linesize - 1) & ~(ci->ci_l1inst.linesize - 1)) - va; 263 264 sva = PHYS_TO_XKPHYS(0, CCA_CACHED); 265 offs = ci->ci_l1inst.setsize; 266 nsets = ci->ci_l1inst.sets; 267 /* keep only the index bits */ 268 sva |= va & (offs - 1); 269 eva = sva + sz; 270 271 while (sva != eva) { 272 for (set = nsets, iva = sva; set != 0; set--, iva += offs) 273 cache(IndexInvalidate_I, iva); 274 sva += ci->ci_l1inst.linesize; 275 } 276} 277 278/* 279 * Register a given page for I$ invalidation. 280 */ 281void 282mips64r2_InvalidateICachePage(struct cpu_info *ci, vaddr_t va) 283{ 284 /* this code is too generic to allow for lazy I$ invalidates, yet */ 285 mips64r2_InvalidateICache(ci, va, PAGE_SIZE); 286} 287 288/* 289 * Perform postponed I$ invalidation. 290 */ 291void 292mips64r2_SyncICache(struct cpu_info *ci) 293{ 294} 295 296/* 297 * Writeback D$ for the given page. 298 */ 299void 300mips64r2_SyncDCachePage(struct cpu_info *ci, vaddr_t va, paddr_t pa) 301{ 302 vaddr_t sva, eva, iva; 303 vsize_t line, offs; 304 uint set, nsets; 305 306 line = ci->ci_l1data.linesize; 307 sva = PHYS_TO_XKPHYS(0, CCA_CACHED); 308 offs = ci->ci_l1data.setsize; 309 nsets = ci->ci_l1data.sets; 310 /* keep only the index bits */ 311 sva += va & (offs - 1); 312 eva = sva + PAGE_SIZE; 313 while (sva != eva) { 314 for (set = nsets, iva = sva; set != 0; set--, iva += offs) 315 cache(IndexWBInvalidate_D, iva); 316 sva += ci->ci_l1data.linesize; 317 } 318} 319 320/* 321 * Writeback D$ for the given page, which is expected to be currently 322 * mapped, allowing the use of `Hit' operations. This is less aggressive 323 * than using `Index' operations. 324 */ 325 326void 327mips64r2_HitSyncDCachePage(struct cpu_info *ci, vaddr_t va, paddr_t pa) 328{ 329 mips64r2_hitwbinv_primary(va, PAGE_SIZE, ci->ci_l1data.linesize); 330} 331 332/* 333 * Writeback D$ for the given range. Range is expected to be currently 334 * mapped, allowing the use of `Hit' operations. This is less aggressive 335 * than using `Index' operations. 336 */ 337 338void 339mips64r2_HitSyncDCache(struct cpu_info *ci, vaddr_t _va, size_t _sz) 340{ 341 vaddr_t va; 342 vsize_t sz; 343 344 /* extend the range to integral cache lines */ 345 va = _va & ~(ci->ci_l1data.linesize - 1); 346 sz = ((_va + _sz + ci->ci_l1data.linesize - 1) & ~(ci->ci_l1data.linesize - 1)) - va; 347 mips64r2_hitwbinv_primary(va, sz, ci->ci_l1data.linesize); 348} 349 350/* 351 * Invalidate D$ for the given range. Range is expected to be currently 352 * mapped, allowing the use of `Hit' operations. This is less aggressive 353 * than using `Index' operations. 354 */ 355 356void 357mips64r2_HitInvalidateDCache(struct cpu_info *ci, vaddr_t _va, size_t _sz) 358{ 359 vaddr_t va; 360 vsize_t sz; 361 362 /* extend the range to integral cache lines */ 363 va = _va & ~(ci->ci_l1data.linesize - 1); 364 sz = ((_va + _sz + ci->ci_l1data.linesize - 1) & ~(ci->ci_l1data.linesize - 1)) - va; 365 mips64r2_hitinv_primary(va, sz, ci->ci_l1data.linesize); 366} 367 368/* 369 * Backend for bus_dmamap_sync(). Enforce coherency of the given range 370 * by performing the necessary cache writeback and/or invalidate 371 * operations. 372 */ 373void 374mips64r2_IOSyncDCache(struct cpu_info *ci, vaddr_t _va, size_t _sz, int how) 375{ 376 vaddr_t va; 377 vsize_t sz; 378 int partial_start, partial_end; 379 380 /* 381 * L1 382 */ 383 384 /* extend the range to integral cache lines */ 385 va = _va & ~(ci->ci_l1data.linesize - 1); 386 sz = ((_va + _sz + ci->ci_l1data.linesize - 1) & ~(ci->ci_l1data.linesize - 1)) - va; 387 388 switch (how) { 389 case CACHE_SYNC_R: 390 /* writeback partial cachelines */ 391 if (((_va | _sz) & (ci->ci_l1data.linesize - 1)) != 0) { 392 partial_start = va != _va; 393 partial_end = va + sz != _va + _sz; 394 } else { 395 partial_start = partial_end = 0; 396 } 397 if (partial_start) { 398 cache(HitWBInvalidate_D, va); 399 va += ci->ci_l1data.linesize; 400 sz -= ci->ci_l1data.linesize; 401 } 402 if (sz != 0 && partial_end) { 403 sz -= ci->ci_l1data.linesize; 404 cache(HitWBInvalidate_D, va + sz); 405 } 406 if (sz != 0) 407 mips64r2_hitinv_primary(va, sz, ci->ci_l1data.linesize); 408 break; 409 case CACHE_SYNC_X: 410 case CACHE_SYNC_W: 411 mips64r2_hitwbinv_primary(va, sz, ci->ci_l1data.linesize); 412 break; 413 } 414 415 /* 416 * L2 417 */ 418 419 if (ci->ci_l2.size != 0) { 420 /* extend the range to integral cache lines */ 421 va = _va & ~(ci->ci_l2.linesize - 1); 422 sz = ((_va + _sz + ci->ci_l2.linesize - 1) & ~(ci->ci_l2.linesize - 1)) - va; 423 424 switch (how) { 425 case CACHE_SYNC_R: 426 /* writeback partial cachelines */ 427 if (((_va | _sz) & (ci->ci_l2.linesize - 1)) != 0) { 428 partial_start = va != _va; 429 partial_end = va + sz != _va + _sz; 430 } else { 431 partial_start = partial_end = 0; 432 } 433 if (partial_start) { 434 cache(HitWBInvalidate_S, va); 435 va += ci->ci_l2.linesize; 436 sz -= ci->ci_l2.linesize; 437 } 438 if (sz != 0 && partial_end) { 439 sz -= ci->ci_l2.linesize; 440 cache(HitWBInvalidate_S, va + sz); 441 } 442 if (sz != 0) 443 mips64r2_hitinv_secondary(va, sz, ci->ci_l2.linesize); 444 break; 445 case CACHE_SYNC_X: 446 case CACHE_SYNC_W: 447 mips64r2_hitwbinv_secondary(va, sz, ci->ci_l2.linesize); 448 break; 449 } 450 } 451 452 /* 453 * L3 454 */ 455 456 if (ci->ci_l3.size != 0) { 457 /* extend the range to integral cache lines */ 458 va = _va & ~(ci->ci_l3.linesize - 1); 459 sz = ((_va + _sz + ci->ci_l3.linesize - 1) & ~(ci->ci_l3.linesize - 1)) - va; 460 461 switch (how) { 462 case CACHE_SYNC_R: 463 /* writeback partial cachelines */ 464 if (((_va | _sz) & (ci->ci_l3.linesize - 1)) != 0) { 465 partial_start = va != _va; 466 partial_end = va + sz != _va + _sz; 467 } else { 468 partial_start = partial_end = 0; 469 } 470 if (partial_start) { 471 cache(HitWBInvalidate_S, va); 472 va += ci->ci_l3.linesize; 473 sz -= ci->ci_l3.linesize; 474 } 475 if (sz != 0 && partial_end) { 476 sz -= ci->ci_l3.linesize; 477 cache(HitWBInvalidate_S, va + sz); 478 } 479 if (sz != 0) 480 mips64r2_hitinv_ternary(va, sz, ci->ci_l3.linesize); 481 break; 482 case CACHE_SYNC_X: 483 case CACHE_SYNC_W: 484 mips64r2_hitwbinv_ternary(va, sz, ci->ci_l3.linesize); 485 break; 486 } 487 } 488} 489