1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26#include <sys/types.h> 27#include <sys/systm.h> 28#include <sys/archsystm.h> 29#include <sys/machparam.h> 30#include <sys/machsystm.h> 31#include <sys/cpu.h> 32#include <sys/elf_SPARC.h> 33#include <vm/hat_sfmmu.h> 34#include <vm/page.h> 35#include <vm/vm_dep.h> 36#include <sys/cpuvar.h> 37#include <sys/async.h> 38#include <sys/cmn_err.h> 39#include <sys/debug.h> 40#include <sys/dditypes.h> 41#include <sys/sunddi.h> 42#include <sys/cpu_module.h> 43#include <sys/prom_debug.h> 44#include <sys/vmsystm.h> 45#include <sys/prom_plat.h> 46#include <sys/sysmacros.h> 47#include <sys/intreg.h> 48#include <sys/machtrap.h> 49#include <sys/ontrap.h> 50#include <sys/ivintr.h> 51#include <sys/atomic.h> 52#include <sys/panic.h> 53#include <sys/dtrace.h> 54#include <sys/simulate.h> 55#include <sys/fault.h> 56#include <sys/niagara2regs.h> 57#include <sys/hsvc.h> 58#include <sys/trapstat.h> 59#include <sys/mutex_impl.h> 60 61uint_t root_phys_addr_lo_mask = 0xffffffffU; 62#if defined(NIAGARA2_IMPL) 63char cpu_module_name[] = "SUNW,UltraSPARC-T2"; 64#elif defined(VFALLS_IMPL) 65char cpu_module_name[] = "SUNW,UltraSPARC-T2+"; 66#elif defined(KT_IMPL) 67char cpu_module_name[] = "SPARC-T3"; 68#endif 69 70/* 71 * Hypervisor services information for the NIAGARA2 and Victoria Falls 72 * CPU module 73 */ 74static boolean_t cpu_hsvc_available = B_TRUE; 75static uint64_t cpu_sup_minor; /* Supported minor number */ 76#if defined(NIAGARA2_IMPL) 77static hsvc_info_t cpu_hsvc = { 78 HSVC_REV_1, NULL, HSVC_GROUP_NIAGARA2_CPU, NIAGARA2_HSVC_MAJOR, 79 NIAGARA2_HSVC_MINOR, cpu_module_name 80}; 81#elif defined(VFALLS_IMPL) 82static hsvc_info_t cpu_hsvc = { 83 HSVC_REV_1, NULL, HSVC_GROUP_VFALLS_CPU, VFALLS_HSVC_MAJOR, 84 VFALLS_HSVC_MINOR, cpu_module_name 85}; 86#elif defined(KT_IMPL) 87static hsvc_info_t cpu_hsvc = { 88 HSVC_REV_1, NULL, HSVC_GROUP_KT_CPU, KT_HSVC_MAJOR, 89 KT_HSVC_MINOR, cpu_module_name 90}; 91#endif 92 93void 94cpu_setup(void) 95{ 96 extern int mmu_exported_pagesize_mask; 97 extern int cpc_has_overflow_intr; 98 extern size_t contig_mem_prealloc_base_size; 99 int status; 100 101 /* 102 * Negotiate the API version for Niagara2 specific hypervisor 103 * services. 104 */ 105 status = hsvc_register(&cpu_hsvc, &cpu_sup_minor); 106 if (status != 0) { 107 cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services " 108 "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d", 109 cpu_hsvc.hsvc_modname, cpu_hsvc.hsvc_group, 110 cpu_hsvc.hsvc_major, cpu_hsvc.hsvc_minor, status); 111 cpu_hsvc_available = B_FALSE; 112 } 113 114 /* 115 * The setup common to all CPU modules is done in cpu_setup_common 116 * routine. 117 */ 118 cpu_setup_common(NULL); 119 120 /* 121 * Initialize the cpu_hwcap_flags for N2 and VF if it is not already 122 * set in cpu_setup_common() by the hwcap MD info. Note that this MD 123 * info may not be available for N2/VF. 124 */ 125 if (cpu_hwcap_flags == 0) { 126#ifdef KT_IMPL 127 /* 128 * This should not happen since hwcap MD info is always 129 * available for KT platforms. 130 */ 131 ASSERT(cpu_hwcap_flags != 0); /* panic in DEBUG mode */ 132 cpu_hwcap_flags |= AV_SPARC_VIS3 | AV_SPARC_HPC | AV_SPARC_FMAF; 133#endif /* KT_IMPL */ 134 cpu_hwcap_flags |= AV_SPARC_VIS | AV_SPARC_VIS2 | 135 AV_SPARC_ASI_BLK_INIT | AV_SPARC_POPC; 136 } 137 138 cache |= (CACHE_PTAG | CACHE_IOCOHERENT); 139 140 if ((mmu_exported_pagesize_mask & 141 DEFAULT_SUN4V_MMU_PAGESIZE_MASK) != 142 DEFAULT_SUN4V_MMU_PAGESIZE_MASK) 143 cmn_err(CE_PANIC, "machine description" 144 " does not have required sun4v page sizes" 145 " 8K, 64K and 4M: MD mask is 0x%x", 146 mmu_exported_pagesize_mask); 147 148 /* 149 * Niagara2 supports a 48-bit subset of the full 64-bit virtual 150 * address space. Virtual addresses between 0x0000800000000000 151 * and 0xffff.7fff.ffff.ffff inclusive lie within a "VA Hole" 152 * and must never be mapped. In addition, software must not use 153 * pages within 4GB of the VA hole as instruction pages to 154 * avoid problems with prefetching into the VA hole. 155 */ 156 hole_start = (caddr_t)((1ull << (va_bits - 1)) - (1ull << 32)); 157 hole_end = (caddr_t)((0ull - (1ull << (va_bits - 1))) + (1ull << 32)); 158 159 /* 160 * Niagara2 has a performance counter overflow interrupt 161 */ 162 cpc_has_overflow_intr = 1; 163 164 /* 165 * Enable 4M pages for OOB. 166 */ 167 max_uheap_lpsize = MMU_PAGESIZE4M; 168 max_ustack_lpsize = MMU_PAGESIZE4M; 169 max_privmap_lpsize = MMU_PAGESIZE4M; 170 171#ifdef SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB 172 /* 173 * Use CPU Makefile specific compile time define (if exists) 174 * to add to the contig preallocation size. 175 */ 176 contig_mem_prealloc_base_size = MB(SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB); 177#endif 178} 179 180/* 181 * Set the magic constants of the implementation. 182 */ 183void 184cpu_fiximp(struct cpu_node *cpunode) 185{ 186 /* 187 * The Cache node is optional in MD. Therefore in case "Cache" 188 * node does not exists in MD, set the default L2 cache associativity, 189 * size, linesize. 190 */ 191 if (cpunode->ecache_size == 0) 192 cpunode->ecache_size = L2CACHE_SIZE; 193 if (cpunode->ecache_linesize == 0) 194 cpunode->ecache_linesize = L2CACHE_LINESIZE; 195 if (cpunode->ecache_associativity == 0) 196 cpunode->ecache_associativity = L2CACHE_ASSOCIATIVITY; 197} 198 199void 200cpu_map_exec_units(struct cpu *cp) 201{ 202 ASSERT(MUTEX_HELD(&cpu_lock)); 203 204 /* 205 * The cpu_ipipe and cpu_fpu fields are initialized based on 206 * the execution unit sharing information from the MD. They 207 * default to the CPU id in the absence of such information. 208 */ 209 cp->cpu_m.cpu_ipipe = cpunodes[cp->cpu_id].exec_unit_mapping; 210 if (cp->cpu_m.cpu_ipipe == NO_EU_MAPPING_FOUND) 211 cp->cpu_m.cpu_ipipe = (id_t)(cp->cpu_id); 212 213 cp->cpu_m.cpu_fpu = cpunodes[cp->cpu_id].fpu_mapping; 214 if (cp->cpu_m.cpu_fpu == NO_EU_MAPPING_FOUND) 215 cp->cpu_m.cpu_fpu = (id_t)(cp->cpu_id); 216 217 /* 218 * Niagara 2 defines the core to be at the FPU level 219 */ 220 cp->cpu_m.cpu_core = cp->cpu_m.cpu_fpu; 221 222 /* 223 * The cpu_chip field is initialized based on the information 224 * in the MD and assume that all cpus within a chip 225 * share the same L2 cache. If no such info is available, we 226 * set the cpu to belong to the defacto chip 0. 227 */ 228 cp->cpu_m.cpu_mpipe = cpunodes[cp->cpu_id].l2_cache_mapping; 229 if (cp->cpu_m.cpu_mpipe == NO_L2_CACHE_MAPPING_FOUND) 230 cp->cpu_m.cpu_mpipe = CPU_L2_CACHEID_INVALID; 231 232 cp->cpu_m.cpu_chip = cpunodes[cp->cpu_id].l2_cache_mapping; 233 if (cp->cpu_m.cpu_chip == NO_L2_CACHE_MAPPING_FOUND) 234 cp->cpu_m.cpu_chip = CPU_CHIPID_INVALID; 235} 236 237static int cpucnt; 238 239void 240cpu_init_private(struct cpu *cp) 241{ 242 extern void niagara_kstat_init(void); 243 244 ASSERT(MUTEX_HELD(&cpu_lock)); 245 246 cpu_map_exec_units(cp); 247 248 if ((cpucnt++ == 0) && (cpu_hsvc_available == B_TRUE)) 249 (void) niagara_kstat_init(); 250 251 mutex_delay = rdccr_delay; 252} 253 254/*ARGSUSED*/ 255void 256cpu_uninit_private(struct cpu *cp) 257{ 258 extern void niagara_kstat_fini(void); 259 260 ASSERT(MUTEX_HELD(&cpu_lock)); 261 if ((--cpucnt == 0) && (cpu_hsvc_available == B_TRUE)) 262 (void) niagara_kstat_fini(); 263} 264 265/* 266 * On Niagara2, any flush will cause all preceding stores to be 267 * synchronized wrt the i$, regardless of address or ASI. In fact, 268 * the address is ignored, so we always flush address 0. 269 */ 270/*ARGSUSED*/ 271void 272dtrace_flush_sec(uintptr_t addr) 273{ 274 doflush(0); 275} 276 277/* 278 * Trapstat support for Niagara2 processor 279 * The Niagara2 provides HWTW support for TSB lookup and with HWTW 280 * enabled no TSB hit information will be available. Therefore setting 281 * the time spent in TLB miss handler for TSB hits to 0. 282 */ 283int 284cpu_trapstat_conf(int cmd) 285{ 286 int status = 0; 287 288 switch (cmd) { 289 case CPU_TSTATCONF_INIT: 290 case CPU_TSTATCONF_FINI: 291 case CPU_TSTATCONF_ENABLE: 292 case CPU_TSTATCONF_DISABLE: 293 break; 294 default: 295 status = EINVAL; 296 break; 297 } 298 return (status); 299} 300 301void 302cpu_trapstat_data(void *buf, uint_t tstat_pgszs) 303{ 304 tstat_pgszdata_t *tstatp = (tstat_pgszdata_t *)buf; 305 int i; 306 307 for (i = 0; i < tstat_pgszs; i++, tstatp++) { 308 tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_count = 0; 309 tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_time = 0; 310 tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_count = 0; 311 tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_time = 0; 312 tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 313 tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 314 tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 315 tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 316 } 317} 318 319/* 320 * Page coloring support for hashed cache index mode 321 */ 322 323/* 324 * Node id bits from machine description (MD). Node id distinguishes 325 * local versus remote memory. Because of MPO, page allocation does 326 * not cross node boundaries. Therefore, remove the node id bits from 327 * the color, since they are fixed. Either bit 30, or 31:30 in 328 * Victoria Falls processors. 329 * The number of node id bits is always 0 in Niagara2. 330 */ 331typedef struct n2color { 332 uchar_t nnbits; /* number of node id bits */ 333 uchar_t nnmask; /* mask for node id bits */ 334 uchar_t lomask; /* mask for bits below node id */ 335 uchar_t lobits; /* number of bits below node id */ 336} n2color_t; 337 338n2color_t n2color[MMU_PAGE_SIZES]; 339static uchar_t nhbits[] = {7, 7, 6, 5, 5, 5}; 340 341/* 342 * Remove node id bits from color bits 32:28. 343 * This will reduce the number of colors. 344 * No change if number of node bits is zero. 345 */ 346static inline uint_t 347n2_hash2color(uint_t color, uchar_t szc) 348{ 349 n2color_t m = n2color[szc]; 350 351 if (m.nnbits > 0) { 352 color = ((color >> m.nnbits) & ~m.lomask) | (color & m.lomask); 353 ASSERT((color & ~(hw_page_array[szc].hp_colors - 1)) == 0); 354 } 355 356 return (color); 357} 358 359/* 360 * Restore node id bits into page color. 361 * This will increase the number of colors to match N2. 362 * No change if number of node bits is zero. 363 */ 364static inline uint_t 365n2_color2hash(uint_t color, uchar_t szc, uint_t node) 366{ 367 n2color_t m = n2color[szc]; 368 369 if (m.nnbits > 0) { 370 color = ((color & ~m.lomask) << m.nnbits) | (color & m.lomask); 371 color |= (node & m.nnmask) << m.lobits; 372 } 373 374 return (color); 375} 376 377/* NI2 L2$ index is pa[32:28]^pa[17:13].pa[19:18]^pa[12:11].pa[10:6] */ 378 379/* 380 * iterator NULL means pfn is VA, do not adjust ra_to_pa 381 * iterator (-1) means pfn is RA, need to convert to PA 382 * iterator non-null means pfn is RA, use ra_to_pa 383 */ 384uint_t 385page_pfn_2_color_cpu(pfn_t pfn, uchar_t szc, void *cookie) 386{ 387 mem_node_iterator_t *it = cookie; 388 uint_t color; 389 390 ASSERT(szc <= TTE256M); 391 392 if (it == ((mem_node_iterator_t *)(-1))) { 393 pfn = plat_rapfn_to_papfn(pfn); 394 } else if (it != NULL) { 395 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 396 pfn = pfn + it->mi_ra_to_pa; 397 } 398 pfn = PFN_BASE(pfn, szc); 399 color = ((pfn >> 15) ^ pfn) & 0x1f; 400 if (szc < TTE4M) { 401 /* 19:18 */ 402 color = (color << 2) | ((pfn >> 5) & 0x3); 403 if (szc > TTE64K) 404 color >>= 1; /* 19 */ 405 } 406 return (n2_hash2color(color, szc)); 407} 408 409static uint_t 410page_papfn_2_color_cpu(pfn_t papfn, uchar_t szc) 411{ 412 uint_t color; 413 414 ASSERT(szc <= TTE256M); 415 416 papfn = PFN_BASE(papfn, szc); 417 color = ((papfn >> 15) ^ papfn) & 0x1f; 418 if (szc < TTE4M) { 419 /* 19:18 */ 420 color = (color << 2) | ((papfn >> 5) & 0x3); 421 if (szc > TTE64K) 422 color >>= 1; /* 19 */ 423 } 424 return (color); 425} 426 427#if TTE256M != 5 428#error TTE256M is not 5 429#endif 430 431uint_t 432page_get_nsz_color_mask_cpu(uchar_t szc, uint_t mask) 433{ 434 static uint_t ni2_color_masks[5] = {0x63, 0x1e, 0x3e, 0x1f, 0x1f}; 435 ASSERT(szc < TTE256M); 436 mask = n2_color2hash(mask, szc, 0); 437 mask &= ni2_color_masks[szc]; 438 if (szc == TTE64K || szc == TTE512K) 439 mask >>= 1; 440 return (n2_hash2color(mask, szc + 1)); 441} 442 443uint_t 444page_get_nsz_color_cpu(uchar_t szc, uint_t color) 445{ 446 ASSERT(szc < TTE256M); 447 color = n2_color2hash(color, szc, 0); 448 if (szc == TTE64K || szc == TTE512K) 449 color >>= 1; 450 return (n2_hash2color(color, szc + 1)); 451} 452 453uint_t 454page_get_color_shift_cpu(uchar_t szc, uchar_t nszc) 455{ 456 uint_t s; 457 ASSERT(nszc >= szc); 458 ASSERT(nszc <= TTE256M); 459 460 s = nhbits[szc] - n2color[szc].nnbits; 461 s -= nhbits[nszc] - n2color[nszc].nnbits; 462 463 return (s); 464} 465 466uint_t 467page_convert_color_cpu(uint_t ncolor, uchar_t szc, uchar_t nszc) 468{ 469 uint_t color; 470 471 ASSERT(nszc > szc); 472 ASSERT(nszc <= TTE256M); 473 ncolor = n2_color2hash(ncolor, nszc, 0); 474 color = ncolor << (nhbits[szc] - nhbits[nszc]); 475 color = n2_hash2color(color, szc); 476 return (color); 477} 478 479#define PAPFN_2_MNODE(pfn) \ 480 (((pfn) & it->mi_mnode_pfn_mask) >> it->mi_mnode_pfn_shift) 481 482/*ARGSUSED*/ 483pfn_t 484page_next_pfn_for_color_cpu(pfn_t pfn, uchar_t szc, uint_t color, 485 uint_t ceq_mask, uint_t color_mask, void *cookie) 486{ 487 mem_node_iterator_t *it = cookie; 488 pfn_t pstep = PNUM_SIZE(szc); 489 pfn_t npfn, pfn_ceq_mask, pfn_color; 490 pfn_t tmpmask, mask = (pfn_t)-1; 491 uint_t pfnmn; 492 493 ASSERT((color & ~ceq_mask) == 0); 494 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 495 496 /* convert RA to PA for accurate color calculation */ 497 if (it->mi_init) { 498 /* first call after it, so cache these values */ 499 it->mi_hash_ceq_mask = 500 n2_color2hash(ceq_mask, szc, it->mi_mnode_mask); 501 it->mi_hash_color = 502 n2_color2hash(color, szc, it->mi_mnode); 503 it->mi_init = 0; 504 } else { 505 ASSERT(it->mi_hash_ceq_mask == 506 n2_color2hash(ceq_mask, szc, it->mi_mnode_mask)); 507 ASSERT(it->mi_hash_color == 508 n2_color2hash(color, szc, it->mi_mnode)); 509 } 510 ceq_mask = it->mi_hash_ceq_mask; 511 color = it->mi_hash_color; 512 pfn += it->mi_ra_to_pa; 513 514 /* restart here when we switch memblocks */ 515next_mem_block: 516 pfnmn = PAPFN_2_MNODE(pfn); 517 if ((((page_papfn_2_color_cpu(pfn, szc) ^ color) & ceq_mask) == 0) && 518 (pfnmn == it->mi_mnode)) { 519 520 /* we start from the page with correct color and mnode */ 521 if (szc >= TTE512K) { 522 if (szc >= TTE4M) { 523 /* page color is PA[32:28] */ 524 pfn_ceq_mask = ceq_mask << 15; 525 } else { 526 /* page color is PA[32:28].PA[19:19] */ 527 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 528 ((ceq_mask >> 1) << 15); 529 } 530 /* 531 * Preserve mnode bits in case they are not part of the 532 * color mask (eg., 8GB interleave, mnode bits 34:33). 533 */ 534 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 535 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 536 goto done; 537 } else { 538 /* 539 * We deal 64K or 8K page. Check if we could the 540 * satisfy the request without changing PA[32:28] 541 */ 542 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 543 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 544 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 545 546 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 547 goto done; 548 549 /* 550 * for next pfn we have to change bits PA[32:28] 551 * set PA[63:28] and PA[19:18] of the next pfn 552 */ 553 npfn = (pfn >> 15) << 15; 554 npfn |= (ceq_mask & color & 3) << 5; 555 pfn_ceq_mask = (szc == TTE8K) ? 0 : 556 (ceq_mask & 0x1c) << 13; 557 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 558 npfn = ADD_MASKED(npfn, (1 << 15), pfn_ceq_mask, mask); 559 560 /* 561 * set bits PA[17:13] to match the color 562 */ 563 npfn |= ((npfn >> 15) ^ (color >> 2)) & (ceq_mask >> 2); 564 goto done; 565 } 566 } 567 568 /* 569 * we start from the page with incorrect color - rare case 570 */ 571 if (szc >= TTE512K) { 572 if (szc >= TTE4M) { 573 /* page color is in bits PA[32:28] */ 574 npfn = ((pfn >> 20) << 20) | (color << 15); 575 pfn_ceq_mask = (ceq_mask << 15) | 0x7fff; 576 } else { 577 /* try get the right color by changing bit PA[19:19] */ 578 npfn = pfn + pstep; 579 pfnmn = PAPFN_2_MNODE(npfn); 580 if ((((page_papfn_2_color_cpu(npfn, szc) ^ color) & 581 ceq_mask) == 0) && (pfnmn == it->mi_mnode)) 582 goto done; 583 584 /* page color is PA[32:28].PA[19:19] */ 585 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 586 ((ceq_mask >> 1) << 15) | (0xff << 7); 587 pfn_color = ((color & 1) << 6) | ((color >> 1) << 15); 588 npfn = ((pfn >> 20) << 20) | pfn_color; 589 } 590 591 /* Fix mnode if necessary */ 592 if ((pfnmn = PAPFN_2_MNODE(npfn)) != it->mi_mnode) 593 npfn += ((it->mi_mnode - pfnmn) & it->mi_mnode_mask) << 594 it->mi_mnode_pfn_shift; 595 596 /* 597 * Preserve mnode bits in case they are not part of the color 598 * mask eg 8GB interleave, mnode bits 34:33). 599 */ 600 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 601 while (npfn <= pfn) { 602 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask); 603 } 604 goto done; 605 } 606 607 /* 608 * We deal 64K or 8K page of incorrect color. 609 * Try correcting color without changing PA[32:28] 610 */ 611 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 612 pfn_color = ((color & 3) << 5) | (color >> 2); 613 if (pfnmn == it->mi_mnode) { 614 npfn = (pfn & ~(pfn_t)0x7f); 615 npfn |= (((pfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 616 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 617 618 if (((page_papfn_2_color_cpu(npfn, szc) ^ color) & 619 ceq_mask) == 0) { 620 /* the color is fixed - find the next page */ 621 pfn_ceq_mask |= it->mi_mnode_pfn_mask; 622 while (npfn <= pfn) { 623 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, 624 mask); 625 } 626 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 627 goto done; 628 } 629 } 630 631 /* to fix the color need to touch PA[32:28] */ 632 npfn = (szc == TTE8K) ? ((pfn >> 15) << 15) : 633 (((pfn >> 18) << 18) | ((color & 0x1c) << 13)); 634 635 /* fix mnode if input pfn is in the wrong mnode. */ 636 if ((pfnmn = PAPFN_2_MNODE(npfn)) != it->mi_mnode) { 637 npfn += ((it->mi_mnode - pfnmn) & it->mi_mnode_mask) << 638 it->mi_mnode_pfn_shift; 639 } 640 641 tmpmask = (szc == TTE8K) ? 0 : (ceq_mask & 0x1c) << 13; 642 tmpmask |= it->mi_mnode_pfn_mask; 643 644 while (npfn <= pfn) { 645 npfn = ADD_MASKED(npfn, (1 << 15), tmpmask, mask); 646 } 647 648 /* set bits PA[19:13] to match the color */ 649 npfn |= (((npfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 650 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 651 652done: 653 ASSERT(((page_papfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0); 654 ASSERT(PAPFN_2_MNODE(npfn) == it->mi_mnode); 655 656 /* PA to RA */ 657 npfn -= it->mi_ra_to_pa; 658 659 /* check for possible memblock switch */ 660 if (npfn > it->mi_mblock_end) { 661 pfn = plat_mem_node_iterator_init(npfn, it->mi_mnode, szc, it, 662 0); 663 if (pfn == (pfn_t)-1) 664 return (pfn); 665 ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); 666 pfn += it->mi_ra_to_pa; 667 goto next_mem_block; 668 } 669 670 return (npfn); 671} 672 673/* 674 * init page coloring 675 * VF encodes node_id for an L-group in either bit 30 or 31:30, 676 * which effectively reduces the number of colors available per mnode. 677 */ 678void 679page_coloring_init_cpu() 680{ 681 int i; 682 uchar_t id; 683 uchar_t lo; 684 uchar_t hi; 685 n2color_t m; 686 mem_node_iterator_t it; 687 static uchar_t idmask[] = {0, 0x7, 0x1f, 0x1f, 0x1f, 0x1f}; 688 689 for (i = 0; i < max_mem_nodes; i++) { 690 memset(&it, 0, sizeof (it)); 691 if (plat_mem_node_iterator_init(0, i, 0, &it, 1) != (pfn_t)-1) 692 break; 693 } 694 ASSERT(i < max_mem_nodes); 695 for (i = 0; i < mmu_page_sizes; i++) { 696 (void) memset(&m, 0, sizeof (m)); 697 id = it.mi_mnode_pfn_mask >> 15; /* node id mask */ 698 id &= idmask[i]; 699 lo = lowbit(id); 700 if (lo > 0) { 701 hi = highbit(id); 702 m.nnbits = hi - lo + 1; 703 m.nnmask = (1 << m.nnbits) - 1; 704 lo += nhbits[i] - 5; 705 m.lomask = (1 << (lo - 1)) - 1; 706 m.lobits = lo - 1; 707 } 708 hw_page_array[i].hp_colors = 1 << (nhbits[i] - m.nnbits); 709 n2color[i] = m; 710 } 711} 712 713/* 714 * group colorequiv colors on N2 by low order bits of the color first 715 */ 716void 717page_set_colorequiv_arr_cpu(void) 718{ 719 static uint_t nequiv_shades_log2[MMU_PAGE_SIZES] = {2, 5, 0, 0, 0, 0}; 720 721 nequiv_shades_log2[1] -= n2color[1].nnbits; 722 if (colorequiv > 1) { 723 int i; 724 uint_t sv_a = lowbit(colorequiv) - 1; 725 726 if (sv_a > 15) 727 sv_a = 15; 728 729 for (i = 0; i < MMU_PAGE_SIZES; i++) { 730 uint_t colors; 731 uint_t a = sv_a; 732 733 if ((colors = hw_page_array[i].hp_colors) <= 1) 734 continue; 735 while ((colors >> a) == 0) 736 a--; 737 if (a > (colorequivszc[i] & 0xf) + 738 (colorequivszc[i] >> 4)) { 739 if (a <= nequiv_shades_log2[i]) { 740 colorequivszc[i] = (uchar_t)a; 741 } else { 742 colorequivszc[i] = 743 ((a - nequiv_shades_log2[i]) << 4) | 744 nequiv_shades_log2[i]; 745 } 746 } 747 } 748 } 749} 750