niagara2.c revision 4318:f8fa23eb4630
1251881Speter/* 2251881Speter * CDDL HEADER START 3251881Speter * 4251881Speter * The contents of this file are subject to the terms of the 5251881Speter * Common Development and Distribution License (the "License"). 6251881Speter * You may not use this file except in compliance with the License. 7251881Speter * 8251881Speter * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9251881Speter * or http://www.opensolaris.org/os/licensing. 10251881Speter * See the License for the specific language governing permissions 11251881Speter * and limitations under the License. 12251881Speter * 13251881Speter * When distributing Covered Code, include this CDDL HEADER in each 14251881Speter * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15251881Speter * If applicable, add the following below this CDDL HEADER, with the 16251881Speter * fields enclosed by brackets "[]" replaced with your own identifying 17251881Speter * information: Portions Copyright [yyyy] [name of copyright owner] 18251881Speter * 19251881Speter * CDDL HEADER END 20251881Speter */ 21251881Speter 22251881Speter/* 23251881Speter * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24251881Speter * Use is subject to license terms. 25251881Speter */ 26251881Speter 27251881Speter#pragma ident "%Z%%M% %I% %E% SMI" 28251881Speter 29251881Speter#include <sys/types.h> 30251881Speter#include <sys/systm.h> 31251881Speter#include <sys/archsystm.h> 32251881Speter#include <sys/machparam.h> 33251881Speter#include <sys/machsystm.h> 34251881Speter#include <sys/cpu.h> 35251881Speter#include <sys/elf_SPARC.h> 36251881Speter#include <vm/hat_sfmmu.h> 37251881Speter#include <vm/page.h> 38251881Speter#include <vm/vm_dep.h> 39251881Speter#include <sys/cpuvar.h> 40251881Speter#include <sys/async.h> 41251881Speter#include <sys/cmn_err.h> 42251881Speter#include <sys/debug.h> 43251881Speter#include <sys/dditypes.h> 44251881Speter#include <sys/sunddi.h> 45251881Speter#include <sys/cpu_module.h> 46251881Speter#include <sys/prom_debug.h> 47251881Speter#include <sys/vmsystm.h> 48251881Speter#include <sys/prom_plat.h> 49251881Speter#include <sys/sysmacros.h> 50251881Speter#include <sys/intreg.h> 51251881Speter#include <sys/machtrap.h> 52251881Speter#include <sys/ontrap.h> 53251881Speter#include <sys/ivintr.h> 54251881Speter#include <sys/atomic.h> 55251881Speter#include <sys/panic.h> 56251881Speter#include <sys/dtrace.h> 57251881Speter#include <sys/simulate.h> 58251881Speter#include <sys/fault.h> 59251881Speter#include <sys/niagara2regs.h> 60251881Speter#include <sys/hsvc.h> 61251881Speter#include <sys/trapstat.h> 62251881Speter 63251881Speteruint_t root_phys_addr_lo_mask = 0xffffffffU; 64251881Speterchar cpu_module_name[] = "SUNW,UltraSPARC-T2"; 65251881Speter 66251881Speter/* 67251881Speter * Hypervisor services information for the NIAGARA2 CPU module 68251881Speter */ 69251881Speterstatic boolean_t niagara2_hsvc_available = B_TRUE; 70251881Speterstatic uint64_t niagara2_sup_minor; /* Supported minor number */ 71251881Speterstatic hsvc_info_t niagara2_hsvc = { 72251881Speter HSVC_REV_1, NULL, HSVC_GROUP_NIAGARA2_CPU, NIAGARA2_HSVC_MAJOR, 73251881Speter NIAGARA2_HSVC_MINOR, cpu_module_name 74251881Speter}; 75251881Speter 76251881Spetervoid 77251881Spetercpu_setup(void) 78251881Speter{ 79251881Speter extern int mmu_exported_pagesize_mask; 80251881Speter extern int cpc_has_overflow_intr; 81251881Speter extern size_t contig_mem_prealloc_base; 82251881Speter int status; 83251881Speter 84251881Speter /* 85251881Speter * Negotiate the API version for Niagara2 specific hypervisor 86251881Speter * services. 87251881Speter */ 88251881Speter status = hsvc_register(&niagara2_hsvc, &niagara2_sup_minor); 89251881Speter if (status != 0) { 90251881Speter cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services " 91251881Speter "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d", 92251881Speter niagara2_hsvc.hsvc_modname, niagara2_hsvc.hsvc_group, 93251881Speter niagara2_hsvc.hsvc_major, niagara2_hsvc.hsvc_minor, status); 94251881Speter niagara2_hsvc_available = B_FALSE; 95251881Speter } 96251881Speter 97251881Speter /* 98251881Speter * The setup common to all CPU modules is done in cpu_setup_common 99251881Speter * routine. 100251881Speter */ 101251881Speter cpu_setup_common(NULL); 102251881Speter 103251881Speter cache |= (CACHE_PTAG | CACHE_IOCOHERENT); 104251881Speter 105251881Speter if ((mmu_exported_pagesize_mask & 106251881Speter DEFAULT_SUN4V_MMU_PAGESIZE_MASK) != 107251881Speter DEFAULT_SUN4V_MMU_PAGESIZE_MASK) 108251881Speter cmn_err(CE_PANIC, "machine description" 109251881Speter " does not have required sun4v page sizes" 110251881Speter " 8K, 64K and 4M: MD mask is 0x%x", 111251881Speter mmu_exported_pagesize_mask); 112251881Speter 113251881Speter cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2 | AV_SPARC_ASI_BLK_INIT; 114251881Speter 115251881Speter /* 116251881Speter * Niagara2 supports a 48-bit subset of the full 64-bit virtual 117251881Speter * address space. Virtual addresses between 0x0000800000000000 118251881Speter * and 0xffff.7fff.ffff.ffff inclusive lie within a "VA Hole" 119251881Speter * and must never be mapped. In addition, software must not use 120251881Speter * pages within 4GB of the VA hole as instruction pages to 121251881Speter * avoid problems with prefetching into the VA hole. 122251881Speter */ 123251881Speter hole_start = (caddr_t)((1ull << (va_bits - 1)) - (1ull << 32)); 124251881Speter hole_end = (caddr_t)((0ull - (1ull << (va_bits - 1))) + (1ull << 32)); 125251881Speter 126251881Speter /* 127251881Speter * Niagara2 has a performance counter overflow interrupt 128251881Speter */ 129251881Speter cpc_has_overflow_intr = 1; 130251881Speter 131251881Speter /* 132251881Speter * Enable 4M pages for OOB. 133 */ 134 max_uheap_lpsize = MMU_PAGESIZE4M; 135 max_ustack_lpsize = MMU_PAGESIZE4M; 136 max_privmap_lpsize = MMU_PAGESIZE4M; 137 138 contig_mem_prealloc_base = NIAGARA2_PREALLOC_BASE; 139} 140 141/* 142 * Set the magic constants of the implementation. 143 */ 144void 145cpu_fiximp(struct cpu_node *cpunode) 146{ 147 /* 148 * The Cache node is optional in MD. Therefore in case "Cache" 149 * node does not exists in MD, set the default L2 cache associativity, 150 * size, linesize. 151 */ 152 if (cpunode->ecache_size == 0) 153 cpunode->ecache_size = L2CACHE_SIZE; 154 if (cpunode->ecache_linesize == 0) 155 cpunode->ecache_linesize = L2CACHE_LINESIZE; 156 if (cpunode->ecache_associativity == 0) 157 cpunode->ecache_associativity = L2CACHE_ASSOCIATIVITY; 158} 159 160void 161cpu_map_exec_units(struct cpu *cp) 162{ 163 ASSERT(MUTEX_HELD(&cpu_lock)); 164 165 /* 166 * The cpu_ipipe and cpu_fpu fields are initialized based on 167 * the execution unit sharing information from the MD. They 168 * default to the CPU id in the absence of such information. 169 */ 170 cp->cpu_m.cpu_ipipe = cpunodes[cp->cpu_id].exec_unit_mapping; 171 if (cp->cpu_m.cpu_ipipe == NO_EU_MAPPING_FOUND) 172 cp->cpu_m.cpu_ipipe = (id_t)(cp->cpu_id); 173 174 cp->cpu_m.cpu_fpu = cpunodes[cp->cpu_id].fpu_mapping; 175 if (cp->cpu_m.cpu_fpu == NO_EU_MAPPING_FOUND) 176 cp->cpu_m.cpu_fpu = (id_t)(cp->cpu_id); 177 178 /* 179 * Niagara 2 defines the core to be at the FPU level 180 */ 181 cp->cpu_m.cpu_core = cp->cpu_m.cpu_fpu; 182} 183 184static int niagara2_cpucnt; 185 186void 187cpu_init_private(struct cpu *cp) 188{ 189 extern void niagara_kstat_init(void); 190 191 ASSERT(MUTEX_HELD(&cpu_lock)); 192 193 cpu_map_exec_units(cp); 194 195 if ((niagara2_cpucnt++ == 0) && (niagara2_hsvc_available == B_TRUE)) 196 niagara_kstat_init(); 197} 198 199/*ARGSUSED*/ 200void 201cpu_uninit_private(struct cpu *cp) 202{ 203 extern void niagara_kstat_fini(void); 204 205 ASSERT(MUTEX_HELD(&cpu_lock)); 206 207 if ((--niagara2_cpucnt == 0) && (niagara2_hsvc_available == B_TRUE)) 208 niagara_kstat_fini(); 209} 210 211/* 212 * On Niagara2, any flush will cause all preceding stores to be 213 * synchronized wrt the i$, regardless of address or ASI. In fact, 214 * the address is ignored, so we always flush address 0. 215 */ 216/*ARGSUSED*/ 217void 218dtrace_flush_sec(uintptr_t addr) 219{ 220 doflush(0); 221} 222 223/* 224 * Trapstat support for Niagara2 processor 225 * The Niagara2 provides HWTW support for TSB lookup and with HWTW 226 * enabled no TSB hit information will be available. Therefore setting 227 * the time spent in TLB miss handler for TSB hits to 0. 228 */ 229int 230cpu_trapstat_conf(int cmd) 231{ 232 int status = 0; 233 234 switch (cmd) { 235 case CPU_TSTATCONF_INIT: 236 case CPU_TSTATCONF_FINI: 237 case CPU_TSTATCONF_ENABLE: 238 case CPU_TSTATCONF_DISABLE: 239 break; 240 default: 241 status = EINVAL; 242 break; 243 } 244 return (status); 245} 246 247void 248cpu_trapstat_data(void *buf, uint_t tstat_pgszs) 249{ 250 tstat_pgszdata_t *tstatp = (tstat_pgszdata_t *)buf; 251 int i; 252 253 for (i = 0; i < tstat_pgszs; i++, tstatp++) { 254 tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_count = 0; 255 tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_time = 0; 256 tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_count = 0; 257 tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_time = 0; 258 tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 259 tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 260 tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_count = 0; 261 tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_time = 0; 262 } 263} 264 265/* NI2 L2$ index is pa[32:28]^pa[17:13].pa[19:18]^pa[12:11].pa[10:6] */ 266uint_t 267page_pfn_2_color_cpu(pfn_t pfn, uchar_t szc) 268{ 269 uint_t color; 270 271 ASSERT(szc <= TTE256M); 272 273 pfn = PFN_BASE(pfn, szc); 274 color = ((pfn >> 15) ^ pfn) & 0x1f; 275 if (szc >= TTE4M) 276 return (color); 277 278 color = (color << 2) | ((pfn >> 5) & 0x3); 279 280 return (szc <= TTE64K ? color : (color >> 1)); 281} 282 283#if TTE256M != 5 284#error TTE256M is not 5 285#endif 286 287uint_t 288page_get_nsz_color_mask_cpu(uchar_t szc, uint_t mask) 289{ 290 static uint_t ni2_color_masks[5] = {0x63, 0x1e, 0x3e, 0x1f, 0x1f}; 291 ASSERT(szc < TTE256M); 292 293 mask &= ni2_color_masks[szc]; 294 return ((szc == TTE64K || szc == TTE512K) ? (mask >> 1) : mask); 295} 296 297uint_t 298page_get_nsz_color_cpu(uchar_t szc, uint_t color) 299{ 300 ASSERT(szc < TTE256M); 301 return ((szc == TTE64K || szc == TTE512K) ? (color >> 1) : color); 302} 303 304uint_t 305page_get_color_shift_cpu(uchar_t szc, uchar_t nszc) 306{ 307 ASSERT(nszc >= szc); 308 ASSERT(nszc <= TTE256M); 309 310 if (szc == nszc) 311 return (0); 312 if (szc <= TTE64K) 313 return ((nszc >= TTE4M) ? 2 : ((nszc >= TTE512K) ? 1 : 0)); 314 if (szc == TTE512K) 315 return (1); 316 317 return (0); 318} 319 320/*ARGSUSED*/ 321pfn_t 322page_next_pfn_for_color_cpu(pfn_t pfn, uchar_t szc, uint_t color, 323 uint_t ceq_mask, uint_t color_mask) 324{ 325 pfn_t pstep = PNUM_SIZE(szc); 326 pfn_t npfn, pfn_ceq_mask, pfn_color; 327 pfn_t tmpmask, mask = (pfn_t)-1; 328 329 ASSERT((color & ~ceq_mask) == 0); 330 331 if (((page_pfn_2_color_cpu(pfn, szc) ^ color) & ceq_mask) == 0) { 332 333 /* we start from the page with correct color */ 334 if (szc >= TTE512K) { 335 if (szc >= TTE4M) { 336 /* page color is PA[32:28] */ 337 pfn_ceq_mask = ceq_mask << 15; 338 } else { 339 /* page color is PA[32:28].PA[19:19] */ 340 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 341 ((ceq_mask >> 1) << 15); 342 } 343 pfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 344 return (pfn); 345 } else { 346 /* 347 * We deal 64K or 8K page. Check if we could the 348 * satisfy the request without changing PA[32:28] 349 */ 350 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 351 npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); 352 353 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 354 return (npfn); 355 356 /* 357 * for next pfn we have to change bits PA[32:28] 358 * set PA[63:28] and PA[19:18] of the next pfn 359 */ 360 npfn = (pfn >> 15) << 15; 361 npfn |= (ceq_mask & color & 3) << 5; 362 pfn_ceq_mask = (szc == TTE8K) ? 0 : 363 (ceq_mask & 0x1c) << 13; 364 npfn = ADD_MASKED(npfn, (1 << 15), pfn_ceq_mask, mask); 365 366 /* 367 * set bits PA[17:13] to match the color 368 */ 369 ceq_mask >>= 2; 370 color = (color >> 2) & ceq_mask; 371 npfn |= ((npfn >> 15) ^ color) & ceq_mask; 372 return (npfn); 373 } 374 } 375 376 /* 377 * we start from the page with incorrect color - rare case 378 */ 379 if (szc >= TTE512K) { 380 if (szc >= TTE4M) { 381 /* page color is in bits PA[32:28] */ 382 npfn = ((pfn >> 20) << 20) | (color << 15); 383 pfn_ceq_mask = (ceq_mask << 15) | 0x7fff; 384 } else { 385 /* try get the right color by changing bit PA[19:19] */ 386 npfn = pfn + pstep; 387 if (((page_pfn_2_color_cpu(npfn, szc) ^ color) & 388 ceq_mask) == 0) 389 return (npfn); 390 391 /* page color is PA[32:28].PA[19:19] */ 392 pfn_ceq_mask = ((ceq_mask & 1) << 6) | 393 ((ceq_mask >> 1) << 15) | (0xff << 7); 394 pfn_color = ((color & 1) << 6) | ((color >> 1) << 15); 395 npfn = ((pfn >> 20) << 20) | pfn_color; 396 } 397 398 while (npfn <= pfn) { 399 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask); 400 } 401 return (npfn); 402 } 403 404 /* 405 * We deal 64K or 8K page of incorrect color. 406 * Try correcting color without changing PA[32:28] 407 */ 408 409 pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); 410 pfn_color = ((color & 3) << 5) | (color >> 2); 411 npfn = (pfn & ~(pfn_t)0x7f); 412 npfn |= (((pfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 413 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 414 415 if (((page_pfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0) { 416 417 /* the color is fixed - find the next page */ 418 while (npfn <= pfn) { 419 npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask); 420 } 421 if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) 422 return (npfn); 423 } 424 425 /* to fix the color need to touch PA[32:28] */ 426 npfn = (szc == TTE8K) ? ((pfn >> 15) << 15) : 427 (((pfn >> 18) << 18) | ((color & 0x1c) << 13)); 428 tmpmask = (szc == TTE8K) ? 0 : (ceq_mask & 0x1c) << 13; 429 430 while (npfn <= pfn) { 431 npfn = ADD_MASKED(npfn, (1 << 15), tmpmask, mask); 432 } 433 434 /* set bits PA[19:13] to match the color */ 435 npfn |= (((npfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; 436 npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; 437 438 ASSERT(((page_pfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0); 439 440 return (npfn); 441} 442 443/* 444 * init page coloring 445 */ 446void 447page_coloring_init_cpu() 448{ 449 int i; 450 451 hw_page_array[0].hp_colors = 1 << 7; 452 hw_page_array[1].hp_colors = 1 << 7; 453 hw_page_array[2].hp_colors = 1 << 6; 454 455 for (i = 3; i < mmu_page_sizes; i++) { 456 hw_page_array[i].hp_colors = 1 << 5; 457 } 458} 459 460/* 461 * group colorequiv colors on N2 by low order bits of the color first 462 */ 463void 464page_set_colorequiv_arr_cpu(void) 465{ 466 static uint_t nequiv_shades_log2[MMU_PAGE_SIZES] = {2, 5, 0, 0, 0, 0}; 467 468 if (colorequiv > 1) { 469 int i; 470 uint_t sv_a = lowbit(colorequiv) - 1; 471 472 if (sv_a > 15) 473 sv_a = 15; 474 475 for (i = 0; i < MMU_PAGE_SIZES; i++) { 476 uint_t colors; 477 uint_t a = sv_a; 478 479 if ((colors = hw_page_array[i].hp_colors) <= 1) 480 continue; 481 while ((colors >> a) == 0) 482 a--; 483 if (a > (colorequivszc[i] & 0xf) + 484 (colorequivszc[i] >> 4)) { 485 if (a <= nequiv_shades_log2[i]) { 486 colorequivszc[i] = a; 487 } else { 488 colorequivszc[i] = 489 ((a - nequiv_shades_log2[i]) << 4) | 490 nequiv_shades_log2[i]; 491 } 492 } 493 } 494 } 495} 496