pl310.c revision 269598
1/*- 2 * Copyright (c) 2012 Olivier Houchard <cognet@FreeBSD.org> 3 * Copyright (c) 2011 4 * Ben Gray <ben.r.gray@gmail.com>. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the company nor the name of the author may be used to 16 * endorse or promote products derived from this software without specific 17 * prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 24 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 25 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 26 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 27 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 28 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD: head/sys/arm/arm/pl310.c 269598 2014-08-05 17:39:58Z ian $"); 33#include <sys/param.h> 34#include <sys/systm.h> 35#include <sys/bus.h> 36#include <sys/kernel.h> 37#include <sys/rman.h> 38#include <sys/module.h> 39#include <sys/lock.h> 40#include <sys/mutex.h> 41#include <machine/intr.h> 42 43#include <machine/bus.h> 44#include <machine/pl310.h> 45 46#include <dev/fdt/fdt_common.h> 47#include <dev/ofw/openfirm.h> 48#include <dev/ofw/ofw_bus.h> 49#include <dev/ofw/ofw_bus_subr.h> 50 51/* 52 * Define this if you need to disable PL310 for debugging purpose 53 * Spec: 54 * http://infocenter.arm.com/help/topic/com.arm.doc.ddi0246e/DDI0246E_l2c310_r3p1_trm.pdf 55 */ 56 57/* 58 * Hardcode errata for now 59 * http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0246b/pr01s02s02.html 60 */ 61#define PL310_ERRATA_588369 62#define PL310_ERRATA_753970 63#define PL310_ERRATA_727915 64 65#define PL310_LOCK(sc) do { \ 66 mtx_lock_spin(&(sc)->sc_mtx); \ 67} while(0); 68 69#define PL310_UNLOCK(sc) do { \ 70 mtx_unlock_spin(&(sc)->sc_mtx); \ 71} while(0); 72 73static int pl310_enabled = 1; 74TUNABLE_INT("hw.pl310.enabled", &pl310_enabled); 75 76static uint32_t g_l2cache_way_mask; 77 78static const uint32_t g_l2cache_line_size = 32; 79static const uint32_t g_l2cache_align_mask = (32 - 1); 80 81static uint32_t g_l2cache_size; 82static uint32_t g_way_size; 83static uint32_t g_ways_assoc; 84 85static struct pl310_softc *pl310_softc; 86 87void 88pl310_print_config(struct pl310_softc *sc) 89{ 90 uint32_t aux, prefetch; 91 const char *dis = "disabled"; 92 const char *ena = "enabled"; 93 94 aux = pl310_read4(sc, PL310_AUX_CTRL); 95 prefetch = pl310_read4(sc, PL310_PREFETCH_CTRL); 96 97 device_printf(sc->sc_dev, "Early BRESP response: %s\n", 98 (aux & AUX_CTRL_EARLY_BRESP) ? ena : dis); 99 device_printf(sc->sc_dev, "Instruction prefetch: %s\n", 100 (aux & AUX_CTRL_INSTR_PREFETCH) ? ena : dis); 101 device_printf(sc->sc_dev, "Data prefetch: %s\n", 102 (aux & AUX_CTRL_DATA_PREFETCH) ? ena : dis); 103 device_printf(sc->sc_dev, "Non-secure interrupt control: %s\n", 104 (aux & AUX_CTRL_NS_INT_CTRL) ? ena : dis); 105 device_printf(sc->sc_dev, "Non-secure lockdown: %s\n", 106 (aux & AUX_CTRL_NS_LOCKDOWN) ? ena : dis); 107 device_printf(sc->sc_dev, "Share override: %s\n", 108 (aux & AUX_CTRL_SHARE_OVERRIDE) ? ena : dis); 109 110 device_printf(sc->sc_dev, "Double linefill: %s\n", 111 (prefetch & PREFETCH_CTRL_DL) ? ena : dis); 112 device_printf(sc->sc_dev, "Instruction prefetch: %s\n", 113 (prefetch & PREFETCH_CTRL_INSTR_PREFETCH) ? ena : dis); 114 device_printf(sc->sc_dev, "Data prefetch: %s\n", 115 (prefetch & PREFETCH_CTRL_DATA_PREFETCH) ? ena : dis); 116 device_printf(sc->sc_dev, "Double linefill on WRAP request: %s\n", 117 (prefetch & PREFETCH_CTRL_DL_ON_WRAP) ? ena : dis); 118 device_printf(sc->sc_dev, "Prefetch drop: %s\n", 119 (prefetch & PREFETCH_CTRL_PREFETCH_DROP) ? ena : dis); 120 device_printf(sc->sc_dev, "Incr double Linefill: %s\n", 121 (prefetch & PREFETCH_CTRL_INCR_DL) ? ena : dis); 122 device_printf(sc->sc_dev, "Not same ID on exclusive sequence: %s\n", 123 (prefetch & PREFETCH_CTRL_NOTSAMEID) ? ena : dis); 124 device_printf(sc->sc_dev, "Prefetch offset: %d\n", 125 (prefetch & PREFETCH_CTRL_OFFSET_MASK)); 126} 127 128void 129pl310_set_ram_latency(struct pl310_softc *sc, uint32_t which_reg, 130 uint32_t read, uint32_t write, uint32_t setup) 131{ 132 uint32_t v; 133 134 KASSERT(which_reg == PL310_TAG_RAM_CTRL || 135 which_reg == PL310_DATA_RAM_CTRL, 136 ("bad pl310 ram latency register address")); 137 138 v = pl310_read4(sc, which_reg); 139 if (setup != 0) { 140 KASSERT(setup <= 8, ("bad pl310 setup latency: %d", setup)); 141 v &= ~RAM_CTRL_SETUP_MASK; 142 v |= (setup - 1) << RAM_CTRL_SETUP_SHIFT; 143 } 144 if (read != 0) { 145 KASSERT(read <= 8, ("bad pl310 read latency: %d", read)); 146 v &= ~RAM_CTRL_READ_MASK; 147 v |= (read - 1) << RAM_CTRL_READ_SHIFT; 148 } 149 if (write != 0) { 150 KASSERT(write <= 8, ("bad pl310 write latency: %d", write)); 151 v &= ~RAM_CTRL_WRITE_MASK; 152 v |= (write - 1) << RAM_CTRL_WRITE_SHIFT; 153 } 154 pl310_write4(sc, which_reg, v); 155} 156 157static int 158pl310_filter(void *arg) 159{ 160 struct pl310_softc *sc = arg; 161 uint32_t intr; 162 163 intr = pl310_read4(sc, PL310_INTR_MASK); 164 165 if (!sc->sc_enabled && (intr & INTR_MASK_ECNTR)) { 166 /* 167 * This is for debug purpose, so be blunt about it 168 * We disable PL310 only when something fishy is going 169 * on and we need to make sure L2 cache is 100% disabled 170 */ 171 panic("pl310: caches disabled but cache event detected\n"); 172 } 173 174 return (FILTER_HANDLED); 175} 176 177static __inline void 178pl310_wait_background_op(uint32_t off, uint32_t mask) 179{ 180 181 while (pl310_read4(pl310_softc, off) & mask) 182 continue; 183} 184 185 186/** 187 * pl310_cache_sync - performs a cache sync operation 188 * 189 * According to the TRM: 190 * 191 * "Before writing to any other register you must perform an explicit 192 * Cache Sync operation. This is particularly important when the cache is 193 * enabled and changes to how the cache allocates new lines are to be made." 194 * 195 * 196 */ 197static __inline void 198pl310_cache_sync(void) 199{ 200 201 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled) 202 return; 203 204#ifdef PL310_ERRATA_753970 205 if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0) 206 /* Write uncached PL310 register */ 207 pl310_write4(pl310_softc, 0x740, 0xffffffff); 208 else 209#endif 210 pl310_write4(pl310_softc, PL310_CACHE_SYNC, 0xffffffff); 211} 212 213 214static void 215pl310_wbinv_all(void) 216{ 217 218 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled) 219 return; 220 221 PL310_LOCK(pl310_softc); 222#ifdef PL310_ERRATA_727915 223 if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r2p0) { 224 int i, j; 225 226 for (i = 0; i < g_ways_assoc; i++) { 227 for (j = 0; j < g_way_size / g_l2cache_line_size; j++) { 228 pl310_write4(pl310_softc, 229 PL310_CLEAN_INV_LINE_IDX, 230 (i << 28 | j << 5)); 231 } 232 } 233 pl310_cache_sync(); 234 PL310_UNLOCK(pl310_softc); 235 return; 236 237 } 238 if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0) 239 platform_pl310_write_debug(pl310_softc, 3); 240#endif 241 pl310_write4(pl310_softc, PL310_CLEAN_INV_WAY, g_l2cache_way_mask); 242 pl310_wait_background_op(PL310_CLEAN_INV_WAY, g_l2cache_way_mask); 243 pl310_cache_sync(); 244#ifdef PL310_ERRATA_727915 245 if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0) 246 platform_pl310_write_debug(pl310_softc, 0); 247#endif 248 PL310_UNLOCK(pl310_softc); 249} 250 251static void 252pl310_wbinv_range(vm_paddr_t start, vm_size_t size) 253{ 254 255 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled) 256 return; 257 258 PL310_LOCK(pl310_softc); 259 if (start & g_l2cache_align_mask) { 260 size += start & g_l2cache_align_mask; 261 start &= ~g_l2cache_align_mask; 262 } 263 if (size & g_l2cache_align_mask) { 264 size &= ~g_l2cache_align_mask; 265 size += g_l2cache_line_size; 266 } 267 268 269#ifdef PL310_ERRATA_727915 270 platform_pl310_write_debug(pl310_softc, 3); 271#endif 272 while (size > 0) { 273#ifdef PL310_ERRATA_588369 274 if (pl310_softc->sc_rtl_revision <= CACHE_ID_RELEASE_r1p0) { 275 /* 276 * Errata 588369 says that clean + inv may keep the 277 * cache line if it was clean, the recommanded 278 * workaround is to clean then invalidate the cache 279 * line, with write-back and cache linefill disabled. 280 */ 281 pl310_write4(pl310_softc, PL310_CLEAN_LINE_PA, start); 282 pl310_write4(pl310_softc, PL310_INV_LINE_PA, start); 283 } else 284#endif 285 pl310_write4(pl310_softc, PL310_CLEAN_INV_LINE_PA, 286 start); 287 start += g_l2cache_line_size; 288 size -= g_l2cache_line_size; 289 } 290#ifdef PL310_ERRATA_727915 291 platform_pl310_write_debug(pl310_softc, 0); 292#endif 293 294 pl310_cache_sync(); 295 PL310_UNLOCK(pl310_softc); 296} 297 298static void 299pl310_wb_range(vm_paddr_t start, vm_size_t size) 300{ 301 302 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled) 303 return; 304 305 PL310_LOCK(pl310_softc); 306 if (start & g_l2cache_align_mask) { 307 size += start & g_l2cache_align_mask; 308 start &= ~g_l2cache_align_mask; 309 } 310 311 if (size & g_l2cache_align_mask) { 312 size &= ~g_l2cache_align_mask; 313 size += g_l2cache_line_size; 314 } 315 316 while (size > 0) { 317 pl310_write4(pl310_softc, PL310_CLEAN_LINE_PA, start); 318 start += g_l2cache_line_size; 319 size -= g_l2cache_line_size; 320 } 321 322 pl310_cache_sync(); 323 PL310_UNLOCK(pl310_softc); 324} 325 326static void 327pl310_inv_range(vm_paddr_t start, vm_size_t size) 328{ 329 330 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled) 331 return; 332 333 PL310_LOCK(pl310_softc); 334 if (start & g_l2cache_align_mask) { 335 size += start & g_l2cache_align_mask; 336 start &= ~g_l2cache_align_mask; 337 } 338 if (size & g_l2cache_align_mask) { 339 size &= ~g_l2cache_align_mask; 340 size += g_l2cache_line_size; 341 } 342 while (size > 0) { 343 pl310_write4(pl310_softc, PL310_INV_LINE_PA, start); 344 start += g_l2cache_line_size; 345 size -= g_l2cache_line_size; 346 } 347 348 pl310_cache_sync(); 349 PL310_UNLOCK(pl310_softc); 350} 351 352static void 353pl310_drain_writebuf(void) 354{ 355 356 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled) 357 return; 358 359 PL310_LOCK(pl310_softc); 360 pl310_cache_sync(); 361 PL310_UNLOCK(pl310_softc); 362} 363 364static void 365pl310_set_way_sizes(struct pl310_softc *sc) 366{ 367 uint32_t aux_value; 368 369 aux_value = pl310_read4(sc, PL310_AUX_CTRL); 370 g_way_size = (aux_value & AUX_CTRL_WAY_SIZE_MASK) >> 371 AUX_CTRL_WAY_SIZE_SHIFT; 372 g_way_size = 1 << (g_way_size + 13); 373 if (aux_value & (1 << AUX_CTRL_ASSOCIATIVITY_SHIFT)) 374 g_ways_assoc = 16; 375 else 376 g_ways_assoc = 8; 377 g_l2cache_way_mask = (1 << g_ways_assoc) - 1; 378 g_l2cache_size = g_way_size * g_ways_assoc; 379} 380 381/* 382 * Setup interrupt handling. This is done only if the cache controller is 383 * disabled, for debugging. We set counters so when a cache event happens we'll 384 * get interrupted and be warned that something is wrong, because no cache 385 * events should happen if we're disabled. 386 */ 387static void 388pl310_config_intr(void *arg) 389{ 390 struct pl310_softc * sc; 391 392 sc = arg; 393 394 /* activate the interrupt */ 395 bus_setup_intr(sc->sc_dev, sc->sc_irq_res, INTR_TYPE_MISC | INTR_MPSAFE, 396 pl310_filter, NULL, sc, &sc->sc_irq_h); 397 398 /* Cache Line Eviction for Counter 0 */ 399 pl310_write4(sc, PL310_EVENT_COUNTER0_CONF, 400 EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_CO); 401 /* Data Read Request for Counter 1 */ 402 pl310_write4(sc, PL310_EVENT_COUNTER1_CONF, 403 EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_DRREQ); 404 405 /* Enable and clear pending interrupts */ 406 pl310_write4(sc, PL310_INTR_CLEAR, INTR_MASK_ECNTR); 407 pl310_write4(sc, PL310_INTR_MASK, INTR_MASK_ALL); 408 409 /* Enable counters and reset C0 and C1 */ 410 pl310_write4(sc, PL310_EVENT_COUNTER_CTRL, 411 EVENT_COUNTER_CTRL_ENABLED | 412 EVENT_COUNTER_CTRL_C0_RESET | 413 EVENT_COUNTER_CTRL_C1_RESET); 414 415 config_intrhook_disestablish(sc->sc_ich); 416 free(sc->sc_ich, M_DEVBUF); 417} 418 419static int 420pl310_probe(device_t dev) 421{ 422 423 if (!ofw_bus_status_okay(dev)) 424 return (ENXIO); 425 426 if (!ofw_bus_is_compatible(dev, "arm,pl310")) 427 return (ENXIO); 428 device_set_desc(dev, "PL310 L2 cache controller"); 429 return (0); 430} 431 432static int 433pl310_attach(device_t dev) 434{ 435 struct pl310_softc *sc = device_get_softc(dev); 436 int rid; 437 uint32_t cache_id, debug_ctrl; 438 439 sc->sc_dev = dev; 440 rid = 0; 441 sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 442 RF_ACTIVE); 443 if (sc->sc_mem_res == NULL) 444 panic("%s: Cannot map registers", device_get_name(dev)); 445 446 /* Allocate an IRQ resource */ 447 rid = 0; 448 sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 449 RF_ACTIVE | RF_SHAREABLE); 450 if (sc->sc_irq_res == NULL) { 451 panic("Cannot allocate IRQ\n"); 452 } 453 454 pl310_softc = sc; 455 mtx_init(&sc->sc_mtx, "pl310lock", NULL, MTX_SPIN); 456 457 cache_id = pl310_read4(sc, PL310_CACHE_ID); 458 sc->sc_rtl_revision = (cache_id >> CACHE_ID_RELEASE_SHIFT) & 459 CACHE_ID_RELEASE_MASK; 460 device_printf(dev, "Part number: 0x%x, release: 0x%x\n", 461 (cache_id >> CACHE_ID_PARTNUM_SHIFT) & CACHE_ID_PARTNUM_MASK, 462 (cache_id >> CACHE_ID_RELEASE_SHIFT) & CACHE_ID_RELEASE_MASK); 463 464 /* 465 * If L2 cache is already enabled then something has violated the rules, 466 * because caches are supposed to be off at kernel entry. The cache 467 * must be disabled to write the configuration registers without 468 * triggering an access error (SLVERR), but there's no documented safe 469 * procedure for disabling the L2 cache in the manual. So we'll try to 470 * invent one: 471 * - Use the debug register to force write-through mode and prevent 472 * linefills (allocation of new lines on read); now anything we do 473 * will not cause new data to come into the L2 cache. 474 * - Writeback and invalidate the current contents. 475 * - Disable the controller. 476 * - Restore the original debug settings. 477 */ 478 if (pl310_read4(sc, PL310_CTRL) & CTRL_ENABLED) { 479 device_printf(dev, "Warning: L2 Cache should not already be " 480 "active; trying to de-activate and re-initialize...\n"); 481 sc->sc_enabled = 1; 482 debug_ctrl = pl310_read4(sc, PL310_DEBUG_CTRL); 483 platform_pl310_write_debug(sc, debug_ctrl | 484 DEBUG_CTRL_DISABLE_WRITEBACK | DEBUG_CTRL_DISABLE_LINEFILL); 485 pl310_set_way_sizes(sc); 486 pl310_wbinv_all(); 487 platform_pl310_write_ctrl(sc, CTRL_DISABLED); 488 platform_pl310_write_debug(sc, debug_ctrl); 489 } 490 sc->sc_enabled = pl310_enabled; 491 492 if (sc->sc_enabled) { 493 platform_pl310_init(sc); 494 pl310_set_way_sizes(sc); /* platform init might change these */ 495 pl310_write4(pl310_softc, PL310_INV_WAY, 0xffff); 496 pl310_wait_background_op(PL310_INV_WAY, 0xffff); 497 platform_pl310_write_ctrl(sc, CTRL_ENABLED); 498 device_printf(dev, "L2 Cache enabled: %uKB/%dB %d ways\n", 499 (g_l2cache_size / 1024), g_l2cache_line_size, g_ways_assoc); 500 if (bootverbose) 501 pl310_print_config(sc); 502 } else { 503 malloc(sizeof(*sc->sc_ich), M_DEVBUF, M_WAITOK); 504 sc->sc_ich->ich_func = pl310_config_intr; 505 sc->sc_ich->ich_arg = sc; 506 if (config_intrhook_establish(sc->sc_ich) != 0) { 507 device_printf(dev, 508 "config_intrhook_establish failed\n"); 509 return(ENXIO); 510 } 511 device_printf(dev, "L2 Cache disabled\n"); 512 } 513 514 /* Set the l2 functions in the set of cpufuncs */ 515 cpufuncs.cf_l2cache_wbinv_all = pl310_wbinv_all; 516 cpufuncs.cf_l2cache_wbinv_range = pl310_wbinv_range; 517 cpufuncs.cf_l2cache_inv_range = pl310_inv_range; 518 cpufuncs.cf_l2cache_wb_range = pl310_wb_range; 519 cpufuncs.cf_l2cache_drain_writebuf = pl310_drain_writebuf; 520 521 return (0); 522} 523 524static device_method_t pl310_methods[] = { 525 DEVMETHOD(device_probe, pl310_probe), 526 DEVMETHOD(device_attach, pl310_attach), 527 DEVMETHOD_END 528}; 529 530static driver_t pl310_driver = { 531 "l2cache", 532 pl310_methods, 533 sizeof(struct pl310_softc), 534}; 535static devclass_t pl310_devclass; 536 537EARLY_DRIVER_MODULE(pl310, simplebus, pl310_driver, pl310_devclass, 0, 0, 538 BUS_PASS_CPU + BUS_PASS_ORDER_MIDDLE); 539 540