1// SPDX-License-Identifier: GPL-2.0 2/* 3 * ARMv6 Performance counter handling code. 4 * 5 * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles 6 * 7 * ARMv6 has 2 configurable performance counters and a single cycle counter. 8 * They all share a single reset bit but can be written to zero so we can use 9 * that for a reset. 10 * 11 * The counters can't be individually enabled or disabled so when we remove 12 * one event and replace it with another we could get spurious counts from the 13 * wrong event. However, we can take advantage of the fact that the 14 * performance counters can export events to the event bus, and the event bus 15 * itself can be monitored. This requires that we *don't* export the events to 16 * the event bus. The procedure for disabling a configurable counter is: 17 * - change the counter to count the ETMEXTOUT[0] signal (0x20). This 18 * effectively stops the counter from counting. 19 * - disable the counter's interrupt generation (each counter has it's 20 * own interrupt enable bit). 21 * Once stopped, the counter value can be written as 0 to reset. 22 * 23 * To enable a counter: 24 * - enable the counter's interrupt generation. 25 * - set the new event type. 26 * 27 * Note: the dedicated cycle counter only counts cycles and can't be 28 * enabled/disabled independently of the others. When we want to disable the 29 * cycle counter, we have to just disable the interrupt reporting and start 30 * ignoring that counter. When re-enabling, we have to reset the value and 31 * enable the interrupt. 32 */ 33 34#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K) 35 36#include <asm/cputype.h> 37#include <asm/irq_regs.h> 38 39#include <linux/of.h> 40#include <linux/perf/arm_pmu.h> 41#include <linux/platform_device.h> 42 43enum armv6_perf_types { 44 ARMV6_PERFCTR_ICACHE_MISS = 0x0, 45 ARMV6_PERFCTR_IBUF_STALL = 0x1, 46 ARMV6_PERFCTR_DDEP_STALL = 0x2, 47 ARMV6_PERFCTR_ITLB_MISS = 0x3, 48 ARMV6_PERFCTR_DTLB_MISS = 0x4, 49 ARMV6_PERFCTR_BR_EXEC = 0x5, 50 ARMV6_PERFCTR_BR_MISPREDICT = 0x6, 51 ARMV6_PERFCTR_INSTR_EXEC = 0x7, 52 ARMV6_PERFCTR_DCACHE_HIT = 0x9, 53 ARMV6_PERFCTR_DCACHE_ACCESS = 0xA, 54 ARMV6_PERFCTR_DCACHE_MISS = 0xB, 55 ARMV6_PERFCTR_DCACHE_WBACK = 0xC, 56 ARMV6_PERFCTR_SW_PC_CHANGE = 0xD, 57 ARMV6_PERFCTR_MAIN_TLB_MISS = 0xF, 58 ARMV6_PERFCTR_EXPL_D_ACCESS = 0x10, 59 ARMV6_PERFCTR_LSU_FULL_STALL = 0x11, 60 ARMV6_PERFCTR_WBUF_DRAINED = 0x12, 61 ARMV6_PERFCTR_CPU_CYCLES = 0xFF, 62 ARMV6_PERFCTR_NOP = 0x20, 63}; 64 65enum armv6_counters { 66 ARMV6_CYCLE_COUNTER = 0, 67 ARMV6_COUNTER0, 68 ARMV6_COUNTER1, 69}; 70 71/* 72 * The hardware events that we support. We do support cache operations but 73 * we have harvard caches and no way to combine instruction and data 74 * accesses/misses in hardware. 75 */ 76static const unsigned armv6_perf_map[PERF_COUNT_HW_MAX] = { 77 PERF_MAP_ALL_UNSUPPORTED, 78 [PERF_COUNT_HW_CPU_CYCLES] = ARMV6_PERFCTR_CPU_CYCLES, 79 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV6_PERFCTR_INSTR_EXEC, 80 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV6_PERFCTR_BR_EXEC, 81 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV6_PERFCTR_BR_MISPREDICT, 82 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV6_PERFCTR_IBUF_STALL, 83 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV6_PERFCTR_LSU_FULL_STALL, 84}; 85 86static const unsigned armv6_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 87 [PERF_COUNT_HW_CACHE_OP_MAX] 88 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 89 PERF_CACHE_MAP_ALL_UNSUPPORTED, 90 91 /* 92 * The performance counters don't differentiate between read and write 93 * accesses/misses so this isn't strictly correct, but it's the best we 94 * can do. Writes and reads get combined. 95 */ 96 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV6_PERFCTR_DCACHE_ACCESS, 97 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_DCACHE_MISS, 98 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV6_PERFCTR_DCACHE_ACCESS, 99 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6_PERFCTR_DCACHE_MISS, 100 101 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_ICACHE_MISS, 102 103 /* 104 * The ARM performance counters can count micro DTLB misses, micro ITLB 105 * misses and main TLB misses. There isn't an event for TLB misses, so 106 * use the micro misses here and if users want the main TLB misses they 107 * can use a raw counter. 108 */ 109 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_DTLB_MISS, 110 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6_PERFCTR_DTLB_MISS, 111 112 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_ITLB_MISS, 113 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6_PERFCTR_ITLB_MISS, 114}; 115 116static inline unsigned long 117armv6_pmcr_read(void) 118{ 119 u32 val; 120 asm volatile("mrc p15, 0, %0, c15, c12, 0" : "=r"(val)); 121 return val; 122} 123 124static inline void 125armv6_pmcr_write(unsigned long val) 126{ 127 asm volatile("mcr p15, 0, %0, c15, c12, 0" : : "r"(val)); 128} 129 130#define ARMV6_PMCR_ENABLE (1 << 0) 131#define ARMV6_PMCR_CTR01_RESET (1 << 1) 132#define ARMV6_PMCR_CCOUNT_RESET (1 << 2) 133#define ARMV6_PMCR_CCOUNT_DIV (1 << 3) 134#define ARMV6_PMCR_COUNT0_IEN (1 << 4) 135#define ARMV6_PMCR_COUNT1_IEN (1 << 5) 136#define ARMV6_PMCR_CCOUNT_IEN (1 << 6) 137#define ARMV6_PMCR_COUNT0_OVERFLOW (1 << 8) 138#define ARMV6_PMCR_COUNT1_OVERFLOW (1 << 9) 139#define ARMV6_PMCR_CCOUNT_OVERFLOW (1 << 10) 140#define ARMV6_PMCR_EVT_COUNT0_SHIFT 20 141#define ARMV6_PMCR_EVT_COUNT0_MASK (0xFF << ARMV6_PMCR_EVT_COUNT0_SHIFT) 142#define ARMV6_PMCR_EVT_COUNT1_SHIFT 12 143#define ARMV6_PMCR_EVT_COUNT1_MASK (0xFF << ARMV6_PMCR_EVT_COUNT1_SHIFT) 144 145#define ARMV6_PMCR_OVERFLOWED_MASK \ 146 (ARMV6_PMCR_COUNT0_OVERFLOW | ARMV6_PMCR_COUNT1_OVERFLOW | \ 147 ARMV6_PMCR_CCOUNT_OVERFLOW) 148 149static inline int 150armv6_pmcr_has_overflowed(unsigned long pmcr) 151{ 152 return pmcr & ARMV6_PMCR_OVERFLOWED_MASK; 153} 154 155static inline int 156armv6_pmcr_counter_has_overflowed(unsigned long pmcr, 157 enum armv6_counters counter) 158{ 159 int ret = 0; 160 161 if (ARMV6_CYCLE_COUNTER == counter) 162 ret = pmcr & ARMV6_PMCR_CCOUNT_OVERFLOW; 163 else if (ARMV6_COUNTER0 == counter) 164 ret = pmcr & ARMV6_PMCR_COUNT0_OVERFLOW; 165 else if (ARMV6_COUNTER1 == counter) 166 ret = pmcr & ARMV6_PMCR_COUNT1_OVERFLOW; 167 else 168 WARN_ONCE(1, "invalid counter number (%d)\n", counter); 169 170 return ret; 171} 172 173static inline u64 armv6pmu_read_counter(struct perf_event *event) 174{ 175 struct hw_perf_event *hwc = &event->hw; 176 int counter = hwc->idx; 177 unsigned long value = 0; 178 179 if (ARMV6_CYCLE_COUNTER == counter) 180 asm volatile("mrc p15, 0, %0, c15, c12, 1" : "=r"(value)); 181 else if (ARMV6_COUNTER0 == counter) 182 asm volatile("mrc p15, 0, %0, c15, c12, 2" : "=r"(value)); 183 else if (ARMV6_COUNTER1 == counter) 184 asm volatile("mrc p15, 0, %0, c15, c12, 3" : "=r"(value)); 185 else 186 WARN_ONCE(1, "invalid counter number (%d)\n", counter); 187 188 return value; 189} 190 191static inline void armv6pmu_write_counter(struct perf_event *event, u64 value) 192{ 193 struct hw_perf_event *hwc = &event->hw; 194 int counter = hwc->idx; 195 196 if (ARMV6_CYCLE_COUNTER == counter) 197 asm volatile("mcr p15, 0, %0, c15, c12, 1" : : "r"(value)); 198 else if (ARMV6_COUNTER0 == counter) 199 asm volatile("mcr p15, 0, %0, c15, c12, 2" : : "r"(value)); 200 else if (ARMV6_COUNTER1 == counter) 201 asm volatile("mcr p15, 0, %0, c15, c12, 3" : : "r"(value)); 202 else 203 WARN_ONCE(1, "invalid counter number (%d)\n", counter); 204} 205 206static void armv6pmu_enable_event(struct perf_event *event) 207{ 208 unsigned long val, mask, evt; 209 struct hw_perf_event *hwc = &event->hw; 210 int idx = hwc->idx; 211 212 if (ARMV6_CYCLE_COUNTER == idx) { 213 mask = 0; 214 evt = ARMV6_PMCR_CCOUNT_IEN; 215 } else if (ARMV6_COUNTER0 == idx) { 216 mask = ARMV6_PMCR_EVT_COUNT0_MASK; 217 evt = (hwc->config_base << ARMV6_PMCR_EVT_COUNT0_SHIFT) | 218 ARMV6_PMCR_COUNT0_IEN; 219 } else if (ARMV6_COUNTER1 == idx) { 220 mask = ARMV6_PMCR_EVT_COUNT1_MASK; 221 evt = (hwc->config_base << ARMV6_PMCR_EVT_COUNT1_SHIFT) | 222 ARMV6_PMCR_COUNT1_IEN; 223 } else { 224 WARN_ONCE(1, "invalid counter number (%d)\n", idx); 225 return; 226 } 227 228 /* 229 * Mask out the current event and set the counter to count the event 230 * that we're interested in. 231 */ 232 val = armv6_pmcr_read(); 233 val &= ~mask; 234 val |= evt; 235 armv6_pmcr_write(val); 236} 237 238static irqreturn_t 239armv6pmu_handle_irq(struct arm_pmu *cpu_pmu) 240{ 241 unsigned long pmcr = armv6_pmcr_read(); 242 struct perf_sample_data data; 243 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); 244 struct pt_regs *regs; 245 int idx; 246 247 if (!armv6_pmcr_has_overflowed(pmcr)) 248 return IRQ_NONE; 249 250 regs = get_irq_regs(); 251 252 /* 253 * The interrupts are cleared by writing the overflow flags back to 254 * the control register. All of the other bits don't have any effect 255 * if they are rewritten, so write the whole value back. 256 */ 257 armv6_pmcr_write(pmcr); 258 259 for (idx = 0; idx < cpu_pmu->num_events; ++idx) { 260 struct perf_event *event = cpuc->events[idx]; 261 struct hw_perf_event *hwc; 262 263 /* Ignore if we don't have an event. */ 264 if (!event) 265 continue; 266 267 /* 268 * We have a single interrupt for all counters. Check that 269 * each counter has overflowed before we process it. 270 */ 271 if (!armv6_pmcr_counter_has_overflowed(pmcr, idx)) 272 continue; 273 274 hwc = &event->hw; 275 armpmu_event_update(event); 276 perf_sample_data_init(&data, 0, hwc->last_period); 277 if (!armpmu_event_set_period(event)) 278 continue; 279 280 if (perf_event_overflow(event, &data, regs)) 281 cpu_pmu->disable(event); 282 } 283 284 /* 285 * Handle the pending perf events. 286 * 287 * Note: this call *must* be run with interrupts disabled. For 288 * platforms that can have the PMU interrupts raised as an NMI, this 289 * will not work. 290 */ 291 irq_work_run(); 292 293 return IRQ_HANDLED; 294} 295 296static void armv6pmu_start(struct arm_pmu *cpu_pmu) 297{ 298 unsigned long val; 299 300 val = armv6_pmcr_read(); 301 val |= ARMV6_PMCR_ENABLE; 302 armv6_pmcr_write(val); 303} 304 305static void armv6pmu_stop(struct arm_pmu *cpu_pmu) 306{ 307 unsigned long val; 308 309 val = armv6_pmcr_read(); 310 val &= ~ARMV6_PMCR_ENABLE; 311 armv6_pmcr_write(val); 312} 313 314static int 315armv6pmu_get_event_idx(struct pmu_hw_events *cpuc, 316 struct perf_event *event) 317{ 318 struct hw_perf_event *hwc = &event->hw; 319 /* Always place a cycle counter into the cycle counter. */ 320 if (ARMV6_PERFCTR_CPU_CYCLES == hwc->config_base) { 321 if (test_and_set_bit(ARMV6_CYCLE_COUNTER, cpuc->used_mask)) 322 return -EAGAIN; 323 324 return ARMV6_CYCLE_COUNTER; 325 } else { 326 /* 327 * For anything other than a cycle counter, try and use 328 * counter0 and counter1. 329 */ 330 if (!test_and_set_bit(ARMV6_COUNTER1, cpuc->used_mask)) 331 return ARMV6_COUNTER1; 332 333 if (!test_and_set_bit(ARMV6_COUNTER0, cpuc->used_mask)) 334 return ARMV6_COUNTER0; 335 336 /* The counters are all in use. */ 337 return -EAGAIN; 338 } 339} 340 341static void armv6pmu_clear_event_idx(struct pmu_hw_events *cpuc, 342 struct perf_event *event) 343{ 344 clear_bit(event->hw.idx, cpuc->used_mask); 345} 346 347static void armv6pmu_disable_event(struct perf_event *event) 348{ 349 unsigned long val, mask, evt; 350 struct hw_perf_event *hwc = &event->hw; 351 int idx = hwc->idx; 352 353 if (ARMV6_CYCLE_COUNTER == idx) { 354 mask = ARMV6_PMCR_CCOUNT_IEN; 355 evt = 0; 356 } else if (ARMV6_COUNTER0 == idx) { 357 mask = ARMV6_PMCR_COUNT0_IEN | ARMV6_PMCR_EVT_COUNT0_MASK; 358 evt = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT0_SHIFT; 359 } else if (ARMV6_COUNTER1 == idx) { 360 mask = ARMV6_PMCR_COUNT1_IEN | ARMV6_PMCR_EVT_COUNT1_MASK; 361 evt = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT1_SHIFT; 362 } else { 363 WARN_ONCE(1, "invalid counter number (%d)\n", idx); 364 return; 365 } 366 367 /* 368 * Mask out the current event and set the counter to count the number 369 * of ETM bus signal assertion cycles. The external reporting should 370 * be disabled and so this should never increment. 371 */ 372 val = armv6_pmcr_read(); 373 val &= ~mask; 374 val |= evt; 375 armv6_pmcr_write(val); 376} 377 378static int armv6_map_event(struct perf_event *event) 379{ 380 return armpmu_map_event(event, &armv6_perf_map, 381 &armv6_perf_cache_map, 0xFF); 382} 383 384static void armv6pmu_init(struct arm_pmu *cpu_pmu) 385{ 386 cpu_pmu->handle_irq = armv6pmu_handle_irq; 387 cpu_pmu->enable = armv6pmu_enable_event; 388 cpu_pmu->disable = armv6pmu_disable_event; 389 cpu_pmu->read_counter = armv6pmu_read_counter; 390 cpu_pmu->write_counter = armv6pmu_write_counter; 391 cpu_pmu->get_event_idx = armv6pmu_get_event_idx; 392 cpu_pmu->clear_event_idx = armv6pmu_clear_event_idx; 393 cpu_pmu->start = armv6pmu_start; 394 cpu_pmu->stop = armv6pmu_stop; 395 cpu_pmu->map_event = armv6_map_event; 396 cpu_pmu->num_events = 3; 397} 398 399static int armv6_1136_pmu_init(struct arm_pmu *cpu_pmu) 400{ 401 armv6pmu_init(cpu_pmu); 402 cpu_pmu->name = "armv6_1136"; 403 return 0; 404} 405 406static int armv6_1156_pmu_init(struct arm_pmu *cpu_pmu) 407{ 408 armv6pmu_init(cpu_pmu); 409 cpu_pmu->name = "armv6_1156"; 410 return 0; 411} 412 413static int armv6_1176_pmu_init(struct arm_pmu *cpu_pmu) 414{ 415 armv6pmu_init(cpu_pmu); 416 cpu_pmu->name = "armv6_1176"; 417 return 0; 418} 419 420static const struct of_device_id armv6_pmu_of_device_ids[] = { 421 {.compatible = "arm,arm1176-pmu", .data = armv6_1176_pmu_init}, 422 {.compatible = "arm,arm1136-pmu", .data = armv6_1136_pmu_init}, 423 { /* sentinel value */ } 424}; 425 426static const struct pmu_probe_info armv6_pmu_probe_table[] = { 427 ARM_PMU_PROBE(ARM_CPU_PART_ARM1136, armv6_1136_pmu_init), 428 ARM_PMU_PROBE(ARM_CPU_PART_ARM1156, armv6_1156_pmu_init), 429 ARM_PMU_PROBE(ARM_CPU_PART_ARM1176, armv6_1176_pmu_init), 430 { /* sentinel value */ } 431}; 432 433static int armv6_pmu_device_probe(struct platform_device *pdev) 434{ 435 return arm_pmu_device_probe(pdev, armv6_pmu_of_device_ids, 436 armv6_pmu_probe_table); 437} 438 439static struct platform_driver armv6_pmu_driver = { 440 .driver = { 441 .name = "armv6-pmu", 442 .of_match_table = armv6_pmu_of_device_ids, 443 }, 444 .probe = armv6_pmu_device_probe, 445}; 446 447builtin_platform_driver(armv6_pmu_driver); 448#endif /* CONFIG_CPU_V6 || CONFIG_CPU_V6K */ 449