1// SPDX-License-Identifier: GPL-2.0 2/* Marvell CN10K DRAM Subsystem (DSS) Performance Monitor Driver 3 * 4 * Copyright (C) 2021 Marvell. 5 */ 6 7#include <linux/init.h> 8#include <linux/io.h> 9#include <linux/module.h> 10#include <linux/of.h> 11#include <linux/perf_event.h> 12#include <linux/hrtimer.h> 13#include <linux/acpi.h> 14#include <linux/platform_device.h> 15 16/* Performance Counters Operating Mode Control Registers */ 17#define DDRC_PERF_CNT_OP_MODE_CTRL 0x8020 18#define OP_MODE_CTRL_VAL_MANNUAL 0x1 19 20/* Performance Counters Start Operation Control Registers */ 21#define DDRC_PERF_CNT_START_OP_CTRL 0x8028 22#define START_OP_CTRL_VAL_START 0x1ULL 23#define START_OP_CTRL_VAL_ACTIVE 0x2 24 25/* Performance Counters End Operation Control Registers */ 26#define DDRC_PERF_CNT_END_OP_CTRL 0x8030 27#define END_OP_CTRL_VAL_END 0x1ULL 28 29/* Performance Counters End Status Registers */ 30#define DDRC_PERF_CNT_END_STATUS 0x8038 31#define END_STATUS_VAL_END_TIMER_MODE_END 0x1 32 33/* Performance Counters Configuration Registers */ 34#define DDRC_PERF_CFG_BASE 0x8040 35 36/* 8 Generic event counter + 2 fixed event counters */ 37#define DDRC_PERF_NUM_GEN_COUNTERS 8 38#define DDRC_PERF_NUM_FIX_COUNTERS 2 39#define DDRC_PERF_READ_COUNTER_IDX DDRC_PERF_NUM_GEN_COUNTERS 40#define DDRC_PERF_WRITE_COUNTER_IDX (DDRC_PERF_NUM_GEN_COUNTERS + 1) 41#define DDRC_PERF_NUM_COUNTERS (DDRC_PERF_NUM_GEN_COUNTERS + \ 42 DDRC_PERF_NUM_FIX_COUNTERS) 43 44/* Generic event counter registers */ 45#define DDRC_PERF_CFG(n) (DDRC_PERF_CFG_BASE + 8 * (n)) 46#define EVENT_ENABLE BIT_ULL(63) 47 48/* Two dedicated event counters for DDR reads and writes */ 49#define EVENT_DDR_READS 101 50#define EVENT_DDR_WRITES 100 51 52/* 53 * programmable events IDs in programmable event counters. 54 * DO NOT change these event-id numbers, they are used to 55 * program event bitmap in h/w. 56 */ 57#define EVENT_OP_IS_ZQLATCH 55 58#define EVENT_OP_IS_ZQSTART 54 59#define EVENT_OP_IS_TCR_MRR 53 60#define EVENT_OP_IS_DQSOSC_MRR 52 61#define EVENT_OP_IS_DQSOSC_MPC 51 62#define EVENT_VISIBLE_WIN_LIMIT_REACHED_WR 50 63#define EVENT_VISIBLE_WIN_LIMIT_REACHED_RD 49 64#define EVENT_BSM_STARVATION 48 65#define EVENT_BSM_ALLOC 47 66#define EVENT_LPR_REQ_WITH_NOCREDIT 46 67#define EVENT_HPR_REQ_WITH_NOCREDIT 45 68#define EVENT_OP_IS_ZQCS 44 69#define EVENT_OP_IS_ZQCL 43 70#define EVENT_OP_IS_LOAD_MODE 42 71#define EVENT_OP_IS_SPEC_REF 41 72#define EVENT_OP_IS_CRIT_REF 40 73#define EVENT_OP_IS_REFRESH 39 74#define EVENT_OP_IS_ENTER_MPSM 35 75#define EVENT_OP_IS_ENTER_POWERDOWN 31 76#define EVENT_OP_IS_ENTER_SELFREF 27 77#define EVENT_WAW_HAZARD 26 78#define EVENT_RAW_HAZARD 25 79#define EVENT_WAR_HAZARD 24 80#define EVENT_WRITE_COMBINE 23 81#define EVENT_RDWR_TRANSITIONS 22 82#define EVENT_PRECHARGE_FOR_OTHER 21 83#define EVENT_PRECHARGE_FOR_RDWR 20 84#define EVENT_OP_IS_PRECHARGE 19 85#define EVENT_OP_IS_MWR 18 86#define EVENT_OP_IS_WR 17 87#define EVENT_OP_IS_RD 16 88#define EVENT_OP_IS_RD_ACTIVATE 15 89#define EVENT_OP_IS_RD_OR_WR 14 90#define EVENT_OP_IS_ACTIVATE 13 91#define EVENT_WR_XACT_WHEN_CRITICAL 12 92#define EVENT_LPR_XACT_WHEN_CRITICAL 11 93#define EVENT_HPR_XACT_WHEN_CRITICAL 10 94#define EVENT_DFI_RD_DATA_CYCLES 9 95#define EVENT_DFI_WR_DATA_CYCLES 8 96#define EVENT_ACT_BYPASS 7 97#define EVENT_READ_BYPASS 6 98#define EVENT_HIF_HI_PRI_RD 5 99#define EVENT_HIF_RMW 4 100#define EVENT_HIF_RD 3 101#define EVENT_HIF_WR 2 102#define EVENT_HIF_RD_OR_WR 1 103 104/* Event counter value registers */ 105#define DDRC_PERF_CNT_VALUE_BASE 0x8080 106#define DDRC_PERF_CNT_VALUE(n) (DDRC_PERF_CNT_VALUE_BASE + 8 * (n)) 107 108/* Fixed event counter enable/disable register */ 109#define DDRC_PERF_CNT_FREERUN_EN 0x80C0 110#define DDRC_PERF_FREERUN_WRITE_EN 0x1 111#define DDRC_PERF_FREERUN_READ_EN 0x2 112 113/* Fixed event counter control register */ 114#define DDRC_PERF_CNT_FREERUN_CTRL 0x80C8 115#define DDRC_FREERUN_WRITE_CNT_CLR 0x1 116#define DDRC_FREERUN_READ_CNT_CLR 0x2 117 118/* Fixed event counter value register */ 119#define DDRC_PERF_CNT_VALUE_WR_OP 0x80D0 120#define DDRC_PERF_CNT_VALUE_RD_OP 0x80D8 121#define DDRC_PERF_CNT_VALUE_OVERFLOW BIT_ULL(48) 122#define DDRC_PERF_CNT_MAX_VALUE GENMASK_ULL(48, 0) 123 124struct cn10k_ddr_pmu { 125 struct pmu pmu; 126 void __iomem *base; 127 unsigned int cpu; 128 struct device *dev; 129 int active_events; 130 struct perf_event *events[DDRC_PERF_NUM_COUNTERS]; 131 struct hrtimer hrtimer; 132 struct hlist_node node; 133}; 134 135#define to_cn10k_ddr_pmu(p) container_of(p, struct cn10k_ddr_pmu, pmu) 136 137static ssize_t cn10k_ddr_pmu_event_show(struct device *dev, 138 struct device_attribute *attr, 139 char *page) 140{ 141 struct perf_pmu_events_attr *pmu_attr; 142 143 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); 144 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); 145 146} 147 148#define CN10K_DDR_PMU_EVENT_ATTR(_name, _id) \ 149 PMU_EVENT_ATTR_ID(_name, cn10k_ddr_pmu_event_show, _id) 150 151static struct attribute *cn10k_ddr_perf_events_attrs[] = { 152 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR), 153 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR), 154 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD), 155 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW), 156 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD), 157 CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS), 158 CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS), 159 CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_wr_data_access, EVENT_DFI_WR_DATA_CYCLES), 160 CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_rd_data_access, EVENT_DFI_RD_DATA_CYCLES), 161 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access, 162 EVENT_HPR_XACT_WHEN_CRITICAL), 163 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access, 164 EVENT_LPR_XACT_WHEN_CRITICAL), 165 CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access, 166 EVENT_WR_XACT_WHEN_CRITICAL), 167 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE), 168 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access, EVENT_OP_IS_RD_OR_WR), 169 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access, EVENT_OP_IS_RD_ACTIVATE), 170 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD), 171 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR), 172 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR), 173 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE), 174 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr, EVENT_PRECHARGE_FOR_RDWR), 175 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other, 176 EVENT_PRECHARGE_FOR_OTHER), 177 CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS), 178 CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE), 179 CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD), 180 CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD), 181 CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD), 182 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF), 183 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown, EVENT_OP_IS_ENTER_POWERDOWN), 184 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_mpsm, EVENT_OP_IS_ENTER_MPSM), 185 CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH), 186 CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF), 187 CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF), 188 CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE), 189 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqcl, EVENT_OP_IS_ZQCL), 190 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_wr_access, EVENT_OP_IS_ZQCS), 191 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpr_req_with_nocredit, 192 EVENT_HPR_REQ_WITH_NOCREDIT), 193 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpr_req_with_nocredit, 194 EVENT_LPR_REQ_WITH_NOCREDIT), 195 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_alloc, EVENT_BSM_ALLOC), 196 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_starvation, EVENT_BSM_STARVATION), 197 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd, 198 EVENT_VISIBLE_WIN_LIMIT_REACHED_RD), 199 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr, 200 EVENT_VISIBLE_WIN_LIMIT_REACHED_WR), 201 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_OP_IS_DQSOSC_MPC), 202 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_OP_IS_DQSOSC_MRR), 203 CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_OP_IS_TCR_MRR), 204 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_OP_IS_ZQSTART), 205 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_OP_IS_ZQLATCH), 206 /* Free run event counters */ 207 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS), 208 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES), 209 NULL 210}; 211 212static struct attribute_group cn10k_ddr_perf_events_attr_group = { 213 .name = "events", 214 .attrs = cn10k_ddr_perf_events_attrs, 215}; 216 217PMU_FORMAT_ATTR(event, "config:0-8"); 218 219static struct attribute *cn10k_ddr_perf_format_attrs[] = { 220 &format_attr_event.attr, 221 NULL, 222}; 223 224static struct attribute_group cn10k_ddr_perf_format_attr_group = { 225 .name = "format", 226 .attrs = cn10k_ddr_perf_format_attrs, 227}; 228 229static ssize_t cn10k_ddr_perf_cpumask_show(struct device *dev, 230 struct device_attribute *attr, 231 char *buf) 232{ 233 struct cn10k_ddr_pmu *pmu = dev_get_drvdata(dev); 234 235 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu)); 236} 237 238static struct device_attribute cn10k_ddr_perf_cpumask_attr = 239 __ATTR(cpumask, 0444, cn10k_ddr_perf_cpumask_show, NULL); 240 241static struct attribute *cn10k_ddr_perf_cpumask_attrs[] = { 242 &cn10k_ddr_perf_cpumask_attr.attr, 243 NULL, 244}; 245 246static struct attribute_group cn10k_ddr_perf_cpumask_attr_group = { 247 .attrs = cn10k_ddr_perf_cpumask_attrs, 248}; 249 250static const struct attribute_group *cn10k_attr_groups[] = { 251 &cn10k_ddr_perf_events_attr_group, 252 &cn10k_ddr_perf_format_attr_group, 253 &cn10k_ddr_perf_cpumask_attr_group, 254 NULL, 255}; 256 257/* Default poll timeout is 100 sec, which is very sufficient for 258 * 48 bit counter incremented max at 5.6 GT/s, which may take many 259 * hours to overflow. 260 */ 261static unsigned long cn10k_ddr_pmu_poll_period_sec = 100; 262module_param_named(poll_period_sec, cn10k_ddr_pmu_poll_period_sec, ulong, 0644); 263 264static ktime_t cn10k_ddr_pmu_timer_period(void) 265{ 266 return ms_to_ktime((u64)cn10k_ddr_pmu_poll_period_sec * USEC_PER_SEC); 267} 268 269static int ddr_perf_get_event_bitmap(int eventid, u64 *event_bitmap) 270{ 271 switch (eventid) { 272 case EVENT_HIF_RD_OR_WR ... EVENT_WAW_HAZARD: 273 case EVENT_OP_IS_REFRESH ... EVENT_OP_IS_ZQLATCH: 274 *event_bitmap = (1ULL << (eventid - 1)); 275 break; 276 case EVENT_OP_IS_ENTER_SELFREF: 277 case EVENT_OP_IS_ENTER_POWERDOWN: 278 case EVENT_OP_IS_ENTER_MPSM: 279 *event_bitmap = (0xFULL << (eventid - 1)); 280 break; 281 default: 282 pr_err("%s Invalid eventid %d\n", __func__, eventid); 283 return -EINVAL; 284 } 285 286 return 0; 287} 288 289static int cn10k_ddr_perf_alloc_counter(struct cn10k_ddr_pmu *pmu, 290 struct perf_event *event) 291{ 292 u8 config = event->attr.config; 293 int i; 294 295 /* DDR read free-run counter index */ 296 if (config == EVENT_DDR_READS) { 297 pmu->events[DDRC_PERF_READ_COUNTER_IDX] = event; 298 return DDRC_PERF_READ_COUNTER_IDX; 299 } 300 301 /* DDR write free-run counter index */ 302 if (config == EVENT_DDR_WRITES) { 303 pmu->events[DDRC_PERF_WRITE_COUNTER_IDX] = event; 304 return DDRC_PERF_WRITE_COUNTER_IDX; 305 } 306 307 /* Allocate DDR generic counters */ 308 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) { 309 if (pmu->events[i] == NULL) { 310 pmu->events[i] = event; 311 return i; 312 } 313 } 314 315 return -ENOENT; 316} 317 318static void cn10k_ddr_perf_free_counter(struct cn10k_ddr_pmu *pmu, int counter) 319{ 320 pmu->events[counter] = NULL; 321} 322 323static int cn10k_ddr_perf_event_init(struct perf_event *event) 324{ 325 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); 326 struct hw_perf_event *hwc = &event->hw; 327 328 if (event->attr.type != event->pmu->type) 329 return -ENOENT; 330 331 if (is_sampling_event(event)) { 332 dev_info(pmu->dev, "Sampling not supported!\n"); 333 return -EOPNOTSUPP; 334 } 335 336 if (event->cpu < 0) { 337 dev_warn(pmu->dev, "Can't provide per-task data!\n"); 338 return -EOPNOTSUPP; 339 } 340 341 /* We must NOT create groups containing mixed PMUs */ 342 if (event->group_leader->pmu != event->pmu && 343 !is_software_event(event->group_leader)) 344 return -EINVAL; 345 346 /* Set ownership of event to one CPU, same event can not be observed 347 * on multiple cpus at same time. 348 */ 349 event->cpu = pmu->cpu; 350 hwc->idx = -1; 351 return 0; 352} 353 354static void cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu *pmu, 355 int counter, bool enable) 356{ 357 u32 reg; 358 u64 val; 359 360 if (counter > DDRC_PERF_NUM_COUNTERS) { 361 pr_err("Error: unsupported counter %d\n", counter); 362 return; 363 } 364 365 if (counter < DDRC_PERF_NUM_GEN_COUNTERS) { 366 reg = DDRC_PERF_CFG(counter); 367 val = readq_relaxed(pmu->base + reg); 368 369 if (enable) 370 val |= EVENT_ENABLE; 371 else 372 val &= ~EVENT_ENABLE; 373 374 writeq_relaxed(val, pmu->base + reg); 375 } else { 376 val = readq_relaxed(pmu->base + DDRC_PERF_CNT_FREERUN_EN); 377 if (enable) { 378 if (counter == DDRC_PERF_READ_COUNTER_IDX) 379 val |= DDRC_PERF_FREERUN_READ_EN; 380 else 381 val |= DDRC_PERF_FREERUN_WRITE_EN; 382 } else { 383 if (counter == DDRC_PERF_READ_COUNTER_IDX) 384 val &= ~DDRC_PERF_FREERUN_READ_EN; 385 else 386 val &= ~DDRC_PERF_FREERUN_WRITE_EN; 387 } 388 writeq_relaxed(val, pmu->base + DDRC_PERF_CNT_FREERUN_EN); 389 } 390} 391 392static u64 cn10k_ddr_perf_read_counter(struct cn10k_ddr_pmu *pmu, int counter) 393{ 394 u64 val; 395 396 if (counter == DDRC_PERF_READ_COUNTER_IDX) 397 return readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE_RD_OP); 398 399 if (counter == DDRC_PERF_WRITE_COUNTER_IDX) 400 return readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE_WR_OP); 401 402 val = readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE(counter)); 403 return val; 404} 405 406static void cn10k_ddr_perf_event_update(struct perf_event *event) 407{ 408 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); 409 struct hw_perf_event *hwc = &event->hw; 410 u64 prev_count, new_count, mask; 411 412 do { 413 prev_count = local64_read(&hwc->prev_count); 414 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx); 415 } while (local64_xchg(&hwc->prev_count, new_count) != prev_count); 416 417 mask = DDRC_PERF_CNT_MAX_VALUE; 418 419 local64_add((new_count - prev_count) & mask, &event->count); 420} 421 422static void cn10k_ddr_perf_event_start(struct perf_event *event, int flags) 423{ 424 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); 425 struct hw_perf_event *hwc = &event->hw; 426 int counter = hwc->idx; 427 428 local64_set(&hwc->prev_count, 0); 429 430 cn10k_ddr_perf_counter_enable(pmu, counter, true); 431 432 hwc->state = 0; 433} 434 435static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags) 436{ 437 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); 438 struct hw_perf_event *hwc = &event->hw; 439 u8 config = event->attr.config; 440 int counter, ret; 441 u32 reg_offset; 442 u64 val; 443 444 counter = cn10k_ddr_perf_alloc_counter(pmu, event); 445 if (counter < 0) 446 return -EAGAIN; 447 448 pmu->active_events++; 449 hwc->idx = counter; 450 451 if (pmu->active_events == 1) 452 hrtimer_start(&pmu->hrtimer, cn10k_ddr_pmu_timer_period(), 453 HRTIMER_MODE_REL_PINNED); 454 455 if (counter < DDRC_PERF_NUM_GEN_COUNTERS) { 456 /* Generic counters, configure event id */ 457 reg_offset = DDRC_PERF_CFG(counter); 458 ret = ddr_perf_get_event_bitmap(config, &val); 459 if (ret) 460 return ret; 461 462 writeq_relaxed(val, pmu->base + reg_offset); 463 } else { 464 /* fixed event counter, clear counter value */ 465 if (counter == DDRC_PERF_READ_COUNTER_IDX) 466 val = DDRC_FREERUN_READ_CNT_CLR; 467 else 468 val = DDRC_FREERUN_WRITE_CNT_CLR; 469 470 writeq_relaxed(val, pmu->base + DDRC_PERF_CNT_FREERUN_CTRL); 471 } 472 473 hwc->state |= PERF_HES_STOPPED; 474 475 if (flags & PERF_EF_START) 476 cn10k_ddr_perf_event_start(event, flags); 477 478 return 0; 479} 480 481static void cn10k_ddr_perf_event_stop(struct perf_event *event, int flags) 482{ 483 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); 484 struct hw_perf_event *hwc = &event->hw; 485 int counter = hwc->idx; 486 487 cn10k_ddr_perf_counter_enable(pmu, counter, false); 488 489 if (flags & PERF_EF_UPDATE) 490 cn10k_ddr_perf_event_update(event); 491 492 hwc->state |= PERF_HES_STOPPED; 493} 494 495static void cn10k_ddr_perf_event_del(struct perf_event *event, int flags) 496{ 497 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); 498 struct hw_perf_event *hwc = &event->hw; 499 int counter = hwc->idx; 500 501 cn10k_ddr_perf_event_stop(event, PERF_EF_UPDATE); 502 503 cn10k_ddr_perf_free_counter(pmu, counter); 504 pmu->active_events--; 505 hwc->idx = -1; 506 507 /* Cancel timer when no events to capture */ 508 if (pmu->active_events == 0) 509 hrtimer_cancel(&pmu->hrtimer); 510} 511 512static void cn10k_ddr_perf_pmu_enable(struct pmu *pmu) 513{ 514 struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu); 515 516 writeq_relaxed(START_OP_CTRL_VAL_START, ddr_pmu->base + 517 DDRC_PERF_CNT_START_OP_CTRL); 518} 519 520static void cn10k_ddr_perf_pmu_disable(struct pmu *pmu) 521{ 522 struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu); 523 524 writeq_relaxed(END_OP_CTRL_VAL_END, ddr_pmu->base + 525 DDRC_PERF_CNT_END_OP_CTRL); 526} 527 528static void cn10k_ddr_perf_event_update_all(struct cn10k_ddr_pmu *pmu) 529{ 530 struct hw_perf_event *hwc; 531 int i; 532 533 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) { 534 if (pmu->events[i] == NULL) 535 continue; 536 537 cn10k_ddr_perf_event_update(pmu->events[i]); 538 } 539 540 /* Reset previous count as h/w counter are reset */ 541 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) { 542 if (pmu->events[i] == NULL) 543 continue; 544 545 hwc = &pmu->events[i]->hw; 546 local64_set(&hwc->prev_count, 0); 547 } 548} 549 550static irqreturn_t cn10k_ddr_pmu_overflow_handler(struct cn10k_ddr_pmu *pmu) 551{ 552 struct perf_event *event; 553 struct hw_perf_event *hwc; 554 u64 prev_count, new_count; 555 u64 value; 556 int i; 557 558 event = pmu->events[DDRC_PERF_READ_COUNTER_IDX]; 559 if (event) { 560 hwc = &event->hw; 561 prev_count = local64_read(&hwc->prev_count); 562 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx); 563 564 /* Overflow condition is when new count less than 565 * previous count 566 */ 567 if (new_count < prev_count) 568 cn10k_ddr_perf_event_update(event); 569 } 570 571 event = pmu->events[DDRC_PERF_WRITE_COUNTER_IDX]; 572 if (event) { 573 hwc = &event->hw; 574 prev_count = local64_read(&hwc->prev_count); 575 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx); 576 577 /* Overflow condition is when new count less than 578 * previous count 579 */ 580 if (new_count < prev_count) 581 cn10k_ddr_perf_event_update(event); 582 } 583 584 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) { 585 if (pmu->events[i] == NULL) 586 continue; 587 588 value = cn10k_ddr_perf_read_counter(pmu, i); 589 if (value == DDRC_PERF_CNT_MAX_VALUE) { 590 pr_info("Counter-(%d) reached max value\n", i); 591 cn10k_ddr_perf_event_update_all(pmu); 592 cn10k_ddr_perf_pmu_disable(&pmu->pmu); 593 cn10k_ddr_perf_pmu_enable(&pmu->pmu); 594 } 595 } 596 597 return IRQ_HANDLED; 598} 599 600static enum hrtimer_restart cn10k_ddr_pmu_timer_handler(struct hrtimer *hrtimer) 601{ 602 struct cn10k_ddr_pmu *pmu = container_of(hrtimer, struct cn10k_ddr_pmu, 603 hrtimer); 604 unsigned long flags; 605 606 local_irq_save(flags); 607 cn10k_ddr_pmu_overflow_handler(pmu); 608 local_irq_restore(flags); 609 610 hrtimer_forward_now(hrtimer, cn10k_ddr_pmu_timer_period()); 611 return HRTIMER_RESTART; 612} 613 614static int cn10k_ddr_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 615{ 616 struct cn10k_ddr_pmu *pmu = hlist_entry_safe(node, struct cn10k_ddr_pmu, 617 node); 618 unsigned int target; 619 620 if (cpu != pmu->cpu) 621 return 0; 622 623 target = cpumask_any_but(cpu_online_mask, cpu); 624 if (target >= nr_cpu_ids) 625 return 0; 626 627 perf_pmu_migrate_context(&pmu->pmu, cpu, target); 628 pmu->cpu = target; 629 return 0; 630} 631 632static int cn10k_ddr_perf_probe(struct platform_device *pdev) 633{ 634 struct cn10k_ddr_pmu *ddr_pmu; 635 struct resource *res; 636 void __iomem *base; 637 char *name; 638 int ret; 639 640 ddr_pmu = devm_kzalloc(&pdev->dev, sizeof(*ddr_pmu), GFP_KERNEL); 641 if (!ddr_pmu) 642 return -ENOMEM; 643 644 ddr_pmu->dev = &pdev->dev; 645 platform_set_drvdata(pdev, ddr_pmu); 646 647 base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); 648 if (IS_ERR(base)) 649 return PTR_ERR(base); 650 651 ddr_pmu->base = base; 652 653 /* Setup the PMU counter to work in manual mode */ 654 writeq_relaxed(OP_MODE_CTRL_VAL_MANNUAL, ddr_pmu->base + 655 DDRC_PERF_CNT_OP_MODE_CTRL); 656 657 ddr_pmu->pmu = (struct pmu) { 658 .module = THIS_MODULE, 659 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 660 .task_ctx_nr = perf_invalid_context, 661 .attr_groups = cn10k_attr_groups, 662 .event_init = cn10k_ddr_perf_event_init, 663 .add = cn10k_ddr_perf_event_add, 664 .del = cn10k_ddr_perf_event_del, 665 .start = cn10k_ddr_perf_event_start, 666 .stop = cn10k_ddr_perf_event_stop, 667 .read = cn10k_ddr_perf_event_update, 668 .pmu_enable = cn10k_ddr_perf_pmu_enable, 669 .pmu_disable = cn10k_ddr_perf_pmu_disable, 670 }; 671 672 /* Choose this cpu to collect perf data */ 673 ddr_pmu->cpu = raw_smp_processor_id(); 674 675 name = devm_kasprintf(ddr_pmu->dev, GFP_KERNEL, "mrvl_ddr_pmu_%llx", 676 res->start); 677 if (!name) 678 return -ENOMEM; 679 680 hrtimer_init(&ddr_pmu->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 681 ddr_pmu->hrtimer.function = cn10k_ddr_pmu_timer_handler; 682 683 cpuhp_state_add_instance_nocalls( 684 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE, 685 &ddr_pmu->node); 686 687 ret = perf_pmu_register(&ddr_pmu->pmu, name, -1); 688 if (ret) 689 goto error; 690 691 pr_info("CN10K DDR PMU Driver for ddrc@%llx\n", res->start); 692 return 0; 693error: 694 cpuhp_state_remove_instance_nocalls( 695 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE, 696 &ddr_pmu->node); 697 return ret; 698} 699 700static void cn10k_ddr_perf_remove(struct platform_device *pdev) 701{ 702 struct cn10k_ddr_pmu *ddr_pmu = platform_get_drvdata(pdev); 703 704 cpuhp_state_remove_instance_nocalls( 705 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE, 706 &ddr_pmu->node); 707 708 perf_pmu_unregister(&ddr_pmu->pmu); 709} 710 711#ifdef CONFIG_OF 712static const struct of_device_id cn10k_ddr_pmu_of_match[] = { 713 { .compatible = "marvell,cn10k-ddr-pmu", }, 714 { }, 715}; 716MODULE_DEVICE_TABLE(of, cn10k_ddr_pmu_of_match); 717#endif 718 719#ifdef CONFIG_ACPI 720static const struct acpi_device_id cn10k_ddr_pmu_acpi_match[] = { 721 {"MRVL000A", 0}, 722 {}, 723}; 724MODULE_DEVICE_TABLE(acpi, cn10k_ddr_pmu_acpi_match); 725#endif 726 727static struct platform_driver cn10k_ddr_pmu_driver = { 728 .driver = { 729 .name = "cn10k-ddr-pmu", 730 .of_match_table = of_match_ptr(cn10k_ddr_pmu_of_match), 731 .acpi_match_table = ACPI_PTR(cn10k_ddr_pmu_acpi_match), 732 .suppress_bind_attrs = true, 733 }, 734 .probe = cn10k_ddr_perf_probe, 735 .remove_new = cn10k_ddr_perf_remove, 736}; 737 738static int __init cn10k_ddr_pmu_init(void) 739{ 740 int ret; 741 742 ret = cpuhp_setup_state_multi( 743 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE, 744 "perf/marvell/cn10k/ddr:online", NULL, 745 cn10k_ddr_pmu_offline_cpu); 746 if (ret) 747 return ret; 748 749 ret = platform_driver_register(&cn10k_ddr_pmu_driver); 750 if (ret) 751 cpuhp_remove_multi_state( 752 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE); 753 return ret; 754} 755 756static void __exit cn10k_ddr_pmu_exit(void) 757{ 758 platform_driver_unregister(&cn10k_ddr_pmu_driver); 759 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE); 760} 761 762module_init(cn10k_ddr_pmu_init); 763module_exit(cn10k_ddr_pmu_exit); 764 765MODULE_AUTHOR("Bharat Bhushan <bbhushan2@marvell.com>"); 766MODULE_LICENSE("GPL v2"); 767