1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright(C) 2016 Linaro Limited. All rights reserved. 4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org> 5 */ 6 7#include <linux/atomic.h> 8#include <linux/circ_buf.h> 9#include <linux/coresight.h> 10#include <linux/perf_event.h> 11#include <linux/slab.h> 12#include "coresight-priv.h" 13#include "coresight-tmc.h" 14#include "coresight-etm-perf.h" 15 16static int tmc_set_etf_buffer(struct coresight_device *csdev, 17 struct perf_output_handle *handle); 18 19static int __tmc_etb_enable_hw(struct tmc_drvdata *drvdata) 20{ 21 int rc = 0; 22 23 CS_UNLOCK(drvdata->base); 24 25 /* Wait for TMCSReady bit to be set */ 26 rc = tmc_wait_for_tmcready(drvdata); 27 if (rc) { 28 dev_err(&drvdata->csdev->dev, 29 "Failed to enable: TMC not ready\n"); 30 CS_LOCK(drvdata->base); 31 return rc; 32 } 33 34 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE); 35 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI | 36 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT | 37 TMC_FFCR_TRIGON_TRIGIN, 38 drvdata->base + TMC_FFCR); 39 40 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG); 41 tmc_enable_hw(drvdata); 42 43 CS_LOCK(drvdata->base); 44 return rc; 45} 46 47static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata) 48{ 49 int rc = coresight_claim_device(drvdata->csdev); 50 51 if (rc) 52 return rc; 53 54 rc = __tmc_etb_enable_hw(drvdata); 55 if (rc) 56 coresight_disclaim_device(drvdata->csdev); 57 return rc; 58} 59 60static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata) 61{ 62 char *bufp; 63 u32 read_data, lost; 64 65 /* Check if the buffer wrapped around. */ 66 lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL; 67 bufp = drvdata->buf; 68 drvdata->len = 0; 69 while (1) { 70 read_data = readl_relaxed(drvdata->base + TMC_RRD); 71 if (read_data == 0xFFFFFFFF) 72 break; 73 memcpy(bufp, &read_data, 4); 74 bufp += 4; 75 drvdata->len += 4; 76 } 77 78 if (lost) 79 coresight_insert_barrier_packet(drvdata->buf); 80 return; 81} 82 83static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata) 84{ 85 CS_UNLOCK(drvdata->base); 86 87 tmc_flush_and_stop(drvdata); 88 /* 89 * When operating in sysFS mode the content of the buffer needs to be 90 * read before the TMC is disabled. 91 */ 92 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) 93 tmc_etb_dump_hw(drvdata); 94 tmc_disable_hw(drvdata); 95 96 CS_LOCK(drvdata->base); 97} 98 99static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata) 100{ 101 __tmc_etb_disable_hw(drvdata); 102 coresight_disclaim_device(drvdata->csdev); 103} 104 105static int __tmc_etf_enable_hw(struct tmc_drvdata *drvdata) 106{ 107 int rc = 0; 108 109 CS_UNLOCK(drvdata->base); 110 111 /* Wait for TMCSReady bit to be set */ 112 rc = tmc_wait_for_tmcready(drvdata); 113 if (rc) { 114 dev_err(&drvdata->csdev->dev, 115 "Failed to enable : TMC is not ready\n"); 116 CS_LOCK(drvdata->base); 117 return rc; 118 } 119 120 writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE); 121 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI, 122 drvdata->base + TMC_FFCR); 123 writel_relaxed(0x0, drvdata->base + TMC_BUFWM); 124 tmc_enable_hw(drvdata); 125 126 CS_LOCK(drvdata->base); 127 return rc; 128} 129 130static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata) 131{ 132 int rc = coresight_claim_device(drvdata->csdev); 133 134 if (rc) 135 return rc; 136 137 rc = __tmc_etf_enable_hw(drvdata); 138 if (rc) 139 coresight_disclaim_device(drvdata->csdev); 140 return rc; 141} 142 143static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata) 144{ 145 struct coresight_device *csdev = drvdata->csdev; 146 147 CS_UNLOCK(drvdata->base); 148 149 tmc_flush_and_stop(drvdata); 150 tmc_disable_hw(drvdata); 151 coresight_disclaim_device_unlocked(csdev); 152 CS_LOCK(drvdata->base); 153} 154 155/* 156 * Return the available trace data in the buffer from @pos, with 157 * a maximum limit of @len, updating the @bufpp on where to 158 * find it. 159 */ 160ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata, 161 loff_t pos, size_t len, char **bufpp) 162{ 163 ssize_t actual = len; 164 165 /* Adjust the len to available size @pos */ 166 if (pos + actual > drvdata->len) 167 actual = drvdata->len - pos; 168 if (actual > 0) 169 *bufpp = drvdata->buf + pos; 170 return actual; 171} 172 173static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev) 174{ 175 int ret = 0; 176 bool used = false; 177 char *buf = NULL; 178 unsigned long flags; 179 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 180 181 /* 182 * If we don't have a buffer release the lock and allocate memory. 183 * Otherwise keep the lock and move along. 184 */ 185 spin_lock_irqsave(&drvdata->spinlock, flags); 186 if (!drvdata->buf) { 187 spin_unlock_irqrestore(&drvdata->spinlock, flags); 188 189 /* Allocating the memory here while outside of the spinlock */ 190 buf = kzalloc(drvdata->size, GFP_KERNEL); 191 if (!buf) 192 return -ENOMEM; 193 194 /* Let's try again */ 195 spin_lock_irqsave(&drvdata->spinlock, flags); 196 } 197 198 if (drvdata->reading) { 199 ret = -EBUSY; 200 goto out; 201 } 202 203 /* 204 * In sysFS mode we can have multiple writers per sink. Since this 205 * sink is already enabled no memory is needed and the HW need not be 206 * touched. 207 */ 208 if (coresight_get_mode(csdev) == CS_MODE_SYSFS) { 209 csdev->refcnt++; 210 goto out; 211 } 212 213 /* 214 * If drvdata::buf isn't NULL, memory was allocated for a previous 215 * trace run but wasn't read. If so simply zero-out the memory. 216 * Otherwise use the memory allocated above. 217 * 218 * The memory is freed when users read the buffer using the 219 * /dev/xyz.{etf|etb} interface. See tmc_read_unprepare_etf() for 220 * details. 221 */ 222 if (drvdata->buf) { 223 memset(drvdata->buf, 0, drvdata->size); 224 } else { 225 used = true; 226 drvdata->buf = buf; 227 } 228 229 ret = tmc_etb_enable_hw(drvdata); 230 if (!ret) { 231 coresight_set_mode(csdev, CS_MODE_SYSFS); 232 csdev->refcnt++; 233 } else { 234 /* Free up the buffer if we failed to enable */ 235 used = false; 236 } 237out: 238 spin_unlock_irqrestore(&drvdata->spinlock, flags); 239 240 /* Free memory outside the spinlock if need be */ 241 if (!used) 242 kfree(buf); 243 244 return ret; 245} 246 247static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data) 248{ 249 int ret = 0; 250 pid_t pid; 251 unsigned long flags; 252 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 253 struct perf_output_handle *handle = data; 254 struct cs_buffers *buf = etm_perf_sink_config(handle); 255 256 spin_lock_irqsave(&drvdata->spinlock, flags); 257 do { 258 ret = -EINVAL; 259 if (drvdata->reading) 260 break; 261 /* 262 * No need to continue if the ETB/ETF is already operated 263 * from sysFS. 264 */ 265 if (coresight_get_mode(csdev) == CS_MODE_SYSFS) { 266 ret = -EBUSY; 267 break; 268 } 269 270 /* Get a handle on the pid of the process to monitor */ 271 pid = buf->pid; 272 273 if (drvdata->pid != -1 && drvdata->pid != pid) { 274 ret = -EBUSY; 275 break; 276 } 277 278 ret = tmc_set_etf_buffer(csdev, handle); 279 if (ret) 280 break; 281 282 /* 283 * No HW configuration is needed if the sink is already in 284 * use for this session. 285 */ 286 if (drvdata->pid == pid) { 287 csdev->refcnt++; 288 break; 289 } 290 291 ret = tmc_etb_enable_hw(drvdata); 292 if (!ret) { 293 /* Associate with monitored process. */ 294 drvdata->pid = pid; 295 coresight_set_mode(csdev, CS_MODE_PERF); 296 csdev->refcnt++; 297 } 298 } while (0); 299 spin_unlock_irqrestore(&drvdata->spinlock, flags); 300 301 return ret; 302} 303 304static int tmc_enable_etf_sink(struct coresight_device *csdev, 305 enum cs_mode mode, void *data) 306{ 307 int ret; 308 309 switch (mode) { 310 case CS_MODE_SYSFS: 311 ret = tmc_enable_etf_sink_sysfs(csdev); 312 break; 313 case CS_MODE_PERF: 314 ret = tmc_enable_etf_sink_perf(csdev, data); 315 break; 316 /* We shouldn't be here */ 317 default: 318 ret = -EINVAL; 319 break; 320 } 321 322 if (ret) 323 return ret; 324 325 dev_dbg(&csdev->dev, "TMC-ETB/ETF enabled\n"); 326 return 0; 327} 328 329static int tmc_disable_etf_sink(struct coresight_device *csdev) 330{ 331 unsigned long flags; 332 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 333 334 spin_lock_irqsave(&drvdata->spinlock, flags); 335 336 if (drvdata->reading) { 337 spin_unlock_irqrestore(&drvdata->spinlock, flags); 338 return -EBUSY; 339 } 340 341 csdev->refcnt--; 342 if (csdev->refcnt) { 343 spin_unlock_irqrestore(&drvdata->spinlock, flags); 344 return -EBUSY; 345 } 346 347 /* Complain if we (somehow) got out of sync */ 348 WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED); 349 tmc_etb_disable_hw(drvdata); 350 /* Dissociate from monitored process. */ 351 drvdata->pid = -1; 352 coresight_set_mode(csdev, CS_MODE_DISABLED); 353 354 spin_unlock_irqrestore(&drvdata->spinlock, flags); 355 356 dev_dbg(&csdev->dev, "TMC-ETB/ETF disabled\n"); 357 return 0; 358} 359 360static int tmc_enable_etf_link(struct coresight_device *csdev, 361 struct coresight_connection *in, 362 struct coresight_connection *out) 363{ 364 int ret = 0; 365 unsigned long flags; 366 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 367 bool first_enable = false; 368 369 spin_lock_irqsave(&drvdata->spinlock, flags); 370 if (drvdata->reading) { 371 spin_unlock_irqrestore(&drvdata->spinlock, flags); 372 return -EBUSY; 373 } 374 375 if (csdev->refcnt == 0) { 376 ret = tmc_etf_enable_hw(drvdata); 377 if (!ret) { 378 coresight_set_mode(csdev, CS_MODE_SYSFS); 379 first_enable = true; 380 } 381 } 382 if (!ret) 383 csdev->refcnt++; 384 spin_unlock_irqrestore(&drvdata->spinlock, flags); 385 386 if (first_enable) 387 dev_dbg(&csdev->dev, "TMC-ETF enabled\n"); 388 return ret; 389} 390 391static void tmc_disable_etf_link(struct coresight_device *csdev, 392 struct coresight_connection *in, 393 struct coresight_connection *out) 394{ 395 unsigned long flags; 396 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 397 bool last_disable = false; 398 399 spin_lock_irqsave(&drvdata->spinlock, flags); 400 if (drvdata->reading) { 401 spin_unlock_irqrestore(&drvdata->spinlock, flags); 402 return; 403 } 404 405 csdev->refcnt--; 406 if (csdev->refcnt == 0) { 407 tmc_etf_disable_hw(drvdata); 408 coresight_set_mode(csdev, CS_MODE_DISABLED); 409 last_disable = true; 410 } 411 spin_unlock_irqrestore(&drvdata->spinlock, flags); 412 413 if (last_disable) 414 dev_dbg(&csdev->dev, "TMC-ETF disabled\n"); 415} 416 417static void *tmc_alloc_etf_buffer(struct coresight_device *csdev, 418 struct perf_event *event, void **pages, 419 int nr_pages, bool overwrite) 420{ 421 int node; 422 struct cs_buffers *buf; 423 424 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu); 425 426 /* Allocate memory structure for interaction with Perf */ 427 buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node); 428 if (!buf) 429 return NULL; 430 431 buf->pid = task_pid_nr(event->owner); 432 buf->snapshot = overwrite; 433 buf->nr_pages = nr_pages; 434 buf->data_pages = pages; 435 436 return buf; 437} 438 439static void tmc_free_etf_buffer(void *config) 440{ 441 struct cs_buffers *buf = config; 442 443 kfree(buf); 444} 445 446static int tmc_set_etf_buffer(struct coresight_device *csdev, 447 struct perf_output_handle *handle) 448{ 449 int ret = 0; 450 unsigned long head; 451 struct cs_buffers *buf = etm_perf_sink_config(handle); 452 453 if (!buf) 454 return -EINVAL; 455 456 /* wrap head around to the amount of space we have */ 457 head = handle->head & (((unsigned long)buf->nr_pages << PAGE_SHIFT) - 1); 458 459 /* find the page to write to */ 460 buf->cur = head / PAGE_SIZE; 461 462 /* and offset within that page */ 463 buf->offset = head % PAGE_SIZE; 464 465 local_set(&buf->data_size, 0); 466 467 return ret; 468} 469 470static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev, 471 struct perf_output_handle *handle, 472 void *sink_config) 473{ 474 bool lost = false; 475 int i, cur; 476 const u32 *barrier; 477 u32 *buf_ptr; 478 u64 read_ptr, write_ptr; 479 u32 status; 480 unsigned long offset, to_read = 0, flags; 481 struct cs_buffers *buf = sink_config; 482 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); 483 484 if (!buf) 485 return 0; 486 487 /* This shouldn't happen */ 488 if (WARN_ON_ONCE(coresight_get_mode(csdev) != CS_MODE_PERF)) 489 return 0; 490 491 spin_lock_irqsave(&drvdata->spinlock, flags); 492 493 /* Don't do anything if another tracer is using this sink */ 494 if (csdev->refcnt != 1) 495 goto out; 496 497 CS_UNLOCK(drvdata->base); 498 499 tmc_flush_and_stop(drvdata); 500 501 read_ptr = tmc_read_rrp(drvdata); 502 write_ptr = tmc_read_rwp(drvdata); 503 504 /* 505 * Get a hold of the status register and see if a wrap around 506 * has occurred. If so adjust things accordingly. 507 */ 508 status = readl_relaxed(drvdata->base + TMC_STS); 509 if (status & TMC_STS_FULL) { 510 lost = true; 511 to_read = drvdata->size; 512 } else { 513 to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size); 514 } 515 516 /* 517 * The TMC RAM buffer may be bigger than the space available in the 518 * perf ring buffer (handle->size). If so advance the RRP so that we 519 * get the latest trace data. In snapshot mode none of that matters 520 * since we are expected to clobber stale data in favour of the latest 521 * traces. 522 */ 523 if (!buf->snapshot && to_read > handle->size) { 524 u32 mask = tmc_get_memwidth_mask(drvdata); 525 526 /* 527 * Make sure the new size is aligned in accordance with the 528 * requirement explained in function tmc_get_memwidth_mask(). 529 */ 530 to_read = handle->size & mask; 531 /* Move the RAM read pointer up */ 532 read_ptr = (write_ptr + drvdata->size) - to_read; 533 /* Make sure we are still within our limits */ 534 if (read_ptr > (drvdata->size - 1)) 535 read_ptr -= drvdata->size; 536 /* Tell the HW */ 537 tmc_write_rrp(drvdata, read_ptr); 538 lost = true; 539 } 540 541 /* 542 * Don't set the TRUNCATED flag in snapshot mode because 1) the 543 * captured buffer is expected to be truncated and 2) a full buffer 544 * prevents the event from being re-enabled by the perf core, 545 * resulting in stale data being send to user space. 546 */ 547 if (!buf->snapshot && lost) 548 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); 549 550 cur = buf->cur; 551 offset = buf->offset; 552 barrier = coresight_barrier_pkt; 553 554 /* for every byte to read */ 555 for (i = 0; i < to_read; i += 4) { 556 buf_ptr = buf->data_pages[cur] + offset; 557 *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD); 558 559 if (lost && i < CORESIGHT_BARRIER_PKT_SIZE) { 560 *buf_ptr = *barrier; 561 barrier++; 562 } 563 564 offset += 4; 565 if (offset >= PAGE_SIZE) { 566 offset = 0; 567 cur++; 568 /* wrap around at the end of the buffer */ 569 cur &= buf->nr_pages - 1; 570 } 571 } 572 573 /* 574 * In snapshot mode we simply increment the head by the number of byte 575 * that were written. User space will figure out how many bytes to get 576 * from the AUX buffer based on the position of the head. 577 */ 578 if (buf->snapshot) 579 handle->head += to_read; 580 581 /* 582 * CS_LOCK() contains mb() so it can ensure visibility of the AUX trace 583 * data before the aux_head is updated via perf_aux_output_end(), which 584 * is expected by the perf ring buffer. 585 */ 586 CS_LOCK(drvdata->base); 587out: 588 spin_unlock_irqrestore(&drvdata->spinlock, flags); 589 590 return to_read; 591} 592 593static const struct coresight_ops_sink tmc_etf_sink_ops = { 594 .enable = tmc_enable_etf_sink, 595 .disable = tmc_disable_etf_sink, 596 .alloc_buffer = tmc_alloc_etf_buffer, 597 .free_buffer = tmc_free_etf_buffer, 598 .update_buffer = tmc_update_etf_buffer, 599}; 600 601static const struct coresight_ops_link tmc_etf_link_ops = { 602 .enable = tmc_enable_etf_link, 603 .disable = tmc_disable_etf_link, 604}; 605 606const struct coresight_ops tmc_etb_cs_ops = { 607 .sink_ops = &tmc_etf_sink_ops, 608}; 609 610const struct coresight_ops tmc_etf_cs_ops = { 611 .sink_ops = &tmc_etf_sink_ops, 612 .link_ops = &tmc_etf_link_ops, 613}; 614 615int tmc_read_prepare_etb(struct tmc_drvdata *drvdata) 616{ 617 enum tmc_mode mode; 618 int ret = 0; 619 unsigned long flags; 620 621 /* config types are set a boot time and never change */ 622 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB && 623 drvdata->config_type != TMC_CONFIG_TYPE_ETF)) 624 return -EINVAL; 625 626 spin_lock_irqsave(&drvdata->spinlock, flags); 627 628 if (drvdata->reading) { 629 ret = -EBUSY; 630 goto out; 631 } 632 633 /* Don't interfere if operated from Perf */ 634 if (coresight_get_mode(drvdata->csdev) == CS_MODE_PERF) { 635 ret = -EINVAL; 636 goto out; 637 } 638 639 /* If drvdata::buf is NULL the trace data has been read already */ 640 if (drvdata->buf == NULL) { 641 ret = -EINVAL; 642 goto out; 643 } 644 645 /* Disable the TMC if need be */ 646 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) { 647 /* There is no point in reading a TMC in HW FIFO mode */ 648 mode = readl_relaxed(drvdata->base + TMC_MODE); 649 if (mode != TMC_MODE_CIRCULAR_BUFFER) { 650 ret = -EINVAL; 651 goto out; 652 } 653 __tmc_etb_disable_hw(drvdata); 654 } 655 656 drvdata->reading = true; 657out: 658 spin_unlock_irqrestore(&drvdata->spinlock, flags); 659 660 return ret; 661} 662 663int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata) 664{ 665 char *buf = NULL; 666 enum tmc_mode mode; 667 unsigned long flags; 668 int rc = 0; 669 670 /* config types are set a boot time and never change */ 671 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB && 672 drvdata->config_type != TMC_CONFIG_TYPE_ETF)) 673 return -EINVAL; 674 675 spin_lock_irqsave(&drvdata->spinlock, flags); 676 677 /* Re-enable the TMC if need be */ 678 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) { 679 /* There is no point in reading a TMC in HW FIFO mode */ 680 mode = readl_relaxed(drvdata->base + TMC_MODE); 681 if (mode != TMC_MODE_CIRCULAR_BUFFER) { 682 spin_unlock_irqrestore(&drvdata->spinlock, flags); 683 return -EINVAL; 684 } 685 /* 686 * The trace run will continue with the same allocated trace 687 * buffer. As such zero-out the buffer so that we don't end 688 * up with stale data. 689 * 690 * Since the tracer is still enabled drvdata::buf 691 * can't be NULL. 692 */ 693 memset(drvdata->buf, 0, drvdata->size); 694 rc = __tmc_etb_enable_hw(drvdata); 695 if (rc) { 696 spin_unlock_irqrestore(&drvdata->spinlock, flags); 697 return rc; 698 } 699 } else { 700 /* 701 * The ETB/ETF is not tracing and the buffer was just read. 702 * As such prepare to free the trace buffer. 703 */ 704 buf = drvdata->buf; 705 drvdata->buf = NULL; 706 } 707 708 drvdata->reading = false; 709 spin_unlock_irqrestore(&drvdata->spinlock, flags); 710 711 /* 712 * Free allocated memory outside of the spinlock. There is no need 713 * to assert the validity of 'buf' since calling kfree(NULL) is safe. 714 */ 715 kfree(buf); 716 717 return 0; 718} 719