1// SPDX-License-Identifier: MIT 2/* 3 * Copyright �� 2023-2024 Intel Corporation 4 */ 5 6#include <linux/anon_inodes.h> 7#include <linux/delay.h> 8#include <linux/nospec.h> 9#include <linux/poll.h> 10 11#include <drm/drm_drv.h> 12#include <drm/drm_managed.h> 13#include <drm/xe_drm.h> 14 15#include "abi/guc_actions_slpc_abi.h" 16#include "instructions/xe_mi_commands.h" 17#include "regs/xe_engine_regs.h" 18#include "regs/xe_gt_regs.h" 19#include "regs/xe_lrc_layout.h" 20#include "regs/xe_oa_regs.h" 21#include "xe_assert.h" 22#include "xe_bb.h" 23#include "xe_bo.h" 24#include "xe_device.h" 25#include "xe_exec_queue.h" 26#include "xe_force_wake.h" 27#include "xe_gt.h" 28#include "xe_gt_mcr.h" 29#include "xe_gt_printk.h" 30#include "xe_guc_pc.h" 31#include "xe_lrc.h" 32#include "xe_macros.h" 33#include "xe_mmio.h" 34#include "xe_oa.h" 35#include "xe_observation.h" 36#include "xe_pm.h" 37#include "xe_sched_job.h" 38#include "xe_sriov.h" 39 40#define DEFAULT_POLL_FREQUENCY_HZ 200 41#define DEFAULT_POLL_PERIOD_NS (NSEC_PER_SEC / DEFAULT_POLL_FREQUENCY_HZ) 42#define XE_OA_UNIT_INVALID U32_MAX 43 44struct xe_oa_reg { 45 struct xe_reg addr; 46 u32 value; 47}; 48 49struct xe_oa_config { 50 struct xe_oa *oa; 51 52 char uuid[UUID_STRING_LEN + 1]; 53 int id; 54 55 const struct xe_oa_reg *regs; 56 u32 regs_len; 57 58 struct attribute_group sysfs_metric; 59 struct attribute *attrs[2]; 60 struct kobj_attribute sysfs_metric_id; 61 62 struct kref ref; 63 struct rcu_head rcu; 64}; 65 66struct flex { 67 struct xe_reg reg; 68 u32 offset; 69 u32 value; 70}; 71 72struct xe_oa_open_param { 73 u32 oa_unit_id; 74 bool sample; 75 u32 metric_set; 76 enum xe_oa_format_name oa_format; 77 int period_exponent; 78 bool disabled; 79 int exec_queue_id; 80 int engine_instance; 81 struct xe_exec_queue *exec_q; 82 struct xe_hw_engine *hwe; 83 bool no_preempt; 84}; 85 86struct xe_oa_config_bo { 87 struct llist_node node; 88 89 struct xe_oa_config *oa_config; 90 struct xe_bb *bb; 91}; 92 93#define DRM_FMT(x) DRM_XE_OA_FMT_TYPE_##x 94 95static const struct xe_oa_format oa_formats[] = { 96 [XE_OA_FORMAT_C4_B8] = { 7, 64, DRM_FMT(OAG) }, 97 [XE_OA_FORMAT_A12] = { 0, 64, DRM_FMT(OAG) }, 98 [XE_OA_FORMAT_A12_B8_C8] = { 2, 128, DRM_FMT(OAG) }, 99 [XE_OA_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256, DRM_FMT(OAG) }, 100 [XE_OAR_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256, DRM_FMT(OAR) }, 101 [XE_OA_FORMAT_A24u40_A14u32_B8_C8] = { 5, 256, DRM_FMT(OAG) }, 102 [XE_OAC_FORMAT_A24u64_B8_C8] = { 1, 320, DRM_FMT(OAC), HDR_64_BIT }, 103 [XE_OAC_FORMAT_A22u32_R2u32_B8_C8] = { 2, 192, DRM_FMT(OAC), HDR_64_BIT }, 104 [XE_OAM_FORMAT_MPEC8u64_B8_C8] = { 1, 192, DRM_FMT(OAM_MPEC), HDR_64_BIT }, 105 [XE_OAM_FORMAT_MPEC8u32_B8_C8] = { 2, 128, DRM_FMT(OAM_MPEC), HDR_64_BIT }, 106 [XE_OA_FORMAT_PEC64u64] = { 1, 576, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, 107 [XE_OA_FORMAT_PEC64u64_B8_C8] = { 1, 640, DRM_FMT(PEC), HDR_64_BIT, 1, 1 }, 108 [XE_OA_FORMAT_PEC64u32] = { 1, 320, DRM_FMT(PEC), HDR_64_BIT }, 109 [XE_OA_FORMAT_PEC32u64_G1] = { 5, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, 110 [XE_OA_FORMAT_PEC32u32_G1] = { 5, 192, DRM_FMT(PEC), HDR_64_BIT }, 111 [XE_OA_FORMAT_PEC32u64_G2] = { 6, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, 112 [XE_OA_FORMAT_PEC32u32_G2] = { 6, 192, DRM_FMT(PEC), HDR_64_BIT }, 113 [XE_OA_FORMAT_PEC36u64_G1_32_G2_4] = { 3, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, 114 [XE_OA_FORMAT_PEC36u64_G1_4_G2_32] = { 4, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 }, 115}; 116 117static u32 xe_oa_circ_diff(struct xe_oa_stream *stream, u32 tail, u32 head) 118{ 119 return tail >= head ? tail - head : 120 tail + stream->oa_buffer.circ_size - head; 121} 122 123static u32 xe_oa_circ_incr(struct xe_oa_stream *stream, u32 ptr, u32 n) 124{ 125 return ptr + n >= stream->oa_buffer.circ_size ? 126 ptr + n - stream->oa_buffer.circ_size : ptr + n; 127} 128 129static void xe_oa_config_release(struct kref *ref) 130{ 131 struct xe_oa_config *oa_config = 132 container_of(ref, typeof(*oa_config), ref); 133 134 kfree(oa_config->regs); 135 136 kfree_rcu(oa_config, rcu); 137} 138 139static void xe_oa_config_put(struct xe_oa_config *oa_config) 140{ 141 if (!oa_config) 142 return; 143 144 kref_put(&oa_config->ref, xe_oa_config_release); 145} 146 147static struct xe_oa_config *xe_oa_config_get(struct xe_oa_config *oa_config) 148{ 149 return kref_get_unless_zero(&oa_config->ref) ? oa_config : NULL; 150} 151 152static struct xe_oa_config *xe_oa_get_oa_config(struct xe_oa *oa, int metrics_set) 153{ 154 struct xe_oa_config *oa_config; 155 156 rcu_read_lock(); 157 oa_config = idr_find(&oa->metrics_idr, metrics_set); 158 if (oa_config) 159 oa_config = xe_oa_config_get(oa_config); 160 rcu_read_unlock(); 161 162 return oa_config; 163} 164 165static void free_oa_config_bo(struct xe_oa_config_bo *oa_bo) 166{ 167 xe_oa_config_put(oa_bo->oa_config); 168 xe_bb_free(oa_bo->bb, NULL); 169 kfree(oa_bo); 170} 171 172static const struct xe_oa_regs *__oa_regs(struct xe_oa_stream *stream) 173{ 174 return &stream->hwe->oa_unit->regs; 175} 176 177static u32 xe_oa_hw_tail_read(struct xe_oa_stream *stream) 178{ 179 return xe_mmio_read32(stream->gt, __oa_regs(stream)->oa_tail_ptr) & 180 OAG_OATAILPTR_MASK; 181} 182 183#define oa_report_header_64bit(__s) \ 184 ((__s)->oa_buffer.format->header == HDR_64_BIT) 185 186static u64 oa_report_id(struct xe_oa_stream *stream, void *report) 187{ 188 return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report; 189} 190 191static void oa_report_id_clear(struct xe_oa_stream *stream, u32 *report) 192{ 193 if (oa_report_header_64bit(stream)) 194 *(u64 *)report = 0; 195 else 196 *report = 0; 197} 198 199static u64 oa_timestamp(struct xe_oa_stream *stream, void *report) 200{ 201 return oa_report_header_64bit(stream) ? 202 *((u64 *)report + 1) : 203 *((u32 *)report + 1); 204} 205 206static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 *report) 207{ 208 if (oa_report_header_64bit(stream)) 209 *(u64 *)&report[2] = 0; 210 else 211 report[1] = 0; 212} 213 214static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream) 215{ 216 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); 217 int report_size = stream->oa_buffer.format->size; 218 u32 tail, hw_tail; 219 unsigned long flags; 220 bool pollin; 221 u32 partial_report_size; 222 223 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); 224 225 hw_tail = xe_oa_hw_tail_read(stream); 226 hw_tail -= gtt_offset; 227 228 /* 229 * The tail pointer increases in 64 byte (cacheline size), not in report_size 230 * increments. Also report size may not be a power of 2. Compute potential 231 * partially landed report in OA buffer. 232 */ 233 partial_report_size = xe_oa_circ_diff(stream, hw_tail, stream->oa_buffer.tail); 234 partial_report_size %= report_size; 235 236 /* Subtract partial amount off the tail */ 237 hw_tail = xe_oa_circ_diff(stream, hw_tail, partial_report_size); 238 239 tail = hw_tail; 240 241 /* 242 * Walk the stream backward until we find a report with report id and timestamp 243 * not 0. We can't tell whether a report has fully landed in memory before the 244 * report id and timestamp of the following report have landed. 245 * 246 * This is assuming that the writes of the OA unit land in memory in the order 247 * they were written. If not : (������������������� ��������� 248 */ 249 while (xe_oa_circ_diff(stream, tail, stream->oa_buffer.tail) >= report_size) { 250 void *report = stream->oa_buffer.vaddr + tail; 251 252 if (oa_report_id(stream, report) || oa_timestamp(stream, report)) 253 break; 254 255 tail = xe_oa_circ_diff(stream, tail, report_size); 256 } 257 258 if (xe_oa_circ_diff(stream, hw_tail, tail) > report_size) 259 drm_dbg(&stream->oa->xe->drm, 260 "unlanded report(s) head=0x%x tail=0x%x hw_tail=0x%x\n", 261 stream->oa_buffer.head, tail, hw_tail); 262 263 stream->oa_buffer.tail = tail; 264 265 pollin = xe_oa_circ_diff(stream, stream->oa_buffer.tail, 266 stream->oa_buffer.head) >= report_size; 267 268 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); 269 270 return pollin; 271} 272 273static enum hrtimer_restart xe_oa_poll_check_timer_cb(struct hrtimer *hrtimer) 274{ 275 struct xe_oa_stream *stream = 276 container_of(hrtimer, typeof(*stream), poll_check_timer); 277 278 if (xe_oa_buffer_check_unlocked(stream)) { 279 stream->pollin = true; 280 wake_up(&stream->poll_wq); 281 } 282 283 hrtimer_forward_now(hrtimer, ns_to_ktime(stream->poll_period_ns)); 284 285 return HRTIMER_RESTART; 286} 287 288static int xe_oa_append_report(struct xe_oa_stream *stream, char __user *buf, 289 size_t count, size_t *offset, const u8 *report) 290{ 291 int report_size = stream->oa_buffer.format->size; 292 int report_size_partial; 293 u8 *oa_buf_end; 294 295 if ((count - *offset) < report_size) 296 return -ENOSPC; 297 298 buf += *offset; 299 300 oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size; 301 report_size_partial = oa_buf_end - report; 302 303 if (report_size_partial < report_size) { 304 if (copy_to_user(buf, report, report_size_partial)) 305 return -EFAULT; 306 buf += report_size_partial; 307 308 if (copy_to_user(buf, stream->oa_buffer.vaddr, 309 report_size - report_size_partial)) 310 return -EFAULT; 311 } else if (copy_to_user(buf, report, report_size)) { 312 return -EFAULT; 313 } 314 315 *offset += report_size; 316 317 return 0; 318} 319 320static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf, 321 size_t count, size_t *offset) 322{ 323 int report_size = stream->oa_buffer.format->size; 324 u8 *oa_buf_base = stream->oa_buffer.vaddr; 325 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); 326 size_t start_offset = *offset; 327 unsigned long flags; 328 u32 head, tail; 329 int ret = 0; 330 331 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); 332 head = stream->oa_buffer.head; 333 tail = stream->oa_buffer.tail; 334 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); 335 336 xe_assert(stream->oa->xe, 337 head < stream->oa_buffer.circ_size && tail < stream->oa_buffer.circ_size); 338 339 for (; xe_oa_circ_diff(stream, tail, head); 340 head = xe_oa_circ_incr(stream, head, report_size)) { 341 u8 *report = oa_buf_base + head; 342 343 ret = xe_oa_append_report(stream, buf, count, offset, report); 344 if (ret) 345 break; 346 347 if (!(stream->oa_buffer.circ_size % report_size)) { 348 /* Clear out report id and timestamp to detect unlanded reports */ 349 oa_report_id_clear(stream, (void *)report); 350 oa_timestamp_clear(stream, (void *)report); 351 } else { 352 u8 *oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size; 353 u32 part = oa_buf_end - report; 354 355 /* Zero out the entire report */ 356 if (report_size <= part) { 357 memset(report, 0, report_size); 358 } else { 359 memset(report, 0, part); 360 memset(oa_buf_base, 0, report_size - part); 361 } 362 } 363 } 364 365 if (start_offset != *offset) { 366 struct xe_reg oaheadptr = __oa_regs(stream)->oa_head_ptr; 367 368 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); 369 xe_mmio_write32(stream->gt, oaheadptr, 370 (head + gtt_offset) & OAG_OAHEADPTR_MASK); 371 stream->oa_buffer.head = head; 372 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); 373 } 374 375 return ret; 376} 377 378static void xe_oa_init_oa_buffer(struct xe_oa_stream *stream) 379{ 380 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); 381 u32 oa_buf = gtt_offset | OABUFFER_SIZE_16M | OAG_OABUFFER_MEMORY_SELECT; 382 unsigned long flags; 383 384 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); 385 386 xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_status, 0); 387 xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_head_ptr, 388 gtt_offset & OAG_OAHEADPTR_MASK); 389 stream->oa_buffer.head = 0; 390 /* 391 * PRM says: "This MMIO must be set before the OATAILPTR register and after the 392 * OAHEADPTR register. This is to enable proper functionality of the overflow bit". 393 */ 394 xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_buffer, oa_buf); 395 xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_tail_ptr, 396 gtt_offset & OAG_OATAILPTR_MASK); 397 398 /* Mark that we need updated tail pointer to read from */ 399 stream->oa_buffer.tail = 0; 400 401 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); 402 403 /* Zero out the OA buffer since we rely on zero report id and timestamp fields */ 404 memset(stream->oa_buffer.vaddr, 0, stream->oa_buffer.bo->size); 405} 406 407static u32 __format_to_oactrl(const struct xe_oa_format *format, int counter_sel_mask) 408{ 409 return ((format->counter_select << (ffs(counter_sel_mask) - 1)) & counter_sel_mask) | 410 REG_FIELD_PREP(OA_OACONTROL_REPORT_BC_MASK, format->bc_report) | 411 REG_FIELD_PREP(OA_OACONTROL_COUNTER_SIZE_MASK, format->counter_size); 412} 413 414static u32 __oa_ccs_select(struct xe_oa_stream *stream) 415{ 416 u32 val; 417 418 if (stream->hwe->class != XE_ENGINE_CLASS_COMPUTE) 419 return 0; 420 421 val = REG_FIELD_PREP(OAG_OACONTROL_OA_CCS_SELECT_MASK, stream->hwe->instance); 422 xe_assert(stream->oa->xe, 423 REG_FIELD_GET(OAG_OACONTROL_OA_CCS_SELECT_MASK, val) == stream->hwe->instance); 424 return val; 425} 426 427static void xe_oa_enable(struct xe_oa_stream *stream) 428{ 429 const struct xe_oa_format *format = stream->oa_buffer.format; 430 const struct xe_oa_regs *regs; 431 u32 val; 432 433 /* 434 * BSpec: 46822: Bit 0. Even if stream->sample is 0, for OAR to function, the OA 435 * buffer must be correctly initialized 436 */ 437 xe_oa_init_oa_buffer(stream); 438 439 regs = __oa_regs(stream); 440 val = __format_to_oactrl(format, regs->oa_ctrl_counter_select_mask) | 441 __oa_ccs_select(stream) | OAG_OACONTROL_OA_COUNTER_ENABLE; 442 443 xe_mmio_write32(stream->gt, regs->oa_ctrl, val); 444} 445 446static void xe_oa_disable(struct xe_oa_stream *stream) 447{ 448 xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_ctrl, 0); 449 if (xe_mmio_wait32(stream->gt, __oa_regs(stream)->oa_ctrl, 450 OAG_OACONTROL_OA_COUNTER_ENABLE, 0, 50000, NULL, false)) 451 drm_err(&stream->oa->xe->drm, 452 "wait for OA to be disabled timed out\n"); 453 454 if (GRAPHICS_VERx100(stream->oa->xe) <= 1270 && GRAPHICS_VERx100(stream->oa->xe) != 1260) { 455 /* <= XE_METEORLAKE except XE_PVC */ 456 xe_mmio_write32(stream->gt, OA_TLB_INV_CR, 1); 457 if (xe_mmio_wait32(stream->gt, OA_TLB_INV_CR, 1, 0, 50000, NULL, false)) 458 drm_err(&stream->oa->xe->drm, 459 "wait for OA tlb invalidate timed out\n"); 460 } 461} 462 463static int xe_oa_wait_unlocked(struct xe_oa_stream *stream) 464{ 465 /* We might wait indefinitely if periodic sampling is not enabled */ 466 if (!stream->periodic) 467 return -EINVAL; 468 469 return wait_event_interruptible(stream->poll_wq, 470 xe_oa_buffer_check_unlocked(stream)); 471} 472 473#define OASTATUS_RELEVANT_BITS (OASTATUS_MMIO_TRG_Q_FULL | OASTATUS_COUNTER_OVERFLOW | \ 474 OASTATUS_BUFFER_OVERFLOW | OASTATUS_REPORT_LOST) 475 476static int __xe_oa_read(struct xe_oa_stream *stream, char __user *buf, 477 size_t count, size_t *offset) 478{ 479 /* Only clear our bits to avoid side-effects */ 480 stream->oa_status = xe_mmio_rmw32(stream->gt, __oa_regs(stream)->oa_status, 481 OASTATUS_RELEVANT_BITS, 0); 482 /* 483 * Signal to userspace that there is non-zero OA status to read via 484 * @DRM_XE_OBSERVATION_IOCTL_STATUS observation stream fd ioctl 485 */ 486 if (stream->oa_status & OASTATUS_RELEVANT_BITS) 487 return -EIO; 488 489 return xe_oa_append_reports(stream, buf, count, offset); 490} 491 492static ssize_t xe_oa_read(struct file *file, char __user *buf, 493 size_t count, loff_t *ppos) 494{ 495 struct xe_oa_stream *stream = file->private_data; 496 size_t offset = 0; 497 int ret; 498 499 /* Can't read from disabled streams */ 500 if (!stream->enabled || !stream->sample) 501 return -EINVAL; 502 503 if (!(file->f_flags & O_NONBLOCK)) { 504 do { 505 ret = xe_oa_wait_unlocked(stream); 506 if (ret) 507 return ret; 508 509 mutex_lock(&stream->stream_lock); 510 ret = __xe_oa_read(stream, buf, count, &offset); 511 mutex_unlock(&stream->stream_lock); 512 } while (!offset && !ret); 513 } else { 514 mutex_lock(&stream->stream_lock); 515 ret = __xe_oa_read(stream, buf, count, &offset); 516 mutex_unlock(&stream->stream_lock); 517 } 518 519 /* 520 * Typically we clear pollin here in order to wait for the new hrtimer callback 521 * before unblocking. The exception to this is if __xe_oa_read returns -ENOSPC, 522 * which means that more OA data is available than could fit in the user provided 523 * buffer. In this case we want the next poll() call to not block. 524 * 525 * Also in case of -EIO, we have already waited for data before returning 526 * -EIO, so need to wait again 527 */ 528 if (ret != -ENOSPC && ret != -EIO) 529 stream->pollin = false; 530 531 /* Possible values for ret are 0, -EFAULT, -ENOSPC, -EIO, -EINVAL, ... */ 532 return offset ?: (ret ?: -EAGAIN); 533} 534 535static __poll_t xe_oa_poll_locked(struct xe_oa_stream *stream, 536 struct file *file, poll_table *wait) 537{ 538 __poll_t events = 0; 539 540 poll_wait(file, &stream->poll_wq, wait); 541 542 /* 543 * We don't explicitly check whether there's something to read here since this 544 * path may be hot depending on what else userspace is polling, or on the timeout 545 * in use. We rely on hrtimer xe_oa_poll_check_timer_cb to notify us when there 546 * are samples to read 547 */ 548 if (stream->pollin) 549 events |= EPOLLIN; 550 551 return events; 552} 553 554static __poll_t xe_oa_poll(struct file *file, poll_table *wait) 555{ 556 struct xe_oa_stream *stream = file->private_data; 557 __poll_t ret; 558 559 mutex_lock(&stream->stream_lock); 560 ret = xe_oa_poll_locked(stream, file, wait); 561 mutex_unlock(&stream->stream_lock); 562 563 return ret; 564} 565 566static int xe_oa_submit_bb(struct xe_oa_stream *stream, struct xe_bb *bb) 567{ 568 struct xe_sched_job *job; 569 struct dma_fence *fence; 570 long timeout; 571 int err = 0; 572 573 /* Kernel configuration is issued on stream->k_exec_q, not stream->exec_q */ 574 job = xe_bb_create_job(stream->k_exec_q, bb); 575 if (IS_ERR(job)) { 576 err = PTR_ERR(job); 577 goto exit; 578 } 579 580 xe_sched_job_arm(job); 581 fence = dma_fence_get(&job->drm.s_fence->finished); 582 xe_sched_job_push(job); 583 584 timeout = dma_fence_wait_timeout(fence, false, HZ); 585 dma_fence_put(fence); 586 if (timeout < 0) 587 err = timeout; 588 else if (!timeout) 589 err = -ETIME; 590exit: 591 return err; 592} 593 594static void write_cs_mi_lri(struct xe_bb *bb, const struct xe_oa_reg *reg_data, u32 n_regs) 595{ 596 u32 i; 597 598#define MI_LOAD_REGISTER_IMM_MAX_REGS (126) 599 600 for (i = 0; i < n_regs; i++) { 601 if ((i % MI_LOAD_REGISTER_IMM_MAX_REGS) == 0) { 602 u32 n_lri = min_t(u32, n_regs - i, 603 MI_LOAD_REGISTER_IMM_MAX_REGS); 604 605 bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(n_lri); 606 } 607 bb->cs[bb->len++] = reg_data[i].addr.addr; 608 bb->cs[bb->len++] = reg_data[i].value; 609 } 610} 611 612static int num_lri_dwords(int num_regs) 613{ 614 int count = 0; 615 616 if (num_regs > 0) { 617 count += DIV_ROUND_UP(num_regs, MI_LOAD_REGISTER_IMM_MAX_REGS); 618 count += num_regs * 2; 619 } 620 621 return count; 622} 623 624static void xe_oa_free_oa_buffer(struct xe_oa_stream *stream) 625{ 626 xe_bo_unpin_map_no_vm(stream->oa_buffer.bo); 627} 628 629static void xe_oa_free_configs(struct xe_oa_stream *stream) 630{ 631 struct xe_oa_config_bo *oa_bo, *tmp; 632 633 xe_oa_config_put(stream->oa_config); 634 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node) 635 free_oa_config_bo(oa_bo); 636} 637 638static void xe_oa_store_flex(struct xe_oa_stream *stream, struct xe_lrc *lrc, 639 struct xe_bb *bb, const struct flex *flex, u32 count) 640{ 641 u32 offset = xe_bo_ggtt_addr(lrc->bo); 642 643 do { 644 bb->cs[bb->len++] = MI_STORE_DATA_IMM | BIT(22) /* GGTT */ | 2; 645 bb->cs[bb->len++] = offset + flex->offset * sizeof(u32); 646 bb->cs[bb->len++] = 0; 647 bb->cs[bb->len++] = flex->value; 648 649 } while (flex++, --count); 650} 651 652static int xe_oa_modify_ctx_image(struct xe_oa_stream *stream, struct xe_lrc *lrc, 653 const struct flex *flex, u32 count) 654{ 655 struct xe_bb *bb; 656 int err; 657 658 bb = xe_bb_new(stream->gt, 4 * count, false); 659 if (IS_ERR(bb)) { 660 err = PTR_ERR(bb); 661 goto exit; 662 } 663 664 xe_oa_store_flex(stream, lrc, bb, flex, count); 665 666 err = xe_oa_submit_bb(stream, bb); 667 xe_bb_free(bb, NULL); 668exit: 669 return err; 670} 671 672static int xe_oa_load_with_lri(struct xe_oa_stream *stream, struct xe_oa_reg *reg_lri) 673{ 674 struct xe_bb *bb; 675 int err; 676 677 bb = xe_bb_new(stream->gt, 3, false); 678 if (IS_ERR(bb)) { 679 err = PTR_ERR(bb); 680 goto exit; 681 } 682 683 write_cs_mi_lri(bb, reg_lri, 1); 684 685 err = xe_oa_submit_bb(stream, bb); 686 xe_bb_free(bb, NULL); 687exit: 688 return err; 689} 690 691static int xe_oa_configure_oar_context(struct xe_oa_stream *stream, bool enable) 692{ 693 const struct xe_oa_format *format = stream->oa_buffer.format; 694 struct xe_lrc *lrc = stream->exec_q->lrc[0]; 695 u32 regs_offset = xe_lrc_regs_offset(lrc) / sizeof(u32); 696 u32 oacontrol = __format_to_oactrl(format, OAR_OACONTROL_COUNTER_SEL_MASK) | 697 (enable ? OAR_OACONTROL_COUNTER_ENABLE : 0); 698 699 struct flex regs_context[] = { 700 { 701 OACTXCONTROL(stream->hwe->mmio_base), 702 stream->oa->ctx_oactxctrl_offset[stream->hwe->class] + 1, 703 enable ? OA_COUNTER_RESUME : 0, 704 }, 705 { 706 RING_CONTEXT_CONTROL(stream->hwe->mmio_base), 707 regs_offset + CTX_CONTEXT_CONTROL, 708 _MASKED_FIELD(CTX_CTRL_OAC_CONTEXT_ENABLE, 709 enable ? CTX_CTRL_OAC_CONTEXT_ENABLE : 0) 710 }, 711 }; 712 struct xe_oa_reg reg_lri = { OAR_OACONTROL, oacontrol }; 713 int err; 714 715 /* Modify stream hwe context image with regs_context */ 716 err = xe_oa_modify_ctx_image(stream, stream->exec_q->lrc[0], 717 regs_context, ARRAY_SIZE(regs_context)); 718 if (err) 719 return err; 720 721 /* Apply reg_lri using LRI */ 722 return xe_oa_load_with_lri(stream, ®_lri); 723} 724 725static int xe_oa_configure_oac_context(struct xe_oa_stream *stream, bool enable) 726{ 727 const struct xe_oa_format *format = stream->oa_buffer.format; 728 struct xe_lrc *lrc = stream->exec_q->lrc[0]; 729 u32 regs_offset = xe_lrc_regs_offset(lrc) / sizeof(u32); 730 u32 oacontrol = __format_to_oactrl(format, OAR_OACONTROL_COUNTER_SEL_MASK) | 731 (enable ? OAR_OACONTROL_COUNTER_ENABLE : 0); 732 struct flex regs_context[] = { 733 { 734 OACTXCONTROL(stream->hwe->mmio_base), 735 stream->oa->ctx_oactxctrl_offset[stream->hwe->class] + 1, 736 enable ? OA_COUNTER_RESUME : 0, 737 }, 738 { 739 RING_CONTEXT_CONTROL(stream->hwe->mmio_base), 740 regs_offset + CTX_CONTEXT_CONTROL, 741 _MASKED_FIELD(CTX_CTRL_OAC_CONTEXT_ENABLE, 742 enable ? CTX_CTRL_OAC_CONTEXT_ENABLE : 0) | 743 _MASKED_FIELD(CTX_CTRL_RUN_ALONE, 744 enable ? CTX_CTRL_RUN_ALONE : 0), 745 }, 746 }; 747 struct xe_oa_reg reg_lri = { OAC_OACONTROL, oacontrol }; 748 int err; 749 750 /* Set ccs select to enable programming of OAC_OACONTROL */ 751 xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_ctrl, __oa_ccs_select(stream)); 752 753 /* Modify stream hwe context image with regs_context */ 754 err = xe_oa_modify_ctx_image(stream, stream->exec_q->lrc[0], 755 regs_context, ARRAY_SIZE(regs_context)); 756 if (err) 757 return err; 758 759 /* Apply reg_lri using LRI */ 760 return xe_oa_load_with_lri(stream, ®_lri); 761} 762 763static int xe_oa_configure_oa_context(struct xe_oa_stream *stream, bool enable) 764{ 765 switch (stream->hwe->class) { 766 case XE_ENGINE_CLASS_RENDER: 767 return xe_oa_configure_oar_context(stream, enable); 768 case XE_ENGINE_CLASS_COMPUTE: 769 return xe_oa_configure_oac_context(stream, enable); 770 default: 771 /* Video engines do not support MI_REPORT_PERF_COUNT */ 772 return 0; 773 } 774} 775 776#define HAS_OA_BPC_REPORTING(xe) (GRAPHICS_VERx100(xe) >= 1255) 777 778static u32 oag_configure_mmio_trigger(const struct xe_oa_stream *stream, bool enable) 779{ 780 return _MASKED_FIELD(OAG_OA_DEBUG_DISABLE_MMIO_TRG, 781 enable && stream && stream->sample ? 782 0 : OAG_OA_DEBUG_DISABLE_MMIO_TRG); 783} 784 785static void xe_oa_disable_metric_set(struct xe_oa_stream *stream) 786{ 787 u32 sqcnt1; 788 789 /* 790 * Wa_1508761755:xehpsdv, dg2 791 * Enable thread stall DOP gating and EU DOP gating. 792 */ 793 if (stream->oa->xe->info.platform == XE_DG2) { 794 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN, 795 _MASKED_BIT_DISABLE(STALL_DOP_GATING_DISABLE)); 796 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2, 797 _MASKED_BIT_DISABLE(DISABLE_DOP_GATING)); 798 } 799 800 xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_debug, 801 oag_configure_mmio_trigger(stream, false)); 802 803 /* disable the context save/restore or OAR counters */ 804 if (stream->exec_q) 805 xe_oa_configure_oa_context(stream, false); 806 807 /* Make sure we disable noa to save power. */ 808 xe_mmio_rmw32(stream->gt, RPM_CONFIG1, GT_NOA_ENABLE, 0); 809 810 sqcnt1 = SQCNT1_PMON_ENABLE | 811 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0); 812 813 /* Reset PMON Enable to save power. */ 814 xe_mmio_rmw32(stream->gt, XELPMP_SQCNT1, sqcnt1, 0); 815} 816 817static void xe_oa_stream_destroy(struct xe_oa_stream *stream) 818{ 819 struct xe_oa_unit *u = stream->hwe->oa_unit; 820 struct xe_gt *gt = stream->hwe->gt; 821 822 if (WARN_ON(stream != u->exclusive_stream)) 823 return; 824 825 WRITE_ONCE(u->exclusive_stream, NULL); 826 827 mutex_destroy(&stream->stream_lock); 828 829 xe_oa_disable_metric_set(stream); 830 xe_exec_queue_put(stream->k_exec_q); 831 832 xe_oa_free_oa_buffer(stream); 833 834 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); 835 xe_pm_runtime_put(stream->oa->xe); 836 837 /* Wa_1509372804:pvc: Unset the override of GUCRC mode to enable rc6 */ 838 if (stream->override_gucrc) 839 xe_gt_WARN_ON(gt, xe_guc_pc_unset_gucrc_mode(>->uc.guc.pc)); 840 841 xe_oa_free_configs(stream); 842} 843 844static int xe_oa_alloc_oa_buffer(struct xe_oa_stream *stream) 845{ 846 struct xe_bo *bo; 847 848 BUILD_BUG_ON_NOT_POWER_OF_2(XE_OA_BUFFER_SIZE); 849 BUILD_BUG_ON(XE_OA_BUFFER_SIZE < SZ_128K || XE_OA_BUFFER_SIZE > SZ_16M); 850 851 bo = xe_bo_create_pin_map(stream->oa->xe, stream->gt->tile, NULL, 852 XE_OA_BUFFER_SIZE, ttm_bo_type_kernel, 853 XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT); 854 if (IS_ERR(bo)) 855 return PTR_ERR(bo); 856 857 stream->oa_buffer.bo = bo; 858 /* mmap implementation requires OA buffer to be in system memory */ 859 xe_assert(stream->oa->xe, bo->vmap.is_iomem == 0); 860 stream->oa_buffer.vaddr = bo->vmap.vaddr; 861 return 0; 862} 863 864static struct xe_oa_config_bo * 865__xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config) 866{ 867 struct xe_oa_config_bo *oa_bo; 868 size_t config_length; 869 struct xe_bb *bb; 870 871 oa_bo = kzalloc(sizeof(*oa_bo), GFP_KERNEL); 872 if (!oa_bo) 873 return ERR_PTR(-ENOMEM); 874 875 config_length = num_lri_dwords(oa_config->regs_len); 876 config_length = ALIGN(sizeof(u32) * config_length, XE_PAGE_SIZE) / sizeof(u32); 877 878 bb = xe_bb_new(stream->gt, config_length, false); 879 if (IS_ERR(bb)) 880 goto err_free; 881 882 write_cs_mi_lri(bb, oa_config->regs, oa_config->regs_len); 883 884 oa_bo->bb = bb; 885 oa_bo->oa_config = xe_oa_config_get(oa_config); 886 llist_add(&oa_bo->node, &stream->oa_config_bos); 887 888 return oa_bo; 889err_free: 890 kfree(oa_bo); 891 return ERR_CAST(bb); 892} 893 894static struct xe_oa_config_bo * 895xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config) 896{ 897 struct xe_oa_config_bo *oa_bo; 898 899 /* Look for the buffer in the already allocated BOs attached to the stream */ 900 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) { 901 if (oa_bo->oa_config == oa_config && 902 memcmp(oa_bo->oa_config->uuid, oa_config->uuid, 903 sizeof(oa_config->uuid)) == 0) 904 goto out; 905 } 906 907 oa_bo = __xe_oa_alloc_config_buffer(stream, oa_config); 908out: 909 return oa_bo; 910} 911 912static int xe_oa_emit_oa_config(struct xe_oa_stream *stream, struct xe_oa_config *config) 913{ 914#define NOA_PROGRAM_ADDITIONAL_DELAY_US 500 915 struct xe_oa_config_bo *oa_bo; 916 int err, us = NOA_PROGRAM_ADDITIONAL_DELAY_US; 917 918 oa_bo = xe_oa_alloc_config_buffer(stream, config); 919 if (IS_ERR(oa_bo)) { 920 err = PTR_ERR(oa_bo); 921 goto exit; 922 } 923 924 err = xe_oa_submit_bb(stream, oa_bo->bb); 925 926 /* Additional empirical delay needed for NOA programming after registers are written */ 927 usleep_range(us, 2 * us); 928exit: 929 return err; 930} 931 932static u32 oag_report_ctx_switches(const struct xe_oa_stream *stream) 933{ 934 /* If user didn't require OA reports, ask HW not to emit ctx switch reports */ 935 return _MASKED_FIELD(OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS, 936 stream->sample ? 937 0 : OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS); 938} 939 940static int xe_oa_enable_metric_set(struct xe_oa_stream *stream) 941{ 942 u32 oa_debug, sqcnt1; 943 int ret; 944 945 /* 946 * Wa_1508761755:xehpsdv, dg2 947 * EU NOA signals behave incorrectly if EU clock gating is enabled. 948 * Disable thread stall DOP gating and EU DOP gating. 949 */ 950 if (stream->oa->xe->info.platform == XE_DG2) { 951 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN, 952 _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE)); 953 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2, 954 _MASKED_BIT_ENABLE(DISABLE_DOP_GATING)); 955 } 956 957 /* Disable clk ratio reports */ 958 oa_debug = OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS | 959 OAG_OA_DEBUG_INCLUDE_CLK_RATIO; 960 961 if (GRAPHICS_VER(stream->oa->xe) >= 20) 962 oa_debug |= 963 /* The three bits below are needed to get PEC counters running */ 964 OAG_OA_DEBUG_START_TRIGGER_SCOPE_CONTROL | 965 OAG_OA_DEBUG_DISABLE_START_TRG_2_COUNT_QUAL | 966 OAG_OA_DEBUG_DISABLE_START_TRG_1_COUNT_QUAL; 967 968 xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_debug, 969 _MASKED_BIT_ENABLE(oa_debug) | 970 oag_report_ctx_switches(stream) | 971 oag_configure_mmio_trigger(stream, true)); 972 973 xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_ctx_ctrl, stream->periodic ? 974 (OAG_OAGLBCTXCTRL_COUNTER_RESUME | 975 OAG_OAGLBCTXCTRL_TIMER_ENABLE | 976 REG_FIELD_PREP(OAG_OAGLBCTXCTRL_TIMER_PERIOD_MASK, 977 stream->period_exponent)) : 0); 978 979 /* 980 * Initialize Super Queue Internal Cnt Register 981 * Set PMON Enable in order to collect valid metrics 982 * Enable bytes per clock reporting 983 */ 984 sqcnt1 = SQCNT1_PMON_ENABLE | 985 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0); 986 987 xe_mmio_rmw32(stream->gt, XELPMP_SQCNT1, 0, sqcnt1); 988 989 /* Configure OAR/OAC */ 990 if (stream->exec_q) { 991 ret = xe_oa_configure_oa_context(stream, true); 992 if (ret) 993 return ret; 994 } 995 996 return xe_oa_emit_oa_config(stream, stream->oa_config); 997} 998 999static void xe_oa_stream_enable(struct xe_oa_stream *stream) 1000{ 1001 stream->pollin = false; 1002 1003 xe_oa_enable(stream); 1004 1005 if (stream->sample) 1006 hrtimer_start(&stream->poll_check_timer, 1007 ns_to_ktime(stream->poll_period_ns), 1008 HRTIMER_MODE_REL_PINNED); 1009} 1010 1011static void xe_oa_stream_disable(struct xe_oa_stream *stream) 1012{ 1013 xe_oa_disable(stream); 1014 1015 if (stream->sample) 1016 hrtimer_cancel(&stream->poll_check_timer); 1017} 1018 1019static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream) 1020{ 1021 struct xe_exec_queue *q = stream->exec_q; 1022 int ret1, ret2; 1023 1024 /* Best effort recovery: try to revert both to original, irrespective of error */ 1025 ret1 = q->ops->set_timeslice(q, stream->hwe->eclass->sched_props.timeslice_us); 1026 ret2 = q->ops->set_preempt_timeout(q, stream->hwe->eclass->sched_props.preempt_timeout_us); 1027 if (ret1 || ret2) 1028 goto err; 1029 return 0; 1030err: 1031 drm_dbg(&stream->oa->xe->drm, "%s failed ret1 %d ret2 %d\n", __func__, ret1, ret2); 1032 return ret1 ?: ret2; 1033} 1034 1035static int xe_oa_disable_preempt_timeslice(struct xe_oa_stream *stream) 1036{ 1037 struct xe_exec_queue *q = stream->exec_q; 1038 int ret; 1039 1040 /* Setting values to 0 will disable timeslice and preempt_timeout */ 1041 ret = q->ops->set_timeslice(q, 0); 1042 if (ret) 1043 goto err; 1044 1045 ret = q->ops->set_preempt_timeout(q, 0); 1046 if (ret) 1047 goto err; 1048 1049 return 0; 1050err: 1051 xe_oa_enable_preempt_timeslice(stream); 1052 drm_dbg(&stream->oa->xe->drm, "%s failed %d\n", __func__, ret); 1053 return ret; 1054} 1055 1056static int xe_oa_enable_locked(struct xe_oa_stream *stream) 1057{ 1058 if (stream->enabled) 1059 return 0; 1060 1061 if (stream->no_preempt) { 1062 int ret = xe_oa_disable_preempt_timeslice(stream); 1063 1064 if (ret) 1065 return ret; 1066 } 1067 1068 xe_oa_stream_enable(stream); 1069 1070 stream->enabled = true; 1071 return 0; 1072} 1073 1074static int xe_oa_disable_locked(struct xe_oa_stream *stream) 1075{ 1076 int ret = 0; 1077 1078 if (!stream->enabled) 1079 return 0; 1080 1081 xe_oa_stream_disable(stream); 1082 1083 if (stream->no_preempt) 1084 ret = xe_oa_enable_preempt_timeslice(stream); 1085 1086 stream->enabled = false; 1087 return ret; 1088} 1089 1090static long xe_oa_config_locked(struct xe_oa_stream *stream, u64 arg) 1091{ 1092 struct drm_xe_ext_set_property ext; 1093 long ret = stream->oa_config->id; 1094 struct xe_oa_config *config; 1095 int err; 1096 1097 err = __copy_from_user(&ext, u64_to_user_ptr(arg), sizeof(ext)); 1098 if (XE_IOCTL_DBG(stream->oa->xe, err)) 1099 return -EFAULT; 1100 1101 if (XE_IOCTL_DBG(stream->oa->xe, ext.pad) || 1102 XE_IOCTL_DBG(stream->oa->xe, ext.base.name != DRM_XE_OA_EXTENSION_SET_PROPERTY) || 1103 XE_IOCTL_DBG(stream->oa->xe, ext.base.next_extension) || 1104 XE_IOCTL_DBG(stream->oa->xe, ext.property != DRM_XE_OA_PROPERTY_OA_METRIC_SET)) 1105 return -EINVAL; 1106 1107 config = xe_oa_get_oa_config(stream->oa, ext.value); 1108 if (!config) 1109 return -ENODEV; 1110 1111 if (config != stream->oa_config) { 1112 err = xe_oa_emit_oa_config(stream, config); 1113 if (!err) 1114 config = xchg(&stream->oa_config, config); 1115 else 1116 ret = err; 1117 } 1118 1119 xe_oa_config_put(config); 1120 1121 return ret; 1122} 1123 1124static long xe_oa_status_locked(struct xe_oa_stream *stream, unsigned long arg) 1125{ 1126 struct drm_xe_oa_stream_status status = {}; 1127 void __user *uaddr = (void __user *)arg; 1128 1129 /* Map from register to uapi bits */ 1130 if (stream->oa_status & OASTATUS_REPORT_LOST) 1131 status.oa_status |= DRM_XE_OASTATUS_REPORT_LOST; 1132 if (stream->oa_status & OASTATUS_BUFFER_OVERFLOW) 1133 status.oa_status |= DRM_XE_OASTATUS_BUFFER_OVERFLOW; 1134 if (stream->oa_status & OASTATUS_COUNTER_OVERFLOW) 1135 status.oa_status |= DRM_XE_OASTATUS_COUNTER_OVERFLOW; 1136 if (stream->oa_status & OASTATUS_MMIO_TRG_Q_FULL) 1137 status.oa_status |= DRM_XE_OASTATUS_MMIO_TRG_Q_FULL; 1138 1139 if (copy_to_user(uaddr, &status, sizeof(status))) 1140 return -EFAULT; 1141 1142 return 0; 1143} 1144 1145static long xe_oa_info_locked(struct xe_oa_stream *stream, unsigned long arg) 1146{ 1147 struct drm_xe_oa_stream_info info = { .oa_buf_size = XE_OA_BUFFER_SIZE, }; 1148 void __user *uaddr = (void __user *)arg; 1149 1150 if (copy_to_user(uaddr, &info, sizeof(info))) 1151 return -EFAULT; 1152 1153 return 0; 1154} 1155 1156static long xe_oa_ioctl_locked(struct xe_oa_stream *stream, 1157 unsigned int cmd, 1158 unsigned long arg) 1159{ 1160 switch (cmd) { 1161 case DRM_XE_OBSERVATION_IOCTL_ENABLE: 1162 return xe_oa_enable_locked(stream); 1163 case DRM_XE_OBSERVATION_IOCTL_DISABLE: 1164 return xe_oa_disable_locked(stream); 1165 case DRM_XE_OBSERVATION_IOCTL_CONFIG: 1166 return xe_oa_config_locked(stream, arg); 1167 case DRM_XE_OBSERVATION_IOCTL_STATUS: 1168 return xe_oa_status_locked(stream, arg); 1169 case DRM_XE_OBSERVATION_IOCTL_INFO: 1170 return xe_oa_info_locked(stream, arg); 1171 } 1172 1173 return -EINVAL; 1174} 1175 1176static long xe_oa_ioctl(struct file *file, 1177 unsigned int cmd, 1178 unsigned long arg) 1179{ 1180 struct xe_oa_stream *stream = file->private_data; 1181 long ret; 1182 1183 mutex_lock(&stream->stream_lock); 1184 ret = xe_oa_ioctl_locked(stream, cmd, arg); 1185 mutex_unlock(&stream->stream_lock); 1186 1187 return ret; 1188} 1189 1190static void xe_oa_destroy_locked(struct xe_oa_stream *stream) 1191{ 1192 if (stream->enabled) 1193 xe_oa_disable_locked(stream); 1194 1195 xe_oa_stream_destroy(stream); 1196 1197 if (stream->exec_q) 1198 xe_exec_queue_put(stream->exec_q); 1199 1200 kfree(stream); 1201} 1202 1203static int xe_oa_release(struct inode *inode, struct file *file) 1204{ 1205 struct xe_oa_stream *stream = file->private_data; 1206 struct xe_gt *gt = stream->gt; 1207 1208 mutex_lock(>->oa.gt_lock); 1209 xe_oa_destroy_locked(stream); 1210 mutex_unlock(>->oa.gt_lock); 1211 1212 /* Release the reference the OA stream kept on the driver */ 1213 drm_dev_put(>_to_xe(gt)->drm); 1214 1215 return 0; 1216} 1217 1218static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma) 1219{ 1220 struct xe_oa_stream *stream = file->private_data; 1221 struct xe_bo *bo = stream->oa_buffer.bo; 1222 unsigned long start = vma->vm_start; 1223 int i, ret; 1224 1225 if (xe_observation_paranoid && !perfmon_capable()) { 1226 drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n"); 1227 return -EACCES; 1228 } 1229 1230 /* Can mmap the entire OA buffer or nothing (no partial OA buffer mmaps) */ 1231 if (vma->vm_end - vma->vm_start != XE_OA_BUFFER_SIZE) { 1232 drm_dbg(&stream->oa->xe->drm, "Wrong mmap size, must be OA buffer size\n"); 1233 return -EINVAL; 1234 } 1235 1236 /* 1237 * Only support VM_READ, enforce MAP_PRIVATE by checking for 1238 * VM_MAYSHARE, don't copy the vma on fork 1239 */ 1240 if (vma->vm_flags & (VM_WRITE | VM_EXEC | VM_SHARED | VM_MAYSHARE)) { 1241 drm_dbg(&stream->oa->xe->drm, "mmap must be read only\n"); 1242 return -EINVAL; 1243 } 1244 vm_flags_mod(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY, 1245 VM_MAYWRITE | VM_MAYEXEC); 1246 1247 xe_assert(stream->oa->xe, bo->ttm.ttm->num_pages == 1248 (vma->vm_end - vma->vm_start) >> PAGE_SHIFT); 1249 for (i = 0; i < bo->ttm.ttm->num_pages; i++) { 1250 ret = remap_pfn_range(vma, start, page_to_pfn(bo->ttm.ttm->pages[i]), 1251 PAGE_SIZE, vma->vm_page_prot); 1252 if (ret) 1253 break; 1254 1255 start += PAGE_SIZE; 1256 } 1257 1258 return ret; 1259} 1260 1261static const struct file_operations xe_oa_fops = { 1262 .owner = THIS_MODULE, 1263 .llseek = no_llseek, 1264 .release = xe_oa_release, 1265 .poll = xe_oa_poll, 1266 .read = xe_oa_read, 1267 .unlocked_ioctl = xe_oa_ioctl, 1268 .mmap = xe_oa_mmap, 1269}; 1270 1271static bool engine_supports_mi_query(struct xe_hw_engine *hwe) 1272{ 1273 return hwe->class == XE_ENGINE_CLASS_RENDER || 1274 hwe->class == XE_ENGINE_CLASS_COMPUTE; 1275} 1276 1277static bool xe_oa_find_reg_in_lri(u32 *state, u32 reg, u32 *offset, u32 end) 1278{ 1279 u32 idx = *offset; 1280 u32 len = min(MI_LRI_LEN(state[idx]) + idx, end); 1281 bool found = false; 1282 1283 idx++; 1284 for (; idx < len; idx += 2) { 1285 if (state[idx] == reg) { 1286 found = true; 1287 break; 1288 } 1289 } 1290 1291 *offset = idx; 1292 return found; 1293} 1294 1295#define IS_MI_LRI_CMD(x) (REG_FIELD_GET(MI_OPCODE, (x)) == \ 1296 REG_FIELD_GET(MI_OPCODE, MI_LOAD_REGISTER_IMM)) 1297 1298static u32 xe_oa_context_image_offset(struct xe_oa_stream *stream, u32 reg) 1299{ 1300 struct xe_lrc *lrc = stream->exec_q->lrc[0]; 1301 u32 len = (xe_gt_lrc_size(stream->gt, stream->hwe->class) + 1302 lrc->ring.size) / sizeof(u32); 1303 u32 offset = xe_lrc_regs_offset(lrc) / sizeof(u32); 1304 u32 *state = (u32 *)lrc->bo->vmap.vaddr; 1305 1306 if (drm_WARN_ON(&stream->oa->xe->drm, !state)) 1307 return U32_MAX; 1308 1309 for (; offset < len; ) { 1310 if (IS_MI_LRI_CMD(state[offset])) { 1311 /* 1312 * We expect reg-value pairs in MI_LRI command, so 1313 * MI_LRI_LEN() should be even 1314 */ 1315 drm_WARN_ON(&stream->oa->xe->drm, 1316 MI_LRI_LEN(state[offset]) & 0x1); 1317 1318 if (xe_oa_find_reg_in_lri(state, reg, &offset, len)) 1319 break; 1320 } else { 1321 offset++; 1322 } 1323 } 1324 1325 return offset < len ? offset : U32_MAX; 1326} 1327 1328static int xe_oa_set_ctx_ctrl_offset(struct xe_oa_stream *stream) 1329{ 1330 struct xe_reg reg = OACTXCONTROL(stream->hwe->mmio_base); 1331 u32 offset = stream->oa->ctx_oactxctrl_offset[stream->hwe->class]; 1332 1333 /* Do this only once. Failure is stored as offset of U32_MAX */ 1334 if (offset) 1335 goto exit; 1336 1337 offset = xe_oa_context_image_offset(stream, reg.addr); 1338 stream->oa->ctx_oactxctrl_offset[stream->hwe->class] = offset; 1339 1340 drm_dbg(&stream->oa->xe->drm, "%s oa ctx control at 0x%08x dword offset\n", 1341 stream->hwe->name, offset); 1342exit: 1343 return offset && offset != U32_MAX ? 0 : -ENODEV; 1344} 1345 1346static int xe_oa_stream_init(struct xe_oa_stream *stream, 1347 struct xe_oa_open_param *param) 1348{ 1349 struct xe_oa_unit *u = param->hwe->oa_unit; 1350 struct xe_gt *gt = param->hwe->gt; 1351 int ret; 1352 1353 stream->exec_q = param->exec_q; 1354 stream->poll_period_ns = DEFAULT_POLL_PERIOD_NS; 1355 stream->hwe = param->hwe; 1356 stream->gt = stream->hwe->gt; 1357 stream->oa_buffer.format = &stream->oa->oa_formats[param->oa_format]; 1358 1359 stream->sample = param->sample; 1360 stream->periodic = param->period_exponent > 0; 1361 stream->period_exponent = param->period_exponent; 1362 stream->no_preempt = param->no_preempt; 1363 1364 /* 1365 * For Xe2+, when overrun mode is enabled, there are no partial reports at the end 1366 * of buffer, making the OA buffer effectively a non-power-of-2 size circular 1367 * buffer whose size, circ_size, is a multiple of the report size 1368 */ 1369 if (GRAPHICS_VER(stream->oa->xe) >= 20 && 1370 stream->hwe->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG && stream->sample) 1371 stream->oa_buffer.circ_size = 1372 XE_OA_BUFFER_SIZE - XE_OA_BUFFER_SIZE % stream->oa_buffer.format->size; 1373 else 1374 stream->oa_buffer.circ_size = XE_OA_BUFFER_SIZE; 1375 1376 if (stream->exec_q && engine_supports_mi_query(stream->hwe)) { 1377 /* If we don't find the context offset, just return error */ 1378 ret = xe_oa_set_ctx_ctrl_offset(stream); 1379 if (ret) { 1380 drm_err(&stream->oa->xe->drm, 1381 "xe_oa_set_ctx_ctrl_offset failed for %s\n", 1382 stream->hwe->name); 1383 goto exit; 1384 } 1385 } 1386 1387 stream->oa_config = xe_oa_get_oa_config(stream->oa, param->metric_set); 1388 if (!stream->oa_config) { 1389 drm_dbg(&stream->oa->xe->drm, "Invalid OA config id=%i\n", param->metric_set); 1390 ret = -EINVAL; 1391 goto exit; 1392 } 1393 1394 /* 1395 * Wa_1509372804:pvc 1396 * 1397 * GuC reset of engines causes OA to lose configuration 1398 * state. Prevent this by overriding GUCRC mode. 1399 */ 1400 if (stream->oa->xe->info.platform == XE_PVC) { 1401 ret = xe_guc_pc_override_gucrc_mode(>->uc.guc.pc, 1402 SLPC_GUCRC_MODE_GUCRC_NO_RC6); 1403 if (ret) 1404 goto err_free_configs; 1405 1406 stream->override_gucrc = true; 1407 } 1408 1409 /* Take runtime pm ref and forcewake to disable RC6 */ 1410 xe_pm_runtime_get(stream->oa->xe); 1411 XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL)); 1412 1413 ret = xe_oa_alloc_oa_buffer(stream); 1414 if (ret) 1415 goto err_fw_put; 1416 1417 stream->k_exec_q = xe_exec_queue_create(stream->oa->xe, NULL, 1418 BIT(stream->hwe->logical_instance), 1, 1419 stream->hwe, EXEC_QUEUE_FLAG_KERNEL, 0); 1420 if (IS_ERR(stream->k_exec_q)) { 1421 ret = PTR_ERR(stream->k_exec_q); 1422 drm_err(&stream->oa->xe->drm, "gt%d, hwe %s, xe_exec_queue_create failed=%d", 1423 stream->gt->info.id, stream->hwe->name, ret); 1424 goto err_free_oa_buf; 1425 } 1426 1427 ret = xe_oa_enable_metric_set(stream); 1428 if (ret) { 1429 drm_dbg(&stream->oa->xe->drm, "Unable to enable metric set\n"); 1430 goto err_put_k_exec_q; 1431 } 1432 1433 drm_dbg(&stream->oa->xe->drm, "opening stream oa config uuid=%s\n", 1434 stream->oa_config->uuid); 1435 1436 WRITE_ONCE(u->exclusive_stream, stream); 1437 1438 hrtimer_init(&stream->poll_check_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1439 stream->poll_check_timer.function = xe_oa_poll_check_timer_cb; 1440 init_waitqueue_head(&stream->poll_wq); 1441 1442 spin_lock_init(&stream->oa_buffer.ptr_lock); 1443 mutex_init(&stream->stream_lock); 1444 1445 return 0; 1446 1447err_put_k_exec_q: 1448 xe_oa_disable_metric_set(stream); 1449 xe_exec_queue_put(stream->k_exec_q); 1450err_free_oa_buf: 1451 xe_oa_free_oa_buffer(stream); 1452err_fw_put: 1453 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); 1454 xe_pm_runtime_put(stream->oa->xe); 1455 if (stream->override_gucrc) 1456 xe_gt_WARN_ON(gt, xe_guc_pc_unset_gucrc_mode(>->uc.guc.pc)); 1457err_free_configs: 1458 xe_oa_free_configs(stream); 1459exit: 1460 return ret; 1461} 1462 1463static int xe_oa_stream_open_ioctl_locked(struct xe_oa *oa, 1464 struct xe_oa_open_param *param) 1465{ 1466 struct xe_oa_stream *stream; 1467 int stream_fd; 1468 int ret; 1469 1470 /* We currently only allow exclusive access */ 1471 if (param->hwe->oa_unit->exclusive_stream) { 1472 drm_dbg(&oa->xe->drm, "OA unit already in use\n"); 1473 ret = -EBUSY; 1474 goto exit; 1475 } 1476 1477 stream = kzalloc(sizeof(*stream), GFP_KERNEL); 1478 if (!stream) { 1479 ret = -ENOMEM; 1480 goto exit; 1481 } 1482 1483 stream->oa = oa; 1484 ret = xe_oa_stream_init(stream, param); 1485 if (ret) 1486 goto err_free; 1487 1488 if (!param->disabled) { 1489 ret = xe_oa_enable_locked(stream); 1490 if (ret) 1491 goto err_destroy; 1492 } 1493 1494 stream_fd = anon_inode_getfd("[xe_oa]", &xe_oa_fops, stream, 0); 1495 if (stream_fd < 0) { 1496 ret = stream_fd; 1497 goto err_disable; 1498 } 1499 1500 /* Hold a reference on the drm device till stream_fd is released */ 1501 drm_dev_get(&stream->oa->xe->drm); 1502 1503 return stream_fd; 1504err_disable: 1505 if (!param->disabled) 1506 xe_oa_disable_locked(stream); 1507err_destroy: 1508 xe_oa_stream_destroy(stream); 1509err_free: 1510 kfree(stream); 1511exit: 1512 return ret; 1513} 1514 1515/** 1516 * xe_oa_timestamp_frequency - Return OA timestamp frequency 1517 * @gt: @xe_gt 1518 * 1519 * OA timestamp frequency = CS timestamp frequency in most platforms. On some 1520 * platforms OA unit ignores the CTC_SHIFT and the 2 timestamps differ. In such 1521 * cases, return the adjusted CS timestamp frequency to the user. 1522 */ 1523u32 xe_oa_timestamp_frequency(struct xe_gt *gt) 1524{ 1525 u32 reg, shift; 1526 1527 /* 1528 * Wa_18013179988:dg2 1529 * Wa_14015568240:pvc 1530 * Wa_14015846243:mtl 1531 */ 1532 switch (gt_to_xe(gt)->info.platform) { 1533 case XE_DG2: 1534 case XE_PVC: 1535 case XE_METEORLAKE: 1536 xe_pm_runtime_get(gt_to_xe(gt)); 1537 reg = xe_mmio_read32(gt, RPM_CONFIG0); 1538 xe_pm_runtime_put(gt_to_xe(gt)); 1539 1540 shift = REG_FIELD_GET(RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK, reg); 1541 return gt->info.reference_clock << (3 - shift); 1542 1543 default: 1544 return gt->info.reference_clock; 1545 } 1546} 1547 1548static u64 oa_exponent_to_ns(struct xe_gt *gt, int exponent) 1549{ 1550 u64 nom = (2ULL << exponent) * NSEC_PER_SEC; 1551 u32 den = xe_oa_timestamp_frequency(gt); 1552 1553 return div_u64(nom + den - 1, den); 1554} 1555 1556static bool engine_supports_oa_format(const struct xe_hw_engine *hwe, int type) 1557{ 1558 switch (hwe->oa_unit->type) { 1559 case DRM_XE_OA_UNIT_TYPE_OAG: 1560 return type == DRM_XE_OA_FMT_TYPE_OAG || type == DRM_XE_OA_FMT_TYPE_OAR || 1561 type == DRM_XE_OA_FMT_TYPE_OAC || type == DRM_XE_OA_FMT_TYPE_PEC; 1562 case DRM_XE_OA_UNIT_TYPE_OAM: 1563 return type == DRM_XE_OA_FMT_TYPE_OAM || type == DRM_XE_OA_FMT_TYPE_OAM_MPEC; 1564 default: 1565 return false; 1566 } 1567} 1568 1569static int decode_oa_format(struct xe_oa *oa, u64 fmt, enum xe_oa_format_name *name) 1570{ 1571 u32 counter_size = FIELD_GET(DRM_XE_OA_FORMAT_MASK_COUNTER_SIZE, fmt); 1572 u32 counter_sel = FIELD_GET(DRM_XE_OA_FORMAT_MASK_COUNTER_SEL, fmt); 1573 u32 bc_report = FIELD_GET(DRM_XE_OA_FORMAT_MASK_BC_REPORT, fmt); 1574 u32 type = FIELD_GET(DRM_XE_OA_FORMAT_MASK_FMT_TYPE, fmt); 1575 int idx; 1576 1577 for_each_set_bit(idx, oa->format_mask, __XE_OA_FORMAT_MAX) { 1578 const struct xe_oa_format *f = &oa->oa_formats[idx]; 1579 1580 if (counter_size == f->counter_size && bc_report == f->bc_report && 1581 type == f->type && counter_sel == f->counter_select) { 1582 *name = idx; 1583 return 0; 1584 } 1585 } 1586 1587 return -EINVAL; 1588} 1589 1590/** 1591 * xe_oa_unit_id - Return OA unit ID for a hardware engine 1592 * @hwe: @xe_hw_engine 1593 * 1594 * Return OA unit ID for a hardware engine when available 1595 */ 1596u16 xe_oa_unit_id(struct xe_hw_engine *hwe) 1597{ 1598 return hwe->oa_unit && hwe->oa_unit->num_engines ? 1599 hwe->oa_unit->oa_unit_id : U16_MAX; 1600} 1601 1602static int xe_oa_assign_hwe(struct xe_oa *oa, struct xe_oa_open_param *param) 1603{ 1604 struct xe_gt *gt; 1605 int i, ret = 0; 1606 1607 if (param->exec_q) { 1608 /* When we have an exec_q, get hwe from the exec_q */ 1609 param->hwe = xe_gt_hw_engine(param->exec_q->gt, param->exec_q->class, 1610 param->engine_instance, true); 1611 } else { 1612 struct xe_hw_engine *hwe; 1613 enum xe_hw_engine_id id; 1614 1615 /* Else just get the first hwe attached to the oa unit */ 1616 for_each_gt(gt, oa->xe, i) { 1617 for_each_hw_engine(hwe, gt, id) { 1618 if (xe_oa_unit_id(hwe) == param->oa_unit_id) { 1619 param->hwe = hwe; 1620 goto out; 1621 } 1622 } 1623 } 1624 } 1625out: 1626 if (!param->hwe || xe_oa_unit_id(param->hwe) != param->oa_unit_id) { 1627 drm_dbg(&oa->xe->drm, "Unable to find hwe (%d, %d) for OA unit ID %d\n", 1628 param->exec_q ? param->exec_q->class : -1, 1629 param->engine_instance, param->oa_unit_id); 1630 ret = -EINVAL; 1631 } 1632 1633 return ret; 1634} 1635 1636static int xe_oa_set_prop_oa_unit_id(struct xe_oa *oa, u64 value, 1637 struct xe_oa_open_param *param) 1638{ 1639 if (value >= oa->oa_unit_ids) { 1640 drm_dbg(&oa->xe->drm, "OA unit ID out of range %lld\n", value); 1641 return -EINVAL; 1642 } 1643 param->oa_unit_id = value; 1644 return 0; 1645} 1646 1647static int xe_oa_set_prop_sample_oa(struct xe_oa *oa, u64 value, 1648 struct xe_oa_open_param *param) 1649{ 1650 param->sample = value; 1651 return 0; 1652} 1653 1654static int xe_oa_set_prop_metric_set(struct xe_oa *oa, u64 value, 1655 struct xe_oa_open_param *param) 1656{ 1657 param->metric_set = value; 1658 return 0; 1659} 1660 1661static int xe_oa_set_prop_oa_format(struct xe_oa *oa, u64 value, 1662 struct xe_oa_open_param *param) 1663{ 1664 int ret = decode_oa_format(oa, value, ¶m->oa_format); 1665 1666 if (ret) { 1667 drm_dbg(&oa->xe->drm, "Unsupported OA report format %#llx\n", value); 1668 return ret; 1669 } 1670 return 0; 1671} 1672 1673static int xe_oa_set_prop_oa_exponent(struct xe_oa *oa, u64 value, 1674 struct xe_oa_open_param *param) 1675{ 1676#define OA_EXPONENT_MAX 31 1677 1678 if (value > OA_EXPONENT_MAX) { 1679 drm_dbg(&oa->xe->drm, "OA timer exponent too high (> %u)\n", OA_EXPONENT_MAX); 1680 return -EINVAL; 1681 } 1682 param->period_exponent = value; 1683 return 0; 1684} 1685 1686static int xe_oa_set_prop_disabled(struct xe_oa *oa, u64 value, 1687 struct xe_oa_open_param *param) 1688{ 1689 param->disabled = value; 1690 return 0; 1691} 1692 1693static int xe_oa_set_prop_exec_queue_id(struct xe_oa *oa, u64 value, 1694 struct xe_oa_open_param *param) 1695{ 1696 param->exec_queue_id = value; 1697 return 0; 1698} 1699 1700static int xe_oa_set_prop_engine_instance(struct xe_oa *oa, u64 value, 1701 struct xe_oa_open_param *param) 1702{ 1703 param->engine_instance = value; 1704 return 0; 1705} 1706 1707static int xe_oa_set_no_preempt(struct xe_oa *oa, u64 value, 1708 struct xe_oa_open_param *param) 1709{ 1710 param->no_preempt = value; 1711 return 0; 1712} 1713 1714typedef int (*xe_oa_set_property_fn)(struct xe_oa *oa, u64 value, 1715 struct xe_oa_open_param *param); 1716static const xe_oa_set_property_fn xe_oa_set_property_funcs[] = { 1717 [DRM_XE_OA_PROPERTY_OA_UNIT_ID] = xe_oa_set_prop_oa_unit_id, 1718 [DRM_XE_OA_PROPERTY_SAMPLE_OA] = xe_oa_set_prop_sample_oa, 1719 [DRM_XE_OA_PROPERTY_OA_METRIC_SET] = xe_oa_set_prop_metric_set, 1720 [DRM_XE_OA_PROPERTY_OA_FORMAT] = xe_oa_set_prop_oa_format, 1721 [DRM_XE_OA_PROPERTY_OA_PERIOD_EXPONENT] = xe_oa_set_prop_oa_exponent, 1722 [DRM_XE_OA_PROPERTY_OA_DISABLED] = xe_oa_set_prop_disabled, 1723 [DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID] = xe_oa_set_prop_exec_queue_id, 1724 [DRM_XE_OA_PROPERTY_OA_ENGINE_INSTANCE] = xe_oa_set_prop_engine_instance, 1725 [DRM_XE_OA_PROPERTY_NO_PREEMPT] = xe_oa_set_no_preempt, 1726}; 1727 1728static int xe_oa_user_ext_set_property(struct xe_oa *oa, u64 extension, 1729 struct xe_oa_open_param *param) 1730{ 1731 u64 __user *address = u64_to_user_ptr(extension); 1732 struct drm_xe_ext_set_property ext; 1733 int err; 1734 u32 idx; 1735 1736 err = __copy_from_user(&ext, address, sizeof(ext)); 1737 if (XE_IOCTL_DBG(oa->xe, err)) 1738 return -EFAULT; 1739 1740 if (XE_IOCTL_DBG(oa->xe, ext.property >= ARRAY_SIZE(xe_oa_set_property_funcs)) || 1741 XE_IOCTL_DBG(oa->xe, ext.pad)) 1742 return -EINVAL; 1743 1744 idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_oa_set_property_funcs)); 1745 return xe_oa_set_property_funcs[idx](oa, ext.value, param); 1746} 1747 1748typedef int (*xe_oa_user_extension_fn)(struct xe_oa *oa, u64 extension, 1749 struct xe_oa_open_param *param); 1750static const xe_oa_user_extension_fn xe_oa_user_extension_funcs[] = { 1751 [DRM_XE_OA_EXTENSION_SET_PROPERTY] = xe_oa_user_ext_set_property, 1752}; 1753 1754#define MAX_USER_EXTENSIONS 16 1755static int xe_oa_user_extensions(struct xe_oa *oa, u64 extension, int ext_number, 1756 struct xe_oa_open_param *param) 1757{ 1758 u64 __user *address = u64_to_user_ptr(extension); 1759 struct drm_xe_user_extension ext; 1760 int err; 1761 u32 idx; 1762 1763 if (XE_IOCTL_DBG(oa->xe, ext_number >= MAX_USER_EXTENSIONS)) 1764 return -E2BIG; 1765 1766 err = __copy_from_user(&ext, address, sizeof(ext)); 1767 if (XE_IOCTL_DBG(oa->xe, err)) 1768 return -EFAULT; 1769 1770 if (XE_IOCTL_DBG(oa->xe, ext.pad) || 1771 XE_IOCTL_DBG(oa->xe, ext.name >= ARRAY_SIZE(xe_oa_user_extension_funcs))) 1772 return -EINVAL; 1773 1774 idx = array_index_nospec(ext.name, ARRAY_SIZE(xe_oa_user_extension_funcs)); 1775 err = xe_oa_user_extension_funcs[idx](oa, extension, param); 1776 if (XE_IOCTL_DBG(oa->xe, err)) 1777 return err; 1778 1779 if (ext.next_extension) 1780 return xe_oa_user_extensions(oa, ext.next_extension, ++ext_number, param); 1781 1782 return 0; 1783} 1784 1785/** 1786 * xe_oa_stream_open_ioctl - Opens an OA stream 1787 * @dev: @drm_device 1788 * @data: pointer to struct @drm_xe_oa_config 1789 * @file: @drm_file 1790 * 1791 * The functions opens an OA stream. An OA stream, opened with specified 1792 * properties, enables OA counter samples to be collected, either 1793 * periodically (time based sampling), or on request (using OA queries) 1794 */ 1795int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *file) 1796{ 1797 struct xe_device *xe = to_xe_device(dev); 1798 struct xe_oa *oa = &xe->oa; 1799 struct xe_file *xef = to_xe_file(file); 1800 struct xe_oa_open_param param = {}; 1801 const struct xe_oa_format *f; 1802 bool privileged_op = true; 1803 int ret; 1804 1805 if (!oa->xe) { 1806 drm_dbg(&xe->drm, "xe oa interface not available for this system\n"); 1807 return -ENODEV; 1808 } 1809 1810 ret = xe_oa_user_extensions(oa, data, 0, ¶m); 1811 if (ret) 1812 return ret; 1813 1814 if (param.exec_queue_id > 0) { 1815 param.exec_q = xe_exec_queue_lookup(xef, param.exec_queue_id); 1816 if (XE_IOCTL_DBG(oa->xe, !param.exec_q)) 1817 return -ENOENT; 1818 1819 if (param.exec_q->width > 1) 1820 drm_dbg(&oa->xe->drm, "exec_q->width > 1, programming only exec_q->lrc[0]\n"); 1821 } 1822 1823 /* 1824 * Query based sampling (using MI_REPORT_PERF_COUNT) with OAR/OAC, 1825 * without global stream access, can be an unprivileged operation 1826 */ 1827 if (param.exec_q && !param.sample) 1828 privileged_op = false; 1829 1830 if (param.no_preempt) { 1831 if (!param.exec_q) { 1832 drm_dbg(&oa->xe->drm, "Preemption disable without exec_q!\n"); 1833 ret = -EINVAL; 1834 goto err_exec_q; 1835 } 1836 privileged_op = true; 1837 } 1838 1839 if (privileged_op && xe_observation_paranoid && !perfmon_capable()) { 1840 drm_dbg(&oa->xe->drm, "Insufficient privileges to open xe OA stream\n"); 1841 ret = -EACCES; 1842 goto err_exec_q; 1843 } 1844 1845 if (!param.exec_q && !param.sample) { 1846 drm_dbg(&oa->xe->drm, "Only OA report sampling supported\n"); 1847 ret = -EINVAL; 1848 goto err_exec_q; 1849 } 1850 1851 ret = xe_oa_assign_hwe(oa, ¶m); 1852 if (ret) 1853 goto err_exec_q; 1854 1855 f = &oa->oa_formats[param.oa_format]; 1856 if (!param.oa_format || !f->size || 1857 !engine_supports_oa_format(param.hwe, f->type)) { 1858 drm_dbg(&oa->xe->drm, "Invalid OA format %d type %d size %d for class %d\n", 1859 param.oa_format, f->type, f->size, param.hwe->class); 1860 ret = -EINVAL; 1861 goto err_exec_q; 1862 } 1863 1864 if (param.period_exponent > 0) { 1865 u64 oa_period, oa_freq_hz; 1866 1867 /* Requesting samples from OAG buffer is a privileged operation */ 1868 if (!param.sample) { 1869 drm_dbg(&oa->xe->drm, "OA_EXPONENT specified without SAMPLE_OA\n"); 1870 ret = -EINVAL; 1871 goto err_exec_q; 1872 } 1873 oa_period = oa_exponent_to_ns(param.hwe->gt, param.period_exponent); 1874 oa_freq_hz = div64_u64(NSEC_PER_SEC, oa_period); 1875 drm_dbg(&oa->xe->drm, "Using periodic sampling freq %lld Hz\n", oa_freq_hz); 1876 } 1877 1878 mutex_lock(¶m.hwe->gt->oa.gt_lock); 1879 ret = xe_oa_stream_open_ioctl_locked(oa, ¶m); 1880 mutex_unlock(¶m.hwe->gt->oa.gt_lock); 1881err_exec_q: 1882 if (ret < 0 && param.exec_q) 1883 xe_exec_queue_put(param.exec_q); 1884 return ret; 1885} 1886 1887static bool xe_oa_is_valid_flex_addr(struct xe_oa *oa, u32 addr) 1888{ 1889 static const struct xe_reg flex_eu_regs[] = { 1890 EU_PERF_CNTL0, 1891 EU_PERF_CNTL1, 1892 EU_PERF_CNTL2, 1893 EU_PERF_CNTL3, 1894 EU_PERF_CNTL4, 1895 EU_PERF_CNTL5, 1896 EU_PERF_CNTL6, 1897 }; 1898 int i; 1899 1900 for (i = 0; i < ARRAY_SIZE(flex_eu_regs); i++) { 1901 if (flex_eu_regs[i].addr == addr) 1902 return true; 1903 } 1904 return false; 1905} 1906 1907static bool xe_oa_reg_in_range_table(u32 addr, const struct xe_mmio_range *table) 1908{ 1909 while (table->start && table->end) { 1910 if (addr >= table->start && addr <= table->end) 1911 return true; 1912 1913 table++; 1914 } 1915 1916 return false; 1917} 1918 1919static const struct xe_mmio_range xehp_oa_b_counters[] = { 1920 { .start = 0xdc48, .end = 0xdc48 }, /* OAA_ENABLE_REG */ 1921 { .start = 0xdd00, .end = 0xdd48 }, /* OAG_LCE0_0 - OAA_LENABLE_REG */ 1922 {} 1923}; 1924 1925static const struct xe_mmio_range gen12_oa_b_counters[] = { 1926 { .start = 0x2b2c, .end = 0x2b2c }, /* OAG_OA_PESS */ 1927 { .start = 0xd900, .end = 0xd91c }, /* OAG_OASTARTTRIG[1-8] */ 1928 { .start = 0xd920, .end = 0xd93c }, /* OAG_OAREPORTTRIG1[1-8] */ 1929 { .start = 0xd940, .end = 0xd97c }, /* OAG_CEC[0-7][0-1] */ 1930 { .start = 0xdc00, .end = 0xdc3c }, /* OAG_SCEC[0-7][0-1] */ 1931 { .start = 0xdc40, .end = 0xdc40 }, /* OAG_SPCTR_CNF */ 1932 { .start = 0xdc44, .end = 0xdc44 }, /* OAA_DBG_REG */ 1933 {} 1934}; 1935 1936static const struct xe_mmio_range mtl_oam_b_counters[] = { 1937 { .start = 0x393000, .end = 0x39301c }, /* OAM_STARTTRIG1[1-8] */ 1938 { .start = 0x393020, .end = 0x39303c }, /* OAM_REPORTTRIG1[1-8] */ 1939 { .start = 0x393040, .end = 0x39307c }, /* OAM_CEC[0-7][0-1] */ 1940 { .start = 0x393200, .end = 0x39323C }, /* MPES[0-7] */ 1941 {} 1942}; 1943 1944static const struct xe_mmio_range xe2_oa_b_counters[] = { 1945 { .start = 0x393200, .end = 0x39323C }, /* MPES_0_MPES_SAG - MPES_7_UPPER_MPES_SAG */ 1946 { .start = 0x394200, .end = 0x39423C }, /* MPES_0_MPES_SCMI0 - MPES_7_UPPER_MPES_SCMI0 */ 1947 { .start = 0x394A00, .end = 0x394A3C }, /* MPES_0_MPES_SCMI1 - MPES_7_UPPER_MPES_SCMI1 */ 1948 {}, 1949}; 1950 1951static bool xe_oa_is_valid_b_counter_addr(struct xe_oa *oa, u32 addr) 1952{ 1953 return xe_oa_reg_in_range_table(addr, xehp_oa_b_counters) || 1954 xe_oa_reg_in_range_table(addr, gen12_oa_b_counters) || 1955 xe_oa_reg_in_range_table(addr, mtl_oam_b_counters) || 1956 (GRAPHICS_VER(oa->xe) >= 20 && 1957 xe_oa_reg_in_range_table(addr, xe2_oa_b_counters)); 1958} 1959 1960static const struct xe_mmio_range mtl_oa_mux_regs[] = { 1961 { .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */ 1962 { .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */ 1963 { .start = 0x9840, .end = 0x9840 }, /* GDT_CHICKEN_BITS */ 1964 { .start = 0x9884, .end = 0x9888 }, /* NOA_WRITE */ 1965 { .start = 0x38d100, .end = 0x38d114}, /* VISACTL */ 1966 {} 1967}; 1968 1969static const struct xe_mmio_range gen12_oa_mux_regs[] = { 1970 { .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */ 1971 { .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */ 1972 { .start = 0x9840, .end = 0x9840 }, /* GDT_CHICKEN_BITS */ 1973 { .start = 0x9884, .end = 0x9888 }, /* NOA_WRITE */ 1974 { .start = 0x20cc, .end = 0x20cc }, /* WAIT_FOR_RC6_EXIT */ 1975 {} 1976}; 1977 1978static const struct xe_mmio_range xe2_oa_mux_regs[] = { 1979 { .start = 0x5194, .end = 0x5194 }, /* SYS_MEM_LAT_MEASURE_MERTF_GRP_3D */ 1980 { .start = 0x8704, .end = 0x8704 }, /* LMEM_LAT_MEASURE_MCFG_GRP */ 1981 { .start = 0xB1BC, .end = 0xB1BC }, /* L3_BANK_LAT_MEASURE_LBCF_GFX */ 1982 { .start = 0xE18C, .end = 0xE18C }, /* SAMPLER_MODE */ 1983 { .start = 0xE590, .end = 0xE590 }, /* TDL_LSC_LAT_MEASURE_TDL_GFX */ 1984 { .start = 0x13000, .end = 0x137FC }, /* PES_0_PESL0 - PES_63_UPPER_PESL3 */ 1985 {}, 1986}; 1987 1988static bool xe_oa_is_valid_mux_addr(struct xe_oa *oa, u32 addr) 1989{ 1990 if (GRAPHICS_VER(oa->xe) >= 20) 1991 return xe_oa_reg_in_range_table(addr, xe2_oa_mux_regs); 1992 else if (GRAPHICS_VERx100(oa->xe) >= 1270) 1993 return xe_oa_reg_in_range_table(addr, mtl_oa_mux_regs); 1994 else 1995 return xe_oa_reg_in_range_table(addr, gen12_oa_mux_regs); 1996} 1997 1998static bool xe_oa_is_valid_config_reg_addr(struct xe_oa *oa, u32 addr) 1999{ 2000 return xe_oa_is_valid_flex_addr(oa, addr) || 2001 xe_oa_is_valid_b_counter_addr(oa, addr) || 2002 xe_oa_is_valid_mux_addr(oa, addr); 2003} 2004 2005static struct xe_oa_reg * 2006xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr), 2007 u32 __user *regs, u32 n_regs) 2008{ 2009 struct xe_oa_reg *oa_regs; 2010 int err; 2011 u32 i; 2012 2013 oa_regs = kmalloc_array(n_regs, sizeof(*oa_regs), GFP_KERNEL); 2014 if (!oa_regs) 2015 return ERR_PTR(-ENOMEM); 2016 2017 for (i = 0; i < n_regs; i++) { 2018 u32 addr, value; 2019 2020 err = get_user(addr, regs); 2021 if (err) 2022 goto addr_err; 2023 2024 if (!is_valid(oa, addr)) { 2025 drm_dbg(&oa->xe->drm, "Invalid oa_reg address: %X\n", addr); 2026 err = -EINVAL; 2027 goto addr_err; 2028 } 2029 2030 err = get_user(value, regs + 1); 2031 if (err) 2032 goto addr_err; 2033 2034 oa_regs[i].addr = XE_REG(addr); 2035 oa_regs[i].value = value; 2036 2037 regs += 2; 2038 } 2039 2040 return oa_regs; 2041 2042addr_err: 2043 kfree(oa_regs); 2044 return ERR_PTR(err); 2045} 2046 2047static ssize_t show_dynamic_id(struct kobject *kobj, 2048 struct kobj_attribute *attr, 2049 char *buf) 2050{ 2051 struct xe_oa_config *oa_config = 2052 container_of(attr, typeof(*oa_config), sysfs_metric_id); 2053 2054 return sysfs_emit(buf, "%d\n", oa_config->id); 2055} 2056 2057static int create_dynamic_oa_sysfs_entry(struct xe_oa *oa, 2058 struct xe_oa_config *oa_config) 2059{ 2060 sysfs_attr_init(&oa_config->sysfs_metric_id.attr); 2061 oa_config->sysfs_metric_id.attr.name = "id"; 2062 oa_config->sysfs_metric_id.attr.mode = 0444; 2063 oa_config->sysfs_metric_id.show = show_dynamic_id; 2064 oa_config->sysfs_metric_id.store = NULL; 2065 2066 oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr; 2067 oa_config->attrs[1] = NULL; 2068 2069 oa_config->sysfs_metric.name = oa_config->uuid; 2070 oa_config->sysfs_metric.attrs = oa_config->attrs; 2071 2072 return sysfs_create_group(oa->metrics_kobj, &oa_config->sysfs_metric); 2073} 2074 2075/** 2076 * xe_oa_add_config_ioctl - Adds one OA config 2077 * @dev: @drm_device 2078 * @data: pointer to struct @drm_xe_oa_config 2079 * @file: @drm_file 2080 * 2081 * The functions adds an OA config to the set of OA configs maintained in 2082 * the kernel. The config determines which OA metrics are collected for an 2083 * OA stream. 2084 */ 2085int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file) 2086{ 2087 struct xe_device *xe = to_xe_device(dev); 2088 struct xe_oa *oa = &xe->oa; 2089 struct drm_xe_oa_config param; 2090 struct drm_xe_oa_config *arg = ¶m; 2091 struct xe_oa_config *oa_config, *tmp; 2092 struct xe_oa_reg *regs; 2093 int err, id; 2094 2095 if (!oa->xe) { 2096 drm_dbg(&xe->drm, "xe oa interface not available for this system\n"); 2097 return -ENODEV; 2098 } 2099 2100 if (xe_observation_paranoid && !perfmon_capable()) { 2101 drm_dbg(&oa->xe->drm, "Insufficient privileges to add xe OA config\n"); 2102 return -EACCES; 2103 } 2104 2105 err = __copy_from_user(¶m, u64_to_user_ptr(data), sizeof(param)); 2106 if (XE_IOCTL_DBG(oa->xe, err)) 2107 return -EFAULT; 2108 2109 if (XE_IOCTL_DBG(oa->xe, arg->extensions) || 2110 XE_IOCTL_DBG(oa->xe, !arg->regs_ptr) || 2111 XE_IOCTL_DBG(oa->xe, !arg->n_regs)) 2112 return -EINVAL; 2113 2114 oa_config = kzalloc(sizeof(*oa_config), GFP_KERNEL); 2115 if (!oa_config) 2116 return -ENOMEM; 2117 2118 oa_config->oa = oa; 2119 kref_init(&oa_config->ref); 2120 2121 if (!uuid_is_valid(arg->uuid)) { 2122 drm_dbg(&oa->xe->drm, "Invalid uuid format for OA config\n"); 2123 err = -EINVAL; 2124 goto reg_err; 2125 } 2126 2127 /* Last character in oa_config->uuid will be 0 because oa_config is kzalloc */ 2128 memcpy(oa_config->uuid, arg->uuid, sizeof(arg->uuid)); 2129 2130 oa_config->regs_len = arg->n_regs; 2131 regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg_addr, 2132 u64_to_user_ptr(arg->regs_ptr), 2133 arg->n_regs); 2134 if (IS_ERR(regs)) { 2135 drm_dbg(&oa->xe->drm, "Failed to create OA config for mux_regs\n"); 2136 err = PTR_ERR(regs); 2137 goto reg_err; 2138 } 2139 oa_config->regs = regs; 2140 2141 err = mutex_lock_interruptible(&oa->metrics_lock); 2142 if (err) 2143 goto reg_err; 2144 2145 /* We shouldn't have too many configs, so this iteration shouldn't be too costly */ 2146 idr_for_each_entry(&oa->metrics_idr, tmp, id) { 2147 if (!strcmp(tmp->uuid, oa_config->uuid)) { 2148 drm_dbg(&oa->xe->drm, "OA config already exists with this uuid\n"); 2149 err = -EADDRINUSE; 2150 goto sysfs_err; 2151 } 2152 } 2153 2154 err = create_dynamic_oa_sysfs_entry(oa, oa_config); 2155 if (err) { 2156 drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n"); 2157 goto sysfs_err; 2158 } 2159 2160 oa_config->id = idr_alloc(&oa->metrics_idr, oa_config, 1, 0, GFP_KERNEL); 2161 if (oa_config->id < 0) { 2162 drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n"); 2163 err = oa_config->id; 2164 goto sysfs_err; 2165 } 2166 2167 mutex_unlock(&oa->metrics_lock); 2168 2169 drm_dbg(&oa->xe->drm, "Added config %s id=%i\n", oa_config->uuid, oa_config->id); 2170 2171 return oa_config->id; 2172 2173sysfs_err: 2174 mutex_unlock(&oa->metrics_lock); 2175reg_err: 2176 xe_oa_config_put(oa_config); 2177 drm_dbg(&oa->xe->drm, "Failed to add new OA config\n"); 2178 return err; 2179} 2180 2181/** 2182 * xe_oa_remove_config_ioctl - Removes one OA config 2183 * @dev: @drm_device 2184 * @data: pointer to struct @drm_xe_observation_param 2185 * @file: @drm_file 2186 */ 2187int xe_oa_remove_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file) 2188{ 2189 struct xe_device *xe = to_xe_device(dev); 2190 struct xe_oa *oa = &xe->oa; 2191 struct xe_oa_config *oa_config; 2192 u64 arg, *ptr = u64_to_user_ptr(data); 2193 int ret; 2194 2195 if (!oa->xe) { 2196 drm_dbg(&xe->drm, "xe oa interface not available for this system\n"); 2197 return -ENODEV; 2198 } 2199 2200 if (xe_observation_paranoid && !perfmon_capable()) { 2201 drm_dbg(&oa->xe->drm, "Insufficient privileges to remove xe OA config\n"); 2202 return -EACCES; 2203 } 2204 2205 ret = get_user(arg, ptr); 2206 if (XE_IOCTL_DBG(oa->xe, ret)) 2207 return ret; 2208 2209 ret = mutex_lock_interruptible(&oa->metrics_lock); 2210 if (ret) 2211 return ret; 2212 2213 oa_config = idr_find(&oa->metrics_idr, arg); 2214 if (!oa_config) { 2215 drm_dbg(&oa->xe->drm, "Failed to remove unknown OA config\n"); 2216 ret = -ENOENT; 2217 goto err_unlock; 2218 } 2219 2220 WARN_ON(arg != oa_config->id); 2221 2222 sysfs_remove_group(oa->metrics_kobj, &oa_config->sysfs_metric); 2223 idr_remove(&oa->metrics_idr, arg); 2224 2225 mutex_unlock(&oa->metrics_lock); 2226 2227 drm_dbg(&oa->xe->drm, "Removed config %s id=%i\n", oa_config->uuid, oa_config->id); 2228 2229 xe_oa_config_put(oa_config); 2230 2231 return 0; 2232 2233err_unlock: 2234 mutex_unlock(&oa->metrics_lock); 2235 return ret; 2236} 2237 2238/** 2239 * xe_oa_register - Xe OA registration 2240 * @xe: @xe_device 2241 * 2242 * Exposes the metrics sysfs directory upon completion of module initialization 2243 */ 2244void xe_oa_register(struct xe_device *xe) 2245{ 2246 struct xe_oa *oa = &xe->oa; 2247 2248 if (!oa->xe) 2249 return; 2250 2251 oa->metrics_kobj = kobject_create_and_add("metrics", 2252 &xe->drm.primary->kdev->kobj); 2253} 2254 2255/** 2256 * xe_oa_unregister - Xe OA de-registration 2257 * @xe: @xe_device 2258 */ 2259void xe_oa_unregister(struct xe_device *xe) 2260{ 2261 struct xe_oa *oa = &xe->oa; 2262 2263 if (!oa->metrics_kobj) 2264 return; 2265 2266 kobject_put(oa->metrics_kobj); 2267 oa->metrics_kobj = NULL; 2268} 2269 2270static u32 num_oa_units_per_gt(struct xe_gt *gt) 2271{ 2272 return 1; 2273} 2274 2275static u32 __hwe_oam_unit(struct xe_hw_engine *hwe) 2276{ 2277 if (GRAPHICS_VERx100(gt_to_xe(hwe->gt)) >= 1270) { 2278 /* 2279 * There's 1 SAMEDIA gt and 1 OAM per SAMEDIA gt. All media slices 2280 * within the gt use the same OAM. All MTL/LNL SKUs list 1 SA MEDIA 2281 */ 2282 xe_gt_WARN_ON(hwe->gt, hwe->gt->info.type != XE_GT_TYPE_MEDIA); 2283 2284 return 0; 2285 } 2286 2287 return XE_OA_UNIT_INVALID; 2288} 2289 2290static u32 __hwe_oa_unit(struct xe_hw_engine *hwe) 2291{ 2292 switch (hwe->class) { 2293 case XE_ENGINE_CLASS_RENDER: 2294 case XE_ENGINE_CLASS_COMPUTE: 2295 return 0; 2296 2297 case XE_ENGINE_CLASS_VIDEO_DECODE: 2298 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 2299 return __hwe_oam_unit(hwe); 2300 2301 default: 2302 return XE_OA_UNIT_INVALID; 2303 } 2304} 2305 2306static struct xe_oa_regs __oam_regs(u32 base) 2307{ 2308 return (struct xe_oa_regs) { 2309 base, 2310 OAM_HEAD_POINTER(base), 2311 OAM_TAIL_POINTER(base), 2312 OAM_BUFFER(base), 2313 OAM_CONTEXT_CONTROL(base), 2314 OAM_CONTROL(base), 2315 OAM_DEBUG(base), 2316 OAM_STATUS(base), 2317 OAM_CONTROL_COUNTER_SEL_MASK, 2318 }; 2319} 2320 2321static struct xe_oa_regs __oag_regs(void) 2322{ 2323 return (struct xe_oa_regs) { 2324 0, 2325 OAG_OAHEADPTR, 2326 OAG_OATAILPTR, 2327 OAG_OABUFFER, 2328 OAG_OAGLBCTXCTRL, 2329 OAG_OACONTROL, 2330 OAG_OA_DEBUG, 2331 OAG_OASTATUS, 2332 OAG_OACONTROL_OA_COUNTER_SEL_MASK, 2333 }; 2334} 2335 2336static void __xe_oa_init_oa_units(struct xe_gt *gt) 2337{ 2338 const u32 mtl_oa_base[] = { 0x13000 }; 2339 int i, num_units = gt->oa.num_oa_units; 2340 2341 for (i = 0; i < num_units; i++) { 2342 struct xe_oa_unit *u = >->oa.oa_unit[i]; 2343 2344 if (gt->info.type != XE_GT_TYPE_MEDIA) { 2345 u->regs = __oag_regs(); 2346 u->type = DRM_XE_OA_UNIT_TYPE_OAG; 2347 } else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) { 2348 u->regs = __oam_regs(mtl_oa_base[i]); 2349 u->type = DRM_XE_OA_UNIT_TYPE_OAM; 2350 } 2351 2352 /* Ensure MMIO trigger remains disabled till there is a stream */ 2353 xe_mmio_write32(gt, u->regs.oa_debug, 2354 oag_configure_mmio_trigger(NULL, false)); 2355 2356 /* Set oa_unit_ids now to ensure ids remain contiguous */ 2357 u->oa_unit_id = gt_to_xe(gt)->oa.oa_unit_ids++; 2358 } 2359} 2360 2361static int xe_oa_init_gt(struct xe_gt *gt) 2362{ 2363 u32 num_oa_units = num_oa_units_per_gt(gt); 2364 struct xe_hw_engine *hwe; 2365 enum xe_hw_engine_id id; 2366 struct xe_oa_unit *u; 2367 2368 u = drmm_kcalloc(>_to_xe(gt)->drm, num_oa_units, sizeof(*u), GFP_KERNEL); 2369 if (!u) 2370 return -ENOMEM; 2371 2372 for_each_hw_engine(hwe, gt, id) { 2373 u32 index = __hwe_oa_unit(hwe); 2374 2375 hwe->oa_unit = NULL; 2376 if (index < num_oa_units) { 2377 u[index].num_engines++; 2378 hwe->oa_unit = &u[index]; 2379 } 2380 } 2381 2382 /* 2383 * Fused off engines can result in oa_unit's with num_engines == 0. These units 2384 * will appear in OA unit query, but no OA streams can be opened on them. 2385 */ 2386 gt->oa.num_oa_units = num_oa_units; 2387 gt->oa.oa_unit = u; 2388 2389 __xe_oa_init_oa_units(gt); 2390 2391 drmm_mutex_init(>_to_xe(gt)->drm, >->oa.gt_lock); 2392 2393 return 0; 2394} 2395 2396static int xe_oa_init_oa_units(struct xe_oa *oa) 2397{ 2398 struct xe_gt *gt; 2399 int i, ret; 2400 2401 for_each_gt(gt, oa->xe, i) { 2402 ret = xe_oa_init_gt(gt); 2403 if (ret) 2404 return ret; 2405 } 2406 2407 return 0; 2408} 2409 2410static void oa_format_add(struct xe_oa *oa, enum xe_oa_format_name format) 2411{ 2412 __set_bit(format, oa->format_mask); 2413} 2414 2415static void xe_oa_init_supported_formats(struct xe_oa *oa) 2416{ 2417 if (GRAPHICS_VER(oa->xe) >= 20) { 2418 /* Xe2+ */ 2419 oa_format_add(oa, XE_OAM_FORMAT_MPEC8u64_B8_C8); 2420 oa_format_add(oa, XE_OAM_FORMAT_MPEC8u32_B8_C8); 2421 oa_format_add(oa, XE_OA_FORMAT_PEC64u64); 2422 oa_format_add(oa, XE_OA_FORMAT_PEC64u64_B8_C8); 2423 oa_format_add(oa, XE_OA_FORMAT_PEC64u32); 2424 oa_format_add(oa, XE_OA_FORMAT_PEC32u64_G1); 2425 oa_format_add(oa, XE_OA_FORMAT_PEC32u32_G1); 2426 oa_format_add(oa, XE_OA_FORMAT_PEC32u64_G2); 2427 oa_format_add(oa, XE_OA_FORMAT_PEC32u32_G2); 2428 oa_format_add(oa, XE_OA_FORMAT_PEC36u64_G1_32_G2_4); 2429 oa_format_add(oa, XE_OA_FORMAT_PEC36u64_G1_4_G2_32); 2430 } else if (GRAPHICS_VERx100(oa->xe) >= 1270) { 2431 /* XE_METEORLAKE */ 2432 oa_format_add(oa, XE_OAR_FORMAT_A32u40_A4u32_B8_C8); 2433 oa_format_add(oa, XE_OA_FORMAT_A24u40_A14u32_B8_C8); 2434 oa_format_add(oa, XE_OAC_FORMAT_A24u64_B8_C8); 2435 oa_format_add(oa, XE_OAC_FORMAT_A22u32_R2u32_B8_C8); 2436 oa_format_add(oa, XE_OAM_FORMAT_MPEC8u64_B8_C8); 2437 oa_format_add(oa, XE_OAM_FORMAT_MPEC8u32_B8_C8); 2438 } else if (GRAPHICS_VERx100(oa->xe) >= 1255) { 2439 /* XE_DG2, XE_PVC */ 2440 oa_format_add(oa, XE_OAR_FORMAT_A32u40_A4u32_B8_C8); 2441 oa_format_add(oa, XE_OA_FORMAT_A24u40_A14u32_B8_C8); 2442 oa_format_add(oa, XE_OAC_FORMAT_A24u64_B8_C8); 2443 oa_format_add(oa, XE_OAC_FORMAT_A22u32_R2u32_B8_C8); 2444 } else { 2445 /* Gen12+ */ 2446 xe_assert(oa->xe, GRAPHICS_VER(oa->xe) >= 12); 2447 oa_format_add(oa, XE_OA_FORMAT_A12); 2448 oa_format_add(oa, XE_OA_FORMAT_A12_B8_C8); 2449 oa_format_add(oa, XE_OA_FORMAT_A32u40_A4u32_B8_C8); 2450 oa_format_add(oa, XE_OA_FORMAT_C4_B8); 2451 } 2452} 2453 2454/** 2455 * xe_oa_init - OA initialization during device probe 2456 * @xe: @xe_device 2457 * 2458 * Return: 0 on success or a negative error code on failure 2459 */ 2460int xe_oa_init(struct xe_device *xe) 2461{ 2462 struct xe_oa *oa = &xe->oa; 2463 int ret; 2464 2465 /* Support OA only with GuC submission and Gen12+ */ 2466 if (!xe_device_uc_enabled(xe) || GRAPHICS_VER(xe) < 12) 2467 return 0; 2468 2469 if (IS_SRIOV_VF(xe)) 2470 return 0; 2471 2472 oa->xe = xe; 2473 oa->oa_formats = oa_formats; 2474 2475 drmm_mutex_init(&oa->xe->drm, &oa->metrics_lock); 2476 idr_init_base(&oa->metrics_idr, 1); 2477 2478 ret = xe_oa_init_oa_units(oa); 2479 if (ret) { 2480 drm_err(&xe->drm, "OA initialization failed (%pe)\n", ERR_PTR(ret)); 2481 goto exit; 2482 } 2483 2484 xe_oa_init_supported_formats(oa); 2485 return 0; 2486exit: 2487 oa->xe = NULL; 2488 return ret; 2489} 2490 2491static int destroy_config(int id, void *p, void *data) 2492{ 2493 xe_oa_config_put(p); 2494 return 0; 2495} 2496 2497/** 2498 * xe_oa_fini - OA de-initialization during device remove 2499 * @xe: @xe_device 2500 */ 2501void xe_oa_fini(struct xe_device *xe) 2502{ 2503 struct xe_oa *oa = &xe->oa; 2504 2505 if (!oa->xe) 2506 return; 2507 2508 idr_for_each(&oa->metrics_idr, destroy_config, oa); 2509 idr_destroy(&oa->metrics_idr); 2510 2511 oa->xe = NULL; 2512} 2513