Lines Matching refs:stream
36 * descriptor representing a stream of GPU metrics which can then be read() as
37 * a stream of sample records.
56 * i915 perf file descriptors represent a "stream" instead of an "event"; where
57 * a perf event primarily corresponds to a single 64bit value, while a stream
61 * of related counters. Samples for an i915 perf stream capturing OA metrics
64 * selected by the user opening the stream. Perf has support for grouping
68 * i915 perf stream configurations are provided as an array of u64 (key,value)
343 * struct perf_open_properties - for validated properties given to open a stream
362 * to open a stream of metrics the configuration is built up in the structure
431 struct i915_perf_regs *__oa_regs(struct i915_perf_stream *stream)
433 return &stream->engine->oa_group->regs;
436 static u32 gen12_oa_hw_tail_read(struct i915_perf_stream *stream)
438 struct intel_uncore *uncore = stream->uncore;
440 return intel_uncore_read(uncore, __oa_regs(stream)->oa_tail_ptr) &
444 static u32 gen8_oa_hw_tail_read(struct i915_perf_stream *stream)
446 struct intel_uncore *uncore = stream->uncore;
451 static u32 gen7_oa_hw_tail_read(struct i915_perf_stream *stream)
453 struct intel_uncore *uncore = stream->uncore;
462 static u64 oa_report_id(struct i915_perf_stream *stream, void *report)
464 return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report;
467 static u64 oa_report_reason(struct i915_perf_stream *stream, void *report)
469 return (oa_report_id(stream, report) >> OAREPORT_REASON_SHIFT) &
470 (GRAPHICS_VER(stream->perf->i915) == 12 ?
475 static void oa_report_id_clear(struct i915_perf_stream *stream, u32 *report)
477 if (oa_report_header_64bit(stream))
483 static bool oa_report_ctx_invalid(struct i915_perf_stream *stream, void *report)
485 return !(oa_report_id(stream, report) &
486 stream->perf->gen8_valid_ctx_bit);
489 static u64 oa_timestamp(struct i915_perf_stream *stream, void *report)
491 return oa_report_header_64bit(stream) ?
496 static void oa_timestamp_clear(struct i915_perf_stream *stream, u32 *report)
498 if (oa_report_header_64bit(stream))
504 static u32 oa_context_id(struct i915_perf_stream *stream, u32 *report)
506 u32 ctx_id = oa_report_header_64bit(stream) ? report[4] : report[2];
508 return ctx_id & stream->specific_ctx_id_mask;
511 static void oa_context_id_squash(struct i915_perf_stream *stream, u32 *report)
513 if (oa_report_header_64bit(stream))
521 * @stream: i915 stream instance
537 * only called while the stream is enabled, while the global OA configuration
542 static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream)
544 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
545 int report_size = stream->oa_buffer.format->size;
555 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
557 hw_tail = stream->perf->ops.oa_hw_tail_read(stream);
564 partial_report_size = OA_TAKEN(hw_tail, stream->oa_buffer.tail);
572 /* Walk the stream backward until we find a report with report
583 while (OA_TAKEN(tail, stream->oa_buffer.tail) >= report_size) {
584 void *report = stream->oa_buffer.vaddr + tail;
586 if (oa_report_id(stream, report) ||
587 oa_timestamp(stream, report))
594 __ratelimit(&stream->perf->tail_pointer_race))
595 drm_notice(&stream->uncore->i915->drm,
597 stream->oa_buffer.head, tail, hw_tail);
599 stream->oa_buffer.tail = tail;
601 pollin = OA_TAKEN(stream->oa_buffer.tail,
602 stream->oa_buffer.head) >= report_size;
604 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
611 * @stream: An i915-perf stream opened for OA metrics
624 static int append_oa_status(struct i915_perf_stream *stream,
645 * @stream: An i915-perf stream opened for OA metrics
652 * properties when opening a stream, tracked as `stream->sample_flags`. This
660 static int append_oa_sample(struct i915_perf_stream *stream,
666 int report_size = stream->oa_buffer.format->size;
673 header.size = stream->sample_size;
683 oa_buf_end = stream->oa_buffer.vaddr + OA_BUFFER_SIZE;
691 if (copy_to_user(buf, stream->oa_buffer.vaddr,
706 * @stream: An i915-perf stream opened for OA metrics
724 static int gen8_append_oa_reports(struct i915_perf_stream *stream,
729 struct intel_uncore *uncore = stream->uncore;
730 int report_size = stream->oa_buffer.format->size;
731 u8 *oa_buf_base = stream->oa_buffer.vaddr;
732 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
739 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled))
742 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
744 head = stream->oa_buffer.head;
745 tail = stream->oa_buffer.tail;
747 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
776 reason = oa_report_reason(stream, report);
777 ctx_id = oa_context_id(stream, report32);
819 if (oa_report_ctx_invalid(stream, report) &&
820 GRAPHICS_VER_FULL(stream->engine->i915) < IP_VER(12, 55)) {
822 oa_context_id_squash(stream, report32);
856 if (!stream->ctx ||
857 stream->specific_ctx_id == ctx_id ||
858 stream->oa_buffer.last_ctx_id == stream->specific_ctx_id ||
865 if (stream->ctx &&
866 stream->specific_ctx_id != ctx_id) {
867 oa_context_id_squash(stream, report32);
870 ret = append_oa_sample(stream, buf, count, offset,
875 stream->oa_buffer.last_ctx_id = ctx_id;
883 oa_report_id_clear(stream, report32);
884 oa_timestamp_clear(stream, report32);
886 u8 *oa_buf_end = stream->oa_buffer.vaddr +
903 oaheadptr = GRAPHICS_VER(stream->perf->i915) == 12 ?
904 __oa_regs(stream)->oa_head_ptr :
907 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
915 stream->oa_buffer.head = head;
917 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
925 * @stream: An i915-perf stream opened for OA metrics
943 static int gen8_oa_read(struct i915_perf_stream *stream,
948 struct intel_uncore *uncore = stream->uncore;
953 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr))
956 oastatus_reg = GRAPHICS_VER(stream->perf->i915) == 12 ?
957 __oa_regs(stream)->oa_status :
977 ret = append_oa_status(stream, buf, count, offset,
982 drm_dbg(&stream->perf->i915->drm,
984 stream->period_exponent);
986 stream->perf->ops.oa_disable(stream);
987 stream->perf->ops.oa_enable(stream);
997 ret = append_oa_status(stream, buf, count, offset,
1010 return gen8_append_oa_reports(stream, buf, count, offset);
1016 * @stream: An i915-perf stream opened for OA metrics
1034 static int gen7_append_oa_reports(struct i915_perf_stream *stream,
1039 struct intel_uncore *uncore = stream->uncore;
1040 int report_size = stream->oa_buffer.format->size;
1041 u8 *oa_buf_base = stream->oa_buffer.vaddr;
1042 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
1049 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled))
1052 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1054 head = stream->oa_buffer.head;
1055 tail = stream->oa_buffer.tail;
1057 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1101 if (__ratelimit(&stream->perf->spurious_report_rs))
1107 ret = append_oa_sample(stream, buf, count, offset, report);
1119 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1124 stream->oa_buffer.head = head;
1126 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1134 * @stream: An i915-perf stream opened for OA metrics
1148 static int gen7_oa_read(struct i915_perf_stream *stream,
1153 struct intel_uncore *uncore = stream->uncore;
1157 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr))
1167 oastatus1 &= ~stream->perf->gen7_latched_oastatus1;
1190 ret = append_oa_status(stream, buf, count, offset,
1195 drm_dbg(&stream->perf->i915->drm,
1197 stream->period_exponent);
1199 stream->perf->ops.oa_disable(stream);
1200 stream->perf->ops.oa_enable(stream);
1206 ret = append_oa_status(stream, buf, count, offset,
1210 stream->perf->gen7_latched_oastatus1 |=
1214 return gen7_append_oa_reports(stream, buf, count, offset);
1219 * @stream: An i915-perf stream opened for OA metrics
1221 * Called when userspace tries to read() from a blocking stream FD opened
1231 static int i915_oa_wait_unlocked(struct i915_perf_stream *stream)
1234 if (!stream->periodic)
1237 return wait_event_interruptible(stream->poll_wq,
1238 oa_buffer_check_unlocked(stream));
1242 * i915_oa_poll_wait - call poll_wait() for an OA stream poll()
1243 * @stream: An i915-perf stream opened for OA metrics
1244 * @file: An i915 perf stream file
1247 * For handling userspace polling on an i915 perf stream opened for OA metrics,
1251 static void i915_oa_poll_wait(struct i915_perf_stream *stream,
1255 poll_wait(file, &stream->poll_wq, wait);
1260 * @stream: An i915-perf stream opened for OA metrics
1270 static int i915_oa_read(struct i915_perf_stream *stream,
1275 return stream->perf->ops.read(stream, buf, count, offset);
1278 static struct intel_context *oa_pin_context(struct i915_perf_stream *stream)
1281 struct i915_gem_context *ctx = stream->ctx;
1287 if (ce->engine != stream->engine) /* first match! */
1315 stream->pinned_ctx = ce;
1316 return stream->pinned_ctx;
1410 static int gen12_get_render_context_id(struct i915_perf_stream *stream)
1415 if (intel_engine_uses_guc(stream->engine)) {
1416 ret = gen12_guc_sw_ctx_id(stream->pinned_ctx, &ctx_id);
1422 } else if (GRAPHICS_VER_FULL(stream->engine->i915) >= IP_VER(12, 55)) {
1435 stream->specific_ctx_id = ctx_id & mask;
1436 stream->specific_ctx_id_mask = mask;
1514 * @stream: An i915-perf stream opened for OA metrics
1517 * lifetime of the stream. This ensures that we don't have to worry about
1522 static int oa_get_render_ctx_id(struct i915_perf_stream *stream)
1527 ce = oa_pin_context(stream);
1531 if (engine_supports_mi_query(stream->engine) &&
1532 HAS_LOGICAL_RING_CONTEXTS(stream->perf->i915)) {
1540 drm_err(&stream->perf->i915->drm,
1542 stream->engine->name);
1553 stream->specific_ctx_id = i915_ggtt_offset(ce->state);
1554 stream->specific_ctx_id_mask = 0;
1571 stream->specific_ctx_id = ce->lrc.lrca >> 12;
1577 stream->specific_ctx_id_mask =
1580 stream->specific_ctx_id_mask =
1582 stream->specific_ctx_id = stream->specific_ctx_id_mask;
1588 ret = gen12_get_render_context_id(stream);
1595 ce->tag = stream->specific_ctx_id;
1597 drm_dbg(&stream->perf->i915->drm,
1599 stream->specific_ctx_id,
1600 stream->specific_ctx_id_mask);
1607 * @stream: An i915-perf stream opened for OA metrics
1610 * for the lifetime of the stream, then that can be undone here.
1612 static void oa_put_render_ctx_id(struct i915_perf_stream *stream)
1616 ce = fetch_and_zero(&stream->pinned_ctx);
1622 stream->specific_ctx_id = INVALID_CTX_ID;
1623 stream->specific_ctx_id_mask = 0;
1627 free_oa_buffer(struct i915_perf_stream *stream)
1629 i915_vma_unpin_and_release(&stream->oa_buffer.vma,
1632 stream->oa_buffer.vaddr = NULL;
1636 free_oa_configs(struct i915_perf_stream *stream)
1640 i915_oa_config_put(stream->oa_config);
1641 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node)
1646 free_noa_wait(struct i915_perf_stream *stream)
1648 i915_vma_unpin_and_release(&stream->noa_wait, 0);
1661 static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
1663 struct i915_perf *perf = stream->perf;
1664 struct intel_gt *gt = stream->engine->gt;
1665 struct i915_perf_group *g = stream->engine->oa_group;
1667 if (WARN_ON(stream != g->exclusive_stream))
1677 perf->ops.disable_metric_set(stream);
1679 free_oa_buffer(stream);
1681 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL);
1682 intel_engine_pm_put(stream->engine);
1684 if (stream->ctx)
1685 oa_put_render_ctx_id(stream);
1687 free_oa_configs(stream);
1688 free_noa_wait(stream);
1696 static void gen7_init_oa_buffer(struct i915_perf_stream *stream)
1698 struct intel_uncore *uncore = stream->uncore;
1699 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
1702 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1709 stream->oa_buffer.head = 0;
1717 stream->oa_buffer.tail = 0;
1719 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1725 stream->perf->gen7_latched_oastatus1 = 0;
1730 * when re-enabling a stream or in error/reset paths.
1738 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE);
1741 static void gen8_init_oa_buffer(struct i915_perf_stream *stream)
1743 struct intel_uncore *uncore = stream->uncore;
1744 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
1747 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1751 stream->oa_buffer.head = 0;
1768 stream->oa_buffer.tail = 0;
1775 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID;
1777 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1783 * when re-enabling a stream or in error/reset paths.
1791 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE);
1794 static void gen12_init_oa_buffer(struct i915_perf_stream *stream)
1796 struct intel_uncore *uncore = stream->uncore;
1797 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);
1800 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
1802 intel_uncore_write(uncore, __oa_regs(stream)->oa_status, 0);
1803 intel_uncore_write(uncore, __oa_regs(stream)->oa_head_ptr,
1805 stream->oa_buffer.head = 0;
1815 intel_uncore_write(uncore, __oa_regs(stream)->oa_buffer, gtt_offset |
1817 intel_uncore_write(uncore, __oa_regs(stream)->oa_tail_ptr,
1821 stream->oa_buffer.tail = 0;
1828 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID;
1830 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
1836 * when re-enabling a stream or in error/reset paths.
1844 memset(stream->oa_buffer.vaddr, 0,
1845 stream->oa_buffer.vma->size);
1848 static int alloc_oa_buffer(struct i915_perf_stream *stream)
1850 struct drm_i915_private *i915 = stream->perf->i915;
1851 struct intel_gt *gt = stream->engine->gt;
1856 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.vma))
1862 bo = i915_gem_object_create_shmem(stream->perf->i915, OA_BUFFER_SIZE);
1887 stream->oa_buffer.vma = vma;
1889 stream->oa_buffer.vaddr =
1891 if (IS_ERR(stream->oa_buffer.vaddr)) {
1892 ret = PTR_ERR(stream->oa_buffer.vaddr);
1904 stream->oa_buffer.vaddr = NULL;
1905 stream->oa_buffer.vma = NULL;
1910 static u32 *save_restore_register(struct i915_perf_stream *stream, u32 *cs,
1919 if (GRAPHICS_VER(stream->perf->i915) >= 8)
1925 *cs++ = i915_ggtt_offset(stream->noa_wait) + offset + 4 * d;
1932 static int alloc_noa_wait(struct i915_perf_stream *stream)
1934 struct drm_i915_private *i915 = stream->perf->i915;
1935 struct intel_gt *gt = stream->engine->gt;
1939 intel_gt_ns_to_clock_interval(to_gt(stream->perf->i915),
1940 atomic64_read(&stream->perf->noa_programming_delay));
1941 const u32 base = stream->engine->mmio_base;
1980 * needs to be fixed during the lifetime of the i915/perf stream.
1998 stream->noa_wait = vma;
2006 stream, cs, true /* save */, CS_GPR(i),
2009 stream, cs, true /* save */, mi_predicate_result,
2125 stream, cs, false /* restore */, CS_GPR(i),
2128 stream, cs, false /* restore */, mi_predicate_result,
2189 alloc_oa_config_buffer(struct i915_perf_stream *stream,
2209 obj = i915_gem_object_create_shmem(stream->perf->i915, config_length);
2238 *cs++ = (GRAPHICS_VER(stream->perf->i915) < 8 ?
2241 *cs++ = i915_ggtt_offset(stream->noa_wait);
2248 &stream->engine->gt->ggtt->vm,
2256 llist_add(&oa_bo->node, &stream->oa_config_bos);
2277 get_oa_vma(struct i915_perf_stream *stream, struct i915_oa_config *oa_config)
2283 * to the stream.
2285 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) {
2293 oa_bo = alloc_oa_config_buffer(stream, oa_config);
2302 emit_oa_config(struct i915_perf_stream *stream,
2312 vma = get_oa_vma(stream, oa_config);
2372 static struct intel_context *oa_context(struct i915_perf_stream *stream)
2374 return stream->pinned_ctx ?: stream->engine->kernel_context;
2378 hsw_enable_metric_set(struct i915_perf_stream *stream,
2381 struct intel_uncore *uncore = stream->uncore;
2398 return emit_oa_config(stream,
2399 stream->oa_config, oa_context(stream),
2403 static void hsw_disable_metric_set(struct i915_perf_stream *stream)
2405 struct intel_uncore *uncore = stream->uncore;
2445 const struct i915_perf_stream *stream)
2447 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset;
2448 u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset;
2463 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
2464 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) |
2469 oa_config_flex_reg(stream->oa_config, flex_regs[i]);
2576 static int gen8_configure_context(struct i915_perf_stream *stream,
2606 static int gen12_configure_oar_context(struct i915_perf_stream *stream,
2610 struct intel_context *ce = stream->pinned_ctx;
2611 u32 format = stream->oa_buffer.format->format;
2612 u32 offset = stream->perf->ctx_oactxctrl_offset;
2657 * Manages updating the per-context aspects of the OA stream
2682 oa_configure_all_contexts(struct i915_perf_stream *stream,
2687 struct drm_i915_private *i915 = stream->perf->i915;
2689 struct intel_gt *gt = stream->engine->gt;
2718 err = gen8_configure_context(stream, ctx, regs, num_regs);
2752 gen12_configure_all_contexts(struct i915_perf_stream *stream,
2763 if (stream->engine->class != RENDER_CLASS)
2766 return oa_configure_all_contexts(stream,
2772 lrc_configure_all_contexts(struct i915_perf_stream *stream,
2776 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset;
2778 const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset;
2801 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
2802 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) |
2808 return oa_configure_all_contexts(stream,
2814 gen8_enable_metric_set(struct i915_perf_stream *stream,
2817 struct intel_uncore *uncore = stream->uncore;
2818 struct i915_oa_config *oa_config = stream->oa_config;
2844 if (IS_GRAPHICS_VER(stream->perf->i915, 9, 11)) {
2855 ret = lrc_configure_all_contexts(stream, oa_config, active);
2859 return emit_oa_config(stream,
2860 stream->oa_config, oa_context(stream),
2864 static u32 oag_report_ctx_switches(const struct i915_perf_stream *stream)
2867 (stream->sample_flags & SAMPLE_OA_REPORT) ?
2872 gen12_enable_metric_set(struct i915_perf_stream *stream,
2875 struct drm_i915_private *i915 = stream->perf->i915;
2876 struct intel_uncore *uncore = stream->uncore;
2877 struct i915_oa_config *oa_config = stream->oa_config;
2878 bool periodic = stream->periodic;
2879 u32 period_exponent = stream->period_exponent;
2895 intel_uncore_write(uncore, __oa_regs(stream)->oa_debug,
2903 oag_report_ctx_switches(stream));
2905 intel_uncore_write(uncore, __oa_regs(stream)->oa_ctx_ctrl, periodic ?
2926 ret = gen12_configure_all_contexts(stream, oa_config, active);
2935 if (stream->ctx) {
2936 ret = gen12_configure_oar_context(stream, active);
2941 return emit_oa_config(stream,
2942 stream->oa_config, oa_context(stream),
2946 static void gen8_disable_metric_set(struct i915_perf_stream *stream)
2948 struct intel_uncore *uncore = stream->uncore;
2951 lrc_configure_all_contexts(stream, NULL, NULL);
2956 static void gen11_disable_metric_set(struct i915_perf_stream *stream)
2958 struct intel_uncore *uncore = stream->uncore;
2961 lrc_configure_all_contexts(stream, NULL, NULL);
2967 static void gen12_disable_metric_set(struct i915_perf_stream *stream)
2969 struct intel_uncore *uncore = stream->uncore;
2970 struct drm_i915_private *i915 = stream->perf->i915;
2984 gen12_configure_all_contexts(stream, NULL, NULL);
2987 if (stream->ctx)
2988 gen12_configure_oar_context(stream, NULL);
3000 static void gen7_oa_enable(struct i915_perf_stream *stream)
3002 struct intel_uncore *uncore = stream->uncore;
3003 struct i915_gem_context *ctx = stream->ctx;
3004 u32 ctx_id = stream->specific_ctx_id;
3005 bool periodic = stream->periodic;
3006 u32 period_exponent = stream->period_exponent;
3007 u32 report_format = stream->oa_buffer.format->format;
3018 gen7_init_oa_buffer(stream);
3030 static void gen8_oa_enable(struct i915_perf_stream *stream)
3032 struct intel_uncore *uncore = stream->uncore;
3033 u32 report_format = stream->oa_buffer.format->format;
3044 gen8_init_oa_buffer(stream);
3056 static void gen12_oa_enable(struct i915_perf_stream *stream)
3065 if (!(stream->sample_flags & SAMPLE_OA_REPORT))
3068 gen12_init_oa_buffer(stream);
3070 regs = __oa_regs(stream);
3071 val = (stream->oa_buffer.format->format << regs->oa_ctrl_counter_format_shift) |
3074 intel_uncore_write(stream->uncore, regs->oa_ctrl, val);
3078 * i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream
3079 * @stream: An i915 perf stream opened for OA metrics
3082 * when opening the stream. This also starts a hrtimer that will periodically
3086 static void i915_oa_stream_enable(struct i915_perf_stream *stream)
3088 stream->pollin = false;
3090 stream->perf->ops.oa_enable(stream);
3092 if (stream->sample_flags & SAMPLE_OA_REPORT)
3093 hrtimer_start(&stream->poll_check_timer,
3094 ns_to_ktime(stream->poll_oa_period),
3098 static void gen7_oa_disable(struct i915_perf_stream *stream)
3100 struct intel_uncore *uncore = stream->uncore;
3106 drm_err(&stream->perf->i915->drm,
3110 static void gen8_oa_disable(struct i915_perf_stream *stream)
3112 struct intel_uncore *uncore = stream->uncore;
3118 drm_err(&stream->perf->i915->drm,
3122 static void gen12_oa_disable(struct i915_perf_stream *stream)
3124 struct intel_uncore *uncore = stream->uncore;
3126 intel_uncore_write(uncore, __oa_regs(stream)->oa_ctrl, 0);
3128 __oa_regs(stream)->oa_ctrl,
3131 drm_err(&stream->perf->i915->drm,
3139 drm_err(&stream->perf->i915->drm,
3144 * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream
3145 * @stream: An i915 perf stream opened for OA metrics
3151 static void i915_oa_stream_disable(struct i915_perf_stream *stream)
3153 stream->perf->ops.oa_disable(stream);
3155 if (stream->sample_flags & SAMPLE_OA_REPORT)
3156 hrtimer_cancel(&stream->poll_check_timer);
3168 static int i915_perf_stream_enable_sync(struct i915_perf_stream *stream)
3177 err = stream->perf->ops.enable_metric_set(stream, active);
3244 * i915_oa_stream_init - validate combined props for OA stream and init
3245 * @stream: An i915 perf stream
3247 * @props: The property state that configures stream (individually validated)
3252 * At this point it has been determined that userspace wants a stream of
3261 static int i915_oa_stream_init(struct i915_perf_stream *stream,
3265 struct drm_i915_private *i915 = stream->perf->i915;
3266 struct i915_perf *perf = stream->perf;
3271 drm_dbg(&stream->perf->i915->drm,
3283 drm_dbg(&stream->perf->i915->drm,
3289 (GRAPHICS_VER(perf->i915) < 12 || !stream->ctx)) {
3290 drm_dbg(&stream->perf->i915->drm,
3296 drm_dbg(&stream->perf->i915->drm,
3307 drm_dbg(&stream->perf->i915->drm,
3313 drm_dbg(&stream->perf->i915->drm,
3318 stream->engine = props->engine;
3319 stream->uncore = stream->engine->gt->uncore;
3321 stream->sample_size = sizeof(struct drm_i915_perf_record_header);
3323 stream->oa_buffer.format = &perf->oa_formats[props->oa_format];
3324 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.format->size == 0))
3327 stream->sample_flags = props->sample_flags;
3328 stream->sample_size += stream->oa_buffer.format->size;
3330 stream->hold_preemption = props->hold_preemption;
3332 stream->periodic = props->oa_periodic;
3333 if (stream->periodic)
3334 stream->period_exponent = props->oa_period_exponent;
3336 if (stream->ctx) {
3337 ret = oa_get_render_ctx_id(stream);
3339 drm_dbg(&stream->perf->i915->drm,
3345 ret = alloc_noa_wait(stream);
3347 drm_dbg(&stream->perf->i915->drm,
3352 stream->oa_config = i915_perf_get_oa_config(perf, props->metrics_set);
3353 if (!stream->oa_config) {
3354 drm_dbg(&stream->perf->i915->drm,
3372 intel_engine_pm_get(stream->engine);
3373 intel_uncore_forcewake_get(stream->uncore, FORCEWAKE_ALL);
3375 ret = alloc_oa_buffer(stream);
3379 stream->ops = &i915_oa_stream_ops;
3381 stream->engine->gt->perf.sseu = props->sseu;
3382 WRITE_ONCE(g->exclusive_stream, stream);
3384 ret = i915_perf_stream_enable_sync(stream);
3386 drm_dbg(&stream->perf->i915->drm,
3391 drm_dbg(&stream->perf->i915->drm,
3392 "opening stream oa config uuid=%s\n",
3393 stream->oa_config->uuid);
3395 hrtimer_init(&stream->poll_check_timer,
3397 stream->poll_check_timer.function = oa_poll_check_timer_cb;
3398 init_waitqueue_head(&stream->poll_wq);
3399 spin_lock_init(&stream->oa_buffer.ptr_lock);
3400 mutex_init(&stream->lock);
3406 perf->ops.disable_metric_set(stream);
3408 free_oa_buffer(stream);
3411 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL);
3412 intel_engine_pm_put(stream->engine);
3414 free_oa_configs(stream);
3417 free_noa_wait(stream);
3420 if (stream->ctx)
3421 oa_put_render_ctx_id(stream);
3429 struct i915_perf_stream *stream;
3435 stream = READ_ONCE(engine->oa_group->exclusive_stream);
3436 if (stream && GRAPHICS_VER(stream->perf->i915) < 12)
3437 gen8_update_reg_state_unlocked(ce, stream);
3441 * i915_perf_read - handles read() FOP for i915 perf stream FDs
3442 * @file: An i915 perf stream file
3447 * The entry point for handling a read() on a stream file descriptor from
3449 * &i915_perf_stream_ops->read but to save having stream implementations (of
3452 * We can also consistently treat trying to read from a disabled stream
3453 * as an IO error so implementations can assume the stream is enabled
3463 struct i915_perf_stream *stream = file->private_data;
3468 * disabled stream as an error. In particular it might otherwise lead
3471 if (!stream->enabled || !(stream->sample_flags & SAMPLE_OA_REPORT))
3476 * stream->ops->wait_unlocked.
3483 ret = stream->ops->wait_unlocked(stream);
3487 mutex_lock(&stream->lock);
3488 ret = stream->ops->read(stream, buf, count, &offset);
3489 mutex_unlock(&stream->lock);
3492 mutex_lock(&stream->lock);
3493 ret = stream->ops->read(stream, buf, count, &offset);
3494 mutex_unlock(&stream->lock);
3509 stream->pollin = false;
3517 struct i915_perf_stream *stream =
3518 container_of(hrtimer, typeof(*stream), poll_check_timer);
3520 if (oa_buffer_check_unlocked(stream)) {
3521 stream->pollin = true;
3522 wake_up(&stream->poll_wq);
3526 ns_to_ktime(stream->poll_oa_period));
3532 * i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream
3533 * @stream: An i915 perf stream
3534 * @file: An i915 perf stream file
3537 * For handling userspace polling on an i915 perf stream, this calls through to
3539 * will be woken for new stream data.
3543 static __poll_t i915_perf_poll_locked(struct i915_perf_stream *stream,
3549 stream->ops->poll_wait(stream, file, wait);
3557 if (stream->pollin)
3564 * i915_perf_poll - call poll_wait() with a suitable wait queue for stream
3565 * @file: An i915 perf stream file
3568 * For handling userspace polling on an i915 perf stream, this ensures
3569 * poll_wait() gets called with a wait queue that will be woken for new stream
3578 struct i915_perf_stream *stream = file->private_data;
3581 mutex_lock(&stream->lock);
3582 ret = i915_perf_poll_locked(stream, file, wait);
3583 mutex_unlock(&stream->lock);
3590 * @stream: A disabled i915 perf stream
3592 * [Re]enables the associated capture of data for this stream.
3594 * If a stream was previously enabled then there's currently no intention
3598 static void i915_perf_enable_locked(struct i915_perf_stream *stream)
3600 if (stream->enabled)
3603 /* Allow stream->ops->enable() to refer to this */
3604 stream->enabled = true;
3606 if (stream->ops->enable)
3607 stream->ops->enable(stream);
3609 if (stream->hold_preemption)
3610 intel_context_set_nopreempt(stream->pinned_ctx);
3615 * @stream: An enabled i915 perf stream
3617 * Disables the associated capture of data for this stream.
3619 * The intention is that disabling an re-enabling a stream will ideally be
3620 * cheaper than destroying and re-opening a stream with the same configuration,
3622 * must be retained between disabling and re-enabling a stream.
3624 * Note: while a stream is disabled it's considered an error for userspace
3625 * to attempt to read from the stream (-EIO).
3627 static void i915_perf_disable_locked(struct i915_perf_stream *stream)
3629 if (!stream->enabled)
3632 /* Allow stream->ops->disable() to refer to this */
3633 stream->enabled = false;
3635 if (stream->hold_preemption)
3636 intel_context_clear_nopreempt(stream->pinned_ctx);
3638 if (stream->ops->disable)
3639 stream->ops->disable(stream);
3642 static long i915_perf_config_locked(struct i915_perf_stream *stream,
3646 long ret = stream->oa_config->id;
3648 config = i915_perf_get_oa_config(stream->perf, metrics_set);
3652 if (config != stream->oa_config) {
3664 err = emit_oa_config(stream, config, oa_context(stream), NULL);
3666 config = xchg(&stream->oa_config, config);
3677 * i915_perf_ioctl_locked - support ioctl() usage with i915 perf stream FDs
3678 * @stream: An i915 perf stream
3685 static long i915_perf_ioctl_locked(struct i915_perf_stream *stream,
3691 i915_perf_enable_locked(stream);
3694 i915_perf_disable_locked(stream);
3697 return i915_perf_config_locked(stream, arg);
3704 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
3705 * @file: An i915 perf stream file
3718 struct i915_perf_stream *stream = file->private_data;
3721 mutex_lock(&stream->lock);
3722 ret = i915_perf_ioctl_locked(stream, cmd, arg);
3723 mutex_unlock(&stream->lock);
3729 * i915_perf_destroy_locked - destroy an i915 perf stream
3730 * @stream: An i915 perf stream
3732 * Frees all resources associated with the given i915 perf @stream, disabling
3738 static void i915_perf_destroy_locked(struct i915_perf_stream *stream)
3740 if (stream->enabled)
3741 i915_perf_disable_locked(stream);
3743 if (stream->ops->destroy)
3744 stream->ops->destroy(stream);
3746 if (stream->ctx)
3747 i915_gem_context_put(stream->ctx);
3749 kfree(stream);
3753 * i915_perf_release - handles userspace close() of a stream file
3755 * @file: An i915 perf stream file
3757 * Cleans up any resources associated with an open i915 perf stream file.
3765 struct i915_perf_stream *stream = file->private_data;
3766 struct i915_perf *perf = stream->perf;
3767 struct intel_gt *gt = stream->engine->gt;
3771 * other user of stream->lock. Use the perf lock to destroy the stream
3775 i915_perf_destroy_locked(stream);
3778 /* Release the reference the perf stream kept on the driver. */
3800 * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD
3808 * Implements further stream config validation and stream initialization on
3817 * config validation and stream initialization details will be handled by
3819 * will be relevant to all stream types / backends.
3830 struct i915_perf_stream *stream = NULL;
3843 "Failed to look up context with ID %u for opening perf stream\n",
3866 * doesn't request global stream access (i.e. query based sampling
3901 "Insufficient privileges to open i915 perf stream\n");
3906 stream = kzalloc(sizeof(*stream), GFP_KERNEL);
3907 if (!stream) {
3912 stream->perf = perf;
3913 stream->ctx = specific_ctx;
3914 stream->poll_oa_period = props->poll_oa_period;
3916 ret = i915_oa_stream_init(stream, param, props);
3920 /* we avoid simply assigning stream->sample_flags = props->sample_flags
3924 if (WARN_ON(stream->sample_flags != props->sample_flags)) {
3934 stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags);
3941 i915_perf_enable_locked(stream);
3951 if (stream->ops->destroy)
3952 stream->ops->destroy(stream);
3954 kfree(stream);
3983 * read_properties_unlocked - validate + copy userspace stream open properties
3987 * @props: The stream configuration built up while validating properties
3991 * properties necessary for a particular kind of stream have been set.
4225 * i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD
4230 * Validates the stream open parameters given by userspace including flags
4233 * Very little is assumed up front about the nature of the stream being
4235 * i915-perf stream is expected to be a suitable interface for other forms of
4245 * Return: A newly opened i915 Perf stream file descriptor or negative
4292 * used to open an i915-perf stream.
4786 * and their content will be freed when the stream using the config is closed.
5179 * stream instead of waiting until driver _fini which no one