1#ifdef CONFIG_CPU_SUP_INTEL 2 3enum { 4 LBR_FORMAT_32 = 0x00, 5 LBR_FORMAT_LIP = 0x01, 6 LBR_FORMAT_EIP = 0x02, 7 LBR_FORMAT_EIP_FLAGS = 0x03, 8}; 9 10/* 11 * We only support LBR implementations that have FREEZE_LBRS_ON_PMI 12 * otherwise it becomes near impossible to get a reliable stack. 13 */ 14 15static void __intel_pmu_lbr_enable(void) 16{ 17 u64 debugctl; 18 19 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 20 debugctl |= (DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); 21 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 22} 23 24static void __intel_pmu_lbr_disable(void) 25{ 26 u64 debugctl; 27 28 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 29 debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); 30 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 31} 32 33static void intel_pmu_lbr_reset_32(void) 34{ 35 int i; 36 37 for (i = 0; i < x86_pmu.lbr_nr; i++) 38 wrmsrl(x86_pmu.lbr_from + i, 0); 39} 40 41static void intel_pmu_lbr_reset_64(void) 42{ 43 int i; 44 45 for (i = 0; i < x86_pmu.lbr_nr; i++) { 46 wrmsrl(x86_pmu.lbr_from + i, 0); 47 wrmsrl(x86_pmu.lbr_to + i, 0); 48 } 49} 50 51static void intel_pmu_lbr_reset(void) 52{ 53 if (!x86_pmu.lbr_nr) 54 return; 55 56 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32) 57 intel_pmu_lbr_reset_32(); 58 else 59 intel_pmu_lbr_reset_64(); 60} 61 62static void intel_pmu_lbr_enable(struct perf_event *event) 63{ 64 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 65 66 if (!x86_pmu.lbr_nr) 67 return; 68 69 WARN_ON_ONCE(cpuc->enabled); 70 71 /* 72 * Reset the LBR stack if we changed task context to 73 * avoid data leaks. 74 */ 75 76 if (event->ctx->task && cpuc->lbr_context != event->ctx) { 77 intel_pmu_lbr_reset(); 78 cpuc->lbr_context = event->ctx; 79 } 80 81 cpuc->lbr_users++; 82} 83 84static void intel_pmu_lbr_disable(struct perf_event *event) 85{ 86 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 87 88 if (!x86_pmu.lbr_nr) 89 return; 90 91 cpuc->lbr_users--; 92 WARN_ON_ONCE(cpuc->lbr_users < 0); 93 94 if (cpuc->enabled && !cpuc->lbr_users) 95 __intel_pmu_lbr_disable(); 96} 97 98static void intel_pmu_lbr_enable_all(void) 99{ 100 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 101 102 if (cpuc->lbr_users) 103 __intel_pmu_lbr_enable(); 104} 105 106static void intel_pmu_lbr_disable_all(void) 107{ 108 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 109 110 if (cpuc->lbr_users) 111 __intel_pmu_lbr_disable(); 112} 113 114static inline u64 intel_pmu_lbr_tos(void) 115{ 116 u64 tos; 117 118 rdmsrl(x86_pmu.lbr_tos, tos); 119 120 return tos; 121} 122 123static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) 124{ 125 unsigned long mask = x86_pmu.lbr_nr - 1; 126 u64 tos = intel_pmu_lbr_tos(); 127 int i; 128 129 for (i = 0; i < x86_pmu.lbr_nr; i++) { 130 unsigned long lbr_idx = (tos - i) & mask; 131 union { 132 struct { 133 u32 from; 134 u32 to; 135 }; 136 u64 lbr; 137 } msr_lastbranch; 138 139 rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr); 140 141 cpuc->lbr_entries[i].from = msr_lastbranch.from; 142 cpuc->lbr_entries[i].to = msr_lastbranch.to; 143 cpuc->lbr_entries[i].flags = 0; 144 } 145 cpuc->lbr_stack.nr = i; 146} 147 148#define LBR_FROM_FLAG_MISPRED (1ULL << 63) 149 150/* 151 * Due to lack of segmentation in Linux the effective address (offset) 152 * is the same as the linear address, allowing us to merge the LIP and EIP 153 * LBR formats. 154 */ 155static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) 156{ 157 unsigned long mask = x86_pmu.lbr_nr - 1; 158 int lbr_format = x86_pmu.intel_cap.lbr_format; 159 u64 tos = intel_pmu_lbr_tos(); 160 int i; 161 162 for (i = 0; i < x86_pmu.lbr_nr; i++) { 163 unsigned long lbr_idx = (tos - i) & mask; 164 u64 from, to, flags = 0; 165 166 rdmsrl(x86_pmu.lbr_from + lbr_idx, from); 167 rdmsrl(x86_pmu.lbr_to + lbr_idx, to); 168 169 if (lbr_format == LBR_FORMAT_EIP_FLAGS) { 170 flags = !!(from & LBR_FROM_FLAG_MISPRED); 171 from = (u64)((((s64)from) << 1) >> 1); 172 } 173 174 cpuc->lbr_entries[i].from = from; 175 cpuc->lbr_entries[i].to = to; 176 cpuc->lbr_entries[i].flags = flags; 177 } 178 cpuc->lbr_stack.nr = i; 179} 180 181static void intel_pmu_lbr_read(void) 182{ 183 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 184 185 if (!cpuc->lbr_users) 186 return; 187 188 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32) 189 intel_pmu_lbr_read_32(cpuc); 190 else 191 intel_pmu_lbr_read_64(cpuc); 192} 193 194static void intel_pmu_lbr_init_core(void) 195{ 196 x86_pmu.lbr_nr = 4; 197 x86_pmu.lbr_tos = 0x01c9; 198 x86_pmu.lbr_from = 0x40; 199 x86_pmu.lbr_to = 0x60; 200} 201 202static void intel_pmu_lbr_init_nhm(void) 203{ 204 x86_pmu.lbr_nr = 16; 205 x86_pmu.lbr_tos = 0x01c9; 206 x86_pmu.lbr_from = 0x680; 207 x86_pmu.lbr_to = 0x6c0; 208} 209 210static void intel_pmu_lbr_init_atom(void) 211{ 212 x86_pmu.lbr_nr = 8; 213 x86_pmu.lbr_tos = 0x01c9; 214 x86_pmu.lbr_from = 0x40; 215 x86_pmu.lbr_to = 0x60; 216} 217 218#endif /* CONFIG_CPU_SUP_INTEL */ 219