1/*
2 * Copyright 2020, J��r��me Duval, jerome.duval@gmail.com.
3 * Copyright 2013, Haiku, Inc. All Rights Reserved.
4 * Distributed under the terms of the MIT License.
5 *
6 * Authors:
7 *		Pawe�� Dziepak, <pdziepak@quarnos.org>
8 */
9
10
11#include <cpufreq.h>
12#include <KernelExport.h>
13#include <new>
14
15#include <arch_cpu.h>
16#include <cpu.h>
17#include <smp.h>
18#include <util/AutoLock.h>
19
20
21#define INTEL_PSTATES_MODULE_NAME	CPUFREQ_MODULES_PREFIX "/intel_pstates/v1"
22
23
24const int kMinimalInterval = 50000;
25
26static uint16 sMinPState;
27static uint16 sMaxPState;
28static uint16 sBoostPState;
29static bool sHWPActive;
30static bool sHWPEPP;
31static uint8 sHWPLowest;
32static uint8 sHWPGuaranteed;
33static uint8 sHWPEfficient;
34static uint8 sHWPHighest;
35static bool sHWPPackage;
36
37static bool sAvoidBoost;
38
39
40struct CPUEntry {
41				CPUEntry();
42
43	uint16		fCurrentPState;
44
45	bigtime_t	fLastUpdate;
46
47	uint64		fPrevAperf;
48	uint64		fPrevMperf;
49} CACHE_LINE_ALIGN;
50static CPUEntry* sCPUEntries;
51
52
53CPUEntry::CPUEntry()
54	:
55	fCurrentPState(sMinPState - 1),
56	fLastUpdate(0)
57{
58}
59
60
61static void set_normal_pstate(void* /* dummy */, int cpu);
62
63
64static void
65pstates_set_scheduler_mode(scheduler_mode mode)
66{
67	sAvoidBoost = mode == SCHEDULER_MODE_POWER_SAVING;
68	if (sHWPActive)
69		call_all_cpus(set_normal_pstate, NULL);
70}
71
72
73static int
74measure_pstate(CPUEntry* entry)
75{
76	InterruptsLocker locker;
77
78	uint64 mperf = x86_read_msr(IA32_MSR_MPERF);
79	uint64 aperf = x86_read_msr(IA32_MSR_APERF);
80
81	locker.Unlock();
82
83	if (mperf == 0 || mperf == entry->fPrevMperf)
84		return sMinPState;
85
86	int oldPState = sMaxPState * (aperf - entry->fPrevAperf)
87		/ (mperf - entry->fPrevMperf);
88	oldPState = min_c(max_c(oldPState, sMinPState), sBoostPState);
89	entry->fPrevAperf = aperf;
90	entry->fPrevMperf = mperf;
91
92	return oldPState;
93}
94
95
96static inline void
97set_pstate(uint16 pstate)
98{
99	CPUEntry* entry = &sCPUEntries[smp_get_current_cpu()];
100	pstate = min_c(max_c(sMinPState, pstate), sBoostPState);
101
102	if (entry->fCurrentPState != pstate) {
103		entry->fLastUpdate = system_time();
104		entry->fCurrentPState = pstate;
105
106		x86_write_msr(IA32_MSR_PERF_CTL, pstate << 8);
107	}
108}
109
110
111static status_t
112pstates_increase_performance(int delta)
113{
114	CPUEntry* entry = &sCPUEntries[smp_get_current_cpu()];
115
116	if (sHWPActive)
117		return B_NOT_SUPPORTED;
118
119	if (system_time() - entry->fLastUpdate < kMinimalInterval)
120		return B_OK;
121
122	int pState = measure_pstate(entry);
123	pState += (sBoostPState - pState) * delta / kCPUPerformanceScaleMax;
124
125	if (sAvoidBoost && pState < (sMaxPState + sBoostPState) / 2)
126		pState = min_c(pState, sMaxPState);
127
128	set_pstate(pState);
129	return B_OK;
130}
131
132
133static status_t
134pstates_decrease_performance(int delta)
135{
136	CPUEntry* entry = &sCPUEntries[smp_get_current_cpu()];
137
138	if (sHWPActive)
139		return B_NOT_SUPPORTED;
140
141	if (system_time() - entry->fLastUpdate < kMinimalInterval)
142		return B_OK;
143
144	int pState = measure_pstate(entry);
145	pState -= (pState - sMinPState) * delta / kCPUPerformanceScaleMax;
146
147	set_pstate(pState);
148	return B_OK;
149}
150
151
152static bool
153is_cpu_model_supported(cpu_ent* cpu)
154{
155	uint8 model = cpu->arch.model + (cpu->arch.extended_model << 4);
156
157	if (cpu->arch.vendor != VENDOR_INTEL)
158		return false;
159
160	if (cpu->arch.family != 6)
161		return false;
162
163	if (x86_check_feature(IA32_FEATURE_HWP, FEATURE_6_EAX))
164		return true;
165
166	const uint8 kSupportedFamily6Models[] = {
167		0x2a, 0x2d, 0x3a, 0x3c, 0x3d, 0x3e, 0x3f, 0x45, 0x46, 0x47, 0x4a,
168		0x4d, 0x4e, 0x4f, 0x55, 0x56, 0x57, 0x5a, 0x5c, 0x5e, 0x5f, 0x75,
169		0x7a, 0x85
170	};
171	/* TODO: support Atom Silvermont and Airmont: 0x37, 0x4c */
172	const int kSupportedFamily6ModelsCount
173		= sizeof(kSupportedFamily6Models) / sizeof(uint8);
174
175	int i;
176	for (i = 0; i < kSupportedFamily6ModelsCount; i++) {
177		if (model == kSupportedFamily6Models[i])
178			break;
179	}
180
181	return i != kSupportedFamily6ModelsCount;
182}
183
184
185static void
186set_normal_pstate(void* /* dummy */, int cpu)
187{
188	if (sHWPActive) {
189		if (x86_check_feature(IA32_FEATURE_HWP_NOTIFY, FEATURE_6_EAX))
190			x86_write_msr(IA32_MSR_HWP_INTERRUPT, 0);
191		x86_write_msr(IA32_MSR_PM_ENABLE, 1);
192		uint64 hwpRequest = x86_read_msr(IA32_MSR_HWP_REQUEST);
193		uint64 caps = x86_read_msr(IA32_MSR_HWP_CAPABILITIES);
194		sHWPLowest = IA32_HWP_CAPS_LOWEST_PERFORMANCE(caps);
195		sHWPEfficient = IA32_HWP_CAPS_EFFICIENT_PERFORMANCE(caps);
196		sHWPGuaranteed = IA32_HWP_CAPS_GUARANTEED_PERFORMANCE(caps);
197		sHWPHighest = IA32_HWP_CAPS_HIGHEST_PERFORMANCE(caps);
198
199		hwpRequest &= ~IA32_HWP_REQUEST_DESIRED_PERFORMANCE;
200		hwpRequest &= ~IA32_HWP_REQUEST_ACTIVITY_WINDOW;
201
202		hwpRequest &= ~IA32_HWP_REQUEST_MINIMUM_PERFORMANCE;
203		hwpRequest |= sHWPLowest;
204
205		hwpRequest &= ~IA32_HWP_REQUEST_MAXIMUM_PERFORMANCE;
206		hwpRequest |= sHWPHighest << 8;
207
208		if (x86_check_feature(IA32_FEATURE_HWP_EPP, FEATURE_6_EAX)) {
209			hwpRequest &= ~IA32_HWP_REQUEST_ENERGY_PERFORMANCE_PREFERENCE;
210			hwpRequest |= (sAvoidBoost ? 0x80ULL : 0x0ULL) << 24;
211		} else if (x86_check_feature(IA32_FEATURE_EPB, FEATURE_6_ECX)) {
212			uint64 perfBias = x86_read_msr(IA32_MSR_ENERGY_PERF_BIAS);
213			perfBias &= ~(0xfULL << 0);
214			perfBias |= (sAvoidBoost ? 0xfULL : 0x0ULL) << 0;
215			x86_write_msr(IA32_MSR_ENERGY_PERF_BIAS, perfBias);
216		}
217
218		if (sHWPPackage) {
219			x86_write_msr(IA32_MSR_HWP_REQUEST, hwpRequest
220				| IA32_HWP_REQUEST_PACKAGE_CONTROL);
221			x86_write_msr(IA32_MSR_HWP_REQUEST_PKG, hwpRequest);
222		} else
223			x86_write_msr(IA32_MSR_HWP_REQUEST, hwpRequest);
224	} else {
225		measure_pstate(&sCPUEntries[cpu]);
226		set_pstate(sMaxPState);
227	}
228}
229
230
231static status_t
232init_pstates()
233{
234	if (!x86_check_feature(IA32_FEATURE_MSR, FEATURE_COMMON))
235		return B_ERROR;
236
237	if (!x86_check_feature(IA32_FEATURE_APERFMPERF, FEATURE_6_ECX))
238		return B_ERROR;
239
240	int32 cpuCount = smp_get_num_cpus();
241	for (int32 i = 0; i < cpuCount; i++) {
242		if (!is_cpu_model_supported(&gCPU[i]))
243			return B_ERROR;
244	}
245
246	uint64 platformInfo = x86_read_msr(IA32_MSR_PLATFORM_INFO);
247	sHWPEPP = x86_check_feature(IA32_FEATURE_HWP_EPP, FEATURE_6_EAX);
248	sHWPActive = (x86_check_feature(IA32_FEATURE_HWP, FEATURE_6_EAX)
249		&& sHWPEPP);
250	sMinPState = (platformInfo >> 40) & 0xff;
251	sMaxPState = (platformInfo >> 8) & 0xff;
252	sBoostPState
253		= max_c(x86_read_msr(IA32_MSR_TURBO_RATIO_LIMIT) & 0xff, sMaxPState);
254	/* x86_check_feature(IA32_FEATURE_HWP_PLR, FEATURE_6_EAX)) */
255	sHWPPackage = false;
256
257	dprintf("using Intel P-States: min %" B_PRIu16 ", max %" B_PRIu16
258		", boost %" B_PRIu16 "%s\n", sMinPState, sMaxPState, sBoostPState,
259		sHWPActive ? ", HWP active" : "");
260
261	if (sMaxPState <= sMinPState || sMaxPState == 0) {
262		dprintf("unexpected or invalid Intel P-States limits, aborting\n");
263		return B_ERROR;
264	}
265
266	sCPUEntries = new(std::nothrow) CPUEntry[cpuCount];
267	if (sCPUEntries == NULL)
268		return B_NO_MEMORY;
269
270	pstates_set_scheduler_mode(SCHEDULER_MODE_LOW_LATENCY);
271
272	call_all_cpus_sync(set_normal_pstate, NULL);
273	return B_OK;
274}
275
276
277static status_t
278uninit_pstates()
279{
280	call_all_cpus_sync(set_normal_pstate, NULL);
281	delete[] sCPUEntries;
282
283	return B_OK;
284}
285
286
287static status_t
288std_ops(int32 op, ...)
289{
290	switch (op) {
291		case B_MODULE_INIT:
292			return init_pstates();
293
294		case B_MODULE_UNINIT:
295			uninit_pstates();
296			return B_OK;
297	}
298
299	return B_ERROR;
300}
301
302
303static cpufreq_module_info sIntelPStates = {
304	{
305		INTEL_PSTATES_MODULE_NAME,
306		0,
307		std_ops,
308	},
309
310	1.0f,
311
312	pstates_set_scheduler_mode,
313
314	pstates_increase_performance,
315	pstates_decrease_performance,
316};
317
318
319module_info* modules[] = {
320	(module_info*)&sIntelPStates,
321	NULL
322};
323
324