1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * This file was modified from the coreboot version.
4 *
5 * Copyright (C) 2015-2016 Intel Corp.
6 */
7
8#include <config.h>
9#include <asm/msr-index.h>
10#include <asm/mtrr.h>
11#include <asm/post.h>
12#include <asm/processor.h>
13#include <asm/processor-flags.h>
14
15#define KiB 1024
16
17#define IS_POWER_OF_2(x)	(!((x) & ((x) - 1)))
18
19.global car_init
20car_init:
21	post_code(POST_CAR_START)
22
23	/*
24	 * Use the MTRR default type MSR as a proxy for detecting INIT#.
25	 * Reset the system if any known bits are set in that MSR. That is
26	 * an indication of the CPU not being properly reset.
27	 */
28check_for_clean_reset:
29	mov	$MTRR_DEF_TYPE_MSR, %ecx
30	rdmsr
31	and	$(MTRR_DEF_TYPE_EN | MTRR_DEF_TYPE_FIX_EN), %eax
32	cmp	$0, %eax
33	jz	no_reset
34	/* perform warm reset */
35	movw	$IO_PORT_RESET, %dx
36	movb	$(SYS_RST | RST_CPU), %al
37	outb	%al, %dx
38
39no_reset:
40	post_code(POST_CAR_SIPI)
41
42	/* Clear/disable fixed MTRRs */
43	mov	$fixed_mtrr_list_size, %ebx
44	xor	%eax, %eax
45	xor	%edx, %edx
46
47clear_fixed_mtrr:
48	add	$-2, %ebx
49	movzwl	fixed_mtrr_list(%ebx), %ecx
50	wrmsr
51	jnz	clear_fixed_mtrr
52
53	post_code(POST_CAR_MTRR)
54
55	/* Figure put how many MTRRs we have, and clear them out */
56	mov	$MTRR_CAP_MSR, %ecx
57	rdmsr
58	movzb	%al, %ebx		/* Number of variable MTRRs */
59	mov	$MTRR_PHYS_BASE_MSR(0), %ecx
60	xor	%eax, %eax
61	xor	%edx, %edx
62
63clear_var_mtrr:
64	wrmsr
65	inc	%ecx
66	wrmsr
67	inc	%ecx
68	dec	%ebx
69	jnz	clear_var_mtrr
70
71	post_code(POST_CAR_UNCACHEABLE)
72
73	/* Configure default memory type to uncacheable (UC) */
74	mov	$MTRR_DEF_TYPE_MSR, %ecx
75	rdmsr
76	/* Clear enable bits and set default type to UC */
77	and	$~(MTRR_DEF_TYPE_MASK | MTRR_DEF_TYPE_EN | \
78		 MTRR_DEF_TYPE_FIX_EN), %eax
79	wrmsr
80
81	/*
82	 * Configure MTRR_PHYS_MASK_HIGH for proper addressing above 4GB
83	 * based on the physical address size supported for this processor
84	 * This is based on read from CPUID EAX = 080000008h, EAX bits [7:0]
85	 *
86	 * Examples:
87	 *  MTRR_PHYS_MASK_HIGH = 00000000Fh  For 36 bit addressing
88	 *  MTRR_PHYS_MASK_HIGH = 0000000FFh  For 40 bit addressing
89	 */
90
91	movl	$0x80000008, %eax	/* Address sizes leaf */
92	cpuid
93	sub	$32, %al
94	movzx	%al, %eax
95	xorl	%esi, %esi
96	bts	%eax, %esi
97	dec	%esi			/* esi <- MTRR_PHYS_MASK_HIGH */
98
99	post_code(POST_CAR_BASE_ADDRESS)
100
101#if IS_POWER_OF_2(CONFIG_DCACHE_RAM_SIZE)
102	/* Configure CAR region as write-back (WB) */
103	mov	$MTRR_PHYS_BASE_MSR(0), %ecx
104	mov	$CONFIG_DCACHE_RAM_BASE, %eax
105	or	$MTRR_TYPE_WRBACK, %eax
106	xor	%edx,%edx
107	wrmsr
108
109	/* Configure the MTRR mask for the size region */
110	mov	$MTRR_PHYS_MASK(0), %ecx
111	mov	$CONFIG_DCACHE_RAM_SIZE, %eax	/* size mask */
112	dec	%eax
113	not	%eax
114	or	$MTRR_PHYS_MASK_VALID, %eax
115	movl	%esi, %edx	/* edx <- MTRR_PHYS_MASK_HIGH */
116	wrmsr
117#elif (CONFIG_DCACHE_RAM_SIZE == 768 * KiB) /* 768 KiB */
118	/* Configure CAR region as write-back (WB) */
119	mov	$MTRR_PHYS_BASE_MSR(0), %ecx
120	mov	$CONFIG_DCACHE_RAM_BASE, %eax
121	or	$MTRR_TYPE_WRBACK, %eax
122	xor	%edx,%edx
123	wrmsr
124
125	mov	$MTRR_PHYS_MASK_MSR(0), %ecx
126	mov	$(512 * KiB), %eax	/* size mask */
127	dec	%eax
128	not	%eax
129	or	$MTRR_PHYS_MASK_VALID, %eax
130	movl	%esi, %edx	/* edx <- MTRR_PHYS_MASK_HIGH */
131	wrmsr
132
133	mov	$MTRR_PHYS_BASE_MSR(1), %ecx
134	mov	$(CONFIG_DCACHE_RAM_BASE + 512 * KiB), %eax
135	or	$MTRR_TYPE_WRBACK, %eax
136	xor	%edx,%edx
137	wrmsr
138
139	mov	$MTRR_PHYS_MASK_MSR(1), %ecx
140	mov	$(256 * KiB), %eax	/* size mask */
141	dec	%eax
142	not	%eax
143	or	$MTRR_PHYS_MASK_VALID, %eax
144	movl	%esi, %edx	/* edx <- MTRR_PHYS_MASK_HIGH */
145	wrmsr
146#else
147#error "DCACHE_RAM_SIZE is not a power of 2 and setup code is missing"
148#endif
149	post_code(POST_CAR_FILL)
150
151	/* Enable variable MTRRs */
152	mov	$MTRR_DEF_TYPE_MSR, %ecx
153	rdmsr
154	or	$MTRR_DEF_TYPE_EN, %eax
155	wrmsr
156
157	/* Enable caching */
158	mov	%cr0, %eax
159	and	$~(X86_CR0_CD | X86_CR0_NW), %eax
160	invd
161	mov	%eax, %cr0
162
163#if IS_ENABLED(CONFIG_INTEL_CAR_NEM)
164	jmp	car_nem
165#elif IS_ENABLED(CONFIG_INTEL_CAR_CQOS)
166	jmp	car_cqos
167#elif IS_ENABLED(CONFIG_INTEL_CAR_NEM_ENHANCED)
168	jmp	car_nem_enhanced
169#else
170#error "No CAR mechanism selected:
171#endif
172	jmp	car_init_ret
173
174fixed_mtrr_list:
175	.word	MTRR_FIX_64K_00000_MSR
176	.word	MTRR_FIX_16K_80000_MSR
177	.word	MTRR_FIX_16K_A0000_MSR
178	.word	MTRR_FIX_4K_C0000_MSR
179	.word	MTRR_FIX_4K_C8000_MSR
180	.word	MTRR_FIX_4K_D0000_MSR
181	.word	MTRR_FIX_4K_D8000_MSR
182	.word	MTRR_FIX_4K_E0000_MSR
183	.word	MTRR_FIX_4K_E8000_MSR
184	.word	MTRR_FIX_4K_F0000_MSR
185	.word	MTRR_FIX_4K_F8000_MSR
186fixed_mtrr_list_size = . - fixed_mtrr_list
187
188#if IS_ENABLED(CONFIG_INTEL_CAR_NEM)
189.global car_nem
190car_nem:
191	/* Disable cache eviction (setup stage) */
192	mov	$MSR_EVICT_CTL, %ecx
193	rdmsr
194	or	$0x1, %eax
195	wrmsr
196
197	post_code(0x26)
198
199	/* Clear the cache memory region. This will also fill up the cache */
200	movl	$CONFIG_DCACHE_RAM_BASE, %edi
201	movl	$CONFIG_DCACHE_RAM_SIZE, %ecx
202	shr	$0x02, %ecx
203	xor	%eax, %eax
204	cld
205	rep	stosl
206
207	post_code(0x27)
208
209	/* Disable cache eviction (run stage) */
210	mov	$MSR_EVICT_CTL, %ecx
211	rdmsr
212	or	$0x2, %eax
213	wrmsr
214
215	post_code(0x28)
216
217	jmp	car_init_ret
218
219#elif IS_ENABLED(CONFIG_INTEL_CAR_CQOS)
220.global car_cqos
221car_cqos:
222	/*
223	 * Create CBM_LEN_MASK based on CBM_LEN
224	 * Get CPUID.(EAX=10H, ECX=2H):EAX.CBM_LEN[bits 4:0]
225	 */
226	mov	$0x10, %eax
227	mov	$0x2,  %ecx
228	cpuid
229	and	$0x1f, %eax
230	add	$1, %al
231
232	mov	$1, %ebx
233	mov	%al, %cl
234	shl	%cl, %ebx
235	sub	$1, %ebx
236
237	/* Store the CBM_LEN_MASK in mm3 for later use */
238	movd	%ebx, %mm3
239
240	/*
241	 * Disable both L1 and L2 prefetcher. For yet-to-understood reason,
242	 * prefetchers slow down filling cache with rep stos in CQOS mode.
243	 */
244	mov	$MSR_PREFETCH_CTL, %ecx
245	rdmsr
246	or	$(PREFETCH_L1_DISABLE | PREFETCH_L2_DISABLE), %eax
247	wrmsr
248
249#if (CONFIG_DCACHE_RAM_SIZE == CONFIG_L2_CACHE_SIZE)
250/*
251 * If CAR size is set to full L2 size, mask is calculated as all-zeros.
252 * This is not supported by the CPU/uCode.
253 */
254#error "CQOS CAR may not use whole L2 cache area"
255#endif
256
257	/* Calculate how many bits to be used for CAR */
258	xor	%edx, %edx
259	mov	$CONFIG_DCACHE_RAM_SIZE, %eax	/* dividend */
260	mov	$CONFIG_CACHE_QOS_SIZE_PER_BIT, %ecx	/* divisor */
261	div	%ecx		/* result is in eax */
262	mov	%eax, %ecx	/* save to ecx */
263	mov	$1, %ebx
264	shl	%cl, %ebx
265	sub	$1, %ebx	/* resulting mask is is in ebx */
266
267	/* Set this mask for initial cache fill */
268	mov	$MSR_L2_QOS_MASK(0), %ecx
269	rdmsr
270	mov	%ebx, %eax
271	wrmsr
272
273	/* Set CLOS selector to 0 */
274	mov	$MSR_IA32_PQR_ASSOC, %ecx
275	rdmsr
276	and	$~MSR_IA32_PQR_ASSOC_MASK, %edx	/* select mask 0 */
277	wrmsr
278
279	/* We will need to block CAR region from evicts */
280	mov	$MSR_L2_QOS_MASK(1), %ecx
281	rdmsr
282	/* Invert bits that are to be used for cache */
283	mov	%ebx, %eax
284	xor	$~0, %eax			/* invert 32 bits */
285
286	/*
287	 * Use CBM_LEN_MASK stored in mm3 to set bits based on Capacity Bit
288	 * Mask Length.
289	 */
290	movd	%mm3, %ebx
291	and	%ebx, %eax
292	wrmsr
293
294	post_code(0x26)
295
296	/* Clear the cache memory region. This will also fill up the cache */
297	movl	$CONFIG_DCACHE_RAM_BASE, %edi
298	movl	$CONFIG_DCACHE_RAM_SIZE, %ecx
299	shr	$0x02, %ecx
300	xor	%eax, %eax
301	cld
302	rep	stosl
303
304	post_code(0x27)
305
306	/* Cache is populated. Use mask 1 that will block evicts */
307	mov	$MSR_IA32_PQR_ASSOC, %ecx
308	rdmsr
309	and	$~MSR_IA32_PQR_ASSOC_MASK, %edx	/* clear index bits first */
310	or	$1, %edx			/* select mask 1 */
311	wrmsr
312
313	/* Enable prefetchers */
314	mov	$MSR_PREFETCH_CTL, %ecx
315	rdmsr
316	and	$~(PREFETCH_L1_DISABLE | PREFETCH_L2_DISABLE), %eax
317	wrmsr
318
319	post_code(0x28)
320
321	jmp	car_init_ret
322
323#elif IS_ENABLED(CONFIG_INTEL_CAR_NEM_ENHANCED)
324.global car_nem_enhanced
325car_nem_enhanced:
326	/* Disable cache eviction (setup stage) */
327	mov	$MSR_EVICT_CTL, %ecx
328	rdmsr
329	or	$0x1, %eax
330	wrmsr
331	post_code(0x26)
332
333	/* Create n-way set associativity of cache */
334	xorl	%edi, %edi
335find_llc_subleaf:
336	movl	%edi, %ecx
337	movl	$0x04, %eax
338	cpuid
339	inc	%edi
340	and	$0xe0, %al	/* EAX[7:5] = Cache Level */
341	cmp	$0x60, %al	/* Check to see if it is LLC */
342	jnz	find_llc_subleaf
343
344	/*
345	 * Set MSR 0xC91 IA32_L3_MASK_! = 0xE/0xFE/0xFFE/0xFFFE
346	 * for 4/8/16 way of LLC
347	*/
348	shr	$22, %ebx
349	inc	%ebx
350	/* Calculate n-way associativity of LLC */
351	mov	%bl, %cl
352
353	/*
354	 * Maximizing RO cacheability while locking in the CAR to a
355	 * single way since that particular way won't be victim candidate
356	 * for evictions.
357	 * This has been done after programing LLC_WAY_MASK_1 MSR
358	 * with desired LLC way as mentioned below.
359	 *
360	 * Hence create Code and Data Size as per request
361	 * Code Size (RO) : Up to 16M
362	 * Data Size (RW) : Up to 256K
363	 */
364	movl	$0x01, %eax
365	/*
366	 * LLC Ways -> LLC_WAY_MASK_1:
367	 *  4: 0x000E
368	 *  8: 0x00FE
369	 * 12: 0x0FFE
370	 * 16: 0xFFFE
371	 *
372	 * These MSRs contain one bit per each way of LLC
373	 * - If this bit is '0' - the way is protected from eviction
374	 * - If this bit is '1' - the way is not protected from eviction
375	 */
376	shl	%cl, %eax
377	subl	$0x02, %eax
378	movl	$MSR_IA32_L3_MASK_1, %ecx
379	xorl	%edx, %edx
380	wrmsr
381	/*
382	 * Set MSR 0xC92 IA32_L3_MASK_2 = 0x1
383	 *
384	 * For SKL SOC, data size remains 256K consistently.
385	 * Hence, creating 1-way associative cache for Data
386	*/
387	mov	$MSR_IA32_L3_MASK_2, %ecx
388	mov	$0x01, %eax
389	xorl	%edx, %edx
390	wrmsr
391	/*
392	 * Set MSR_IA32_PQR_ASSOC = 0x02
393	 *
394	 * Possible values:
395	 * 0: Default value, no way mask should be applied
396	 * 1: Apply way mask 1 to LLC
397	 * 2: Apply way mask 2 to LLC
398	 * 3: Shouldn't be use in NEM Mode
399	 */
400	movl	$MSR_IA32_PQR_ASSOC, %ecx
401	movl	$0x02, %eax
402	xorl	%edx, %edx
403	wrmsr
404
405	movl	$CONFIG_DCACHE_RAM_BASE, %edi
406	movl	$CONFIG_DCACHE_RAM_SIZE, %ecx
407	shr	$0x02, %ecx
408	xor	%eax, %eax
409	cld
410	rep	stosl
411	/*
412	 * Set MSR_IA32_PQR_ASSOC = 0x01
413	 * At this stage we apply LLC_WAY_MASK_1 to the cache.
414	 * i.e. way 0 is protected from eviction.
415	*/
416	movl	$MSR_IA32_PQR_ASSOC, %ecx
417	movl	$0x01, %eax
418	xorl	%edx, %edx
419	wrmsr
420
421	post_code(0x27)
422	/*
423	 * Enable No-Eviction Mode Run State by setting
424	 * NO_EVICT_MODE MSR 2E0h bit [1] = '1'.
425	 */
426
427	movl	$MSR_EVICT_CTL, %ecx
428	rdmsr
429	orl	$0x02, %eax
430	wrmsr
431
432	post_code(0x28)
433
434	jmp	car_init_ret
435#endif
436
437#if CONFIG_IS_ENABLED(X86_16BIT_INIT)
438_dt_ucode_base_size:
439	/* These next two fields are filled in by binman */
440.globl ucode_base
441ucode_base:	/* Declared in microcode.h */
442	.long	0			/* microcode base */
443.globl ucode_size
444ucode_size:	/* Declared in microcode.h */
445	.long	0			/* microcode size */
446	.long	CONFIG_SYS_MONITOR_BASE	/* code region base */
447	.long	CONFIG_SYS_MONITOR_LEN	/* code region size */
448#endif
449