1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * FPU data structures:
4 */
5#ifndef _ASM_X86_FPU_H
6#define _ASM_X86_FPU_H
7
8#include <asm/page_types.h>
9
10/*
11 * The legacy x87 FPU state format, as saved by FSAVE and
12 * restored by the FRSTOR instructions:
13 */
14struct fregs_state {
15	u32			cwd;	/* FPU Control Word		*/
16	u32			swd;	/* FPU Status Word		*/
17	u32			twd;	/* FPU Tag Word			*/
18	u32			fip;	/* FPU IP Offset		*/
19	u32			fcs;	/* FPU IP Selector		*/
20	u32			foo;	/* FPU Operand Pointer Offset	*/
21	u32			fos;	/* FPU Operand Pointer Selector	*/
22
23	/* 8*10 bytes for each FP-reg = 80 bytes:			*/
24	u32			st_space[20];
25
26	/* Software status information [not touched by FSAVE]:		*/
27	u32			status;
28};
29
30/*
31 * The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and
32 * restored by the FXRSTOR instructions. It's similar to the FSAVE
33 * format, but differs in some areas, plus has extensions at
34 * the end for the XMM registers.
35 */
36struct fxregs_state {
37	u16			cwd; /* Control Word			*/
38	u16			swd; /* Status Word			*/
39	u16			twd; /* Tag Word			*/
40	u16			fop; /* Last Instruction Opcode		*/
41	union {
42		struct {
43			u64	rip; /* Instruction Pointer		*/
44			u64	rdp; /* Data Pointer			*/
45		};
46		struct {
47			u32	fip; /* FPU IP Offset			*/
48			u32	fcs; /* FPU IP Selector			*/
49			u32	foo; /* FPU Operand Offset		*/
50			u32	fos; /* FPU Operand Selector		*/
51		};
52	};
53	u32			mxcsr;		/* MXCSR Register State */
54	u32			mxcsr_mask;	/* MXCSR Mask		*/
55
56	/* 8*16 bytes for each FP-reg = 128 bytes:			*/
57	u32			st_space[32];
58
59	/* 16*16 bytes for each XMM-reg = 256 bytes:			*/
60	u32			xmm_space[64];
61
62	u32			padding[12];
63
64	union {
65		u32		padding1[12];
66		u32		sw_reserved[12];
67	};
68
69} __attribute__((aligned(16)));
70
71/* Default value for fxregs_state.mxcsr: */
72#define MXCSR_DEFAULT		0x1f80
73
74/* Copy both mxcsr & mxcsr_flags with a single u64 memcpy: */
75#define MXCSR_AND_FLAGS_SIZE sizeof(u64)
76
77/*
78 * Software based FPU emulation state. This is arbitrary really,
79 * it matches the x87 format to make it easier to understand:
80 */
81struct swregs_state {
82	u32			cwd;
83	u32			swd;
84	u32			twd;
85	u32			fip;
86	u32			fcs;
87	u32			foo;
88	u32			fos;
89	/* 8*10 bytes for each FP-reg = 80 bytes: */
90	u32			st_space[20];
91	u8			ftop;
92	u8			changed;
93	u8			lookahead;
94	u8			no_update;
95	u8			rm;
96	u8			alimit;
97	struct math_emu_info	*info;
98	u32			entry_eip;
99};
100
101/*
102 * List of XSAVE features Linux knows about:
103 */
104enum xfeature {
105	XFEATURE_FP,
106	XFEATURE_SSE,
107	/*
108	 * Values above here are "legacy states".
109	 * Those below are "extended states".
110	 */
111	XFEATURE_YMM,
112	XFEATURE_BNDREGS,
113	XFEATURE_BNDCSR,
114	XFEATURE_OPMASK,
115	XFEATURE_ZMM_Hi256,
116	XFEATURE_Hi16_ZMM,
117	XFEATURE_PT_UNIMPLEMENTED_SO_FAR,
118	XFEATURE_PKRU,
119	XFEATURE_PASID,
120	XFEATURE_CET_USER,
121	XFEATURE_CET_KERNEL_UNUSED,
122	XFEATURE_RSRVD_COMP_13,
123	XFEATURE_RSRVD_COMP_14,
124	XFEATURE_LBR,
125	XFEATURE_RSRVD_COMP_16,
126	XFEATURE_XTILE_CFG,
127	XFEATURE_XTILE_DATA,
128
129	XFEATURE_MAX,
130};
131
132#define XFEATURE_MASK_FP		(1 << XFEATURE_FP)
133#define XFEATURE_MASK_SSE		(1 << XFEATURE_SSE)
134#define XFEATURE_MASK_YMM		(1 << XFEATURE_YMM)
135#define XFEATURE_MASK_BNDREGS		(1 << XFEATURE_BNDREGS)
136#define XFEATURE_MASK_BNDCSR		(1 << XFEATURE_BNDCSR)
137#define XFEATURE_MASK_OPMASK		(1 << XFEATURE_OPMASK)
138#define XFEATURE_MASK_ZMM_Hi256		(1 << XFEATURE_ZMM_Hi256)
139#define XFEATURE_MASK_Hi16_ZMM		(1 << XFEATURE_Hi16_ZMM)
140#define XFEATURE_MASK_PT		(1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR)
141#define XFEATURE_MASK_PKRU		(1 << XFEATURE_PKRU)
142#define XFEATURE_MASK_PASID		(1 << XFEATURE_PASID)
143#define XFEATURE_MASK_CET_USER		(1 << XFEATURE_CET_USER)
144#define XFEATURE_MASK_CET_KERNEL	(1 << XFEATURE_CET_KERNEL_UNUSED)
145#define XFEATURE_MASK_LBR		(1 << XFEATURE_LBR)
146#define XFEATURE_MASK_XTILE_CFG		(1 << XFEATURE_XTILE_CFG)
147#define XFEATURE_MASK_XTILE_DATA	(1 << XFEATURE_XTILE_DATA)
148
149#define XFEATURE_MASK_FPSSE		(XFEATURE_MASK_FP | XFEATURE_MASK_SSE)
150#define XFEATURE_MASK_AVX512		(XFEATURE_MASK_OPMASK \
151					 | XFEATURE_MASK_ZMM_Hi256 \
152					 | XFEATURE_MASK_Hi16_ZMM)
153
154#ifdef CONFIG_X86_64
155# define XFEATURE_MASK_XTILE		(XFEATURE_MASK_XTILE_DATA \
156					 | XFEATURE_MASK_XTILE_CFG)
157#else
158# define XFEATURE_MASK_XTILE		(0)
159#endif
160
161#define FIRST_EXTENDED_XFEATURE	XFEATURE_YMM
162
163struct reg_128_bit {
164	u8      regbytes[128/8];
165};
166struct reg_256_bit {
167	u8	regbytes[256/8];
168};
169struct reg_512_bit {
170	u8	regbytes[512/8];
171};
172struct reg_1024_byte {
173	u8	regbytes[1024];
174};
175
176/*
177 * State component 2:
178 *
179 * There are 16x 256-bit AVX registers named YMM0-YMM15.
180 * The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15)
181 * and are stored in 'struct fxregs_state::xmm_space[]' in the
182 * "legacy" area.
183 *
184 * The high 128 bits are stored here.
185 */
186struct ymmh_struct {
187	struct reg_128_bit              hi_ymm[16];
188} __packed;
189
190/* Intel MPX support: */
191
192struct mpx_bndreg {
193	u64				lower_bound;
194	u64				upper_bound;
195} __packed;
196/*
197 * State component 3 is used for the 4 128-bit bounds registers
198 */
199struct mpx_bndreg_state {
200	struct mpx_bndreg		bndreg[4];
201} __packed;
202
203/*
204 * State component 4 is used for the 64-bit user-mode MPX
205 * configuration register BNDCFGU and the 64-bit MPX status
206 * register BNDSTATUS.  We call the pair "BNDCSR".
207 */
208struct mpx_bndcsr {
209	u64				bndcfgu;
210	u64				bndstatus;
211} __packed;
212
213/*
214 * The BNDCSR state is padded out to be 64-bytes in size.
215 */
216struct mpx_bndcsr_state {
217	union {
218		struct mpx_bndcsr		bndcsr;
219		u8				pad_to_64_bytes[64];
220	};
221} __packed;
222
223/* AVX-512 Components: */
224
225/*
226 * State component 5 is used for the 8 64-bit opmask registers
227 * k0-k7 (opmask state).
228 */
229struct avx_512_opmask_state {
230	u64				opmask_reg[8];
231} __packed;
232
233/*
234 * State component 6 is used for the upper 256 bits of the
235 * registers ZMM0-ZMM15. These 16 256-bit values are denoted
236 * ZMM0_H-ZMM15_H (ZMM_Hi256 state).
237 */
238struct avx_512_zmm_uppers_state {
239	struct reg_256_bit		zmm_upper[16];
240} __packed;
241
242/*
243 * State component 7 is used for the 16 512-bit registers
244 * ZMM16-ZMM31 (Hi16_ZMM state).
245 */
246struct avx_512_hi16_state {
247	struct reg_512_bit		hi16_zmm[16];
248} __packed;
249
250/*
251 * State component 9: 32-bit PKRU register.  The state is
252 * 8 bytes long but only 4 bytes is used currently.
253 */
254struct pkru_state {
255	u32				pkru;
256	u32				pad;
257} __packed;
258
259/*
260 * State component 11 is Control-flow Enforcement user states
261 */
262struct cet_user_state {
263	/* user control-flow settings */
264	u64 user_cet;
265	/* user shadow stack pointer */
266	u64 user_ssp;
267};
268
269/*
270 * State component 15: Architectural LBR configuration state.
271 * The size of Arch LBR state depends on the number of LBRs (lbr_depth).
272 */
273
274struct lbr_entry {
275	u64 from;
276	u64 to;
277	u64 info;
278};
279
280struct arch_lbr_state {
281	u64 lbr_ctl;
282	u64 lbr_depth;
283	u64 ler_from;
284	u64 ler_to;
285	u64 ler_info;
286	struct lbr_entry		entries[];
287};
288
289/*
290 * State component 17: 64-byte tile configuration register.
291 */
292struct xtile_cfg {
293	u64				tcfg[8];
294} __packed;
295
296/*
297 * State component 18: 1KB tile data register.
298 * Each register represents 16 64-byte rows of the matrix
299 * data. But the number of registers depends on the actual
300 * implementation.
301 */
302struct xtile_data {
303	struct reg_1024_byte		tmm;
304} __packed;
305
306/*
307 * State component 10 is supervisor state used for context-switching the
308 * PASID state.
309 */
310struct ia32_pasid_state {
311	u64 pasid;
312} __packed;
313
314struct xstate_header {
315	u64				xfeatures;
316	u64				xcomp_bv;
317	u64				reserved[6];
318} __attribute__((packed));
319
320/*
321 * xstate_header.xcomp_bv[63] indicates that the extended_state_area
322 * is in compacted format.
323 */
324#define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63)
325
326/*
327 * This is our most modern FPU state format, as saved by the XSAVE
328 * and restored by the XRSTOR instructions.
329 *
330 * It consists of a legacy fxregs portion, an xstate header and
331 * subsequent areas as defined by the xstate header.  Not all CPUs
332 * support all the extensions, so the size of the extended area
333 * can vary quite a bit between CPUs.
334 */
335struct xregs_state {
336	struct fxregs_state		i387;
337	struct xstate_header		header;
338	u8				extended_state_area[];
339} __attribute__ ((packed, aligned (64)));
340
341/*
342 * This is a union of all the possible FPU state formats
343 * put together, so that we can pick the right one runtime.
344 *
345 * The size of the structure is determined by the largest
346 * member - which is the xsave area.  The padding is there
347 * to ensure that statically-allocated task_structs (just
348 * the init_task today) have enough space.
349 */
350union fpregs_state {
351	struct fregs_state		fsave;
352	struct fxregs_state		fxsave;
353	struct swregs_state		soft;
354	struct xregs_state		xsave;
355	u8 __padding[PAGE_SIZE];
356};
357
358struct fpstate {
359	/* @kernel_size: The size of the kernel register image */
360	unsigned int		size;
361
362	/* @user_size: The size in non-compacted UABI format */
363	unsigned int		user_size;
364
365	/* @xfeatures:		xfeatures for which the storage is sized */
366	u64			xfeatures;
367
368	/* @user_xfeatures:	xfeatures valid in UABI buffers */
369	u64			user_xfeatures;
370
371	/* @xfd:		xfeatures disabled to trap userspace use. */
372	u64			xfd;
373
374	/* @is_valloc:		Indicator for dynamically allocated state */
375	unsigned int		is_valloc	: 1;
376
377	/* @is_guest:		Indicator for guest state (KVM) */
378	unsigned int		is_guest	: 1;
379
380	/*
381	 * @is_confidential:	Indicator for KVM confidential mode.
382	 *			The FPU registers are restored by the
383	 *			vmentry firmware from encrypted guest
384	 *			memory. On vmexit the FPU registers are
385	 *			saved by firmware to encrypted guest memory
386	 *			and the registers are scrubbed before
387	 *			returning to the host. So there is no
388	 *			content which is worth saving and restoring.
389	 *			The fpstate has to be there so that
390	 *			preemption and softirq FPU usage works
391	 *			without special casing.
392	 */
393	unsigned int		is_confidential	: 1;
394
395	/* @in_use:		State is in use */
396	unsigned int		in_use		: 1;
397
398	/* @regs: The register state union for all supported formats */
399	union fpregs_state	regs;
400
401	/* @regs is dynamically sized! Don't add anything after @regs! */
402} __aligned(64);
403
404#define FPU_GUEST_PERM_LOCKED		BIT_ULL(63)
405
406struct fpu_state_perm {
407	/*
408	 * @__state_perm:
409	 *
410	 * This bitmap indicates the permission for state components, which
411	 * are available to a thread group. The permission prctl() sets the
412	 * enabled state bits in thread_group_leader()->thread.fpu.
413	 *
414	 * All run time operations use the per thread information in the
415	 * currently active fpu.fpstate which contains the xfeature masks
416	 * and sizes for kernel and user space.
417	 *
418	 * This master permission field is only to be used when
419	 * task.fpu.fpstate based checks fail to validate whether the task
420	 * is allowed to expand its xfeatures set which requires to
421	 * allocate a larger sized fpstate buffer.
422	 *
423	 * Do not access this field directly.  Use the provided helper
424	 * function. Unlocked access is possible for quick checks.
425	 */
426	u64				__state_perm;
427
428	/*
429	 * @__state_size:
430	 *
431	 * The size required for @__state_perm. Only valid to access
432	 * with sighand locked.
433	 */
434	unsigned int			__state_size;
435
436	/*
437	 * @__user_state_size:
438	 *
439	 * The size required for @__state_perm user part. Only valid to
440	 * access with sighand locked.
441	 */
442	unsigned int			__user_state_size;
443};
444
445/*
446 * Highest level per task FPU state data structure that
447 * contains the FPU register state plus various FPU
448 * state fields:
449 */
450struct fpu {
451	/*
452	 * @last_cpu:
453	 *
454	 * Records the last CPU on which this context was loaded into
455	 * FPU registers. (In the lazy-restore case we might be
456	 * able to reuse FPU registers across multiple context switches
457	 * this way, if no intermediate task used the FPU.)
458	 *
459	 * A value of -1 is used to indicate that the FPU state in context
460	 * memory is newer than the FPU state in registers, and that the
461	 * FPU state should be reloaded next time the task is run.
462	 */
463	unsigned int			last_cpu;
464
465	/*
466	 * @avx512_timestamp:
467	 *
468	 * Records the timestamp of AVX512 use during last context switch.
469	 */
470	unsigned long			avx512_timestamp;
471
472	/*
473	 * @fpstate:
474	 *
475	 * Pointer to the active struct fpstate. Initialized to
476	 * point at @__fpstate below.
477	 */
478	struct fpstate			*fpstate;
479
480	/*
481	 * @__task_fpstate:
482	 *
483	 * Pointer to an inactive struct fpstate. Initialized to NULL. Is
484	 * used only for KVM support to swap out the regular task fpstate.
485	 */
486	struct fpstate			*__task_fpstate;
487
488	/*
489	 * @perm:
490	 *
491	 * Permission related information
492	 */
493	struct fpu_state_perm		perm;
494
495	/*
496	 * @guest_perm:
497	 *
498	 * Permission related information for guest pseudo FPUs
499	 */
500	struct fpu_state_perm		guest_perm;
501
502	/*
503	 * @__fpstate:
504	 *
505	 * Initial in-memory storage for FPU registers which are saved in
506	 * context switch and when the kernel uses the FPU. The registers
507	 * are restored from this storage on return to user space if they
508	 * are not longer containing the tasks FPU register state.
509	 */
510	struct fpstate			__fpstate;
511	/*
512	 * WARNING: '__fpstate' is dynamically-sized.  Do not put
513	 * anything after it here.
514	 */
515};
516
517/*
518 * Guest pseudo FPU container
519 */
520struct fpu_guest {
521	/*
522	 * @xfeatures:			xfeature bitmap of features which are
523	 *				currently enabled for the guest vCPU.
524	 */
525	u64				xfeatures;
526
527	/*
528	 * @perm:			xfeature bitmap of features which are
529	 *				permitted to be enabled for the guest
530	 *				vCPU.
531	 */
532	u64				perm;
533
534	/*
535	 * @xfd_err:			Save the guest value.
536	 */
537	u64				xfd_err;
538
539	/*
540	 * @uabi_size:			Size required for save/restore
541	 */
542	unsigned int			uabi_size;
543
544	/*
545	 * @fpstate:			Pointer to the allocated guest fpstate
546	 */
547	struct fpstate			*fpstate;
548};
549
550/*
551 * FPU state configuration data. Initialized at boot time. Read only after init.
552 */
553struct fpu_state_config {
554	/*
555	 * @max_size:
556	 *
557	 * The maximum size of the register state buffer. Includes all
558	 * supported features except independent managed features.
559	 */
560	unsigned int		max_size;
561
562	/*
563	 * @default_size:
564	 *
565	 * The default size of the register state buffer. Includes all
566	 * supported features except independent managed features and
567	 * features which have to be requested by user space before usage.
568	 */
569	unsigned int		default_size;
570
571	/*
572	 * @max_features:
573	 *
574	 * The maximum supported features bitmap. Does not include
575	 * independent managed features.
576	 */
577	u64 max_features;
578
579	/*
580	 * @default_features:
581	 *
582	 * The default supported features bitmap. Does not include
583	 * independent managed features and features which have to
584	 * be requested by user space before usage.
585	 */
586	u64 default_features;
587	/*
588	 * @legacy_features:
589	 *
590	 * Features which can be reported back to user space
591	 * even without XSAVE support, i.e. legacy features FP + SSE
592	 */
593	u64 legacy_features;
594};
595
596/* FPU state configuration information */
597extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg;
598
599#endif /* _ASM_X86_FPU_H */
600