1/* SPDX-License-Identifier: GPL-2.0 */
2#include <asm/ptrace.h>
3
4#include "bpf_jit_32.h"
5
6#define SAVE_SZ		96
7#define SCRATCH_OFF	72
8#define BE_PTR(label)	be label
9#define SIGN_EXTEND(reg)
10
11#define SKF_MAX_NEG_OFF	(-0x200000) /* SKF_LL_OFF from filter.h */
12
13	.text
14	.globl	bpf_jit_load_word
15bpf_jit_load_word:
16	cmp	r_OFF, 0
17	bl	bpf_slow_path_word_neg
18	 nop
19	.globl	bpf_jit_load_word_positive_offset
20bpf_jit_load_word_positive_offset:
21	sub	r_HEADLEN, r_OFF, r_TMP
22	cmp	r_TMP, 3
23	ble	bpf_slow_path_word
24	 add	r_SKB_DATA, r_OFF, r_TMP
25	andcc	r_TMP, 3, %g0
26	bne	load_word_unaligned
27	 nop
28	retl
29	 ld	[r_TMP], r_A
30load_word_unaligned:
31	ldub	[r_TMP + 0x0], r_OFF
32	ldub	[r_TMP + 0x1], r_TMP2
33	sll	r_OFF, 8, r_OFF
34	or	r_OFF, r_TMP2, r_OFF
35	ldub	[r_TMP + 0x2], r_TMP2
36	sll	r_OFF, 8, r_OFF
37	or	r_OFF, r_TMP2, r_OFF
38	ldub	[r_TMP + 0x3], r_TMP2
39	sll	r_OFF, 8, r_OFF
40	retl
41	 or	r_OFF, r_TMP2, r_A
42
43	.globl	bpf_jit_load_half
44bpf_jit_load_half:
45	cmp	r_OFF, 0
46	bl	bpf_slow_path_half_neg
47	 nop
48	.globl	bpf_jit_load_half_positive_offset
49bpf_jit_load_half_positive_offset:
50	sub	r_HEADLEN, r_OFF, r_TMP
51	cmp	r_TMP, 1
52	ble	bpf_slow_path_half
53	 add	r_SKB_DATA, r_OFF, r_TMP
54	andcc	r_TMP, 1, %g0
55	bne	load_half_unaligned
56	 nop
57	retl
58	 lduh	[r_TMP], r_A
59load_half_unaligned:
60	ldub	[r_TMP + 0x0], r_OFF
61	ldub	[r_TMP + 0x1], r_TMP2
62	sll	r_OFF, 8, r_OFF
63	retl
64	 or	r_OFF, r_TMP2, r_A
65
66	.globl	bpf_jit_load_byte
67bpf_jit_load_byte:
68	cmp	r_OFF, 0
69	bl	bpf_slow_path_byte_neg
70	 nop
71	.globl	bpf_jit_load_byte_positive_offset
72bpf_jit_load_byte_positive_offset:
73	cmp	r_OFF, r_HEADLEN
74	bge	bpf_slow_path_byte
75	 nop
76	retl
77	 ldub	[r_SKB_DATA + r_OFF], r_A
78
79	.globl	bpf_jit_load_byte_msh
80bpf_jit_load_byte_msh:
81	cmp	r_OFF, 0
82	bl	bpf_slow_path_byte_msh_neg
83	 nop
84	.globl	bpf_jit_load_byte_msh_positive_offset
85bpf_jit_load_byte_msh_positive_offset:
86	cmp	r_OFF, r_HEADLEN
87	bge	bpf_slow_path_byte_msh
88	 nop
89	ldub	[r_SKB_DATA + r_OFF], r_OFF
90	and	r_OFF, 0xf, r_OFF
91	retl
92	 sll	r_OFF, 2, r_X
93
94#define bpf_slow_path_common(LEN)	\
95	save	%sp, -SAVE_SZ, %sp;	\
96	mov	%i0, %o0;		\
97	mov	r_OFF, %o1;		\
98	add	%fp, SCRATCH_OFF, %o2;	\
99	call	skb_copy_bits;		\
100	 mov	(LEN), %o3;		\
101	cmp	%o0, 0;			\
102	restore;
103
104bpf_slow_path_word:
105	bpf_slow_path_common(4)
106	bl	bpf_error
107	 ld	[%sp + SCRATCH_OFF], r_A
108	retl
109	 nop
110bpf_slow_path_half:
111	bpf_slow_path_common(2)
112	bl	bpf_error
113	 lduh	[%sp + SCRATCH_OFF], r_A
114	retl
115	 nop
116bpf_slow_path_byte:
117	bpf_slow_path_common(1)
118	bl	bpf_error
119	 ldub	[%sp + SCRATCH_OFF], r_A
120	retl
121	 nop
122bpf_slow_path_byte_msh:
123	bpf_slow_path_common(1)
124	bl	bpf_error
125	 ldub	[%sp + SCRATCH_OFF], r_A
126	and	r_OFF, 0xf, r_OFF
127	retl
128	 sll	r_OFF, 2, r_X
129
130#define bpf_negative_common(LEN)			\
131	save	%sp, -SAVE_SZ, %sp;			\
132	mov	%i0, %o0;				\
133	mov	r_OFF, %o1;				\
134	SIGN_EXTEND(%o1);				\
135	call	bpf_internal_load_pointer_neg_helper;	\
136	 mov	(LEN), %o2;				\
137	mov	%o0, r_TMP;				\
138	cmp	%o0, 0;					\
139	BE_PTR(bpf_error);				\
140	 restore;
141
142bpf_slow_path_word_neg:
143	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
144	cmp	r_OFF, r_TMP
145	bl	bpf_error
146	 nop
147	.globl	bpf_jit_load_word_negative_offset
148bpf_jit_load_word_negative_offset:
149	bpf_negative_common(4)
150	andcc	r_TMP, 3, %g0
151	bne	load_word_unaligned
152	 nop
153	retl
154	 ld	[r_TMP], r_A
155
156bpf_slow_path_half_neg:
157	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
158	cmp	r_OFF, r_TMP
159	bl	bpf_error
160	 nop
161	.globl	bpf_jit_load_half_negative_offset
162bpf_jit_load_half_negative_offset:
163	bpf_negative_common(2)
164	andcc	r_TMP, 1, %g0
165	bne	load_half_unaligned
166	 nop
167	retl
168	 lduh	[r_TMP], r_A
169
170bpf_slow_path_byte_neg:
171	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
172	cmp	r_OFF, r_TMP
173	bl	bpf_error
174	 nop
175	.globl	bpf_jit_load_byte_negative_offset
176bpf_jit_load_byte_negative_offset:
177	bpf_negative_common(1)
178	retl
179	 ldub	[r_TMP], r_A
180
181bpf_slow_path_byte_msh_neg:
182	sethi	%hi(SKF_MAX_NEG_OFF), r_TMP
183	cmp	r_OFF, r_TMP
184	bl	bpf_error
185	 nop
186	.globl	bpf_jit_load_byte_msh_negative_offset
187bpf_jit_load_byte_msh_negative_offset:
188	bpf_negative_common(1)
189	ldub	[r_TMP], r_OFF
190	and	r_OFF, 0xf, r_OFF
191	retl
192	 sll	r_OFF, 2, r_X
193
194bpf_error:
195	/* Make the JIT program return zero.  The JIT epilogue
196	 * stores away the original %o7 into r_saved_O7.  The
197	 * normal leaf function return is to use "retl" which
198	 * would evalute to "jmpl %o7 + 8, %g0" but we want to
199	 * use the saved value thus the sequence you see here.
200	 */
201	jmpl	r_saved_O7 + 8, %g0
202	 clr	%o0
203