1// SPDX-License-Identifier: GPL-2.0 2/* Converted from tools/testing/selftests/bpf/verifier/spill_fill.c */ 3 4#include <linux/bpf.h> 5#include <bpf/bpf_helpers.h> 6#include "bpf_misc.h" 7#include <../../../tools/include/linux/filter.h> 8 9struct { 10 __uint(type, BPF_MAP_TYPE_RINGBUF); 11 __uint(max_entries, 4096); 12} map_ringbuf SEC(".maps"); 13 14SEC("socket") 15__description("check valid spill/fill") 16__success __failure_unpriv __msg_unpriv("R0 leaks addr") 17__retval(POINTER_VALUE) 18__naked void check_valid_spill_fill(void) 19{ 20 asm volatile (" \ 21 /* spill R1(ctx) into stack */ \ 22 *(u64*)(r10 - 8) = r1; \ 23 /* fill it back into R2 */ \ 24 r2 = *(u64*)(r10 - 8); \ 25 /* should be able to access R0 = *(R2 + 8) */ \ 26 /* BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 8), */\ 27 r0 = r2; \ 28 exit; \ 29" ::: __clobber_all); 30} 31 32SEC("socket") 33__description("check valid spill/fill, skb mark") 34__success __success_unpriv __retval(0) 35__naked void valid_spill_fill_skb_mark(void) 36{ 37 asm volatile (" \ 38 r6 = r1; \ 39 *(u64*)(r10 - 8) = r6; \ 40 r0 = *(u64*)(r10 - 8); \ 41 r0 = *(u32*)(r0 + %[__sk_buff_mark]); \ 42 exit; \ 43" : 44 : __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)) 45 : __clobber_all); 46} 47 48SEC("socket") 49__description("check valid spill/fill, ptr to mem") 50__success __success_unpriv __retval(0) 51__naked void spill_fill_ptr_to_mem(void) 52{ 53 asm volatile (" \ 54 /* reserve 8 byte ringbuf memory */ \ 55 r1 = 0; \ 56 *(u64*)(r10 - 8) = r1; \ 57 r1 = %[map_ringbuf] ll; \ 58 r2 = 8; \ 59 r3 = 0; \ 60 call %[bpf_ringbuf_reserve]; \ 61 /* store a pointer to the reserved memory in R6 */\ 62 r6 = r0; \ 63 /* check whether the reservation was successful */\ 64 if r0 == 0 goto l0_%=; \ 65 /* spill R6(mem) into the stack */ \ 66 *(u64*)(r10 - 8) = r6; \ 67 /* fill it back in R7 */ \ 68 r7 = *(u64*)(r10 - 8); \ 69 /* should be able to access *(R7) = 0 */ \ 70 r1 = 0; \ 71 *(u64*)(r7 + 0) = r1; \ 72 /* submit the reserved ringbuf memory */ \ 73 r1 = r7; \ 74 r2 = 0; \ 75 call %[bpf_ringbuf_submit]; \ 76l0_%=: r0 = 0; \ 77 exit; \ 78" : 79 : __imm(bpf_ringbuf_reserve), 80 __imm(bpf_ringbuf_submit), 81 __imm_addr(map_ringbuf) 82 : __clobber_all); 83} 84 85SEC("socket") 86__description("check with invalid reg offset 0") 87__failure __msg("R0 pointer arithmetic on ringbuf_mem_or_null prohibited") 88__failure_unpriv 89__naked void with_invalid_reg_offset_0(void) 90{ 91 asm volatile (" \ 92 /* reserve 8 byte ringbuf memory */ \ 93 r1 = 0; \ 94 *(u64*)(r10 - 8) = r1; \ 95 r1 = %[map_ringbuf] ll; \ 96 r2 = 8; \ 97 r3 = 0; \ 98 call %[bpf_ringbuf_reserve]; \ 99 /* store a pointer to the reserved memory in R6 */\ 100 r6 = r0; \ 101 /* add invalid offset to memory or NULL */ \ 102 r0 += 1; \ 103 /* check whether the reservation was successful */\ 104 if r0 == 0 goto l0_%=; \ 105 /* should not be able to access *(R7) = 0 */ \ 106 r1 = 0; \ 107 *(u32*)(r6 + 0) = r1; \ 108 /* submit the reserved ringbuf memory */ \ 109 r1 = r6; \ 110 r2 = 0; \ 111 call %[bpf_ringbuf_submit]; \ 112l0_%=: r0 = 0; \ 113 exit; \ 114" : 115 : __imm(bpf_ringbuf_reserve), 116 __imm(bpf_ringbuf_submit), 117 __imm_addr(map_ringbuf) 118 : __clobber_all); 119} 120 121SEC("socket") 122__description("check corrupted spill/fill") 123__failure __msg("R0 invalid mem access 'scalar'") 124__msg_unpriv("attempt to corrupt spilled") 125__flag(BPF_F_ANY_ALIGNMENT) 126__naked void check_corrupted_spill_fill(void) 127{ 128 asm volatile (" \ 129 /* spill R1(ctx) into stack */ \ 130 *(u64*)(r10 - 8) = r1; \ 131 /* mess up with R1 pointer on stack */ \ 132 r0 = 0x23; \ 133 *(u8*)(r10 - 7) = r0; \ 134 /* fill back into R0 is fine for priv. \ 135 * R0 now becomes SCALAR_VALUE. \ 136 */ \ 137 r0 = *(u64*)(r10 - 8); \ 138 /* Load from R0 should fail. */ \ 139 r0 = *(u64*)(r0 + 8); \ 140 exit; \ 141" ::: __clobber_all); 142} 143 144SEC("socket") 145__description("check corrupted spill/fill, LSB") 146__success __failure_unpriv __msg_unpriv("attempt to corrupt spilled") 147__retval(POINTER_VALUE) 148__naked void check_corrupted_spill_fill_lsb(void) 149{ 150 asm volatile (" \ 151 *(u64*)(r10 - 8) = r1; \ 152 r0 = 0xcafe; \ 153 *(u16*)(r10 - 8) = r0; \ 154 r0 = *(u64*)(r10 - 8); \ 155 exit; \ 156" ::: __clobber_all); 157} 158 159SEC("socket") 160__description("check corrupted spill/fill, MSB") 161__success __failure_unpriv __msg_unpriv("attempt to corrupt spilled") 162__retval(POINTER_VALUE) 163__naked void check_corrupted_spill_fill_msb(void) 164{ 165 asm volatile (" \ 166 *(u64*)(r10 - 8) = r1; \ 167 r0 = 0x12345678; \ 168 *(u32*)(r10 - 4) = r0; \ 169 r0 = *(u64*)(r10 - 8); \ 170 exit; \ 171" ::: __clobber_all); 172} 173 174SEC("tc") 175__description("Spill and refill a u32 const scalar. Offset to skb->data") 176__success __retval(0) 177__naked void scalar_offset_to_skb_data_1(void) 178{ 179 asm volatile (" \ 180 r2 = *(u32*)(r1 + %[__sk_buff_data]); \ 181 r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ 182 w4 = 20; \ 183 *(u32*)(r10 - 8) = r4; \ 184 r4 = *(u32*)(r10 - 8); \ 185 r0 = r2; \ 186 /* r0 += r4 R0=pkt R2=pkt R3=pkt_end R4=20 */ \ 187 r0 += r4; \ 188 /* if (r0 > r3) R0=pkt,off=20 R2=pkt R3=pkt_end R4=20 */\ 189 if r0 > r3 goto l0_%=; \ 190 /* r0 = *(u32 *)r2 R0=pkt,off=20,r=20 R2=pkt,r=20 R3=pkt_end R4=20 */\ 191 r0 = *(u32*)(r2 + 0); \ 192l0_%=: r0 = 0; \ 193 exit; \ 194" : 195 : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), 196 __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) 197 : __clobber_all); 198} 199 200SEC("socket") 201__description("Spill a u32 const, refill from another half of the uninit u32 from the stack") 202/* in privileged mode reads from uninitialized stack locations are permitted */ 203__success __failure_unpriv 204__msg_unpriv("invalid read from stack off -4+0 size 4") 205__retval(0) 206__naked void uninit_u32_from_the_stack(void) 207{ 208 asm volatile (" \ 209 w4 = 20; \ 210 *(u32*)(r10 - 8) = r4; \ 211 /* r4 = *(u32 *)(r10 -4) fp-8=????rrrr*/ \ 212 r4 = *(u32*)(r10 - 4); \ 213 r0 = 0; \ 214 exit; \ 215" ::: __clobber_all); 216} 217 218SEC("tc") 219__description("Spill a u32 const scalar. Refill as u16. Offset to skb->data") 220__success __retval(0) 221__naked void u16_offset_to_skb_data(void) 222{ 223 asm volatile (" \ 224 r2 = *(u32*)(r1 + %[__sk_buff_data]); \ 225 r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ 226 w4 = 20; \ 227 *(u32*)(r10 - 8) = r4; \ 228 " 229#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 230 "r4 = *(u16*)(r10 - 8);" 231#else 232 "r4 = *(u16*)(r10 - 6);" 233#endif 234 " \ 235 r0 = r2; \ 236 /* r0 += r4 R0=pkt R2=pkt R3=pkt_end R4=20 */\ 237 r0 += r4; \ 238 /* if (r0 > r3) R0=pkt,off=20 R2=pkt R3=pkt_end R4=20 */\ 239 if r0 > r3 goto l0_%=; \ 240 /* r0 = *(u32 *)r2 R0=pkt,off=20 R2=pkt R3=pkt_end R4=20 */\ 241 r0 = *(u32*)(r2 + 0); \ 242l0_%=: r0 = 0; \ 243 exit; \ 244" : 245 : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), 246 __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) 247 : __clobber_all); 248} 249 250SEC("tc") 251__description("Spill u32 const scalars. Refill as u64. Offset to skb->data") 252__failure __msg("math between pkt pointer and register with unbounded min value is not allowed") 253__naked void u64_offset_to_skb_data(void) 254{ 255 asm volatile (" \ 256 r2 = *(u32*)(r1 + %[__sk_buff_data]); \ 257 r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ 258 w6 = 0; \ 259 w7 = 20; \ 260 *(u32*)(r10 - 4) = r6; \ 261 *(u32*)(r10 - 8) = r7; \ 262 r4 = *(u64*)(r10 - 8); \ 263 r0 = r2; \ 264 /* r0 += r4 R0=pkt R2=pkt R3=pkt_end R4= */ \ 265 r0 += r4; \ 266 if r0 > r3 goto l0_%=; \ 267 r0 = *(u32*)(r2 + 0); \ 268l0_%=: r0 = 0; \ 269 exit; \ 270" : 271 : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), 272 __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) 273 : __clobber_all); 274} 275 276SEC("tc") 277__description("Spill a u32 const scalar. Refill as u16 from MSB. Offset to skb->data") 278__failure __msg("invalid access to packet") 279__naked void _6_offset_to_skb_data(void) 280{ 281 asm volatile (" \ 282 r2 = *(u32*)(r1 + %[__sk_buff_data]); \ 283 r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ 284 w4 = 20; \ 285 *(u32*)(r10 - 8) = r4; \ 286 " 287#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 288 "r4 = *(u16*)(r10 - 6);" 289#else 290 "r4 = *(u16*)(r10 - 8);" 291#endif 292 " \ 293 r0 = r2; \ 294 /* r0 += r4 R0=pkt R2=pkt R3=pkt_end R4=umax=65535 */\ 295 r0 += r4; \ 296 /* if (r0 > r3) R0=pkt,umax=65535 R2=pkt R3=pkt_end R4=umax=65535 */\ 297 if r0 > r3 goto l0_%=; \ 298 /* r0 = *(u32 *)r2 R0=pkt,umax=65535 R2=pkt R3=pkt_end R4=20 */\ 299 r0 = *(u32*)(r2 + 0); \ 300l0_%=: r0 = 0; \ 301 exit; \ 302" : 303 : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), 304 __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) 305 : __clobber_all); 306} 307 308SEC("tc") 309__description("Spill and refill a u32 const scalar at non 8byte aligned stack addr. Offset to skb->data") 310__failure __msg("invalid access to packet") 311__naked void addr_offset_to_skb_data(void) 312{ 313 asm volatile (" \ 314 r2 = *(u32*)(r1 + %[__sk_buff_data]); \ 315 r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ 316 w4 = 20; \ 317 *(u32*)(r10 - 8) = r4; \ 318 *(u32*)(r10 - 4) = r4; \ 319 r4 = *(u32*)(r10 - 4); \ 320 r0 = r2; \ 321 /* r0 += r4 R0=pkt R2=pkt R3=pkt_end R4=umax=U32_MAX */\ 322 r0 += r4; \ 323 /* if (r0 > r3) R0=pkt,umax=U32_MAX R2=pkt R3=pkt_end R4= */\ 324 if r0 > r3 goto l0_%=; \ 325 /* r0 = *(u32 *)r2 R0=pkt,umax=U32_MAX R2=pkt R3=pkt_end R4= */\ 326 r0 = *(u32*)(r2 + 0); \ 327l0_%=: r0 = 0; \ 328 exit; \ 329" : 330 : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), 331 __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) 332 : __clobber_all); 333} 334 335SEC("tc") 336__description("Spill and refill a umax=40 bounded scalar. Offset to skb->data") 337__success __retval(0) 338__naked void scalar_offset_to_skb_data_2(void) 339{ 340 asm volatile (" \ 341 r2 = *(u32*)(r1 + %[__sk_buff_data]); \ 342 r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ 343 r4 = *(u64*)(r1 + %[__sk_buff_tstamp]); \ 344 if r4 <= 40 goto l0_%=; \ 345 r0 = 0; \ 346 exit; \ 347l0_%=: /* *(u32 *)(r10 -8) = r4 R4=umax=40 */ \ 348 *(u32*)(r10 - 8) = r4; \ 349 /* r4 = (*u32 *)(r10 - 8) */ \ 350 r4 = *(u32*)(r10 - 8); \ 351 /* r2 += r4 R2=pkt R4=umax=40 */ \ 352 r2 += r4; \ 353 /* r0 = r2 R2=pkt,umax=40 R4=umax=40 */ \ 354 r0 = r2; \ 355 /* r2 += 20 R0=pkt,umax=40 R2=pkt,umax=40 */ \ 356 r2 += 20; \ 357 /* if (r2 > r3) R0=pkt,umax=40 R2=pkt,off=20,umax=40 */\ 358 if r2 > r3 goto l1_%=; \ 359 /* r0 = *(u32 *)r0 R0=pkt,r=20,umax=40 R2=pkt,off=20,r=20,umax=40 */\ 360 r0 = *(u32*)(r0 + 0); \ 361l1_%=: r0 = 0; \ 362 exit; \ 363" : 364 : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), 365 __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), 366 __imm_const(__sk_buff_tstamp, offsetof(struct __sk_buff, tstamp)) 367 : __clobber_all); 368} 369 370SEC("tc") 371__description("Spill a u32 scalar at fp-4 and then at fp-8") 372__success __retval(0) 373__naked void and_then_at_fp_8(void) 374{ 375 asm volatile (" \ 376 w4 = 4321; \ 377 *(u32*)(r10 - 4) = r4; \ 378 *(u32*)(r10 - 8) = r4; \ 379 r4 = *(u64*)(r10 - 8); \ 380 r0 = 0; \ 381 exit; \ 382" ::: __clobber_all); 383} 384 385SEC("xdp") 386__description("32-bit spill of 64-bit reg should clear ID") 387__failure __msg("math between ctx pointer and 4294967295 is not allowed") 388__naked void spill_32bit_of_64bit_fail(void) 389{ 390 asm volatile (" \ 391 r6 = r1; \ 392 /* Roll one bit to force the verifier to track both branches. */\ 393 call %[bpf_get_prandom_u32]; \ 394 r0 &= 0x8; \ 395 /* Put a large number into r1. */ \ 396 r1 = 0xffffffff; \ 397 r1 <<= 32; \ 398 r1 += r0; \ 399 /* Assign an ID to r1. */ \ 400 r2 = r1; \ 401 /* 32-bit spill r1 to stack - should clear the ID! */\ 402 *(u32*)(r10 - 8) = r1; \ 403 /* 32-bit fill r2 from stack. */ \ 404 r2 = *(u32*)(r10 - 8); \ 405 /* Compare r2 with another register to trigger find_equal_scalars.\ 406 * Having one random bit is important here, otherwise the verifier cuts\ 407 * the corners. If the ID was mistakenly preserved on spill, this would\ 408 * cause the verifier to think that r1 is also equal to zero in one of\ 409 * the branches, and equal to eight on the other branch.\ 410 */ \ 411 r3 = 0; \ 412 if r2 != r3 goto l0_%=; \ 413l0_%=: r1 >>= 32; \ 414 /* At this point, if the verifier thinks that r1 is 0, an out-of-bounds\ 415 * read will happen, because it actually contains 0xffffffff.\ 416 */ \ 417 r6 += r1; \ 418 r0 = *(u32*)(r6 + 0); \ 419 exit; \ 420" : 421 : __imm(bpf_get_prandom_u32) 422 : __clobber_all); 423} 424 425SEC("xdp") 426__description("16-bit spill of 32-bit reg should clear ID") 427__failure __msg("dereference of modified ctx ptr R6 off=65535 disallowed") 428__naked void spill_16bit_of_32bit_fail(void) 429{ 430 asm volatile (" \ 431 r6 = r1; \ 432 /* Roll one bit to force the verifier to track both branches. */\ 433 call %[bpf_get_prandom_u32]; \ 434 r0 &= 0x8; \ 435 /* Put a large number into r1. */ \ 436 w1 = 0xffff0000; \ 437 r1 += r0; \ 438 /* Assign an ID to r1. */ \ 439 r2 = r1; \ 440 /* 16-bit spill r1 to stack - should clear the ID! */\ 441 *(u16*)(r10 - 8) = r1; \ 442 /* 16-bit fill r2 from stack. */ \ 443 r2 = *(u16*)(r10 - 8); \ 444 /* Compare r2 with another register to trigger find_equal_scalars.\ 445 * Having one random bit is important here, otherwise the verifier cuts\ 446 * the corners. If the ID was mistakenly preserved on spill, this would\ 447 * cause the verifier to think that r1 is also equal to zero in one of\ 448 * the branches, and equal to eight on the other branch.\ 449 */ \ 450 r3 = 0; \ 451 if r2 != r3 goto l0_%=; \ 452l0_%=: r1 >>= 16; \ 453 /* At this point, if the verifier thinks that r1 is 0, an out-of-bounds\ 454 * read will happen, because it actually contains 0xffff.\ 455 */ \ 456 r6 += r1; \ 457 r0 = *(u32*)(r6 + 0); \ 458 exit; \ 459" : 460 : __imm(bpf_get_prandom_u32) 461 : __clobber_all); 462} 463 464SEC("raw_tp") 465__log_level(2) 466__success 467__msg("fp-8=0m??scalar()") 468__msg("fp-16=00mm??scalar()") 469__msg("fp-24=00mm???scalar()") 470__naked void spill_subregs_preserve_stack_zero(void) 471{ 472 asm volatile ( 473 "call %[bpf_get_prandom_u32];" 474 475 /* 32-bit subreg spill with ZERO, MISC, and INVALID */ 476 ".8byte %[fp1_u8_st_zero];" /* ZERO, LLVM-18+: *(u8 *)(r10 -1) = 0; */ 477 "*(u8 *)(r10 -2) = r0;" /* MISC */ 478 /* fp-3 and fp-4 stay INVALID */ 479 "*(u32 *)(r10 -8) = r0;" 480 481 /* 16-bit subreg spill with ZERO, MISC, and INVALID */ 482 ".8byte %[fp10_u16_st_zero];" /* ZERO, LLVM-18+: *(u16 *)(r10 -10) = 0; */ 483 "*(u16 *)(r10 -12) = r0;" /* MISC */ 484 /* fp-13 and fp-14 stay INVALID */ 485 "*(u16 *)(r10 -16) = r0;" 486 487 /* 8-bit subreg spill with ZERO, MISC, and INVALID */ 488 ".8byte %[fp18_u16_st_zero];" /* ZERO, LLVM-18+: *(u16 *)(r18 -10) = 0; */ 489 "*(u16 *)(r10 -20) = r0;" /* MISC */ 490 /* fp-21, fp-22, and fp-23 stay INVALID */ 491 "*(u8 *)(r10 -24) = r0;" 492 493 "r0 = 0;" 494 "exit;" 495 : 496 : __imm(bpf_get_prandom_u32), 497 __imm_insn(fp1_u8_st_zero, BPF_ST_MEM(BPF_B, BPF_REG_FP, -1, 0)), 498 __imm_insn(fp10_u16_st_zero, BPF_ST_MEM(BPF_H, BPF_REG_FP, -10, 0)), 499 __imm_insn(fp18_u16_st_zero, BPF_ST_MEM(BPF_H, BPF_REG_FP, -18, 0)) 500 : __clobber_all); 501} 502 503char single_byte_buf[1] SEC(".data.single_byte_buf"); 504 505SEC("raw_tp") 506__log_level(2) 507__success 508/* fp-8 is spilled IMPRECISE value zero (represented by a zero value fake reg) */ 509__msg("2: (7a) *(u64 *)(r10 -8) = 0 ; R10=fp0 fp-8_w=0") 510/* but fp-16 is spilled IMPRECISE zero const reg */ 511__msg("4: (7b) *(u64 *)(r10 -16) = r0 ; R0_w=0 R10=fp0 fp-16_w=0") 512/* validate that assigning R2 from STACK_SPILL with zero value doesn't mark register 513 * precise immediately; if necessary, it will be marked precise later 514 */ 515__msg("6: (71) r2 = *(u8 *)(r10 -1) ; R2_w=0 R10=fp0 fp-8_w=0") 516/* similarly, when R2 is assigned from spilled register, it is initially 517 * imprecise, but will be marked precise later once it is used in precise context 518 */ 519__msg("10: (71) r2 = *(u8 *)(r10 -9) ; R2_w=0 R10=fp0 fp-16_w=0") 520__msg("11: (0f) r1 += r2") 521__msg("mark_precise: frame0: last_idx 11 first_idx 0 subseq_idx -1") 522__msg("mark_precise: frame0: regs=r2 stack= before 10: (71) r2 = *(u8 *)(r10 -9)") 523__msg("mark_precise: frame0: regs= stack=-16 before 9: (bf) r1 = r6") 524__msg("mark_precise: frame0: regs= stack=-16 before 8: (73) *(u8 *)(r1 +0) = r2") 525__msg("mark_precise: frame0: regs= stack=-16 before 7: (0f) r1 += r2") 526__msg("mark_precise: frame0: regs= stack=-16 before 6: (71) r2 = *(u8 *)(r10 -1)") 527__msg("mark_precise: frame0: regs= stack=-16 before 5: (bf) r1 = r6") 528__msg("mark_precise: frame0: regs= stack=-16 before 4: (7b) *(u64 *)(r10 -16) = r0") 529__msg("mark_precise: frame0: regs=r0 stack= before 3: (b7) r0 = 0") 530__naked void partial_stack_load_preserves_zeros(void) 531{ 532 asm volatile ( 533 /* fp-8 is value zero (represented by a zero value fake reg) */ 534 ".8byte %[fp8_st_zero];" /* LLVM-18+: *(u64 *)(r10 -8) = 0; */ 535 536 /* fp-16 is const zero register */ 537 "r0 = 0;" 538 "*(u64 *)(r10 -16) = r0;" 539 540 /* load single U8 from non-aligned spilled value zero slot */ 541 "r1 = %[single_byte_buf];" 542 "r2 = *(u8 *)(r10 -1);" 543 "r1 += r2;" 544 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 545 546 /* load single U8 from non-aligned ZERO REG slot */ 547 "r1 = %[single_byte_buf];" 548 "r2 = *(u8 *)(r10 -9);" 549 "r1 += r2;" 550 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 551 552 /* load single U16 from non-aligned spilled value zero slot */ 553 "r1 = %[single_byte_buf];" 554 "r2 = *(u16 *)(r10 -2);" 555 "r1 += r2;" 556 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 557 558 /* load single U16 from non-aligned ZERO REG slot */ 559 "r1 = %[single_byte_buf];" 560 "r2 = *(u16 *)(r10 -10);" 561 "r1 += r2;" 562 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 563 564 /* load single U32 from non-aligned spilled value zero slot */ 565 "r1 = %[single_byte_buf];" 566 "r2 = *(u32 *)(r10 -4);" 567 "r1 += r2;" 568 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 569 570 /* load single U32 from non-aligned ZERO REG slot */ 571 "r1 = %[single_byte_buf];" 572 "r2 = *(u32 *)(r10 -12);" 573 "r1 += r2;" 574 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 575 576 /* for completeness, load U64 from STACK_ZERO slot */ 577 "r1 = %[single_byte_buf];" 578 "r2 = *(u64 *)(r10 -8);" 579 "r1 += r2;" 580 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 581 582 /* for completeness, load U64 from ZERO REG slot */ 583 "r1 = %[single_byte_buf];" 584 "r2 = *(u64 *)(r10 -16);" 585 "r1 += r2;" 586 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 587 588 "r0 = 0;" 589 "exit;" 590 : 591 : __imm_ptr(single_byte_buf), 592 __imm_insn(fp8_st_zero, BPF_ST_MEM(BPF_DW, BPF_REG_FP, -8, 0)) 593 : __clobber_common); 594} 595 596SEC("raw_tp") 597__log_level(2) 598__success 599/* fp-4 is STACK_ZERO */ 600__msg("2: (62) *(u32 *)(r10 -4) = 0 ; R10=fp0 fp-8=0000????") 601__msg("4: (71) r2 = *(u8 *)(r10 -1) ; R2_w=0 R10=fp0 fp-8=0000????") 602__msg("5: (0f) r1 += r2") 603__msg("mark_precise: frame0: last_idx 5 first_idx 0 subseq_idx -1") 604__msg("mark_precise: frame0: regs=r2 stack= before 4: (71) r2 = *(u8 *)(r10 -1)") 605__naked void partial_stack_load_preserves_partial_zeros(void) 606{ 607 asm volatile ( 608 /* fp-4 is value zero */ 609 ".8byte %[fp4_st_zero];" /* LLVM-18+: *(u32 *)(r10 -4) = 0; */ 610 611 /* load single U8 from non-aligned stack zero slot */ 612 "r1 = %[single_byte_buf];" 613 "r2 = *(u8 *)(r10 -1);" 614 "r1 += r2;" 615 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 616 617 /* load single U16 from non-aligned stack zero slot */ 618 "r1 = %[single_byte_buf];" 619 "r2 = *(u16 *)(r10 -2);" 620 "r1 += r2;" 621 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 622 623 /* load single U32 from non-aligned stack zero slot */ 624 "r1 = %[single_byte_buf];" 625 "r2 = *(u32 *)(r10 -4);" 626 "r1 += r2;" 627 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 628 629 "r0 = 0;" 630 "exit;" 631 : 632 : __imm_ptr(single_byte_buf), 633 __imm_insn(fp4_st_zero, BPF_ST_MEM(BPF_W, BPF_REG_FP, -4, 0)) 634 : __clobber_common); 635} 636 637char two_byte_buf[2] SEC(".data.two_byte_buf"); 638 639SEC("raw_tp") 640__log_level(2) __flag(BPF_F_TEST_STATE_FREQ) 641__success 642/* make sure fp-8 is IMPRECISE fake register spill */ 643__msg("3: (7a) *(u64 *)(r10 -8) = 1 ; R10=fp0 fp-8_w=1") 644/* and fp-16 is spilled IMPRECISE const reg */ 645__msg("5: (7b) *(u64 *)(r10 -16) = r0 ; R0_w=1 R10=fp0 fp-16_w=1") 646/* validate load from fp-8, which was initialized using BPF_ST_MEM */ 647__msg("8: (79) r2 = *(u64 *)(r10 -8) ; R2_w=1 R10=fp0 fp-8=1") 648__msg("9: (0f) r1 += r2") 649__msg("mark_precise: frame0: last_idx 9 first_idx 7 subseq_idx -1") 650__msg("mark_precise: frame0: regs=r2 stack= before 8: (79) r2 = *(u64 *)(r10 -8)") 651__msg("mark_precise: frame0: regs= stack=-8 before 7: (bf) r1 = r6") 652/* note, fp-8 is precise, fp-16 is not yet precise, we'll get there */ 653__msg("mark_precise: frame0: parent state regs= stack=-8: R0_w=1 R1=ctx() R6_r=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8_rw=P1 fp-16_w=1") 654__msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") 655__msg("mark_precise: frame0: regs= stack=-8 before 6: (05) goto pc+0") 656__msg("mark_precise: frame0: regs= stack=-8 before 5: (7b) *(u64 *)(r10 -16) = r0") 657__msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r0 = 1") 658__msg("mark_precise: frame0: regs= stack=-8 before 3: (7a) *(u64 *)(r10 -8) = 1") 659__msg("10: R1_w=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2_w=1") 660/* validate load from fp-16, which was initialized using BPF_STX_MEM */ 661__msg("12: (79) r2 = *(u64 *)(r10 -16) ; R2_w=1 R10=fp0 fp-16=1") 662__msg("13: (0f) r1 += r2") 663__msg("mark_precise: frame0: last_idx 13 first_idx 7 subseq_idx -1") 664__msg("mark_precise: frame0: regs=r2 stack= before 12: (79) r2 = *(u64 *)(r10 -16)") 665__msg("mark_precise: frame0: regs= stack=-16 before 11: (bf) r1 = r6") 666__msg("mark_precise: frame0: regs= stack=-16 before 10: (73) *(u8 *)(r1 +0) = r2") 667__msg("mark_precise: frame0: regs= stack=-16 before 9: (0f) r1 += r2") 668__msg("mark_precise: frame0: regs= stack=-16 before 8: (79) r2 = *(u64 *)(r10 -8)") 669__msg("mark_precise: frame0: regs= stack=-16 before 7: (bf) r1 = r6") 670/* now both fp-8 and fp-16 are precise, very good */ 671__msg("mark_precise: frame0: parent state regs= stack=-16: R0_w=1 R1=ctx() R6_r=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8_rw=P1 fp-16_rw=P1") 672__msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") 673__msg("mark_precise: frame0: regs= stack=-16 before 6: (05) goto pc+0") 674__msg("mark_precise: frame0: regs= stack=-16 before 5: (7b) *(u64 *)(r10 -16) = r0") 675__msg("mark_precise: frame0: regs=r0 stack= before 4: (b7) r0 = 1") 676__msg("14: R1_w=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2_w=1") 677__naked void stack_load_preserves_const_precision(void) 678{ 679 asm volatile ( 680 /* establish checkpoint with state that has no stack slots; 681 * if we bubble up to this state without finding desired stack 682 * slot, then it's a bug and should be caught 683 */ 684 "goto +0;" 685 686 /* fp-8 is const 1 *fake* register */ 687 ".8byte %[fp8_st_one];" /* LLVM-18+: *(u64 *)(r10 -8) = 1; */ 688 689 /* fp-16 is const 1 register */ 690 "r0 = 1;" 691 "*(u64 *)(r10 -16) = r0;" 692 693 /* force checkpoint to check precision marks preserved in parent states */ 694 "goto +0;" 695 696 /* load single U64 from aligned FAKE_REG=1 slot */ 697 "r1 = %[two_byte_buf];" 698 "r2 = *(u64 *)(r10 -8);" 699 "r1 += r2;" 700 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 701 702 /* load single U64 from aligned REG=1 slot */ 703 "r1 = %[two_byte_buf];" 704 "r2 = *(u64 *)(r10 -16);" 705 "r1 += r2;" 706 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 707 708 "r0 = 0;" 709 "exit;" 710 : 711 : __imm_ptr(two_byte_buf), 712 __imm_insn(fp8_st_one, BPF_ST_MEM(BPF_DW, BPF_REG_FP, -8, 1)) 713 : __clobber_common); 714} 715 716SEC("raw_tp") 717__log_level(2) __flag(BPF_F_TEST_STATE_FREQ) 718__success 719/* make sure fp-8 is 32-bit FAKE subregister spill */ 720__msg("3: (62) *(u32 *)(r10 -8) = 1 ; R10=fp0 fp-8=????1") 721/* but fp-16 is spilled IMPRECISE zero const reg */ 722__msg("5: (63) *(u32 *)(r10 -16) = r0 ; R0_w=1 R10=fp0 fp-16=????1") 723/* validate load from fp-8, which was initialized using BPF_ST_MEM */ 724__msg("8: (61) r2 = *(u32 *)(r10 -8) ; R2_w=1 R10=fp0 fp-8=????1") 725__msg("9: (0f) r1 += r2") 726__msg("mark_precise: frame0: last_idx 9 first_idx 7 subseq_idx -1") 727__msg("mark_precise: frame0: regs=r2 stack= before 8: (61) r2 = *(u32 *)(r10 -8)") 728__msg("mark_precise: frame0: regs= stack=-8 before 7: (bf) r1 = r6") 729__msg("mark_precise: frame0: parent state regs= stack=-8: R0_w=1 R1=ctx() R6_r=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8_r=????P1 fp-16=????1") 730__msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") 731__msg("mark_precise: frame0: regs= stack=-8 before 6: (05) goto pc+0") 732__msg("mark_precise: frame0: regs= stack=-8 before 5: (63) *(u32 *)(r10 -16) = r0") 733__msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r0 = 1") 734__msg("mark_precise: frame0: regs= stack=-8 before 3: (62) *(u32 *)(r10 -8) = 1") 735__msg("10: R1_w=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2_w=1") 736/* validate load from fp-16, which was initialized using BPF_STX_MEM */ 737__msg("12: (61) r2 = *(u32 *)(r10 -16) ; R2_w=1 R10=fp0 fp-16=????1") 738__msg("13: (0f) r1 += r2") 739__msg("mark_precise: frame0: last_idx 13 first_idx 7 subseq_idx -1") 740__msg("mark_precise: frame0: regs=r2 stack= before 12: (61) r2 = *(u32 *)(r10 -16)") 741__msg("mark_precise: frame0: regs= stack=-16 before 11: (bf) r1 = r6") 742__msg("mark_precise: frame0: regs= stack=-16 before 10: (73) *(u8 *)(r1 +0) = r2") 743__msg("mark_precise: frame0: regs= stack=-16 before 9: (0f) r1 += r2") 744__msg("mark_precise: frame0: regs= stack=-16 before 8: (61) r2 = *(u32 *)(r10 -8)") 745__msg("mark_precise: frame0: regs= stack=-16 before 7: (bf) r1 = r6") 746__msg("mark_precise: frame0: parent state regs= stack=-16: R0_w=1 R1=ctx() R6_r=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8_r=????P1 fp-16_r=????P1") 747__msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") 748__msg("mark_precise: frame0: regs= stack=-16 before 6: (05) goto pc+0") 749__msg("mark_precise: frame0: regs= stack=-16 before 5: (63) *(u32 *)(r10 -16) = r0") 750__msg("mark_precise: frame0: regs=r0 stack= before 4: (b7) r0 = 1") 751__msg("14: R1_w=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2_w=1") 752__naked void stack_load_preserves_const_precision_subreg(void) 753{ 754 asm volatile ( 755 /* establish checkpoint with state that has no stack slots; 756 * if we bubble up to this state without finding desired stack 757 * slot, then it's a bug and should be caught 758 */ 759 "goto +0;" 760 761 /* fp-8 is const 1 *fake* SUB-register */ 762 ".8byte %[fp8_st_one];" /* LLVM-18+: *(u32 *)(r10 -8) = 1; */ 763 764 /* fp-16 is const 1 SUB-register */ 765 "r0 = 1;" 766 "*(u32 *)(r10 -16) = r0;" 767 768 /* force checkpoint to check precision marks preserved in parent states */ 769 "goto +0;" 770 771 /* load single U32 from aligned FAKE_REG=1 slot */ 772 "r1 = %[two_byte_buf];" 773 "r2 = *(u32 *)(r10 -8);" 774 "r1 += r2;" 775 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 776 777 /* load single U32 from aligned REG=1 slot */ 778 "r1 = %[two_byte_buf];" 779 "r2 = *(u32 *)(r10 -16);" 780 "r1 += r2;" 781 "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ 782 783 "r0 = 0;" 784 "exit;" 785 : 786 : __imm_ptr(two_byte_buf), 787 __imm_insn(fp8_st_one, BPF_ST_MEM(BPF_W, BPF_REG_FP, -8, 1)) /* 32-bit spill */ 788 : __clobber_common); 789} 790 791SEC("xdp") 792__description("32-bit spilled reg range should be tracked") 793__success __retval(0) 794__naked void spill_32bit_range_track(void) 795{ 796 asm volatile(" \ 797 call %[bpf_ktime_get_ns]; \ 798 /* Make r0 bounded. */ \ 799 r0 &= 65535; \ 800 /* Assign an ID to r0. */ \ 801 r1 = r0; \ 802 /* 32-bit spill r0 to stack. */ \ 803 *(u32*)(r10 - 8) = r0; \ 804 /* Boundary check on r0. */ \ 805 if r0 < 1 goto l0_%=; \ 806 /* 32-bit fill r1 from stack. */ \ 807 r1 = *(u32*)(r10 - 8); \ 808 /* r1 == r0 => r1 >= 1 always. */ \ 809 if r1 >= 1 goto l0_%=; \ 810 /* Dead branch: the verifier should prune it. \ 811 * Do an invalid memory access if the verifier \ 812 * follows it. \ 813 */ \ 814 r0 = *(u64*)(r9 + 0); \ 815l0_%=: r0 = 0; \ 816 exit; \ 817" : 818 : __imm(bpf_ktime_get_ns) 819 : __clobber_all); 820} 821 822SEC("xdp") 823__description("64-bit spill of 64-bit reg should assign ID") 824__success __retval(0) 825__naked void spill_64bit_of_64bit_ok(void) 826{ 827 asm volatile (" \ 828 /* Roll one bit to make the register inexact. */\ 829 call %[bpf_get_prandom_u32]; \ 830 r0 &= 0x80000000; \ 831 r0 <<= 32; \ 832 /* 64-bit spill r0 to stack - should assign an ID. */\ 833 *(u64*)(r10 - 8) = r0; \ 834 /* 64-bit fill r1 from stack - should preserve the ID. */\ 835 r1 = *(u64*)(r10 - 8); \ 836 /* Compare r1 with another register to trigger find_equal_scalars.\ 837 * Having one random bit is important here, otherwise the verifier cuts\ 838 * the corners. \ 839 */ \ 840 r2 = 0; \ 841 if r1 != r2 goto l0_%=; \ 842 /* The result of this comparison is predefined. */\ 843 if r0 == r2 goto l0_%=; \ 844 /* Dead branch: the verifier should prune it. Do an invalid memory\ 845 * access if the verifier follows it. \ 846 */ \ 847 r0 = *(u64*)(r9 + 0); \ 848 exit; \ 849l0_%=: r0 = 0; \ 850 exit; \ 851" : 852 : __imm(bpf_get_prandom_u32) 853 : __clobber_all); 854} 855 856SEC("xdp") 857__description("32-bit spill of 32-bit reg should assign ID") 858__success __retval(0) 859__naked void spill_32bit_of_32bit_ok(void) 860{ 861 asm volatile (" \ 862 /* Roll one bit to make the register inexact. */\ 863 call %[bpf_get_prandom_u32]; \ 864 w0 &= 0x80000000; \ 865 /* 32-bit spill r0 to stack - should assign an ID. */\ 866 *(u32*)(r10 - 8) = r0; \ 867 /* 32-bit fill r1 from stack - should preserve the ID. */\ 868 r1 = *(u32*)(r10 - 8); \ 869 /* Compare r1 with another register to trigger find_equal_scalars.\ 870 * Having one random bit is important here, otherwise the verifier cuts\ 871 * the corners. \ 872 */ \ 873 r2 = 0; \ 874 if r1 != r2 goto l0_%=; \ 875 /* The result of this comparison is predefined. */\ 876 if r0 == r2 goto l0_%=; \ 877 /* Dead branch: the verifier should prune it. Do an invalid memory\ 878 * access if the verifier follows it. \ 879 */ \ 880 r0 = *(u64*)(r9 + 0); \ 881 exit; \ 882l0_%=: r0 = 0; \ 883 exit; \ 884" : 885 : __imm(bpf_get_prandom_u32) 886 : __clobber_all); 887} 888 889SEC("xdp") 890__description("16-bit spill of 16-bit reg should assign ID") 891__success __retval(0) 892__naked void spill_16bit_of_16bit_ok(void) 893{ 894 asm volatile (" \ 895 /* Roll one bit to make the register inexact. */\ 896 call %[bpf_get_prandom_u32]; \ 897 r0 &= 0x8000; \ 898 /* 16-bit spill r0 to stack - should assign an ID. */\ 899 *(u16*)(r10 - 8) = r0; \ 900 /* 16-bit fill r1 from stack - should preserve the ID. */\ 901 r1 = *(u16*)(r10 - 8); \ 902 /* Compare r1 with another register to trigger find_equal_scalars.\ 903 * Having one random bit is important here, otherwise the verifier cuts\ 904 * the corners. \ 905 */ \ 906 r2 = 0; \ 907 if r1 != r2 goto l0_%=; \ 908 /* The result of this comparison is predefined. */\ 909 if r0 == r2 goto l0_%=; \ 910 /* Dead branch: the verifier should prune it. Do an invalid memory\ 911 * access if the verifier follows it. \ 912 */ \ 913 r0 = *(u64*)(r9 + 0); \ 914 exit; \ 915l0_%=: r0 = 0; \ 916 exit; \ 917" : 918 : __imm(bpf_get_prandom_u32) 919 : __clobber_all); 920} 921 922SEC("xdp") 923__description("8-bit spill of 8-bit reg should assign ID") 924__success __retval(0) 925__naked void spill_8bit_of_8bit_ok(void) 926{ 927 asm volatile (" \ 928 /* Roll one bit to make the register inexact. */\ 929 call %[bpf_get_prandom_u32]; \ 930 r0 &= 0x80; \ 931 /* 8-bit spill r0 to stack - should assign an ID. */\ 932 *(u8*)(r10 - 8) = r0; \ 933 /* 8-bit fill r1 from stack - should preserve the ID. */\ 934 r1 = *(u8*)(r10 - 8); \ 935 /* Compare r1 with another register to trigger find_equal_scalars.\ 936 * Having one random bit is important here, otherwise the verifier cuts\ 937 * the corners. \ 938 */ \ 939 r2 = 0; \ 940 if r1 != r2 goto l0_%=; \ 941 /* The result of this comparison is predefined. */\ 942 if r0 == r2 goto l0_%=; \ 943 /* Dead branch: the verifier should prune it. Do an invalid memory\ 944 * access if the verifier follows it. \ 945 */ \ 946 r0 = *(u64*)(r9 + 0); \ 947 exit; \ 948l0_%=: r0 = 0; \ 949 exit; \ 950" : 951 : __imm(bpf_get_prandom_u32) 952 : __clobber_all); 953} 954 955SEC("xdp") 956__description("spill unbounded reg, then range check src") 957__success __retval(0) 958__naked void spill_unbounded(void) 959{ 960 asm volatile (" \ 961 /* Produce an unbounded scalar. */ \ 962 call %[bpf_get_prandom_u32]; \ 963 /* Spill r0 to stack. */ \ 964 *(u64*)(r10 - 8) = r0; \ 965 /* Boundary check on r0. */ \ 966 if r0 > 16 goto l0_%=; \ 967 /* Fill r0 from stack. */ \ 968 r0 = *(u64*)(r10 - 8); \ 969 /* Boundary check on r0 with predetermined result. */\ 970 if r0 <= 16 goto l0_%=; \ 971 /* Dead branch: the verifier should prune it. Do an invalid memory\ 972 * access if the verifier follows it. \ 973 */ \ 974 r0 = *(u64*)(r9 + 0); \ 975l0_%=: r0 = 0; \ 976 exit; \ 977" : 978 : __imm(bpf_get_prandom_u32) 979 : __clobber_all); 980} 981 982SEC("xdp") 983__description("32-bit fill after 64-bit spill") 984__success __retval(0) 985__naked void fill_32bit_after_spill_64bit(void) 986{ 987 asm volatile(" \ 988 /* Randomize the upper 32 bits. */ \ 989 call %[bpf_get_prandom_u32]; \ 990 r0 <<= 32; \ 991 /* 64-bit spill r0 to stack. */ \ 992 *(u64*)(r10 - 8) = r0; \ 993 /* 32-bit fill r0 from stack. */ \ 994 " 995#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 996 "r0 = *(u32*)(r10 - 8);" 997#else 998 "r0 = *(u32*)(r10 - 4);" 999#endif 1000 " \ 1001 /* Boundary check on r0 with predetermined result. */\ 1002 if r0 == 0 goto l0_%=; \ 1003 /* Dead branch: the verifier should prune it. Do an invalid memory\ 1004 * access if the verifier follows it. \ 1005 */ \ 1006 r0 = *(u64*)(r9 + 0); \ 1007l0_%=: exit; \ 1008" : 1009 : __imm(bpf_get_prandom_u32) 1010 : __clobber_all); 1011} 1012 1013SEC("xdp") 1014__description("32-bit fill after 64-bit spill of 32-bit value should preserve ID") 1015__success __retval(0) 1016__naked void fill_32bit_after_spill_64bit_preserve_id(void) 1017{ 1018 asm volatile (" \ 1019 /* Randomize the lower 32 bits. */ \ 1020 call %[bpf_get_prandom_u32]; \ 1021 w0 &= 0xffffffff; \ 1022 /* 64-bit spill r0 to stack - should assign an ID. */\ 1023 *(u64*)(r10 - 8) = r0; \ 1024 /* 32-bit fill r1 from stack - should preserve the ID. */\ 1025 " 1026#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1027 "r1 = *(u32*)(r10 - 8);" 1028#else 1029 "r1 = *(u32*)(r10 - 4);" 1030#endif 1031 " \ 1032 /* Compare r1 with another register to trigger find_equal_scalars. */\ 1033 r2 = 0; \ 1034 if r1 != r2 goto l0_%=; \ 1035 /* The result of this comparison is predefined. */\ 1036 if r0 == r2 goto l0_%=; \ 1037 /* Dead branch: the verifier should prune it. Do an invalid memory\ 1038 * access if the verifier follows it. \ 1039 */ \ 1040 r0 = *(u64*)(r9 + 0); \ 1041 exit; \ 1042l0_%=: r0 = 0; \ 1043 exit; \ 1044" : 1045 : __imm(bpf_get_prandom_u32) 1046 : __clobber_all); 1047} 1048 1049SEC("xdp") 1050__description("32-bit fill after 64-bit spill should clear ID") 1051__failure __msg("math between ctx pointer and 4294967295 is not allowed") 1052__naked void fill_32bit_after_spill_64bit_clear_id(void) 1053{ 1054 asm volatile (" \ 1055 r6 = r1; \ 1056 /* Roll one bit to force the verifier to track both branches. */\ 1057 call %[bpf_get_prandom_u32]; \ 1058 r0 &= 0x8; \ 1059 /* Put a large number into r1. */ \ 1060 r1 = 0xffffffff; \ 1061 r1 <<= 32; \ 1062 r1 += r0; \ 1063 /* 64-bit spill r1 to stack - should assign an ID. */\ 1064 *(u64*)(r10 - 8) = r1; \ 1065 /* 32-bit fill r2 from stack - should clear the ID. */\ 1066 " 1067#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1068 "r2 = *(u32*)(r10 - 8);" 1069#else 1070 "r2 = *(u32*)(r10 - 4);" 1071#endif 1072 " \ 1073 /* Compare r2 with another register to trigger find_equal_scalars.\ 1074 * Having one random bit is important here, otherwise the verifier cuts\ 1075 * the corners. If the ID was mistakenly preserved on fill, this would\ 1076 * cause the verifier to think that r1 is also equal to zero in one of\ 1077 * the branches, and equal to eight on the other branch.\ 1078 */ \ 1079 r3 = 0; \ 1080 if r2 != r3 goto l0_%=; \ 1081l0_%=: r1 >>= 32; \ 1082 /* The verifier shouldn't propagate r2's range to r1, so it should\ 1083 * still remember r1 = 0xffffffff and reject the below.\ 1084 */ \ 1085 r6 += r1; \ 1086 r0 = *(u32*)(r6 + 0); \ 1087 exit; \ 1088" : 1089 : __imm(bpf_get_prandom_u32) 1090 : __clobber_all); 1091} 1092 1093/* stacksafe(): check if stack spill of an imprecise scalar in old state 1094 * is considered equivalent to STACK_{MISC,INVALID} in cur state. 1095 */ 1096SEC("socket") 1097__success __log_level(2) 1098__msg("8: (79) r1 = *(u64 *)(r10 -8)") 1099__msg("8: safe") 1100__msg("processed 11 insns") 1101/* STACK_INVALID should prevent verifier in unpriv mode from 1102 * considering states equivalent and force an error on second 1103 * verification path (entry - label 1 - label 2). 1104 */ 1105__failure_unpriv 1106__msg_unpriv("8: (79) r1 = *(u64 *)(r10 -8)") 1107__msg_unpriv("9: (95) exit") 1108__msg_unpriv("8: (79) r1 = *(u64 *)(r10 -8)") 1109__msg_unpriv("invalid read from stack off -8+2 size 8") 1110__flag(BPF_F_TEST_STATE_FREQ) 1111__naked void old_imprecise_scalar_vs_cur_stack_misc(void) 1112{ 1113 asm volatile( 1114 /* get a random value for branching */ 1115 "call %[bpf_ktime_get_ns];" 1116 "if r0 == 0 goto 1f;" 1117 /* conjure scalar at fp-8 */ 1118 "r0 = 42;" 1119 "*(u64*)(r10 - 8) = r0;" 1120 "goto 2f;" 1121"1:" 1122 /* conjure STACK_{MISC,INVALID} at fp-8 */ 1123 "call %[bpf_ktime_get_ns];" 1124 "*(u16*)(r10 - 8) = r0;" 1125 "*(u16*)(r10 - 4) = r0;" 1126"2:" 1127 /* read fp-8, should be considered safe on second visit */ 1128 "r1 = *(u64*)(r10 - 8);" 1129 "exit;" 1130 : 1131 : __imm(bpf_ktime_get_ns) 1132 : __clobber_all); 1133} 1134 1135/* stacksafe(): check that stack spill of a precise scalar in old state 1136 * is not considered equivalent to STACK_MISC in cur state. 1137 */ 1138SEC("socket") 1139__success __log_level(2) 1140/* verifier should visit 'if r1 == 0x2a ...' two times: 1141 * - once for path entry - label 2; 1142 * - once for path entry - label 1 - label 2. 1143 */ 1144__msg("if r1 == 0x2a goto pc+0") 1145__msg("if r1 == 0x2a goto pc+0") 1146__msg("processed 15 insns") 1147__flag(BPF_F_TEST_STATE_FREQ) 1148__naked void old_precise_scalar_vs_cur_stack_misc(void) 1149{ 1150 asm volatile( 1151 /* get a random value for branching */ 1152 "call %[bpf_ktime_get_ns];" 1153 "if r0 == 0 goto 1f;" 1154 /* conjure scalar at fp-8 */ 1155 "r0 = 42;" 1156 "*(u64*)(r10 - 8) = r0;" 1157 "goto 2f;" 1158"1:" 1159 /* conjure STACK_MISC at fp-8 */ 1160 "call %[bpf_ktime_get_ns];" 1161 "*(u64*)(r10 - 8) = r0;" 1162 "*(u32*)(r10 - 4) = r0;" 1163"2:" 1164 /* read fp-8, should not be considered safe on second visit */ 1165 "r1 = *(u64*)(r10 - 8);" 1166 /* use r1 in precise context */ 1167 "if r1 == 42 goto +0;" 1168 "exit;" 1169 : 1170 : __imm(bpf_ktime_get_ns) 1171 : __clobber_all); 1172} 1173 1174/* stacksafe(): check if STACK_MISC in old state is considered 1175 * equivalent to stack spill of a scalar in cur state. 1176 */ 1177SEC("socket") 1178__success __log_level(2) 1179__msg("8: (79) r0 = *(u64 *)(r10 -8)") 1180__msg("8: safe") 1181__msg("processed 11 insns") 1182__flag(BPF_F_TEST_STATE_FREQ) 1183__naked void old_stack_misc_vs_cur_scalar(void) 1184{ 1185 asm volatile( 1186 /* get a random value for branching */ 1187 "call %[bpf_ktime_get_ns];" 1188 "if r0 == 0 goto 1f;" 1189 /* conjure STACK_{MISC,INVALID} at fp-8 */ 1190 "call %[bpf_ktime_get_ns];" 1191 "*(u16*)(r10 - 8) = r0;" 1192 "*(u16*)(r10 - 4) = r0;" 1193 "goto 2f;" 1194"1:" 1195 /* conjure scalar at fp-8 */ 1196 "r0 = 42;" 1197 "*(u64*)(r10 - 8) = r0;" 1198"2:" 1199 /* read fp-8, should be considered safe on second visit */ 1200 "r0 = *(u64*)(r10 - 8);" 1201 "exit;" 1202 : 1203 : __imm(bpf_ktime_get_ns) 1204 : __clobber_all); 1205} 1206 1207/* stacksafe(): check that STACK_MISC in old state is not considered 1208 * equivalent to stack spill of a non-scalar in cur state. 1209 */ 1210SEC("socket") 1211__success __log_level(2) 1212/* verifier should process exit instructions twice: 1213 * - once for path entry - label 2; 1214 * - once for path entry - label 1 - label 2. 1215 */ 1216__msg("r1 = *(u64 *)(r10 -8)") 1217__msg("exit") 1218__msg("r1 = *(u64 *)(r10 -8)") 1219__msg("exit") 1220__msg("processed 11 insns") 1221__flag(BPF_F_TEST_STATE_FREQ) 1222__naked void old_stack_misc_vs_cur_ctx_ptr(void) 1223{ 1224 asm volatile( 1225 /* remember context pointer in r9 */ 1226 "r9 = r1;" 1227 /* get a random value for branching */ 1228 "call %[bpf_ktime_get_ns];" 1229 "if r0 == 0 goto 1f;" 1230 /* conjure STACK_MISC at fp-8 */ 1231 "call %[bpf_ktime_get_ns];" 1232 "*(u64*)(r10 - 8) = r0;" 1233 "*(u32*)(r10 - 4) = r0;" 1234 "goto 2f;" 1235"1:" 1236 /* conjure context pointer in fp-8 */ 1237 "*(u64*)(r10 - 8) = r9;" 1238"2:" 1239 /* read fp-8, should not be considered safe on second visit */ 1240 "r1 = *(u64*)(r10 - 8);" 1241 "exit;" 1242 : 1243 : __imm(bpf_ktime_get_ns) 1244 : __clobber_all); 1245} 1246 1247char _license[] SEC("license") = "GPL"; 1248