1// SPDX-License-Identifier: GPL-2.0 2#include <unistd.h> 3#include <test_progs.h> 4#include <network_helpers.h> 5#include "tailcall_poke.skel.h" 6 7 8/* test_tailcall_1 checks basic functionality by patching multiple locations 9 * in a single program for a single tail call slot with nop->jmp, jmp->nop 10 * and jmp->jmp rewrites. Also checks for nop->nop. 11 */ 12static void test_tailcall_1(void) 13{ 14 int err, map_fd, prog_fd, main_fd, i, j; 15 struct bpf_map *prog_array; 16 struct bpf_program *prog; 17 struct bpf_object *obj; 18 char prog_name[32]; 19 char buff[128] = {}; 20 LIBBPF_OPTS(bpf_test_run_opts, topts, 21 .data_in = buff, 22 .data_size_in = sizeof(buff), 23 .repeat = 1, 24 ); 25 26 err = bpf_prog_test_load("tailcall1.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, 27 &prog_fd); 28 if (CHECK_FAIL(err)) 29 return; 30 31 prog = bpf_object__find_program_by_name(obj, "entry"); 32 if (CHECK_FAIL(!prog)) 33 goto out; 34 35 main_fd = bpf_program__fd(prog); 36 if (CHECK_FAIL(main_fd < 0)) 37 goto out; 38 39 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 40 if (CHECK_FAIL(!prog_array)) 41 goto out; 42 43 map_fd = bpf_map__fd(prog_array); 44 if (CHECK_FAIL(map_fd < 0)) 45 goto out; 46 47 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 48 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 49 50 prog = bpf_object__find_program_by_name(obj, prog_name); 51 if (CHECK_FAIL(!prog)) 52 goto out; 53 54 prog_fd = bpf_program__fd(prog); 55 if (CHECK_FAIL(prog_fd < 0)) 56 goto out; 57 58 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 59 if (CHECK_FAIL(err)) 60 goto out; 61 } 62 63 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 64 err = bpf_prog_test_run_opts(main_fd, &topts); 65 ASSERT_OK(err, "tailcall"); 66 ASSERT_EQ(topts.retval, i, "tailcall retval"); 67 68 err = bpf_map_delete_elem(map_fd, &i); 69 if (CHECK_FAIL(err)) 70 goto out; 71 } 72 73 err = bpf_prog_test_run_opts(main_fd, &topts); 74 ASSERT_OK(err, "tailcall"); 75 ASSERT_EQ(topts.retval, 3, "tailcall retval"); 76 77 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 78 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 79 80 prog = bpf_object__find_program_by_name(obj, prog_name); 81 if (CHECK_FAIL(!prog)) 82 goto out; 83 84 prog_fd = bpf_program__fd(prog); 85 if (CHECK_FAIL(prog_fd < 0)) 86 goto out; 87 88 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 89 if (CHECK_FAIL(err)) 90 goto out; 91 } 92 93 err = bpf_prog_test_run_opts(main_fd, &topts); 94 ASSERT_OK(err, "tailcall"); 95 ASSERT_OK(topts.retval, "tailcall retval"); 96 97 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 98 j = bpf_map__max_entries(prog_array) - 1 - i; 99 snprintf(prog_name, sizeof(prog_name), "classifier_%d", j); 100 101 prog = bpf_object__find_program_by_name(obj, prog_name); 102 if (CHECK_FAIL(!prog)) 103 goto out; 104 105 prog_fd = bpf_program__fd(prog); 106 if (CHECK_FAIL(prog_fd < 0)) 107 goto out; 108 109 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 110 if (CHECK_FAIL(err)) 111 goto out; 112 } 113 114 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 115 j = bpf_map__max_entries(prog_array) - 1 - i; 116 117 err = bpf_prog_test_run_opts(main_fd, &topts); 118 ASSERT_OK(err, "tailcall"); 119 ASSERT_EQ(topts.retval, j, "tailcall retval"); 120 121 err = bpf_map_delete_elem(map_fd, &i); 122 if (CHECK_FAIL(err)) 123 goto out; 124 } 125 126 err = bpf_prog_test_run_opts(main_fd, &topts); 127 ASSERT_OK(err, "tailcall"); 128 ASSERT_EQ(topts.retval, 3, "tailcall retval"); 129 130 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 131 err = bpf_map_delete_elem(map_fd, &i); 132 if (CHECK_FAIL(err >= 0 || errno != ENOENT)) 133 goto out; 134 135 err = bpf_prog_test_run_opts(main_fd, &topts); 136 ASSERT_OK(err, "tailcall"); 137 ASSERT_EQ(topts.retval, 3, "tailcall retval"); 138 } 139 140out: 141 bpf_object__close(obj); 142} 143 144/* test_tailcall_2 checks that patching multiple programs for a single 145 * tail call slot works. It also jumps through several programs and tests 146 * the tail call limit counter. 147 */ 148static void test_tailcall_2(void) 149{ 150 int err, map_fd, prog_fd, main_fd, i; 151 struct bpf_map *prog_array; 152 struct bpf_program *prog; 153 struct bpf_object *obj; 154 char prog_name[32]; 155 char buff[128] = {}; 156 LIBBPF_OPTS(bpf_test_run_opts, topts, 157 .data_in = buff, 158 .data_size_in = sizeof(buff), 159 .repeat = 1, 160 ); 161 162 err = bpf_prog_test_load("tailcall2.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, 163 &prog_fd); 164 if (CHECK_FAIL(err)) 165 return; 166 167 prog = bpf_object__find_program_by_name(obj, "entry"); 168 if (CHECK_FAIL(!prog)) 169 goto out; 170 171 main_fd = bpf_program__fd(prog); 172 if (CHECK_FAIL(main_fd < 0)) 173 goto out; 174 175 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 176 if (CHECK_FAIL(!prog_array)) 177 goto out; 178 179 map_fd = bpf_map__fd(prog_array); 180 if (CHECK_FAIL(map_fd < 0)) 181 goto out; 182 183 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 184 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 185 186 prog = bpf_object__find_program_by_name(obj, prog_name); 187 if (CHECK_FAIL(!prog)) 188 goto out; 189 190 prog_fd = bpf_program__fd(prog); 191 if (CHECK_FAIL(prog_fd < 0)) 192 goto out; 193 194 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 195 if (CHECK_FAIL(err)) 196 goto out; 197 } 198 199 err = bpf_prog_test_run_opts(main_fd, &topts); 200 ASSERT_OK(err, "tailcall"); 201 ASSERT_EQ(topts.retval, 2, "tailcall retval"); 202 203 i = 2; 204 err = bpf_map_delete_elem(map_fd, &i); 205 if (CHECK_FAIL(err)) 206 goto out; 207 208 err = bpf_prog_test_run_opts(main_fd, &topts); 209 ASSERT_OK(err, "tailcall"); 210 ASSERT_EQ(topts.retval, 1, "tailcall retval"); 211 212 i = 0; 213 err = bpf_map_delete_elem(map_fd, &i); 214 if (CHECK_FAIL(err)) 215 goto out; 216 217 err = bpf_prog_test_run_opts(main_fd, &topts); 218 ASSERT_OK(err, "tailcall"); 219 ASSERT_EQ(topts.retval, 3, "tailcall retval"); 220out: 221 bpf_object__close(obj); 222} 223 224static void test_tailcall_count(const char *which, bool test_fentry, 225 bool test_fexit) 226{ 227 struct bpf_object *obj = NULL, *fentry_obj = NULL, *fexit_obj = NULL; 228 struct bpf_link *fentry_link = NULL, *fexit_link = NULL; 229 int err, map_fd, prog_fd, main_fd, data_fd, i, val; 230 struct bpf_map *prog_array, *data_map; 231 struct bpf_program *prog; 232 char buff[128] = {}; 233 LIBBPF_OPTS(bpf_test_run_opts, topts, 234 .data_in = buff, 235 .data_size_in = sizeof(buff), 236 .repeat = 1, 237 ); 238 239 err = bpf_prog_test_load(which, BPF_PROG_TYPE_SCHED_CLS, &obj, 240 &prog_fd); 241 if (CHECK_FAIL(err)) 242 return; 243 244 prog = bpf_object__find_program_by_name(obj, "entry"); 245 if (CHECK_FAIL(!prog)) 246 goto out; 247 248 main_fd = bpf_program__fd(prog); 249 if (CHECK_FAIL(main_fd < 0)) 250 goto out; 251 252 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 253 if (CHECK_FAIL(!prog_array)) 254 goto out; 255 256 map_fd = bpf_map__fd(prog_array); 257 if (CHECK_FAIL(map_fd < 0)) 258 goto out; 259 260 prog = bpf_object__find_program_by_name(obj, "classifier_0"); 261 if (CHECK_FAIL(!prog)) 262 goto out; 263 264 prog_fd = bpf_program__fd(prog); 265 if (CHECK_FAIL(prog_fd < 0)) 266 goto out; 267 268 i = 0; 269 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 270 if (CHECK_FAIL(err)) 271 goto out; 272 273 if (test_fentry) { 274 fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_fentry.bpf.o", 275 NULL); 276 if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file")) 277 goto out; 278 279 prog = bpf_object__find_program_by_name(fentry_obj, "fentry"); 280 if (!ASSERT_OK_PTR(prog, "find fentry prog")) 281 goto out; 282 283 err = bpf_program__set_attach_target(prog, prog_fd, 284 "subprog_tail"); 285 if (!ASSERT_OK(err, "set_attach_target subprog_tail")) 286 goto out; 287 288 err = bpf_object__load(fentry_obj); 289 if (!ASSERT_OK(err, "load fentry_obj")) 290 goto out; 291 292 fentry_link = bpf_program__attach_trace(prog); 293 if (!ASSERT_OK_PTR(fentry_link, "attach_trace")) 294 goto out; 295 } 296 297 if (test_fexit) { 298 fexit_obj = bpf_object__open_file("tailcall_bpf2bpf_fexit.bpf.o", 299 NULL); 300 if (!ASSERT_OK_PTR(fexit_obj, "open fexit_obj file")) 301 goto out; 302 303 prog = bpf_object__find_program_by_name(fexit_obj, "fexit"); 304 if (!ASSERT_OK_PTR(prog, "find fexit prog")) 305 goto out; 306 307 err = bpf_program__set_attach_target(prog, prog_fd, 308 "subprog_tail"); 309 if (!ASSERT_OK(err, "set_attach_target subprog_tail")) 310 goto out; 311 312 err = bpf_object__load(fexit_obj); 313 if (!ASSERT_OK(err, "load fexit_obj")) 314 goto out; 315 316 fexit_link = bpf_program__attach_trace(prog); 317 if (!ASSERT_OK_PTR(fexit_link, "attach_trace")) 318 goto out; 319 } 320 321 err = bpf_prog_test_run_opts(main_fd, &topts); 322 ASSERT_OK(err, "tailcall"); 323 ASSERT_EQ(topts.retval, 1, "tailcall retval"); 324 325 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 326 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 327 goto out; 328 329 data_fd = bpf_map__fd(data_map); 330 if (CHECK_FAIL(data_fd < 0)) 331 goto out; 332 333 i = 0; 334 err = bpf_map_lookup_elem(data_fd, &i, &val); 335 ASSERT_OK(err, "tailcall count"); 336 ASSERT_EQ(val, 33, "tailcall count"); 337 338 if (test_fentry) { 339 data_map = bpf_object__find_map_by_name(fentry_obj, ".bss"); 340 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 341 "find tailcall_bpf2bpf_fentry.bss map")) 342 goto out; 343 344 data_fd = bpf_map__fd(data_map); 345 if (!ASSERT_FALSE(data_fd < 0, 346 "find tailcall_bpf2bpf_fentry.bss map fd")) 347 goto out; 348 349 i = 0; 350 err = bpf_map_lookup_elem(data_fd, &i, &val); 351 ASSERT_OK(err, "fentry count"); 352 ASSERT_EQ(val, 33, "fentry count"); 353 } 354 355 if (test_fexit) { 356 data_map = bpf_object__find_map_by_name(fexit_obj, ".bss"); 357 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 358 "find tailcall_bpf2bpf_fexit.bss map")) 359 goto out; 360 361 data_fd = bpf_map__fd(data_map); 362 if (!ASSERT_FALSE(data_fd < 0, 363 "find tailcall_bpf2bpf_fexit.bss map fd")) 364 goto out; 365 366 i = 0; 367 err = bpf_map_lookup_elem(data_fd, &i, &val); 368 ASSERT_OK(err, "fexit count"); 369 ASSERT_EQ(val, 33, "fexit count"); 370 } 371 372 i = 0; 373 err = bpf_map_delete_elem(map_fd, &i); 374 if (CHECK_FAIL(err)) 375 goto out; 376 377 err = bpf_prog_test_run_opts(main_fd, &topts); 378 ASSERT_OK(err, "tailcall"); 379 ASSERT_OK(topts.retval, "tailcall retval"); 380out: 381 bpf_link__destroy(fentry_link); 382 bpf_link__destroy(fexit_link); 383 bpf_object__close(fentry_obj); 384 bpf_object__close(fexit_obj); 385 bpf_object__close(obj); 386} 387 388/* test_tailcall_3 checks that the count value of the tail call limit 389 * enforcement matches with expectations. JIT uses direct jump. 390 */ 391static void test_tailcall_3(void) 392{ 393 test_tailcall_count("tailcall3.bpf.o", false, false); 394} 395 396/* test_tailcall_6 checks that the count value of the tail call limit 397 * enforcement matches with expectations. JIT uses indirect jump. 398 */ 399static void test_tailcall_6(void) 400{ 401 test_tailcall_count("tailcall6.bpf.o", false, false); 402} 403 404/* test_tailcall_4 checks that the kernel properly selects indirect jump 405 * for the case where the key is not known. Latter is passed via global 406 * data to select different targets we can compare return value of. 407 */ 408static void test_tailcall_4(void) 409{ 410 int err, map_fd, prog_fd, main_fd, data_fd, i; 411 struct bpf_map *prog_array, *data_map; 412 struct bpf_program *prog; 413 struct bpf_object *obj; 414 static const int zero = 0; 415 char buff[128] = {}; 416 char prog_name[32]; 417 LIBBPF_OPTS(bpf_test_run_opts, topts, 418 .data_in = buff, 419 .data_size_in = sizeof(buff), 420 .repeat = 1, 421 ); 422 423 err = bpf_prog_test_load("tailcall4.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, 424 &prog_fd); 425 if (CHECK_FAIL(err)) 426 return; 427 428 prog = bpf_object__find_program_by_name(obj, "entry"); 429 if (CHECK_FAIL(!prog)) 430 goto out; 431 432 main_fd = bpf_program__fd(prog); 433 if (CHECK_FAIL(main_fd < 0)) 434 goto out; 435 436 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 437 if (CHECK_FAIL(!prog_array)) 438 goto out; 439 440 map_fd = bpf_map__fd(prog_array); 441 if (CHECK_FAIL(map_fd < 0)) 442 goto out; 443 444 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 445 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 446 goto out; 447 448 data_fd = bpf_map__fd(data_map); 449 if (CHECK_FAIL(data_fd < 0)) 450 goto out; 451 452 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 453 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 454 455 prog = bpf_object__find_program_by_name(obj, prog_name); 456 if (CHECK_FAIL(!prog)) 457 goto out; 458 459 prog_fd = bpf_program__fd(prog); 460 if (CHECK_FAIL(prog_fd < 0)) 461 goto out; 462 463 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 464 if (CHECK_FAIL(err)) 465 goto out; 466 } 467 468 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 469 err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY); 470 if (CHECK_FAIL(err)) 471 goto out; 472 473 err = bpf_prog_test_run_opts(main_fd, &topts); 474 ASSERT_OK(err, "tailcall"); 475 ASSERT_EQ(topts.retval, i, "tailcall retval"); 476 } 477 478 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 479 err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY); 480 if (CHECK_FAIL(err)) 481 goto out; 482 483 err = bpf_map_delete_elem(map_fd, &i); 484 if (CHECK_FAIL(err)) 485 goto out; 486 487 err = bpf_prog_test_run_opts(main_fd, &topts); 488 ASSERT_OK(err, "tailcall"); 489 ASSERT_EQ(topts.retval, 3, "tailcall retval"); 490 } 491out: 492 bpf_object__close(obj); 493} 494 495/* test_tailcall_5 probes similarly to test_tailcall_4 that the kernel generates 496 * an indirect jump when the keys are const but different from different branches. 497 */ 498static void test_tailcall_5(void) 499{ 500 int err, map_fd, prog_fd, main_fd, data_fd, i, key[] = { 1111, 1234, 5678 }; 501 struct bpf_map *prog_array, *data_map; 502 struct bpf_program *prog; 503 struct bpf_object *obj; 504 static const int zero = 0; 505 char buff[128] = {}; 506 char prog_name[32]; 507 LIBBPF_OPTS(bpf_test_run_opts, topts, 508 .data_in = buff, 509 .data_size_in = sizeof(buff), 510 .repeat = 1, 511 ); 512 513 err = bpf_prog_test_load("tailcall5.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, 514 &prog_fd); 515 if (CHECK_FAIL(err)) 516 return; 517 518 prog = bpf_object__find_program_by_name(obj, "entry"); 519 if (CHECK_FAIL(!prog)) 520 goto out; 521 522 main_fd = bpf_program__fd(prog); 523 if (CHECK_FAIL(main_fd < 0)) 524 goto out; 525 526 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 527 if (CHECK_FAIL(!prog_array)) 528 goto out; 529 530 map_fd = bpf_map__fd(prog_array); 531 if (CHECK_FAIL(map_fd < 0)) 532 goto out; 533 534 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 535 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 536 goto out; 537 538 data_fd = bpf_map__fd(data_map); 539 if (CHECK_FAIL(data_fd < 0)) 540 goto out; 541 542 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 543 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 544 545 prog = bpf_object__find_program_by_name(obj, prog_name); 546 if (CHECK_FAIL(!prog)) 547 goto out; 548 549 prog_fd = bpf_program__fd(prog); 550 if (CHECK_FAIL(prog_fd < 0)) 551 goto out; 552 553 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 554 if (CHECK_FAIL(err)) 555 goto out; 556 } 557 558 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 559 err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY); 560 if (CHECK_FAIL(err)) 561 goto out; 562 563 err = bpf_prog_test_run_opts(main_fd, &topts); 564 ASSERT_OK(err, "tailcall"); 565 ASSERT_EQ(topts.retval, i, "tailcall retval"); 566 } 567 568 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 569 err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY); 570 if (CHECK_FAIL(err)) 571 goto out; 572 573 err = bpf_map_delete_elem(map_fd, &i); 574 if (CHECK_FAIL(err)) 575 goto out; 576 577 err = bpf_prog_test_run_opts(main_fd, &topts); 578 ASSERT_OK(err, "tailcall"); 579 ASSERT_EQ(topts.retval, 3, "tailcall retval"); 580 } 581out: 582 bpf_object__close(obj); 583} 584 585/* test_tailcall_bpf2bpf_1 purpose is to make sure that tailcalls are working 586 * correctly in correlation with BPF subprograms 587 */ 588static void test_tailcall_bpf2bpf_1(void) 589{ 590 int err, map_fd, prog_fd, main_fd, i; 591 struct bpf_map *prog_array; 592 struct bpf_program *prog; 593 struct bpf_object *obj; 594 char prog_name[32]; 595 LIBBPF_OPTS(bpf_test_run_opts, topts, 596 .data_in = &pkt_v4, 597 .data_size_in = sizeof(pkt_v4), 598 .repeat = 1, 599 ); 600 601 err = bpf_prog_test_load("tailcall_bpf2bpf1.bpf.o", BPF_PROG_TYPE_SCHED_CLS, 602 &obj, &prog_fd); 603 if (CHECK_FAIL(err)) 604 return; 605 606 prog = bpf_object__find_program_by_name(obj, "entry"); 607 if (CHECK_FAIL(!prog)) 608 goto out; 609 610 main_fd = bpf_program__fd(prog); 611 if (CHECK_FAIL(main_fd < 0)) 612 goto out; 613 614 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 615 if (CHECK_FAIL(!prog_array)) 616 goto out; 617 618 map_fd = bpf_map__fd(prog_array); 619 if (CHECK_FAIL(map_fd < 0)) 620 goto out; 621 622 /* nop -> jmp */ 623 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 624 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 625 626 prog = bpf_object__find_program_by_name(obj, prog_name); 627 if (CHECK_FAIL(!prog)) 628 goto out; 629 630 prog_fd = bpf_program__fd(prog); 631 if (CHECK_FAIL(prog_fd < 0)) 632 goto out; 633 634 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 635 if (CHECK_FAIL(err)) 636 goto out; 637 } 638 639 err = bpf_prog_test_run_opts(main_fd, &topts); 640 ASSERT_OK(err, "tailcall"); 641 ASSERT_EQ(topts.retval, 1, "tailcall retval"); 642 643 /* jmp -> nop, call subprog that will do tailcall */ 644 i = 1; 645 err = bpf_map_delete_elem(map_fd, &i); 646 if (CHECK_FAIL(err)) 647 goto out; 648 649 err = bpf_prog_test_run_opts(main_fd, &topts); 650 ASSERT_OK(err, "tailcall"); 651 ASSERT_OK(topts.retval, "tailcall retval"); 652 653 /* make sure that subprog can access ctx and entry prog that 654 * called this subprog can properly return 655 */ 656 i = 0; 657 err = bpf_map_delete_elem(map_fd, &i); 658 if (CHECK_FAIL(err)) 659 goto out; 660 661 err = bpf_prog_test_run_opts(main_fd, &topts); 662 ASSERT_OK(err, "tailcall"); 663 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 2, "tailcall retval"); 664out: 665 bpf_object__close(obj); 666} 667 668/* test_tailcall_bpf2bpf_2 checks that the count value of the tail call limit 669 * enforcement matches with expectations when tailcall is preceded with 670 * bpf2bpf call. 671 */ 672static void test_tailcall_bpf2bpf_2(void) 673{ 674 int err, map_fd, prog_fd, main_fd, data_fd, i, val; 675 struct bpf_map *prog_array, *data_map; 676 struct bpf_program *prog; 677 struct bpf_object *obj; 678 char buff[128] = {}; 679 LIBBPF_OPTS(bpf_test_run_opts, topts, 680 .data_in = buff, 681 .data_size_in = sizeof(buff), 682 .repeat = 1, 683 ); 684 685 err = bpf_prog_test_load("tailcall_bpf2bpf2.bpf.o", BPF_PROG_TYPE_SCHED_CLS, 686 &obj, &prog_fd); 687 if (CHECK_FAIL(err)) 688 return; 689 690 prog = bpf_object__find_program_by_name(obj, "entry"); 691 if (CHECK_FAIL(!prog)) 692 goto out; 693 694 main_fd = bpf_program__fd(prog); 695 if (CHECK_FAIL(main_fd < 0)) 696 goto out; 697 698 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 699 if (CHECK_FAIL(!prog_array)) 700 goto out; 701 702 map_fd = bpf_map__fd(prog_array); 703 if (CHECK_FAIL(map_fd < 0)) 704 goto out; 705 706 prog = bpf_object__find_program_by_name(obj, "classifier_0"); 707 if (CHECK_FAIL(!prog)) 708 goto out; 709 710 prog_fd = bpf_program__fd(prog); 711 if (CHECK_FAIL(prog_fd < 0)) 712 goto out; 713 714 i = 0; 715 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 716 if (CHECK_FAIL(err)) 717 goto out; 718 719 err = bpf_prog_test_run_opts(main_fd, &topts); 720 ASSERT_OK(err, "tailcall"); 721 ASSERT_EQ(topts.retval, 1, "tailcall retval"); 722 723 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 724 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 725 goto out; 726 727 data_fd = bpf_map__fd(data_map); 728 if (CHECK_FAIL(data_fd < 0)) 729 goto out; 730 731 i = 0; 732 err = bpf_map_lookup_elem(data_fd, &i, &val); 733 ASSERT_OK(err, "tailcall count"); 734 ASSERT_EQ(val, 33, "tailcall count"); 735 736 i = 0; 737 err = bpf_map_delete_elem(map_fd, &i); 738 if (CHECK_FAIL(err)) 739 goto out; 740 741 err = bpf_prog_test_run_opts(main_fd, &topts); 742 ASSERT_OK(err, "tailcall"); 743 ASSERT_OK(topts.retval, "tailcall retval"); 744out: 745 bpf_object__close(obj); 746} 747 748/* test_tailcall_bpf2bpf_3 checks that non-trivial amount of stack (up to 749 * 256 bytes) can be used within bpf subprograms that have the tailcalls 750 * in them 751 */ 752static void test_tailcall_bpf2bpf_3(void) 753{ 754 int err, map_fd, prog_fd, main_fd, i; 755 struct bpf_map *prog_array; 756 struct bpf_program *prog; 757 struct bpf_object *obj; 758 char prog_name[32]; 759 LIBBPF_OPTS(bpf_test_run_opts, topts, 760 .data_in = &pkt_v4, 761 .data_size_in = sizeof(pkt_v4), 762 .repeat = 1, 763 ); 764 765 err = bpf_prog_test_load("tailcall_bpf2bpf3.bpf.o", BPF_PROG_TYPE_SCHED_CLS, 766 &obj, &prog_fd); 767 if (CHECK_FAIL(err)) 768 return; 769 770 prog = bpf_object__find_program_by_name(obj, "entry"); 771 if (CHECK_FAIL(!prog)) 772 goto out; 773 774 main_fd = bpf_program__fd(prog); 775 if (CHECK_FAIL(main_fd < 0)) 776 goto out; 777 778 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 779 if (CHECK_FAIL(!prog_array)) 780 goto out; 781 782 map_fd = bpf_map__fd(prog_array); 783 if (CHECK_FAIL(map_fd < 0)) 784 goto out; 785 786 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 787 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 788 789 prog = bpf_object__find_program_by_name(obj, prog_name); 790 if (CHECK_FAIL(!prog)) 791 goto out; 792 793 prog_fd = bpf_program__fd(prog); 794 if (CHECK_FAIL(prog_fd < 0)) 795 goto out; 796 797 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 798 if (CHECK_FAIL(err)) 799 goto out; 800 } 801 802 err = bpf_prog_test_run_opts(main_fd, &topts); 803 ASSERT_OK(err, "tailcall"); 804 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 3, "tailcall retval"); 805 806 i = 1; 807 err = bpf_map_delete_elem(map_fd, &i); 808 if (CHECK_FAIL(err)) 809 goto out; 810 811 err = bpf_prog_test_run_opts(main_fd, &topts); 812 ASSERT_OK(err, "tailcall"); 813 ASSERT_EQ(topts.retval, sizeof(pkt_v4), "tailcall retval"); 814 815 i = 0; 816 err = bpf_map_delete_elem(map_fd, &i); 817 if (CHECK_FAIL(err)) 818 goto out; 819 820 err = bpf_prog_test_run_opts(main_fd, &topts); 821 ASSERT_OK(err, "tailcall"); 822 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 2, "tailcall retval"); 823out: 824 bpf_object__close(obj); 825} 826 827#include "tailcall_bpf2bpf4.skel.h" 828 829/* test_tailcall_bpf2bpf_4 checks that tailcall counter is correctly preserved 830 * across tailcalls combined with bpf2bpf calls. for making sure that tailcall 831 * counter behaves correctly, bpf program will go through following flow: 832 * 833 * entry -> entry_subprog -> tailcall0 -> bpf_func0 -> subprog0 -> 834 * -> tailcall1 -> bpf_func1 -> subprog1 -> tailcall2 -> bpf_func2 -> 835 * subprog2 [here bump global counter] --------^ 836 * 837 * We go through first two tailcalls and start counting from the subprog2 where 838 * the loop begins. At the end of the test make sure that the global counter is 839 * equal to 31, because tailcall counter includes the first two tailcalls 840 * whereas global counter is incremented only on loop presented on flow above. 841 * 842 * The noise parameter is used to insert bpf_map_update calls into the logic 843 * to force verifier to patch instructions. This allows us to ensure jump 844 * logic remains correct with instruction movement. 845 */ 846static void test_tailcall_bpf2bpf_4(bool noise) 847{ 848 int err, map_fd, prog_fd, main_fd, data_fd, i; 849 struct tailcall_bpf2bpf4__bss val; 850 struct bpf_map *prog_array, *data_map; 851 struct bpf_program *prog; 852 struct bpf_object *obj; 853 char prog_name[32]; 854 LIBBPF_OPTS(bpf_test_run_opts, topts, 855 .data_in = &pkt_v4, 856 .data_size_in = sizeof(pkt_v4), 857 .repeat = 1, 858 ); 859 860 err = bpf_prog_test_load("tailcall_bpf2bpf4.bpf.o", BPF_PROG_TYPE_SCHED_CLS, 861 &obj, &prog_fd); 862 if (CHECK_FAIL(err)) 863 return; 864 865 prog = bpf_object__find_program_by_name(obj, "entry"); 866 if (CHECK_FAIL(!prog)) 867 goto out; 868 869 main_fd = bpf_program__fd(prog); 870 if (CHECK_FAIL(main_fd < 0)) 871 goto out; 872 873 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 874 if (CHECK_FAIL(!prog_array)) 875 goto out; 876 877 map_fd = bpf_map__fd(prog_array); 878 if (CHECK_FAIL(map_fd < 0)) 879 goto out; 880 881 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 882 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 883 884 prog = bpf_object__find_program_by_name(obj, prog_name); 885 if (CHECK_FAIL(!prog)) 886 goto out; 887 888 prog_fd = bpf_program__fd(prog); 889 if (CHECK_FAIL(prog_fd < 0)) 890 goto out; 891 892 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 893 if (CHECK_FAIL(err)) 894 goto out; 895 } 896 897 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 898 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 899 goto out; 900 901 data_fd = bpf_map__fd(data_map); 902 if (CHECK_FAIL(data_fd < 0)) 903 goto out; 904 905 i = 0; 906 val.noise = noise; 907 val.count = 0; 908 err = bpf_map_update_elem(data_fd, &i, &val, BPF_ANY); 909 if (CHECK_FAIL(err)) 910 goto out; 911 912 err = bpf_prog_test_run_opts(main_fd, &topts); 913 ASSERT_OK(err, "tailcall"); 914 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 3, "tailcall retval"); 915 916 i = 0; 917 err = bpf_map_lookup_elem(data_fd, &i, &val); 918 ASSERT_OK(err, "tailcall count"); 919 ASSERT_EQ(val.count, 31, "tailcall count"); 920 921out: 922 bpf_object__close(obj); 923} 924 925#include "tailcall_bpf2bpf6.skel.h" 926 927/* Tail call counting works even when there is data on stack which is 928 * not aligned to 8 bytes. 929 */ 930static void test_tailcall_bpf2bpf_6(void) 931{ 932 struct tailcall_bpf2bpf6 *obj; 933 int err, map_fd, prog_fd, main_fd, data_fd, i, val; 934 LIBBPF_OPTS(bpf_test_run_opts, topts, 935 .data_in = &pkt_v4, 936 .data_size_in = sizeof(pkt_v4), 937 .repeat = 1, 938 ); 939 940 obj = tailcall_bpf2bpf6__open_and_load(); 941 if (!ASSERT_OK_PTR(obj, "open and load")) 942 return; 943 944 main_fd = bpf_program__fd(obj->progs.entry); 945 if (!ASSERT_GE(main_fd, 0, "entry prog fd")) 946 goto out; 947 948 map_fd = bpf_map__fd(obj->maps.jmp_table); 949 if (!ASSERT_GE(map_fd, 0, "jmp_table map fd")) 950 goto out; 951 952 prog_fd = bpf_program__fd(obj->progs.classifier_0); 953 if (!ASSERT_GE(prog_fd, 0, "classifier_0 prog fd")) 954 goto out; 955 956 i = 0; 957 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 958 if (!ASSERT_OK(err, "jmp_table map update")) 959 goto out; 960 961 err = bpf_prog_test_run_opts(main_fd, &topts); 962 ASSERT_OK(err, "entry prog test run"); 963 ASSERT_EQ(topts.retval, 0, "tailcall retval"); 964 965 data_fd = bpf_map__fd(obj->maps.bss); 966 if (!ASSERT_GE(data_fd, 0, "bss map fd")) 967 goto out; 968 969 i = 0; 970 err = bpf_map_lookup_elem(data_fd, &i, &val); 971 ASSERT_OK(err, "bss map lookup"); 972 ASSERT_EQ(val, 1, "done flag is set"); 973 974out: 975 tailcall_bpf2bpf6__destroy(obj); 976} 977 978/* test_tailcall_bpf2bpf_fentry checks that the count value of the tail call 979 * limit enforcement matches with expectations when tailcall is preceded with 980 * bpf2bpf call, and the bpf2bpf call is traced by fentry. 981 */ 982static void test_tailcall_bpf2bpf_fentry(void) 983{ 984 test_tailcall_count("tailcall_bpf2bpf2.bpf.o", true, false); 985} 986 987/* test_tailcall_bpf2bpf_fexit checks that the count value of the tail call 988 * limit enforcement matches with expectations when tailcall is preceded with 989 * bpf2bpf call, and the bpf2bpf call is traced by fexit. 990 */ 991static void test_tailcall_bpf2bpf_fexit(void) 992{ 993 test_tailcall_count("tailcall_bpf2bpf2.bpf.o", false, true); 994} 995 996/* test_tailcall_bpf2bpf_fentry_fexit checks that the count value of the tail 997 * call limit enforcement matches with expectations when tailcall is preceded 998 * with bpf2bpf call, and the bpf2bpf call is traced by both fentry and fexit. 999 */ 1000static void test_tailcall_bpf2bpf_fentry_fexit(void) 1001{ 1002 test_tailcall_count("tailcall_bpf2bpf2.bpf.o", true, true); 1003} 1004 1005/* test_tailcall_bpf2bpf_fentry_entry checks that the count value of the tail 1006 * call limit enforcement matches with expectations when tailcall is preceded 1007 * with bpf2bpf call, and the bpf2bpf caller is traced by fentry. 1008 */ 1009static void test_tailcall_bpf2bpf_fentry_entry(void) 1010{ 1011 struct bpf_object *tgt_obj = NULL, *fentry_obj = NULL; 1012 int err, map_fd, prog_fd, data_fd, i, val; 1013 struct bpf_map *prog_array, *data_map; 1014 struct bpf_link *fentry_link = NULL; 1015 struct bpf_program *prog; 1016 char buff[128] = {}; 1017 1018 LIBBPF_OPTS(bpf_test_run_opts, topts, 1019 .data_in = buff, 1020 .data_size_in = sizeof(buff), 1021 .repeat = 1, 1022 ); 1023 1024 err = bpf_prog_test_load("tailcall_bpf2bpf2.bpf.o", 1025 BPF_PROG_TYPE_SCHED_CLS, 1026 &tgt_obj, &prog_fd); 1027 if (!ASSERT_OK(err, "load tgt_obj")) 1028 return; 1029 1030 prog_array = bpf_object__find_map_by_name(tgt_obj, "jmp_table"); 1031 if (!ASSERT_OK_PTR(prog_array, "find jmp_table map")) 1032 goto out; 1033 1034 map_fd = bpf_map__fd(prog_array); 1035 if (!ASSERT_FALSE(map_fd < 0, "find jmp_table map fd")) 1036 goto out; 1037 1038 prog = bpf_object__find_program_by_name(tgt_obj, "classifier_0"); 1039 if (!ASSERT_OK_PTR(prog, "find classifier_0 prog")) 1040 goto out; 1041 1042 prog_fd = bpf_program__fd(prog); 1043 if (!ASSERT_FALSE(prog_fd < 0, "find classifier_0 prog fd")) 1044 goto out; 1045 1046 i = 0; 1047 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 1048 if (!ASSERT_OK(err, "update jmp_table")) 1049 goto out; 1050 1051 fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_fentry.bpf.o", 1052 NULL); 1053 if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file")) 1054 goto out; 1055 1056 prog = bpf_object__find_program_by_name(fentry_obj, "fentry"); 1057 if (!ASSERT_OK_PTR(prog, "find fentry prog")) 1058 goto out; 1059 1060 err = bpf_program__set_attach_target(prog, prog_fd, "classifier_0"); 1061 if (!ASSERT_OK(err, "set_attach_target classifier_0")) 1062 goto out; 1063 1064 err = bpf_object__load(fentry_obj); 1065 if (!ASSERT_OK(err, "load fentry_obj")) 1066 goto out; 1067 1068 fentry_link = bpf_program__attach_trace(prog); 1069 if (!ASSERT_OK_PTR(fentry_link, "attach_trace")) 1070 goto out; 1071 1072 err = bpf_prog_test_run_opts(prog_fd, &topts); 1073 ASSERT_OK(err, "tailcall"); 1074 ASSERT_EQ(topts.retval, 1, "tailcall retval"); 1075 1076 data_map = bpf_object__find_map_by_name(tgt_obj, "tailcall.bss"); 1077 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 1078 "find tailcall.bss map")) 1079 goto out; 1080 1081 data_fd = bpf_map__fd(data_map); 1082 if (!ASSERT_FALSE(data_fd < 0, "find tailcall.bss map fd")) 1083 goto out; 1084 1085 i = 0; 1086 err = bpf_map_lookup_elem(data_fd, &i, &val); 1087 ASSERT_OK(err, "tailcall count"); 1088 ASSERT_EQ(val, 34, "tailcall count"); 1089 1090 data_map = bpf_object__find_map_by_name(fentry_obj, ".bss"); 1091 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 1092 "find tailcall_bpf2bpf_fentry.bss map")) 1093 goto out; 1094 1095 data_fd = bpf_map__fd(data_map); 1096 if (!ASSERT_FALSE(data_fd < 0, 1097 "find tailcall_bpf2bpf_fentry.bss map fd")) 1098 goto out; 1099 1100 i = 0; 1101 err = bpf_map_lookup_elem(data_fd, &i, &val); 1102 ASSERT_OK(err, "fentry count"); 1103 ASSERT_EQ(val, 1, "fentry count"); 1104 1105out: 1106 bpf_link__destroy(fentry_link); 1107 bpf_object__close(fentry_obj); 1108 bpf_object__close(tgt_obj); 1109} 1110 1111#define JMP_TABLE "/sys/fs/bpf/jmp_table" 1112 1113static int poke_thread_exit; 1114 1115static void *poke_update(void *arg) 1116{ 1117 __u32 zero = 0, prog1_fd, prog2_fd, map_fd; 1118 struct tailcall_poke *call = arg; 1119 1120 map_fd = bpf_map__fd(call->maps.jmp_table); 1121 prog1_fd = bpf_program__fd(call->progs.call1); 1122 prog2_fd = bpf_program__fd(call->progs.call2); 1123 1124 while (!poke_thread_exit) { 1125 bpf_map_update_elem(map_fd, &zero, &prog1_fd, BPF_ANY); 1126 bpf_map_update_elem(map_fd, &zero, &prog2_fd, BPF_ANY); 1127 } 1128 1129 return NULL; 1130} 1131 1132/* 1133 * We are trying to hit prog array update during another program load 1134 * that shares the same prog array map. 1135 * 1136 * For that we share the jmp_table map between two skeleton instances 1137 * by pinning the jmp_table to same path. Then first skeleton instance 1138 * periodically updates jmp_table in 'poke update' thread while we load 1139 * the second skeleton instance in the main thread. 1140 */ 1141static void test_tailcall_poke(void) 1142{ 1143 struct tailcall_poke *call, *test; 1144 int err, cnt = 10; 1145 pthread_t thread; 1146 1147 unlink(JMP_TABLE); 1148 1149 call = tailcall_poke__open_and_load(); 1150 if (!ASSERT_OK_PTR(call, "tailcall_poke__open")) 1151 return; 1152 1153 err = bpf_map__pin(call->maps.jmp_table, JMP_TABLE); 1154 if (!ASSERT_OK(err, "bpf_map__pin")) 1155 goto out; 1156 1157 err = pthread_create(&thread, NULL, poke_update, call); 1158 if (!ASSERT_OK(err, "new toggler")) 1159 goto out; 1160 1161 while (cnt--) { 1162 test = tailcall_poke__open(); 1163 if (!ASSERT_OK_PTR(test, "tailcall_poke__open")) 1164 break; 1165 1166 err = bpf_map__set_pin_path(test->maps.jmp_table, JMP_TABLE); 1167 if (!ASSERT_OK(err, "bpf_map__pin")) { 1168 tailcall_poke__destroy(test); 1169 break; 1170 } 1171 1172 bpf_program__set_autoload(test->progs.test, true); 1173 bpf_program__set_autoload(test->progs.call1, false); 1174 bpf_program__set_autoload(test->progs.call2, false); 1175 1176 err = tailcall_poke__load(test); 1177 tailcall_poke__destroy(test); 1178 if (!ASSERT_OK(err, "tailcall_poke__load")) 1179 break; 1180 } 1181 1182 poke_thread_exit = 1; 1183 ASSERT_OK(pthread_join(thread, NULL), "pthread_join"); 1184 1185out: 1186 bpf_map__unpin(call->maps.jmp_table, JMP_TABLE); 1187 tailcall_poke__destroy(call); 1188} 1189 1190void test_tailcalls(void) 1191{ 1192 if (test__start_subtest("tailcall_1")) 1193 test_tailcall_1(); 1194 if (test__start_subtest("tailcall_2")) 1195 test_tailcall_2(); 1196 if (test__start_subtest("tailcall_3")) 1197 test_tailcall_3(); 1198 if (test__start_subtest("tailcall_4")) 1199 test_tailcall_4(); 1200 if (test__start_subtest("tailcall_5")) 1201 test_tailcall_5(); 1202 if (test__start_subtest("tailcall_6")) 1203 test_tailcall_6(); 1204 if (test__start_subtest("tailcall_bpf2bpf_1")) 1205 test_tailcall_bpf2bpf_1(); 1206 if (test__start_subtest("tailcall_bpf2bpf_2")) 1207 test_tailcall_bpf2bpf_2(); 1208 if (test__start_subtest("tailcall_bpf2bpf_3")) 1209 test_tailcall_bpf2bpf_3(); 1210 if (test__start_subtest("tailcall_bpf2bpf_4")) 1211 test_tailcall_bpf2bpf_4(false); 1212 if (test__start_subtest("tailcall_bpf2bpf_5")) 1213 test_tailcall_bpf2bpf_4(true); 1214 if (test__start_subtest("tailcall_bpf2bpf_6")) 1215 test_tailcall_bpf2bpf_6(); 1216 if (test__start_subtest("tailcall_bpf2bpf_fentry")) 1217 test_tailcall_bpf2bpf_fentry(); 1218 if (test__start_subtest("tailcall_bpf2bpf_fexit")) 1219 test_tailcall_bpf2bpf_fexit(); 1220 if (test__start_subtest("tailcall_bpf2bpf_fentry_fexit")) 1221 test_tailcall_bpf2bpf_fentry_fexit(); 1222 if (test__start_subtest("tailcall_bpf2bpf_fentry_entry")) 1223 test_tailcall_bpf2bpf_fentry_entry(); 1224 if (test__start_subtest("tailcall_poke")) 1225 test_tailcall_poke(); 1226} 1227