1// SPDX-License-Identifier: GPL-2.0 2/* Copyright (c) 2019 Facebook */ 3 4#include <linux/err.h> 5#include <netinet/tcp.h> 6#include <test_progs.h> 7#include "network_helpers.h" 8#include "bpf_dctcp.skel.h" 9#include "bpf_cubic.skel.h" 10#include "bpf_tcp_nogpl.skel.h" 11#include "tcp_ca_update.skel.h" 12#include "bpf_dctcp_release.skel.h" 13#include "tcp_ca_write_sk_pacing.skel.h" 14#include "tcp_ca_incompl_cong_ops.skel.h" 15#include "tcp_ca_unsupp_cong_op.skel.h" 16 17#ifndef ENOTSUPP 18#define ENOTSUPP 524 19#endif 20 21static const unsigned int total_bytes = 10 * 1024 * 1024; 22static int expected_stg = 0xeB9F; 23static int stop; 24 25static int settcpca(int fd, const char *tcp_ca) 26{ 27 int err; 28 29 err = setsockopt(fd, IPPROTO_TCP, TCP_CONGESTION, tcp_ca, strlen(tcp_ca)); 30 if (!ASSERT_NEQ(err, -1, "setsockopt")) 31 return -1; 32 33 return 0; 34} 35 36static void *server(void *arg) 37{ 38 int lfd = (int)(long)arg, err = 0, fd; 39 ssize_t nr_sent = 0, bytes = 0; 40 char batch[1500]; 41 42 fd = accept(lfd, NULL, NULL); 43 while (fd == -1) { 44 if (errno == EINTR) 45 continue; 46 err = -errno; 47 goto done; 48 } 49 50 if (settimeo(fd, 0)) { 51 err = -errno; 52 goto done; 53 } 54 55 while (bytes < total_bytes && !READ_ONCE(stop)) { 56 nr_sent = send(fd, &batch, 57 MIN(total_bytes - bytes, sizeof(batch)), 0); 58 if (nr_sent == -1 && errno == EINTR) 59 continue; 60 if (nr_sent == -1) { 61 err = -errno; 62 break; 63 } 64 bytes += nr_sent; 65 } 66 67 ASSERT_EQ(bytes, total_bytes, "send"); 68 69done: 70 if (fd >= 0) 71 close(fd); 72 if (err) { 73 WRITE_ONCE(stop, 1); 74 return ERR_PTR(err); 75 } 76 return NULL; 77} 78 79static void do_test(const char *tcp_ca, const struct bpf_map *sk_stg_map) 80{ 81 struct sockaddr_in6 sa6 = {}; 82 ssize_t nr_recv = 0, bytes = 0; 83 int lfd = -1, fd = -1; 84 pthread_t srv_thread; 85 socklen_t addrlen = sizeof(sa6); 86 void *thread_ret; 87 char batch[1500]; 88 int err; 89 90 WRITE_ONCE(stop, 0); 91 92 lfd = socket(AF_INET6, SOCK_STREAM, 0); 93 if (!ASSERT_NEQ(lfd, -1, "socket")) 94 return; 95 96 fd = socket(AF_INET6, SOCK_STREAM, 0); 97 if (!ASSERT_NEQ(fd, -1, "socket")) { 98 close(lfd); 99 return; 100 } 101 102 if (settcpca(lfd, tcp_ca) || settcpca(fd, tcp_ca) || 103 settimeo(lfd, 0) || settimeo(fd, 0)) 104 goto done; 105 106 /* bind, listen and start server thread to accept */ 107 sa6.sin6_family = AF_INET6; 108 sa6.sin6_addr = in6addr_loopback; 109 err = bind(lfd, (struct sockaddr *)&sa6, addrlen); 110 if (!ASSERT_NEQ(err, -1, "bind")) 111 goto done; 112 113 err = getsockname(lfd, (struct sockaddr *)&sa6, &addrlen); 114 if (!ASSERT_NEQ(err, -1, "getsockname")) 115 goto done; 116 117 err = listen(lfd, 1); 118 if (!ASSERT_NEQ(err, -1, "listen")) 119 goto done; 120 121 if (sk_stg_map) { 122 err = bpf_map_update_elem(bpf_map__fd(sk_stg_map), &fd, 123 &expected_stg, BPF_NOEXIST); 124 if (!ASSERT_OK(err, "bpf_map_update_elem(sk_stg_map)")) 125 goto done; 126 } 127 128 /* connect to server */ 129 err = connect(fd, (struct sockaddr *)&sa6, addrlen); 130 if (!ASSERT_NEQ(err, -1, "connect")) 131 goto done; 132 133 if (sk_stg_map) { 134 int tmp_stg; 135 136 err = bpf_map_lookup_elem(bpf_map__fd(sk_stg_map), &fd, 137 &tmp_stg); 138 if (!ASSERT_ERR(err, "bpf_map_lookup_elem(sk_stg_map)") || 139 !ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem(sk_stg_map)")) 140 goto done; 141 } 142 143 err = pthread_create(&srv_thread, NULL, server, (void *)(long)lfd); 144 if (!ASSERT_OK(err, "pthread_create")) 145 goto done; 146 147 /* recv total_bytes */ 148 while (bytes < total_bytes && !READ_ONCE(stop)) { 149 nr_recv = recv(fd, &batch, 150 MIN(total_bytes - bytes, sizeof(batch)), 0); 151 if (nr_recv == -1 && errno == EINTR) 152 continue; 153 if (nr_recv == -1) 154 break; 155 bytes += nr_recv; 156 } 157 158 ASSERT_EQ(bytes, total_bytes, "recv"); 159 160 WRITE_ONCE(stop, 1); 161 pthread_join(srv_thread, &thread_ret); 162 ASSERT_OK(IS_ERR(thread_ret), "thread_ret"); 163 164done: 165 close(lfd); 166 close(fd); 167} 168 169static void test_cubic(void) 170{ 171 struct bpf_cubic *cubic_skel; 172 struct bpf_link *link; 173 174 cubic_skel = bpf_cubic__open_and_load(); 175 if (!ASSERT_OK_PTR(cubic_skel, "bpf_cubic__open_and_load")) 176 return; 177 178 link = bpf_map__attach_struct_ops(cubic_skel->maps.cubic); 179 if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) { 180 bpf_cubic__destroy(cubic_skel); 181 return; 182 } 183 184 do_test("bpf_cubic", NULL); 185 186 ASSERT_EQ(cubic_skel->bss->bpf_cubic_acked_called, 1, "pkts_acked called"); 187 188 bpf_link__destroy(link); 189 bpf_cubic__destroy(cubic_skel); 190} 191 192static void test_dctcp(void) 193{ 194 struct bpf_dctcp *dctcp_skel; 195 struct bpf_link *link; 196 197 dctcp_skel = bpf_dctcp__open_and_load(); 198 if (!ASSERT_OK_PTR(dctcp_skel, "bpf_dctcp__open_and_load")) 199 return; 200 201 link = bpf_map__attach_struct_ops(dctcp_skel->maps.dctcp); 202 if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) { 203 bpf_dctcp__destroy(dctcp_skel); 204 return; 205 } 206 207 do_test("bpf_dctcp", dctcp_skel->maps.sk_stg_map); 208 ASSERT_EQ(dctcp_skel->bss->stg_result, expected_stg, "stg_result"); 209 210 bpf_link__destroy(link); 211 bpf_dctcp__destroy(dctcp_skel); 212} 213 214static char *err_str; 215static bool found; 216 217static int libbpf_debug_print(enum libbpf_print_level level, 218 const char *format, va_list args) 219{ 220 const char *prog_name, *log_buf; 221 222 if (level != LIBBPF_WARN || 223 !strstr(format, "-- BEGIN PROG LOAD LOG --")) { 224 vprintf(format, args); 225 return 0; 226 } 227 228 prog_name = va_arg(args, char *); 229 log_buf = va_arg(args, char *); 230 if (!log_buf) 231 goto out; 232 if (err_str && strstr(log_buf, err_str) != NULL) 233 found = true; 234out: 235 printf(format, prog_name, log_buf); 236 return 0; 237} 238 239static void test_invalid_license(void) 240{ 241 libbpf_print_fn_t old_print_fn; 242 struct bpf_tcp_nogpl *skel; 243 244 err_str = "struct ops programs must have a GPL compatible license"; 245 found = false; 246 old_print_fn = libbpf_set_print(libbpf_debug_print); 247 248 skel = bpf_tcp_nogpl__open_and_load(); 249 ASSERT_NULL(skel, "bpf_tcp_nogpl"); 250 ASSERT_EQ(found, true, "expected_err_msg"); 251 252 bpf_tcp_nogpl__destroy(skel); 253 libbpf_set_print(old_print_fn); 254} 255 256static void test_dctcp_fallback(void) 257{ 258 int err, lfd = -1, cli_fd = -1, srv_fd = -1; 259 struct network_helper_opts opts = { 260 .cc = "cubic", 261 }; 262 struct bpf_dctcp *dctcp_skel; 263 struct bpf_link *link = NULL; 264 char srv_cc[16]; 265 socklen_t cc_len = sizeof(srv_cc); 266 267 dctcp_skel = bpf_dctcp__open(); 268 if (!ASSERT_OK_PTR(dctcp_skel, "dctcp_skel")) 269 return; 270 strcpy(dctcp_skel->rodata->fallback, "cubic"); 271 if (!ASSERT_OK(bpf_dctcp__load(dctcp_skel), "bpf_dctcp__load")) 272 goto done; 273 274 link = bpf_map__attach_struct_ops(dctcp_skel->maps.dctcp); 275 if (!ASSERT_OK_PTR(link, "dctcp link")) 276 goto done; 277 278 lfd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0); 279 if (!ASSERT_GE(lfd, 0, "lfd") || 280 !ASSERT_OK(settcpca(lfd, "bpf_dctcp"), "lfd=>bpf_dctcp")) 281 goto done; 282 283 cli_fd = connect_to_fd_opts(lfd, &opts); 284 if (!ASSERT_GE(cli_fd, 0, "cli_fd")) 285 goto done; 286 287 srv_fd = accept(lfd, NULL, 0); 288 if (!ASSERT_GE(srv_fd, 0, "srv_fd")) 289 goto done; 290 ASSERT_STREQ(dctcp_skel->bss->cc_res, "cubic", "cc_res"); 291 ASSERT_EQ(dctcp_skel->bss->tcp_cdg_res, -ENOTSUPP, "tcp_cdg_res"); 292 /* All setsockopt(TCP_CONGESTION) in the recurred 293 * bpf_dctcp->init() should fail with -EBUSY. 294 */ 295 ASSERT_EQ(dctcp_skel->bss->ebusy_cnt, 3, "ebusy_cnt"); 296 297 err = getsockopt(srv_fd, SOL_TCP, TCP_CONGESTION, srv_cc, &cc_len); 298 if (!ASSERT_OK(err, "getsockopt(srv_fd, TCP_CONGESTION)")) 299 goto done; 300 ASSERT_STREQ(srv_cc, "cubic", "srv_fd cc"); 301 302done: 303 bpf_link__destroy(link); 304 bpf_dctcp__destroy(dctcp_skel); 305 if (lfd != -1) 306 close(lfd); 307 if (srv_fd != -1) 308 close(srv_fd); 309 if (cli_fd != -1) 310 close(cli_fd); 311} 312 313static void test_rel_setsockopt(void) 314{ 315 struct bpf_dctcp_release *rel_skel; 316 libbpf_print_fn_t old_print_fn; 317 318 err_str = "unknown func bpf_setsockopt"; 319 found = false; 320 321 old_print_fn = libbpf_set_print(libbpf_debug_print); 322 rel_skel = bpf_dctcp_release__open_and_load(); 323 libbpf_set_print(old_print_fn); 324 325 ASSERT_ERR_PTR(rel_skel, "rel_skel"); 326 ASSERT_TRUE(found, "expected_err_msg"); 327 328 bpf_dctcp_release__destroy(rel_skel); 329} 330 331static void test_write_sk_pacing(void) 332{ 333 struct tcp_ca_write_sk_pacing *skel; 334 struct bpf_link *link; 335 336 skel = tcp_ca_write_sk_pacing__open_and_load(); 337 if (!ASSERT_OK_PTR(skel, "open_and_load")) 338 return; 339 340 link = bpf_map__attach_struct_ops(skel->maps.write_sk_pacing); 341 ASSERT_OK_PTR(link, "attach_struct_ops"); 342 343 bpf_link__destroy(link); 344 tcp_ca_write_sk_pacing__destroy(skel); 345} 346 347static void test_incompl_cong_ops(void) 348{ 349 struct tcp_ca_incompl_cong_ops *skel; 350 struct bpf_link *link; 351 352 skel = tcp_ca_incompl_cong_ops__open_and_load(); 353 if (!ASSERT_OK_PTR(skel, "open_and_load")) 354 return; 355 356 /* That cong_avoid() and cong_control() are missing is only reported at 357 * this point: 358 */ 359 link = bpf_map__attach_struct_ops(skel->maps.incompl_cong_ops); 360 ASSERT_ERR_PTR(link, "attach_struct_ops"); 361 362 bpf_link__destroy(link); 363 tcp_ca_incompl_cong_ops__destroy(skel); 364} 365 366static void test_unsupp_cong_op(void) 367{ 368 libbpf_print_fn_t old_print_fn; 369 struct tcp_ca_unsupp_cong_op *skel; 370 371 err_str = "attach to unsupported member get_info"; 372 found = false; 373 old_print_fn = libbpf_set_print(libbpf_debug_print); 374 375 skel = tcp_ca_unsupp_cong_op__open_and_load(); 376 ASSERT_NULL(skel, "open_and_load"); 377 ASSERT_EQ(found, true, "expected_err_msg"); 378 379 tcp_ca_unsupp_cong_op__destroy(skel); 380 libbpf_set_print(old_print_fn); 381} 382 383static void test_update_ca(void) 384{ 385 struct tcp_ca_update *skel; 386 struct bpf_link *link; 387 int saved_ca1_cnt; 388 int err; 389 390 skel = tcp_ca_update__open_and_load(); 391 if (!ASSERT_OK_PTR(skel, "open")) 392 return; 393 394 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 395 ASSERT_OK_PTR(link, "attach_struct_ops"); 396 397 do_test("tcp_ca_update", NULL); 398 saved_ca1_cnt = skel->bss->ca1_cnt; 399 ASSERT_GT(saved_ca1_cnt, 0, "ca1_ca1_cnt"); 400 401 err = bpf_link__update_map(link, skel->maps.ca_update_2); 402 ASSERT_OK(err, "update_map"); 403 404 do_test("tcp_ca_update", NULL); 405 ASSERT_EQ(skel->bss->ca1_cnt, saved_ca1_cnt, "ca2_ca1_cnt"); 406 ASSERT_GT(skel->bss->ca2_cnt, 0, "ca2_ca2_cnt"); 407 408 bpf_link__destroy(link); 409 tcp_ca_update__destroy(skel); 410} 411 412static void test_update_wrong(void) 413{ 414 struct tcp_ca_update *skel; 415 struct bpf_link *link; 416 int saved_ca1_cnt; 417 int err; 418 419 skel = tcp_ca_update__open_and_load(); 420 if (!ASSERT_OK_PTR(skel, "open")) 421 return; 422 423 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 424 ASSERT_OK_PTR(link, "attach_struct_ops"); 425 426 do_test("tcp_ca_update", NULL); 427 saved_ca1_cnt = skel->bss->ca1_cnt; 428 ASSERT_GT(saved_ca1_cnt, 0, "ca1_ca1_cnt"); 429 430 err = bpf_link__update_map(link, skel->maps.ca_wrong); 431 ASSERT_ERR(err, "update_map"); 432 433 do_test("tcp_ca_update", NULL); 434 ASSERT_GT(skel->bss->ca1_cnt, saved_ca1_cnt, "ca2_ca1_cnt"); 435 436 bpf_link__destroy(link); 437 tcp_ca_update__destroy(skel); 438} 439 440static void test_mixed_links(void) 441{ 442 struct tcp_ca_update *skel; 443 struct bpf_link *link, *link_nl; 444 int err; 445 446 skel = tcp_ca_update__open_and_load(); 447 if (!ASSERT_OK_PTR(skel, "open")) 448 return; 449 450 link_nl = bpf_map__attach_struct_ops(skel->maps.ca_no_link); 451 ASSERT_OK_PTR(link_nl, "attach_struct_ops_nl"); 452 453 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 454 ASSERT_OK_PTR(link, "attach_struct_ops"); 455 456 do_test("tcp_ca_update", NULL); 457 ASSERT_GT(skel->bss->ca1_cnt, 0, "ca1_ca1_cnt"); 458 459 err = bpf_link__update_map(link, skel->maps.ca_no_link); 460 ASSERT_ERR(err, "update_map"); 461 462 bpf_link__destroy(link); 463 bpf_link__destroy(link_nl); 464 tcp_ca_update__destroy(skel); 465} 466 467static void test_multi_links(void) 468{ 469 struct tcp_ca_update *skel; 470 struct bpf_link *link; 471 472 skel = tcp_ca_update__open_and_load(); 473 if (!ASSERT_OK_PTR(skel, "open")) 474 return; 475 476 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 477 ASSERT_OK_PTR(link, "attach_struct_ops_1st"); 478 bpf_link__destroy(link); 479 480 /* A map should be able to be used to create links multiple 481 * times. 482 */ 483 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 484 ASSERT_OK_PTR(link, "attach_struct_ops_2nd"); 485 bpf_link__destroy(link); 486 487 tcp_ca_update__destroy(skel); 488} 489 490static void test_link_replace(void) 491{ 492 DECLARE_LIBBPF_OPTS(bpf_link_update_opts, opts); 493 struct tcp_ca_update *skel; 494 struct bpf_link *link; 495 int err; 496 497 skel = tcp_ca_update__open_and_load(); 498 if (!ASSERT_OK_PTR(skel, "open")) 499 return; 500 501 link = bpf_map__attach_struct_ops(skel->maps.ca_update_1); 502 ASSERT_OK_PTR(link, "attach_struct_ops_1st"); 503 bpf_link__destroy(link); 504 505 link = bpf_map__attach_struct_ops(skel->maps.ca_update_2); 506 ASSERT_OK_PTR(link, "attach_struct_ops_2nd"); 507 508 /* BPF_F_REPLACE with a wrong old map Fd. It should fail! 509 * 510 * With BPF_F_REPLACE, the link should be updated only if the 511 * old map fd given here matches the map backing the link. 512 */ 513 opts.old_map_fd = bpf_map__fd(skel->maps.ca_update_1); 514 opts.flags = BPF_F_REPLACE; 515 err = bpf_link_update(bpf_link__fd(link), 516 bpf_map__fd(skel->maps.ca_update_1), 517 &opts); 518 ASSERT_ERR(err, "bpf_link_update_fail"); 519 520 /* BPF_F_REPLACE with a correct old map Fd. It should success! */ 521 opts.old_map_fd = bpf_map__fd(skel->maps.ca_update_2); 522 err = bpf_link_update(bpf_link__fd(link), 523 bpf_map__fd(skel->maps.ca_update_1), 524 &opts); 525 ASSERT_OK(err, "bpf_link_update_success"); 526 527 bpf_link__destroy(link); 528 529 tcp_ca_update__destroy(skel); 530} 531 532void test_bpf_tcp_ca(void) 533{ 534 if (test__start_subtest("dctcp")) 535 test_dctcp(); 536 if (test__start_subtest("cubic")) 537 test_cubic(); 538 if (test__start_subtest("invalid_license")) 539 test_invalid_license(); 540 if (test__start_subtest("dctcp_fallback")) 541 test_dctcp_fallback(); 542 if (test__start_subtest("rel_setsockopt")) 543 test_rel_setsockopt(); 544 if (test__start_subtest("write_sk_pacing")) 545 test_write_sk_pacing(); 546 if (test__start_subtest("incompl_cong_ops")) 547 test_incompl_cong_ops(); 548 if (test__start_subtest("unsupp_cong_op")) 549 test_unsupp_cong_op(); 550 if (test__start_subtest("update_ca")) 551 test_update_ca(); 552 if (test__start_subtest("update_wrong")) 553 test_update_wrong(); 554 if (test__start_subtest("mixed_links")) 555 test_mixed_links(); 556 if (test__start_subtest("multi_links")) 557 test_multi_links(); 558 if (test__start_subtest("link_replace")) 559 test_link_replace(); 560} 561