1// SPDX-License-Identifier: GPL-2.0 2// Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io 3#include <stdio.h> 4#include <stdlib.h> 5#include <sys/socket.h> 6#include <sys/ioctl.h> 7#include <sys/select.h> 8#include <netinet/in.h> 9#include <arpa/inet.h> 10#include <unistd.h> 11#include <string.h> 12#include <errno.h> 13#include <stdbool.h> 14#include <signal.h> 15#include <fcntl.h> 16#include <sys/wait.h> 17#include <time.h> 18#include <sched.h> 19 20#include <sys/time.h> 21#include <sys/types.h> 22#include <sys/sendfile.h> 23 24#include <linux/netlink.h> 25#include <linux/socket.h> 26#include <linux/sock_diag.h> 27#include <linux/bpf.h> 28#include <linux/if_link.h> 29#include <linux/tls.h> 30#include <assert.h> 31#include <libgen.h> 32 33#include <getopt.h> 34 35#include <bpf/bpf.h> 36#include <bpf/libbpf.h> 37 38#include "bpf_util.h" 39#include "cgroup_helpers.h" 40 41int running; 42static void running_handler(int a); 43 44#ifndef TCP_ULP 45# define TCP_ULP 31 46#endif 47#ifndef SOL_TLS 48# define SOL_TLS 282 49#endif 50 51/* randomly selected ports for testing on lo */ 52#define S1_PORT 10000 53#define S2_PORT 10001 54 55#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o" 56#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o" 57#define CG_PATH "/sockmap" 58 59/* global sockets */ 60int s1, s2, c1, c2, p1, p2; 61int test_cnt; 62int passed; 63int failed; 64int map_fd[9]; 65struct bpf_map *maps[9]; 66int prog_fd[11]; 67 68int txmsg_pass; 69int txmsg_redir; 70int txmsg_drop; 71int txmsg_apply; 72int txmsg_cork; 73int txmsg_start; 74int txmsg_end; 75int txmsg_start_push; 76int txmsg_end_push; 77int txmsg_start_pop; 78int txmsg_pop; 79int txmsg_ingress; 80int txmsg_redir_skb; 81int txmsg_ktls_skb; 82int txmsg_ktls_skb_drop; 83int txmsg_ktls_skb_redir; 84int ktls; 85int peek_flag; 86int skb_use_parser; 87int txmsg_omit_skb_parser; 88 89static const struct option long_options[] = { 90 {"help", no_argument, NULL, 'h' }, 91 {"cgroup", required_argument, NULL, 'c' }, 92 {"rate", required_argument, NULL, 'r' }, 93 {"verbose", optional_argument, NULL, 'v' }, 94 {"iov_count", required_argument, NULL, 'i' }, 95 {"length", required_argument, NULL, 'l' }, 96 {"test", required_argument, NULL, 't' }, 97 {"data_test", no_argument, NULL, 'd' }, 98 {"txmsg", no_argument, &txmsg_pass, 1 }, 99 {"txmsg_redir", no_argument, &txmsg_redir, 1 }, 100 {"txmsg_drop", no_argument, &txmsg_drop, 1 }, 101 {"txmsg_apply", required_argument, NULL, 'a'}, 102 {"txmsg_cork", required_argument, NULL, 'k'}, 103 {"txmsg_start", required_argument, NULL, 's'}, 104 {"txmsg_end", required_argument, NULL, 'e'}, 105 {"txmsg_start_push", required_argument, NULL, 'p'}, 106 {"txmsg_end_push", required_argument, NULL, 'q'}, 107 {"txmsg_start_pop", required_argument, NULL, 'w'}, 108 {"txmsg_pop", required_argument, NULL, 'x'}, 109 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, 110 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 }, 111 {"ktls", no_argument, &ktls, 1 }, 112 {"peek", no_argument, &peek_flag, 1 }, 113 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1}, 114 {"whitelist", required_argument, NULL, 'n' }, 115 {"blacklist", required_argument, NULL, 'b' }, 116 {0, 0, NULL, 0 } 117}; 118 119struct test_env { 120 const char *type; 121 const char *subtest; 122 const char *prepend; 123 124 int test_num; 125 int subtest_num; 126 127 int succ_cnt; 128 int fail_cnt; 129 int fail_last; 130}; 131 132struct test_env env; 133 134struct sockmap_options { 135 int verbose; 136 bool base; 137 bool sendpage; 138 bool data_test; 139 bool drop_expected; 140 bool check_recved_len; 141 bool tx_wait_mem; 142 int iov_count; 143 int iov_length; 144 int rate; 145 char *map; 146 char *whitelist; 147 char *blacklist; 148 char *prepend; 149}; 150 151struct _test { 152 char *title; 153 void (*tester)(int cg_fd, struct sockmap_options *opt); 154}; 155 156static void test_start(void) 157{ 158 env.subtest_num++; 159} 160 161static void test_fail(void) 162{ 163 env.fail_cnt++; 164} 165 166static void test_pass(void) 167{ 168 env.succ_cnt++; 169} 170 171static void test_reset(void) 172{ 173 txmsg_start = txmsg_end = 0; 174 txmsg_start_pop = txmsg_pop = 0; 175 txmsg_start_push = txmsg_end_push = 0; 176 txmsg_pass = txmsg_drop = txmsg_redir = 0; 177 txmsg_apply = txmsg_cork = 0; 178 txmsg_ingress = txmsg_redir_skb = 0; 179 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0; 180 txmsg_omit_skb_parser = 0; 181 skb_use_parser = 0; 182} 183 184static int test_start_subtest(const struct _test *t, struct sockmap_options *o) 185{ 186 env.type = o->map; 187 env.subtest = t->title; 188 env.prepend = o->prepend; 189 env.test_num++; 190 env.subtest_num = 0; 191 env.fail_last = env.fail_cnt; 192 test_reset(); 193 return 0; 194} 195 196static void test_end_subtest(void) 197{ 198 int error = env.fail_cnt - env.fail_last; 199 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME); 200 201 if (!error) 202 test_pass(); 203 204 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n", 205 env.test_num, env.subtest_num, 206 !type ? "sockmap" : "sockhash", 207 env.prepend ? : "", 208 env.subtest, error ? "FAIL" : "OK"); 209} 210 211static void test_print_results(void) 212{ 213 fprintf(stdout, "Pass: %d Fail: %d\n", 214 env.succ_cnt, env.fail_cnt); 215} 216 217static void usage(char *argv[]) 218{ 219 int i; 220 221 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]); 222 printf(" options:\n"); 223 for (i = 0; long_options[i].name != 0; i++) { 224 printf(" --%-12s", long_options[i].name); 225 if (long_options[i].flag != NULL) 226 printf(" flag (internal value:%d)\n", 227 *long_options[i].flag); 228 else 229 printf(" -%c\n", long_options[i].val); 230 } 231 printf("\n"); 232} 233 234char *sock_to_string(int s) 235{ 236 if (s == c1) 237 return "client1"; 238 else if (s == c2) 239 return "client2"; 240 else if (s == s1) 241 return "server1"; 242 else if (s == s2) 243 return "server2"; 244 else if (s == p1) 245 return "peer1"; 246 else if (s == p2) 247 return "peer2"; 248 else 249 return "unknown"; 250} 251 252static int sockmap_init_ktls(int verbose, int s) 253{ 254 struct tls12_crypto_info_aes_gcm_128 tls_tx = { 255 .info = { 256 .version = TLS_1_2_VERSION, 257 .cipher_type = TLS_CIPHER_AES_GCM_128, 258 }, 259 }; 260 struct tls12_crypto_info_aes_gcm_128 tls_rx = { 261 .info = { 262 .version = TLS_1_2_VERSION, 263 .cipher_type = TLS_CIPHER_AES_GCM_128, 264 }, 265 }; 266 int so_buf = 6553500; 267 int err; 268 269 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls")); 270 if (err) { 271 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err); 272 return -EINVAL; 273 } 274 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx)); 275 if (err) { 276 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err); 277 return -EINVAL; 278 } 279 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx)); 280 if (err) { 281 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err); 282 return -EINVAL; 283 } 284 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf)); 285 if (err) { 286 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err); 287 return -EINVAL; 288 } 289 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf)); 290 if (err) { 291 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err); 292 return -EINVAL; 293 } 294 295 if (verbose) 296 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s)); 297 return 0; 298} 299static int sockmap_init_sockets(int verbose) 300{ 301 int i, err, one = 1; 302 struct sockaddr_in addr; 303 int *fds[4] = {&s1, &s2, &c1, &c2}; 304 305 s1 = s2 = p1 = p2 = c1 = c2 = 0; 306 307 /* Init sockets */ 308 for (i = 0; i < 4; i++) { 309 *fds[i] = socket(AF_INET, SOCK_STREAM, 0); 310 if (*fds[i] < 0) { 311 perror("socket s1 failed()"); 312 return errno; 313 } 314 } 315 316 /* Allow reuse */ 317 for (i = 0; i < 2; i++) { 318 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, 319 (char *)&one, sizeof(one)); 320 if (err) { 321 perror("setsockopt failed()"); 322 return errno; 323 } 324 } 325 326 /* Non-blocking sockets */ 327 for (i = 0; i < 2; i++) { 328 err = ioctl(*fds[i], FIONBIO, (char *)&one); 329 if (err < 0) { 330 perror("ioctl s1 failed()"); 331 return errno; 332 } 333 } 334 335 /* Bind server sockets */ 336 memset(&addr, 0, sizeof(struct sockaddr_in)); 337 addr.sin_family = AF_INET; 338 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 339 340 addr.sin_port = htons(S1_PORT); 341 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); 342 if (err < 0) { 343 perror("bind s1 failed()"); 344 return errno; 345 } 346 347 addr.sin_port = htons(S2_PORT); 348 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); 349 if (err < 0) { 350 perror("bind s2 failed()"); 351 return errno; 352 } 353 354 /* Listen server sockets */ 355 addr.sin_port = htons(S1_PORT); 356 err = listen(s1, 32); 357 if (err < 0) { 358 perror("listen s1 failed()"); 359 return errno; 360 } 361 362 addr.sin_port = htons(S2_PORT); 363 err = listen(s2, 32); 364 if (err < 0) { 365 perror("listen s1 failed()"); 366 return errno; 367 } 368 369 /* Initiate Connect */ 370 addr.sin_port = htons(S1_PORT); 371 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); 372 if (err < 0 && errno != EINPROGRESS) { 373 perror("connect c1 failed()"); 374 return errno; 375 } 376 377 addr.sin_port = htons(S2_PORT); 378 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); 379 if (err < 0 && errno != EINPROGRESS) { 380 perror("connect c2 failed()"); 381 return errno; 382 } else if (err < 0) { 383 err = 0; 384 } 385 386 /* Accept Connecrtions */ 387 p1 = accept(s1, NULL, NULL); 388 if (p1 < 0) { 389 perror("accept s1 failed()"); 390 return errno; 391 } 392 393 p2 = accept(s2, NULL, NULL); 394 if (p2 < 0) { 395 perror("accept s1 failed()"); 396 return errno; 397 } 398 399 if (verbose > 1) { 400 printf("connected sockets: c1 <-> p1, c2 <-> p2\n"); 401 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n", 402 c1, s1, c2, s2); 403 } 404 return 0; 405} 406 407struct msg_stats { 408 size_t bytes_sent; 409 size_t bytes_recvd; 410 struct timespec start; 411 struct timespec end; 412}; 413 414static int msg_loop_sendpage(int fd, int iov_length, int cnt, 415 struct msg_stats *s, 416 struct sockmap_options *opt) 417{ 418 bool drop = opt->drop_expected; 419 unsigned char k = 0; 420 FILE *file; 421 int i, fp; 422 423 file = tmpfile(); 424 if (!file) { 425 perror("create file for sendpage"); 426 return 1; 427 } 428 for (i = 0; i < iov_length * cnt; i++, k++) 429 fwrite(&k, sizeof(char), 1, file); 430 fflush(file); 431 fseek(file, 0, SEEK_SET); 432 433 fp = fileno(file); 434 435 clock_gettime(CLOCK_MONOTONIC, &s->start); 436 for (i = 0; i < cnt; i++) { 437 int sent; 438 439 errno = 0; 440 sent = sendfile(fd, fp, NULL, iov_length); 441 442 if (!drop && sent < 0) { 443 perror("sendpage loop error"); 444 fclose(file); 445 return sent; 446 } else if (drop && sent >= 0) { 447 printf("sendpage loop error expected: %i errno %i\n", 448 sent, errno); 449 fclose(file); 450 return -EIO; 451 } 452 453 if (sent > 0) 454 s->bytes_sent += sent; 455 } 456 clock_gettime(CLOCK_MONOTONIC, &s->end); 457 fclose(file); 458 return 0; 459} 460 461static void msg_free_iov(struct msghdr *msg) 462{ 463 int i; 464 465 for (i = 0; i < msg->msg_iovlen; i++) 466 free(msg->msg_iov[i].iov_base); 467 free(msg->msg_iov); 468 msg->msg_iov = NULL; 469 msg->msg_iovlen = 0; 470} 471 472static int msg_alloc_iov(struct msghdr *msg, 473 int iov_count, int iov_length, 474 bool data, bool xmit) 475{ 476 unsigned char k = 0; 477 struct iovec *iov; 478 int i; 479 480 iov = calloc(iov_count, sizeof(struct iovec)); 481 if (!iov) 482 return errno; 483 484 for (i = 0; i < iov_count; i++) { 485 unsigned char *d = calloc(iov_length, sizeof(char)); 486 487 if (!d) { 488 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count); 489 goto unwind_iov; 490 } 491 iov[i].iov_base = d; 492 iov[i].iov_len = iov_length; 493 494 if (data && xmit) { 495 int j; 496 497 for (j = 0; j < iov_length; j++) 498 d[j] = k++; 499 } 500 } 501 502 msg->msg_iov = iov; 503 msg->msg_iovlen = iov_count; 504 505 return 0; 506unwind_iov: 507 for (i--; i >= 0 ; i--) 508 free(msg->msg_iov[i].iov_base); 509 return -ENOMEM; 510} 511 512static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz) 513{ 514 int i, j = 0, bytes_cnt = 0; 515 unsigned char k = 0; 516 517 for (i = 0; i < msg->msg_iovlen; i++) { 518 unsigned char *d = msg->msg_iov[i].iov_base; 519 520 /* Special case test for skb ingress + ktls */ 521 if (i == 0 && txmsg_ktls_skb) { 522 if (msg->msg_iov[i].iov_len < 4) 523 return -EIO; 524 if (memcmp(d, "PASS", 4) != 0) { 525 fprintf(stderr, 526 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n", 527 i, 0, d[0], d[1], d[2], d[3]); 528 return -EIO; 529 } 530 j = 4; /* advance index past PASS header */ 531 } 532 533 for (; j < msg->msg_iov[i].iov_len && size; j++) { 534 if (d[j] != k++) { 535 fprintf(stderr, 536 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n", 537 i, j, d[j], k - 1, d[j+1], k); 538 return -EIO; 539 } 540 bytes_cnt++; 541 if (bytes_cnt == chunk_sz) { 542 k = 0; 543 bytes_cnt = 0; 544 } 545 size--; 546 } 547 } 548 return 0; 549} 550 551static int msg_loop(int fd, int iov_count, int iov_length, int cnt, 552 struct msg_stats *s, bool tx, 553 struct sockmap_options *opt) 554{ 555 struct msghdr msg = {0}, msg_peek = {0}; 556 int err, i, flags = MSG_NOSIGNAL; 557 bool drop = opt->drop_expected; 558 bool data = opt->data_test; 559 int iov_alloc_length = iov_length; 560 561 if (!tx && opt->check_recved_len) 562 iov_alloc_length *= 2; 563 564 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx); 565 if (err) 566 goto out_errno; 567 if (peek_flag) { 568 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx); 569 if (err) 570 goto out_errno; 571 } 572 573 if (tx) { 574 clock_gettime(CLOCK_MONOTONIC, &s->start); 575 for (i = 0; i < cnt; i++) { 576 int sent; 577 578 errno = 0; 579 sent = sendmsg(fd, &msg, flags); 580 581 if (!drop && sent < 0) { 582 if (opt->tx_wait_mem && errno == EACCES) { 583 errno = 0; 584 goto out_errno; 585 } 586 perror("sendmsg loop error"); 587 goto out_errno; 588 } else if (drop && sent >= 0) { 589 fprintf(stderr, 590 "sendmsg loop error expected: %i errno %i\n", 591 sent, errno); 592 errno = -EIO; 593 goto out_errno; 594 } 595 if (sent > 0) 596 s->bytes_sent += sent; 597 } 598 clock_gettime(CLOCK_MONOTONIC, &s->end); 599 } else { 600 int slct, recvp = 0, recv, max_fd = fd; 601 float total_bytes, txmsg_pop_total; 602 int fd_flags = O_NONBLOCK; 603 struct timeval timeout; 604 fd_set w; 605 606 fcntl(fd, fd_flags); 607 /* Account for pop bytes noting each iteration of apply will 608 * call msg_pop_data helper so we need to account for this 609 * by calculating the number of apply iterations. Note user 610 * of the tool can create cases where no data is sent by 611 * manipulating pop/push/pull/etc. For example txmsg_apply 1 612 * with txmsg_pop 1 will try to apply 1B at a time but each 613 * iteration will then pop 1B so no data will ever be sent. 614 * This is really only useful for testing edge cases in code 615 * paths. 616 */ 617 total_bytes = (float)iov_count * (float)iov_length * (float)cnt; 618 if (txmsg_apply) 619 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply); 620 else 621 txmsg_pop_total = txmsg_pop * cnt; 622 total_bytes -= txmsg_pop_total; 623 err = clock_gettime(CLOCK_MONOTONIC, &s->start); 624 if (err < 0) 625 perror("recv start time"); 626 while (s->bytes_recvd < total_bytes) { 627 if (txmsg_cork) { 628 timeout.tv_sec = 0; 629 timeout.tv_usec = 300000; 630 } else { 631 timeout.tv_sec = 3; 632 timeout.tv_usec = 0; 633 } 634 635 /* FD sets */ 636 FD_ZERO(&w); 637 FD_SET(fd, &w); 638 639 slct = select(max_fd + 1, &w, NULL, NULL, &timeout); 640 if (slct == -1) { 641 perror("select()"); 642 clock_gettime(CLOCK_MONOTONIC, &s->end); 643 goto out_errno; 644 } else if (!slct) { 645 if (opt->verbose) 646 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total); 647 errno = -EIO; 648 clock_gettime(CLOCK_MONOTONIC, &s->end); 649 goto out_errno; 650 } 651 652 if (opt->tx_wait_mem) { 653 FD_ZERO(&w); 654 FD_SET(fd, &w); 655 slct = select(max_fd + 1, NULL, NULL, &w, &timeout); 656 errno = 0; 657 close(fd); 658 goto out_errno; 659 } 660 661 errno = 0; 662 if (peek_flag) { 663 flags |= MSG_PEEK; 664 recvp = recvmsg(fd, &msg_peek, flags); 665 if (recvp < 0) { 666 if (errno != EWOULDBLOCK) { 667 clock_gettime(CLOCK_MONOTONIC, &s->end); 668 goto out_errno; 669 } 670 } 671 flags = 0; 672 } 673 674 recv = recvmsg(fd, &msg, flags); 675 if (recv < 0) { 676 if (errno != EWOULDBLOCK) { 677 clock_gettime(CLOCK_MONOTONIC, &s->end); 678 perror("recv failed()"); 679 goto out_errno; 680 } 681 } 682 683 s->bytes_recvd += recv; 684 685 if (opt->check_recved_len && s->bytes_recvd > total_bytes) { 686 errno = EMSGSIZE; 687 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n", 688 s->bytes_recvd, total_bytes); 689 goto out_errno; 690 } 691 692 if (data) { 693 int chunk_sz = opt->sendpage ? 694 iov_length * cnt : 695 iov_length * iov_count; 696 697 errno = msg_verify_data(&msg, recv, chunk_sz); 698 if (errno) { 699 perror("data verify msg failed"); 700 goto out_errno; 701 } 702 if (recvp) { 703 errno = msg_verify_data(&msg_peek, 704 recvp, 705 chunk_sz); 706 if (errno) { 707 perror("data verify msg_peek failed"); 708 goto out_errno; 709 } 710 } 711 } 712 } 713 clock_gettime(CLOCK_MONOTONIC, &s->end); 714 } 715 716 msg_free_iov(&msg); 717 msg_free_iov(&msg_peek); 718 return err; 719out_errno: 720 msg_free_iov(&msg); 721 msg_free_iov(&msg_peek); 722 return errno; 723} 724 725static float giga = 1000000000; 726 727static inline float sentBps(struct msg_stats s) 728{ 729 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); 730} 731 732static inline float recvdBps(struct msg_stats s) 733{ 734 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); 735} 736 737static int sendmsg_test(struct sockmap_options *opt) 738{ 739 float sent_Bps = 0, recvd_Bps = 0; 740 int rx_fd, txpid, rxpid, err = 0; 741 struct msg_stats s = {0}; 742 int iov_count = opt->iov_count; 743 int iov_buf = opt->iov_length; 744 int rx_status, tx_status; 745 int cnt = opt->rate; 746 747 errno = 0; 748 749 if (opt->base) 750 rx_fd = p1; 751 else 752 rx_fd = p2; 753 754 if (ktls) { 755 /* Redirecting into non-TLS socket which sends into a TLS 756 * socket is not a valid test. So in this case lets not 757 * enable kTLS but still run the test. 758 */ 759 if (!txmsg_redir || txmsg_ingress) { 760 err = sockmap_init_ktls(opt->verbose, rx_fd); 761 if (err) 762 return err; 763 } 764 err = sockmap_init_ktls(opt->verbose, c1); 765 if (err) 766 return err; 767 } 768 769 if (opt->tx_wait_mem) { 770 struct timeval timeout; 771 int rxtx_buf_len = 1024; 772 773 timeout.tv_sec = 3; 774 timeout.tv_usec = 0; 775 776 err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval)); 777 err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int)); 778 err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int)); 779 if (err) { 780 perror("setsockopt failed()"); 781 return errno; 782 } 783 } 784 785 rxpid = fork(); 786 if (rxpid == 0) { 787 if (txmsg_pop || txmsg_start_pop) 788 iov_buf -= (txmsg_pop - txmsg_start_pop + 1); 789 if (opt->drop_expected || txmsg_ktls_skb_drop) 790 _exit(0); 791 792 if (!iov_buf) /* zero bytes sent case */ 793 _exit(0); 794 795 if (opt->sendpage) 796 iov_count = 1; 797 err = msg_loop(rx_fd, iov_count, iov_buf, 798 cnt, &s, false, opt); 799 if (opt->verbose > 1) 800 fprintf(stderr, 801 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n", 802 iov_count, iov_buf, cnt, err); 803 if (s.end.tv_sec - s.start.tv_sec) { 804 sent_Bps = sentBps(s); 805 recvd_Bps = recvdBps(s); 806 } 807 if (opt->verbose > 1) 808 fprintf(stdout, 809 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n", 810 s.bytes_sent, sent_Bps, sent_Bps/giga, 811 s.bytes_recvd, recvd_Bps, recvd_Bps/giga, 812 peek_flag ? "(peek_msg)" : ""); 813 if (err && txmsg_cork) 814 err = 0; 815 exit(err ? 1 : 0); 816 } else if (rxpid == -1) { 817 perror("msg_loop_rx"); 818 return errno; 819 } 820 821 if (opt->tx_wait_mem) 822 close(c2); 823 824 txpid = fork(); 825 if (txpid == 0) { 826 if (opt->sendpage) 827 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt); 828 else 829 err = msg_loop(c1, iov_count, iov_buf, 830 cnt, &s, true, opt); 831 832 if (err) 833 fprintf(stderr, 834 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n", 835 iov_count, iov_buf, cnt, err); 836 if (s.end.tv_sec - s.start.tv_sec) { 837 sent_Bps = sentBps(s); 838 recvd_Bps = recvdBps(s); 839 } 840 if (opt->verbose > 1) 841 fprintf(stdout, 842 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n", 843 s.bytes_sent, sent_Bps, sent_Bps/giga, 844 s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 845 exit(err ? 1 : 0); 846 } else if (txpid == -1) { 847 perror("msg_loop_tx"); 848 return errno; 849 } 850 851 assert(waitpid(rxpid, &rx_status, 0) == rxpid); 852 assert(waitpid(txpid, &tx_status, 0) == txpid); 853 if (WIFEXITED(rx_status)) { 854 err = WEXITSTATUS(rx_status); 855 if (err) { 856 fprintf(stderr, "rx thread exited with err %d.\n", err); 857 goto out; 858 } 859 } 860 if (WIFEXITED(tx_status)) { 861 err = WEXITSTATUS(tx_status); 862 if (err) 863 fprintf(stderr, "tx thread exited with err %d.\n", err); 864 } 865out: 866 return err; 867} 868 869static int forever_ping_pong(int rate, struct sockmap_options *opt) 870{ 871 struct timeval timeout; 872 char buf[1024] = {0}; 873 int sc; 874 875 timeout.tv_sec = 10; 876 timeout.tv_usec = 0; 877 878 /* Ping/Pong data from client to server */ 879 sc = send(c1, buf, sizeof(buf), 0); 880 if (sc < 0) { 881 perror("send failed()"); 882 return sc; 883 } 884 885 do { 886 int s, rc, i, max_fd = p2; 887 fd_set w; 888 889 /* FD sets */ 890 FD_ZERO(&w); 891 FD_SET(c1, &w); 892 FD_SET(c2, &w); 893 FD_SET(p1, &w); 894 FD_SET(p2, &w); 895 896 s = select(max_fd + 1, &w, NULL, NULL, &timeout); 897 if (s == -1) { 898 perror("select()"); 899 break; 900 } else if (!s) { 901 fprintf(stderr, "unexpected timeout\n"); 902 break; 903 } 904 905 for (i = 0; i <= max_fd && s > 0; ++i) { 906 if (!FD_ISSET(i, &w)) 907 continue; 908 909 s--; 910 911 rc = recv(i, buf, sizeof(buf), 0); 912 if (rc < 0) { 913 if (errno != EWOULDBLOCK) { 914 perror("recv failed()"); 915 return rc; 916 } 917 } 918 919 if (rc == 0) { 920 close(i); 921 break; 922 } 923 924 sc = send(i, buf, rc, 0); 925 if (sc < 0) { 926 perror("send failed()"); 927 return sc; 928 } 929 } 930 931 if (rate) 932 sleep(rate); 933 934 if (opt->verbose) { 935 printf("."); 936 fflush(stdout); 937 938 } 939 } while (running); 940 941 return 0; 942} 943 944enum { 945 SELFTESTS, 946 PING_PONG, 947 SENDMSG, 948 BASE, 949 BASE_SENDPAGE, 950 SENDPAGE, 951}; 952 953static int run_options(struct sockmap_options *options, int cg_fd, int test) 954{ 955 int i, key, next_key, err, tx_prog_fd = -1, zero = 0; 956 957 /* If base test skip BPF setup */ 958 if (test == BASE || test == BASE_SENDPAGE) 959 goto run; 960 961 /* Attach programs to sockmap */ 962 if (!txmsg_omit_skb_parser) { 963 err = bpf_prog_attach(prog_fd[0], map_fd[0], 964 BPF_SK_SKB_STREAM_PARSER, 0); 965 if (err) { 966 fprintf(stderr, 967 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n", 968 prog_fd[0], map_fd[0], err, strerror(errno)); 969 return err; 970 } 971 } 972 973 err = bpf_prog_attach(prog_fd[1], map_fd[0], 974 BPF_SK_SKB_STREAM_VERDICT, 0); 975 if (err) { 976 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n", 977 err, strerror(errno)); 978 return err; 979 } 980 981 /* Attach programs to TLS sockmap */ 982 if (txmsg_ktls_skb) { 983 if (!txmsg_omit_skb_parser) { 984 err = bpf_prog_attach(prog_fd[0], map_fd[8], 985 BPF_SK_SKB_STREAM_PARSER, 0); 986 if (err) { 987 fprintf(stderr, 988 "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n", 989 prog_fd[0], map_fd[8], err, strerror(errno)); 990 return err; 991 } 992 } 993 994 err = bpf_prog_attach(prog_fd[2], map_fd[8], 995 BPF_SK_SKB_STREAM_VERDICT, 0); 996 if (err) { 997 fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n", 998 err, strerror(errno)); 999 return err; 1000 } 1001 } 1002 1003 /* Attach to cgroups */ 1004 err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0); 1005 if (err) { 1006 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", 1007 err, strerror(errno)); 1008 return err; 1009 } 1010 1011run: 1012 err = sockmap_init_sockets(options->verbose); 1013 if (err) { 1014 fprintf(stderr, "ERROR: test socket failed: %d\n", err); 1015 goto out; 1016 } 1017 1018 /* Attach txmsg program to sockmap */ 1019 if (txmsg_pass) 1020 tx_prog_fd = prog_fd[4]; 1021 else if (txmsg_redir) 1022 tx_prog_fd = prog_fd[5]; 1023 else if (txmsg_apply) 1024 tx_prog_fd = prog_fd[6]; 1025 else if (txmsg_cork) 1026 tx_prog_fd = prog_fd[7]; 1027 else if (txmsg_drop) 1028 tx_prog_fd = prog_fd[8]; 1029 else 1030 tx_prog_fd = 0; 1031 1032 if (tx_prog_fd) { 1033 int redir_fd, i = 0; 1034 1035 err = bpf_prog_attach(tx_prog_fd, 1036 map_fd[1], BPF_SK_MSG_VERDICT, 0); 1037 if (err) { 1038 fprintf(stderr, 1039 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n", 1040 err, strerror(errno)); 1041 goto out; 1042 } 1043 1044 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); 1045 if (err) { 1046 fprintf(stderr, 1047 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 1048 err, strerror(errno)); 1049 goto out; 1050 } 1051 1052 if (txmsg_redir) 1053 redir_fd = c2; 1054 else 1055 redir_fd = c1; 1056 1057 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); 1058 if (err) { 1059 fprintf(stderr, 1060 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 1061 err, strerror(errno)); 1062 goto out; 1063 } 1064 1065 if (txmsg_apply) { 1066 err = bpf_map_update_elem(map_fd[3], 1067 &i, &txmsg_apply, BPF_ANY); 1068 if (err) { 1069 fprintf(stderr, 1070 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n", 1071 err, strerror(errno)); 1072 goto out; 1073 } 1074 } 1075 1076 if (txmsg_cork) { 1077 err = bpf_map_update_elem(map_fd[4], 1078 &i, &txmsg_cork, BPF_ANY); 1079 if (err) { 1080 fprintf(stderr, 1081 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n", 1082 err, strerror(errno)); 1083 goto out; 1084 } 1085 } 1086 1087 if (txmsg_start) { 1088 err = bpf_map_update_elem(map_fd[5], 1089 &i, &txmsg_start, BPF_ANY); 1090 if (err) { 1091 fprintf(stderr, 1092 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n", 1093 err, strerror(errno)); 1094 goto out; 1095 } 1096 } 1097 1098 if (txmsg_end) { 1099 i = 1; 1100 err = bpf_map_update_elem(map_fd[5], 1101 &i, &txmsg_end, BPF_ANY); 1102 if (err) { 1103 fprintf(stderr, 1104 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n", 1105 err, strerror(errno)); 1106 goto out; 1107 } 1108 } 1109 1110 if (txmsg_start_push) { 1111 i = 2; 1112 err = bpf_map_update_elem(map_fd[5], 1113 &i, &txmsg_start_push, BPF_ANY); 1114 if (err) { 1115 fprintf(stderr, 1116 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n", 1117 err, strerror(errno)); 1118 goto out; 1119 } 1120 } 1121 1122 if (txmsg_end_push) { 1123 i = 3; 1124 err = bpf_map_update_elem(map_fd[5], 1125 &i, &txmsg_end_push, BPF_ANY); 1126 if (err) { 1127 fprintf(stderr, 1128 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n", 1129 txmsg_end_push, i, err, strerror(errno)); 1130 goto out; 1131 } 1132 } 1133 1134 if (txmsg_start_pop) { 1135 i = 4; 1136 err = bpf_map_update_elem(map_fd[5], 1137 &i, &txmsg_start_pop, BPF_ANY); 1138 if (err) { 1139 fprintf(stderr, 1140 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n", 1141 txmsg_start_pop, i, err, strerror(errno)); 1142 goto out; 1143 } 1144 } else { 1145 i = 4; 1146 bpf_map_update_elem(map_fd[5], 1147 &i, &txmsg_start_pop, BPF_ANY); 1148 } 1149 1150 if (txmsg_pop) { 1151 i = 5; 1152 err = bpf_map_update_elem(map_fd[5], 1153 &i, &txmsg_pop, BPF_ANY); 1154 if (err) { 1155 fprintf(stderr, 1156 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n", 1157 txmsg_pop, i, err, strerror(errno)); 1158 goto out; 1159 } 1160 } else { 1161 i = 5; 1162 bpf_map_update_elem(map_fd[5], 1163 &i, &txmsg_pop, BPF_ANY); 1164 1165 } 1166 1167 if (txmsg_ingress) { 1168 int in = BPF_F_INGRESS; 1169 1170 i = 0; 1171 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); 1172 if (err) { 1173 fprintf(stderr, 1174 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1175 err, strerror(errno)); 1176 } 1177 i = 1; 1178 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); 1179 if (err) { 1180 fprintf(stderr, 1181 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n", 1182 err, strerror(errno)); 1183 } 1184 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); 1185 if (err) { 1186 fprintf(stderr, 1187 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n", 1188 err, strerror(errno)); 1189 } 1190 1191 i = 2; 1192 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); 1193 if (err) { 1194 fprintf(stderr, 1195 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n", 1196 err, strerror(errno)); 1197 } 1198 } 1199 1200 if (txmsg_ktls_skb) { 1201 int ingress = BPF_F_INGRESS; 1202 1203 i = 0; 1204 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY); 1205 if (err) { 1206 fprintf(stderr, 1207 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1208 err, strerror(errno)); 1209 } 1210 1211 if (txmsg_ktls_skb_redir) { 1212 i = 1; 1213 err = bpf_map_update_elem(map_fd[7], 1214 &i, &ingress, BPF_ANY); 1215 if (err) { 1216 fprintf(stderr, 1217 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1218 err, strerror(errno)); 1219 } 1220 } 1221 1222 if (txmsg_ktls_skb_drop) { 1223 i = 1; 1224 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY); 1225 } 1226 } 1227 1228 if (txmsg_redir_skb) { 1229 int skb_fd = (test == SENDMSG || test == SENDPAGE) ? 1230 p2 : p1; 1231 int ingress = BPF_F_INGRESS; 1232 1233 i = 0; 1234 err = bpf_map_update_elem(map_fd[7], 1235 &i, &ingress, BPF_ANY); 1236 if (err) { 1237 fprintf(stderr, 1238 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1239 err, strerror(errno)); 1240 } 1241 1242 i = 3; 1243 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY); 1244 if (err) { 1245 fprintf(stderr, 1246 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1247 err, strerror(errno)); 1248 } 1249 } 1250 } 1251 1252 if (skb_use_parser) { 1253 i = 2; 1254 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY); 1255 } 1256 1257 if (txmsg_drop) 1258 options->drop_expected = true; 1259 1260 if (test == PING_PONG) 1261 err = forever_ping_pong(options->rate, options); 1262 else if (test == SENDMSG) { 1263 options->base = false; 1264 options->sendpage = false; 1265 err = sendmsg_test(options); 1266 } else if (test == SENDPAGE) { 1267 options->base = false; 1268 options->sendpage = true; 1269 err = sendmsg_test(options); 1270 } else if (test == BASE) { 1271 options->base = true; 1272 options->sendpage = false; 1273 err = sendmsg_test(options); 1274 } else if (test == BASE_SENDPAGE) { 1275 options->base = true; 1276 options->sendpage = true; 1277 err = sendmsg_test(options); 1278 } else 1279 fprintf(stderr, "unknown test\n"); 1280out: 1281 /* Detatch and zero all the maps */ 1282 bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS); 1283 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER); 1284 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT); 1285 bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER); 1286 bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT); 1287 1288 if (tx_prog_fd >= 0) 1289 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT); 1290 1291 for (i = 0; i < 8; i++) { 1292 key = next_key = 0; 1293 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1294 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { 1295 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1296 key = next_key; 1297 } 1298 } 1299 1300 close(s1); 1301 close(s2); 1302 close(p1); 1303 close(p2); 1304 close(c1); 1305 close(c2); 1306 return err; 1307} 1308 1309static char *test_to_str(int test) 1310{ 1311 switch (test) { 1312 case SENDMSG: 1313 return "sendmsg"; 1314 case SENDPAGE: 1315 return "sendpage"; 1316 } 1317 return "unknown"; 1318} 1319 1320static void append_str(char *dst, const char *src, size_t dst_cap) 1321{ 1322 size_t avail = dst_cap - strlen(dst); 1323 1324 if (avail <= 1) /* just zero byte could be written */ 1325 return; 1326 1327 strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */ 1328} 1329 1330#define OPTSTRING 60 1331static void test_options(char *options) 1332{ 1333 char tstr[OPTSTRING]; 1334 1335 memset(options, 0, OPTSTRING); 1336 1337 if (txmsg_pass) 1338 append_str(options, "pass,", OPTSTRING); 1339 if (txmsg_redir) 1340 append_str(options, "redir,", OPTSTRING); 1341 if (txmsg_drop) 1342 append_str(options, "drop,", OPTSTRING); 1343 if (txmsg_apply) { 1344 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply); 1345 append_str(options, tstr, OPTSTRING); 1346 } 1347 if (txmsg_cork) { 1348 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork); 1349 append_str(options, tstr, OPTSTRING); 1350 } 1351 if (txmsg_start) { 1352 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start); 1353 append_str(options, tstr, OPTSTRING); 1354 } 1355 if (txmsg_end) { 1356 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end); 1357 append_str(options, tstr, OPTSTRING); 1358 } 1359 if (txmsg_start_pop) { 1360 snprintf(tstr, OPTSTRING, "pop (%d,%d),", 1361 txmsg_start_pop, txmsg_start_pop + txmsg_pop); 1362 append_str(options, tstr, OPTSTRING); 1363 } 1364 if (txmsg_ingress) 1365 append_str(options, "ingress,", OPTSTRING); 1366 if (txmsg_redir_skb) 1367 append_str(options, "redir_skb,", OPTSTRING); 1368 if (txmsg_ktls_skb) 1369 append_str(options, "ktls_skb,", OPTSTRING); 1370 if (ktls) 1371 append_str(options, "ktls,", OPTSTRING); 1372 if (peek_flag) 1373 append_str(options, "peek,", OPTSTRING); 1374} 1375 1376static int __test_exec(int cgrp, int test, struct sockmap_options *opt) 1377{ 1378 char *options = calloc(OPTSTRING, sizeof(char)); 1379 int err; 1380 1381 if (test == SENDPAGE) 1382 opt->sendpage = true; 1383 else 1384 opt->sendpage = false; 1385 1386 if (txmsg_drop) 1387 opt->drop_expected = true; 1388 else 1389 opt->drop_expected = false; 1390 1391 test_options(options); 1392 1393 if (opt->verbose) { 1394 fprintf(stdout, 1395 " [TEST %i]: (%i, %i, %i, %s, %s): ", 1396 test_cnt, opt->rate, opt->iov_count, opt->iov_length, 1397 test_to_str(test), options); 1398 fflush(stdout); 1399 } 1400 err = run_options(opt, cgrp, test); 1401 if (opt->verbose) 1402 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED"); 1403 test_cnt++; 1404 !err ? passed++ : failed++; 1405 free(options); 1406 return err; 1407} 1408 1409static void test_exec(int cgrp, struct sockmap_options *opt) 1410{ 1411 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME); 1412 int err; 1413 1414 if (type == 0) { 1415 test_start(); 1416 err = __test_exec(cgrp, SENDMSG, opt); 1417 if (err) 1418 test_fail(); 1419 } else { 1420 test_start(); 1421 err = __test_exec(cgrp, SENDPAGE, opt); 1422 if (err) 1423 test_fail(); 1424 } 1425} 1426 1427static void test_send_one(struct sockmap_options *opt, int cgrp) 1428{ 1429 opt->iov_length = 1; 1430 opt->iov_count = 1; 1431 opt->rate = 1; 1432 test_exec(cgrp, opt); 1433 1434 opt->iov_length = 1; 1435 opt->iov_count = 1024; 1436 opt->rate = 1; 1437 test_exec(cgrp, opt); 1438 1439 opt->iov_length = 1024; 1440 opt->iov_count = 1; 1441 opt->rate = 1; 1442 test_exec(cgrp, opt); 1443 1444} 1445 1446static void test_send_many(struct sockmap_options *opt, int cgrp) 1447{ 1448 opt->iov_length = 3; 1449 opt->iov_count = 1; 1450 opt->rate = 512; 1451 test_exec(cgrp, opt); 1452 1453 opt->rate = 100; 1454 opt->iov_count = 1; 1455 opt->iov_length = 5; 1456 test_exec(cgrp, opt); 1457} 1458 1459static void test_send_large(struct sockmap_options *opt, int cgrp) 1460{ 1461 opt->iov_length = 256; 1462 opt->iov_count = 1024; 1463 opt->rate = 2; 1464 test_exec(cgrp, opt); 1465} 1466 1467static void test_send(struct sockmap_options *opt, int cgrp) 1468{ 1469 test_send_one(opt, cgrp); 1470 test_send_many(opt, cgrp); 1471 test_send_large(opt, cgrp); 1472 sched_yield(); 1473} 1474 1475static void test_txmsg_pass(int cgrp, struct sockmap_options *opt) 1476{ 1477 /* Test small and large iov_count values with pass/redir/apply/cork */ 1478 txmsg_pass = 1; 1479 test_send(opt, cgrp); 1480} 1481 1482static void test_txmsg_redir(int cgrp, struct sockmap_options *opt) 1483{ 1484 txmsg_redir = 1; 1485 test_send(opt, cgrp); 1486} 1487 1488static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt) 1489{ 1490 txmsg_redir = 1; 1491 opt->tx_wait_mem = true; 1492 test_send_large(opt, cgrp); 1493 opt->tx_wait_mem = false; 1494} 1495 1496static void test_txmsg_drop(int cgrp, struct sockmap_options *opt) 1497{ 1498 txmsg_drop = 1; 1499 test_send(opt, cgrp); 1500} 1501 1502static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt) 1503{ 1504 txmsg_pass = txmsg_drop = 0; 1505 txmsg_ingress = txmsg_redir = 1; 1506 test_send(opt, cgrp); 1507} 1508 1509static void test_txmsg_skb(int cgrp, struct sockmap_options *opt) 1510{ 1511 bool data = opt->data_test; 1512 int k = ktls; 1513 1514 opt->data_test = true; 1515 ktls = 1; 1516 1517 txmsg_pass = txmsg_drop = 0; 1518 txmsg_ingress = txmsg_redir = 0; 1519 txmsg_ktls_skb = 1; 1520 txmsg_pass = 1; 1521 1522 /* Using data verification so ensure iov layout is 1523 * expected from test receiver side. e.g. has enough 1524 * bytes to write test code. 1525 */ 1526 opt->iov_length = 100; 1527 opt->iov_count = 1; 1528 opt->rate = 1; 1529 test_exec(cgrp, opt); 1530 1531 txmsg_ktls_skb_drop = 1; 1532 test_exec(cgrp, opt); 1533 1534 txmsg_ktls_skb_drop = 0; 1535 txmsg_ktls_skb_redir = 1; 1536 test_exec(cgrp, opt); 1537 txmsg_ktls_skb_redir = 0; 1538 1539 /* Tests that omit skb_parser */ 1540 txmsg_omit_skb_parser = 1; 1541 ktls = 0; 1542 txmsg_ktls_skb = 0; 1543 test_exec(cgrp, opt); 1544 1545 txmsg_ktls_skb_drop = 1; 1546 test_exec(cgrp, opt); 1547 txmsg_ktls_skb_drop = 0; 1548 1549 txmsg_ktls_skb_redir = 1; 1550 test_exec(cgrp, opt); 1551 1552 ktls = 1; 1553 test_exec(cgrp, opt); 1554 txmsg_omit_skb_parser = 0; 1555 1556 opt->data_test = data; 1557 ktls = k; 1558} 1559 1560/* Test cork with hung data. This tests poor usage patterns where 1561 * cork can leave data on the ring if user program is buggy and 1562 * doesn't flush them somehow. They do take some time however 1563 * because they wait for a timeout. Test pass, redir and cork with 1564 * apply logic. Use cork size of 4097 with send_large to avoid 1565 * aligning cork size with send size. 1566 */ 1567static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt) 1568{ 1569 txmsg_pass = 1; 1570 txmsg_redir = 0; 1571 txmsg_cork = 4097; 1572 txmsg_apply = 4097; 1573 test_send_large(opt, cgrp); 1574 1575 txmsg_pass = 0; 1576 txmsg_redir = 1; 1577 txmsg_apply = 0; 1578 txmsg_cork = 4097; 1579 test_send_large(opt, cgrp); 1580 1581 txmsg_pass = 0; 1582 txmsg_redir = 1; 1583 txmsg_apply = 4097; 1584 txmsg_cork = 4097; 1585 test_send_large(opt, cgrp); 1586} 1587 1588static void test_txmsg_pull(int cgrp, struct sockmap_options *opt) 1589{ 1590 /* Test basic start/end */ 1591 txmsg_start = 1; 1592 txmsg_end = 2; 1593 test_send(opt, cgrp); 1594 1595 /* Test >4k pull */ 1596 txmsg_start = 4096; 1597 txmsg_end = 9182; 1598 test_send_large(opt, cgrp); 1599 1600 /* Test pull + redirect */ 1601 txmsg_redir = 0; 1602 txmsg_start = 1; 1603 txmsg_end = 2; 1604 test_send(opt, cgrp); 1605 1606 /* Test pull + cork */ 1607 txmsg_redir = 0; 1608 txmsg_cork = 512; 1609 txmsg_start = 1; 1610 txmsg_end = 2; 1611 test_send_many(opt, cgrp); 1612 1613 /* Test pull + cork + redirect */ 1614 txmsg_redir = 1; 1615 txmsg_cork = 512; 1616 txmsg_start = 1; 1617 txmsg_end = 2; 1618 test_send_many(opt, cgrp); 1619} 1620 1621static void test_txmsg_pop(int cgrp, struct sockmap_options *opt) 1622{ 1623 /* Test basic pop */ 1624 txmsg_start_pop = 1; 1625 txmsg_pop = 2; 1626 test_send_many(opt, cgrp); 1627 1628 /* Test pop with >4k */ 1629 txmsg_start_pop = 4096; 1630 txmsg_pop = 4096; 1631 test_send_large(opt, cgrp); 1632 1633 /* Test pop + redirect */ 1634 txmsg_redir = 1; 1635 txmsg_start_pop = 1; 1636 txmsg_pop = 2; 1637 test_send_many(opt, cgrp); 1638 1639 /* Test pop + cork */ 1640 txmsg_redir = 0; 1641 txmsg_cork = 512; 1642 txmsg_start_pop = 1; 1643 txmsg_pop = 2; 1644 test_send_many(opt, cgrp); 1645 1646 /* Test pop + redirect + cork */ 1647 txmsg_redir = 1; 1648 txmsg_cork = 4; 1649 txmsg_start_pop = 1; 1650 txmsg_pop = 2; 1651 test_send_many(opt, cgrp); 1652} 1653 1654static void test_txmsg_push(int cgrp, struct sockmap_options *opt) 1655{ 1656 /* Test basic push */ 1657 txmsg_start_push = 1; 1658 txmsg_end_push = 1; 1659 test_send(opt, cgrp); 1660 1661 /* Test push 4kB >4k */ 1662 txmsg_start_push = 4096; 1663 txmsg_end_push = 4096; 1664 test_send_large(opt, cgrp); 1665 1666 /* Test push + redirect */ 1667 txmsg_redir = 1; 1668 txmsg_start_push = 1; 1669 txmsg_end_push = 2; 1670 test_send_many(opt, cgrp); 1671 1672 /* Test push + cork */ 1673 txmsg_redir = 0; 1674 txmsg_cork = 512; 1675 txmsg_start_push = 1; 1676 txmsg_end_push = 2; 1677 test_send_many(opt, cgrp); 1678} 1679 1680static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt) 1681{ 1682 txmsg_start_push = 1; 1683 txmsg_end_push = 10; 1684 txmsg_start_pop = 5; 1685 txmsg_pop = 4; 1686 test_send_large(opt, cgrp); 1687} 1688 1689static void test_txmsg_apply(int cgrp, struct sockmap_options *opt) 1690{ 1691 txmsg_pass = 1; 1692 txmsg_redir = 0; 1693 txmsg_ingress = 0; 1694 txmsg_apply = 1; 1695 txmsg_cork = 0; 1696 test_send_one(opt, cgrp); 1697 1698 txmsg_pass = 0; 1699 txmsg_redir = 1; 1700 txmsg_ingress = 0; 1701 txmsg_apply = 1; 1702 txmsg_cork = 0; 1703 test_send_one(opt, cgrp); 1704 1705 txmsg_pass = 0; 1706 txmsg_redir = 1; 1707 txmsg_ingress = 1; 1708 txmsg_apply = 1; 1709 txmsg_cork = 0; 1710 test_send_one(opt, cgrp); 1711 1712 txmsg_pass = 1; 1713 txmsg_redir = 0; 1714 txmsg_ingress = 0; 1715 txmsg_apply = 1024; 1716 txmsg_cork = 0; 1717 test_send_large(opt, cgrp); 1718 1719 txmsg_pass = 0; 1720 txmsg_redir = 1; 1721 txmsg_ingress = 0; 1722 txmsg_apply = 1024; 1723 txmsg_cork = 0; 1724 test_send_large(opt, cgrp); 1725 1726 txmsg_pass = 0; 1727 txmsg_redir = 1; 1728 txmsg_ingress = 1; 1729 txmsg_apply = 1024; 1730 txmsg_cork = 0; 1731 test_send_large(opt, cgrp); 1732} 1733 1734static void test_txmsg_cork(int cgrp, struct sockmap_options *opt) 1735{ 1736 txmsg_pass = 1; 1737 txmsg_redir = 0; 1738 txmsg_apply = 0; 1739 txmsg_cork = 1; 1740 test_send(opt, cgrp); 1741 1742 txmsg_pass = 1; 1743 txmsg_redir = 0; 1744 txmsg_apply = 1; 1745 txmsg_cork = 1; 1746 test_send(opt, cgrp); 1747} 1748 1749static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt) 1750{ 1751 txmsg_pass = 1; 1752 skb_use_parser = 512; 1753 if (ktls == 1) 1754 skb_use_parser = 570; 1755 opt->iov_length = 256; 1756 opt->iov_count = 1; 1757 opt->rate = 2; 1758 test_exec(cgrp, opt); 1759} 1760 1761static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt) 1762{ 1763 if (ktls == 1) 1764 return; 1765 skb_use_parser = 10; 1766 opt->iov_length = 20; 1767 opt->iov_count = 1; 1768 opt->rate = 1; 1769 opt->check_recved_len = true; 1770 test_exec(cgrp, opt); 1771 opt->check_recved_len = false; 1772} 1773 1774char *map_names[] = { 1775 "sock_map", 1776 "sock_map_txmsg", 1777 "sock_map_redir", 1778 "sock_apply_bytes", 1779 "sock_cork_bytes", 1780 "sock_bytes", 1781 "sock_redir_flags", 1782 "sock_skb_opts", 1783 "tls_sock_map", 1784}; 1785 1786int prog_attach_type[] = { 1787 BPF_SK_SKB_STREAM_PARSER, 1788 BPF_SK_SKB_STREAM_VERDICT, 1789 BPF_SK_SKB_STREAM_VERDICT, 1790 BPF_CGROUP_SOCK_OPS, 1791 BPF_SK_MSG_VERDICT, 1792 BPF_SK_MSG_VERDICT, 1793 BPF_SK_MSG_VERDICT, 1794 BPF_SK_MSG_VERDICT, 1795 BPF_SK_MSG_VERDICT, 1796 BPF_SK_MSG_VERDICT, 1797 BPF_SK_MSG_VERDICT, 1798}; 1799 1800int prog_type[] = { 1801 BPF_PROG_TYPE_SK_SKB, 1802 BPF_PROG_TYPE_SK_SKB, 1803 BPF_PROG_TYPE_SK_SKB, 1804 BPF_PROG_TYPE_SOCK_OPS, 1805 BPF_PROG_TYPE_SK_MSG, 1806 BPF_PROG_TYPE_SK_MSG, 1807 BPF_PROG_TYPE_SK_MSG, 1808 BPF_PROG_TYPE_SK_MSG, 1809 BPF_PROG_TYPE_SK_MSG, 1810 BPF_PROG_TYPE_SK_MSG, 1811 BPF_PROG_TYPE_SK_MSG, 1812}; 1813 1814static int populate_progs(char *bpf_file) 1815{ 1816 struct bpf_program *prog; 1817 struct bpf_object *obj; 1818 int i = 0; 1819 long err; 1820 1821 obj = bpf_object__open(bpf_file); 1822 err = libbpf_get_error(obj); 1823 if (err) { 1824 char err_buf[256]; 1825 1826 libbpf_strerror(err, err_buf, sizeof(err_buf)); 1827 printf("Unable to load eBPF objects in file '%s' : %s\n", 1828 bpf_file, err_buf); 1829 return -1; 1830 } 1831 1832 bpf_object__for_each_program(prog, obj) { 1833 bpf_program__set_type(prog, prog_type[i]); 1834 bpf_program__set_expected_attach_type(prog, 1835 prog_attach_type[i]); 1836 i++; 1837 } 1838 1839 i = bpf_object__load(obj); 1840 i = 0; 1841 bpf_object__for_each_program(prog, obj) { 1842 prog_fd[i] = bpf_program__fd(prog); 1843 i++; 1844 } 1845 1846 for (i = 0; i < ARRAY_SIZE(map_fd); i++) { 1847 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); 1848 map_fd[i] = bpf_map__fd(maps[i]); 1849 if (map_fd[i] < 0) { 1850 fprintf(stderr, "load_bpf_file: (%i) %s\n", 1851 map_fd[i], strerror(errno)); 1852 return -1; 1853 } 1854 } 1855 1856 return 0; 1857} 1858 1859struct _test test[] = { 1860 {"txmsg test passthrough", test_txmsg_pass}, 1861 {"txmsg test redirect", test_txmsg_redir}, 1862 {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem}, 1863 {"txmsg test drop", test_txmsg_drop}, 1864 {"txmsg test ingress redirect", test_txmsg_ingress_redir}, 1865 {"txmsg test skb", test_txmsg_skb}, 1866 {"txmsg test apply", test_txmsg_apply}, 1867 {"txmsg test cork", test_txmsg_cork}, 1868 {"txmsg test hanging corks", test_txmsg_cork_hangs}, 1869 {"txmsg test push_data", test_txmsg_push}, 1870 {"txmsg test pull-data", test_txmsg_pull}, 1871 {"txmsg test pop-data", test_txmsg_pop}, 1872 {"txmsg test push/pop data", test_txmsg_push_pop}, 1873 {"txmsg test ingress parser", test_txmsg_ingress_parser}, 1874 {"txmsg test ingress parser2", test_txmsg_ingress_parser2}, 1875}; 1876 1877static int check_whitelist(struct _test *t, struct sockmap_options *opt) 1878{ 1879 char *entry, *ptr; 1880 1881 if (!opt->whitelist) 1882 return 0; 1883 ptr = strdup(opt->whitelist); 1884 if (!ptr) 1885 return -ENOMEM; 1886 entry = strtok(ptr, ","); 1887 while (entry) { 1888 if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 1889 strstr(opt->map, entry) != 0 || 1890 strstr(t->title, entry) != 0) 1891 return 0; 1892 entry = strtok(NULL, ","); 1893 } 1894 return -EINVAL; 1895} 1896 1897static int check_blacklist(struct _test *t, struct sockmap_options *opt) 1898{ 1899 char *entry, *ptr; 1900 1901 if (!opt->blacklist) 1902 return -EINVAL; 1903 ptr = strdup(opt->blacklist); 1904 if (!ptr) 1905 return -ENOMEM; 1906 entry = strtok(ptr, ","); 1907 while (entry) { 1908 if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 1909 strstr(opt->map, entry) != 0 || 1910 strstr(t->title, entry) != 0) 1911 return 0; 1912 entry = strtok(NULL, ","); 1913 } 1914 return -EINVAL; 1915} 1916 1917static int __test_selftests(int cg_fd, struct sockmap_options *opt) 1918{ 1919 int i, err; 1920 1921 err = populate_progs(opt->map); 1922 if (err < 0) { 1923 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err); 1924 return err; 1925 } 1926 1927 /* Tests basic commands and APIs */ 1928 for (i = 0; i < ARRAY_SIZE(test); i++) { 1929 struct _test t = test[i]; 1930 1931 if (check_whitelist(&t, opt) != 0) 1932 continue; 1933 if (check_blacklist(&t, opt) == 0) 1934 continue; 1935 1936 test_start_subtest(&t, opt); 1937 t.tester(cg_fd, opt); 1938 test_end_subtest(); 1939 } 1940 1941 return err; 1942} 1943 1944static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt) 1945{ 1946 opt->map = BPF_SOCKMAP_FILENAME; 1947 __test_selftests(cg_fd, opt); 1948} 1949 1950static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt) 1951{ 1952 opt->map = BPF_SOCKHASH_FILENAME; 1953 __test_selftests(cg_fd, opt); 1954} 1955 1956static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt) 1957{ 1958 opt->map = BPF_SOCKHASH_FILENAME; 1959 opt->prepend = "ktls"; 1960 ktls = 1; 1961 __test_selftests(cg_fd, opt); 1962 ktls = 0; 1963} 1964 1965static int test_selftest(int cg_fd, struct sockmap_options *opt) 1966{ 1967 1968 test_selftests_sockmap(cg_fd, opt); 1969 test_selftests_sockhash(cg_fd, opt); 1970 test_selftests_ktls(cg_fd, opt); 1971 test_print_results(); 1972 return 0; 1973} 1974 1975int main(int argc, char **argv) 1976{ 1977 int iov_count = 1, length = 1024, rate = 1; 1978 struct sockmap_options options = {0}; 1979 int opt, longindex, err, cg_fd = 0; 1980 char *bpf_file = BPF_SOCKMAP_FILENAME; 1981 int test = SELFTESTS; 1982 bool cg_created = 0; 1983 1984 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:", 1985 long_options, &longindex)) != -1) { 1986 switch (opt) { 1987 case 's': 1988 txmsg_start = atoi(optarg); 1989 break; 1990 case 'e': 1991 txmsg_end = atoi(optarg); 1992 break; 1993 case 'p': 1994 txmsg_start_push = atoi(optarg); 1995 break; 1996 case 'q': 1997 txmsg_end_push = atoi(optarg); 1998 break; 1999 case 'w': 2000 txmsg_start_pop = atoi(optarg); 2001 break; 2002 case 'x': 2003 txmsg_pop = atoi(optarg); 2004 break; 2005 case 'a': 2006 txmsg_apply = atoi(optarg); 2007 break; 2008 case 'k': 2009 txmsg_cork = atoi(optarg); 2010 break; 2011 case 'c': 2012 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); 2013 if (cg_fd < 0) { 2014 fprintf(stderr, 2015 "ERROR: (%i) open cg path failed: %s\n", 2016 cg_fd, optarg); 2017 return cg_fd; 2018 } 2019 break; 2020 case 'r': 2021 rate = atoi(optarg); 2022 break; 2023 case 'v': 2024 options.verbose = 1; 2025 if (optarg) 2026 options.verbose = atoi(optarg); 2027 break; 2028 case 'i': 2029 iov_count = atoi(optarg); 2030 break; 2031 case 'l': 2032 length = atoi(optarg); 2033 break; 2034 case 'd': 2035 options.data_test = true; 2036 break; 2037 case 't': 2038 if (strcmp(optarg, "ping") == 0) { 2039 test = PING_PONG; 2040 } else if (strcmp(optarg, "sendmsg") == 0) { 2041 test = SENDMSG; 2042 } else if (strcmp(optarg, "base") == 0) { 2043 test = BASE; 2044 } else if (strcmp(optarg, "base_sendpage") == 0) { 2045 test = BASE_SENDPAGE; 2046 } else if (strcmp(optarg, "sendpage") == 0) { 2047 test = SENDPAGE; 2048 } else { 2049 usage(argv); 2050 return -1; 2051 } 2052 break; 2053 case 'n': 2054 options.whitelist = strdup(optarg); 2055 if (!options.whitelist) 2056 return -ENOMEM; 2057 break; 2058 case 'b': 2059 options.blacklist = strdup(optarg); 2060 if (!options.blacklist) 2061 return -ENOMEM; 2062 case 0: 2063 break; 2064 case 'h': 2065 default: 2066 usage(argv); 2067 return -1; 2068 } 2069 } 2070 2071 if (!cg_fd) { 2072 cg_fd = cgroup_setup_and_join(CG_PATH); 2073 if (cg_fd < 0) 2074 return cg_fd; 2075 cg_created = 1; 2076 } 2077 2078 /* Use libbpf 1.0 API mode */ 2079 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 2080 2081 if (test == SELFTESTS) { 2082 err = test_selftest(cg_fd, &options); 2083 goto out; 2084 } 2085 2086 err = populate_progs(bpf_file); 2087 if (err) { 2088 fprintf(stderr, "populate program: (%s) %s\n", 2089 bpf_file, strerror(errno)); 2090 return 1; 2091 } 2092 running = 1; 2093 2094 /* catch SIGINT */ 2095 signal(SIGINT, running_handler); 2096 2097 options.iov_count = iov_count; 2098 options.iov_length = length; 2099 options.rate = rate; 2100 2101 err = run_options(&options, cg_fd, test); 2102out: 2103 if (options.whitelist) 2104 free(options.whitelist); 2105 if (options.blacklist) 2106 free(options.blacklist); 2107 if (cg_created) 2108 cleanup_cgroup_environment(); 2109 close(cg_fd); 2110 return err; 2111} 2112 2113void running_handler(int a) 2114{ 2115 running = 0; 2116} 2117