1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright 2018 Google Inc. 4 * Author: Eric Dumazet (edumazet@google.com) 5 * 6 * Reference program demonstrating tcp mmap() usage, 7 * and SO_RCVLOWAT hints for receiver. 8 * 9 * Note : NIC with header split is needed to use mmap() on TCP : 10 * Each incoming frame must be a multiple of PAGE_SIZE bytes of TCP payload. 11 * 12 * How to use on loopback interface : 13 * 14 * ifconfig lo mtu 61512 # 15*4096 + 40 (ipv6 header) + 32 (TCP with TS option header) 15 * tcp_mmap -s -z & 16 * tcp_mmap -H ::1 -z 17 * 18 * Or leave default lo mtu, but use -M option to set TCP_MAXSEG option to (4096 + 12) 19 * (4096 : page size on x86, 12: TCP TS option length) 20 * tcp_mmap -s -z -M $((4096+12)) & 21 * tcp_mmap -H ::1 -z -M $((4096+12)) 22 * 23 * Note: -z option on sender uses MSG_ZEROCOPY, which forces a copy when packets go through loopback interface. 24 * We might use sendfile() instead, but really this test program is about mmap(), for receivers ;) 25 * 26 * $ ./tcp_mmap -s & # Without mmap() 27 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done 28 * received 32768 MB (0 % mmap'ed) in 14.1157 s, 19.4732 Gbit 29 * cpu usage user:0.057 sys:7.815, 240.234 usec per MB, 65531 c-switches 30 * received 32768 MB (0 % mmap'ed) in 14.6833 s, 18.7204 Gbit 31 * cpu usage user:0.043 sys:8.103, 248.596 usec per MB, 65524 c-switches 32 * received 32768 MB (0 % mmap'ed) in 11.143 s, 24.6682 Gbit 33 * cpu usage user:0.044 sys:6.576, 202.026 usec per MB, 65519 c-switches 34 * received 32768 MB (0 % mmap'ed) in 14.9056 s, 18.4413 Gbit 35 * cpu usage user:0.036 sys:8.193, 251.129 usec per MB, 65530 c-switches 36 * $ kill %1 # kill tcp_mmap server 37 * 38 * $ ./tcp_mmap -s -z & # With mmap() 39 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done 40 * received 32768 MB (99.9939 % mmap'ed) in 6.73792 s, 40.7956 Gbit 41 * cpu usage user:0.045 sys:2.827, 87.6465 usec per MB, 65532 c-switches 42 * received 32768 MB (99.9939 % mmap'ed) in 7.26732 s, 37.8238 Gbit 43 * cpu usage user:0.037 sys:3.087, 95.3369 usec per MB, 65532 c-switches 44 * received 32768 MB (99.9939 % mmap'ed) in 7.61661 s, 36.0893 Gbit 45 * cpu usage user:0.046 sys:3.559, 110.016 usec per MB, 65529 c-switches 46 * received 32768 MB (99.9939 % mmap'ed) in 7.43764 s, 36.9577 Gbit 47 * cpu usage user:0.035 sys:3.467, 106.873 usec per MB, 65530 c-switches 48 */ 49#define _GNU_SOURCE 50#include <pthread.h> 51#include <sys/types.h> 52#include <fcntl.h> 53#include <error.h> 54#include <sys/socket.h> 55#include <sys/mman.h> 56#include <sys/resource.h> 57#include <unistd.h> 58#include <string.h> 59#include <stdlib.h> 60#include <stdio.h> 61#include <errno.h> 62#include <time.h> 63#include <sys/time.h> 64#include <netinet/in.h> 65#include <arpa/inet.h> 66#include <poll.h> 67#include <linux/tcp.h> 68#include <assert.h> 69#include <openssl/pem.h> 70 71#ifndef MSG_ZEROCOPY 72#define MSG_ZEROCOPY 0x4000000 73#endif 74 75#ifndef min 76#define min(a, b) ((a) < (b) ? (a) : (b)) 77#endif 78 79#define FILE_SZ (1ULL << 35) 80static int cfg_family = AF_INET6; 81static socklen_t cfg_alen = sizeof(struct sockaddr_in6); 82static int cfg_port = 8787; 83 84static int rcvbuf; /* Default: autotuning. Can be set with -r <integer> option */ 85static int sndbuf; /* Default: autotuning. Can be set with -w <integer> option */ 86static int zflg; /* zero copy option. (MSG_ZEROCOPY for sender, mmap() for receiver */ 87static int xflg; /* hash received data (simple xor) (-h option) */ 88static int keepflag; /* -k option: receiver shall keep all received file in memory (no munmap() calls) */ 89static int integrity; /* -i option: sender and receiver compute sha256 over the data.*/ 90 91static size_t chunk_size = 512*1024; 92 93static size_t map_align; 94 95unsigned long htotal; 96unsigned int digest_len; 97 98static inline void prefetch(const void *x) 99{ 100#if defined(__x86_64__) 101 asm volatile("prefetcht0 %P0" : : "m" (*(const char *)x)); 102#endif 103} 104 105void hash_zone(void *zone, unsigned int length) 106{ 107 unsigned long temp = htotal; 108 109 while (length >= 8*sizeof(long)) { 110 prefetch(zone + 384); 111 temp ^= *(unsigned long *)zone; 112 temp ^= *(unsigned long *)(zone + sizeof(long)); 113 temp ^= *(unsigned long *)(zone + 2*sizeof(long)); 114 temp ^= *(unsigned long *)(zone + 3*sizeof(long)); 115 temp ^= *(unsigned long *)(zone + 4*sizeof(long)); 116 temp ^= *(unsigned long *)(zone + 5*sizeof(long)); 117 temp ^= *(unsigned long *)(zone + 6*sizeof(long)); 118 temp ^= *(unsigned long *)(zone + 7*sizeof(long)); 119 zone += 8*sizeof(long); 120 length -= 8*sizeof(long); 121 } 122 while (length >= 1) { 123 temp ^= *(unsigned char *)zone; 124 zone += 1; 125 length--; 126 } 127 htotal = temp; 128} 129 130#define ALIGN_UP(x, align_to) (((x) + ((align_to)-1)) & ~((align_to)-1)) 131#define ALIGN_PTR_UP(p, ptr_align_to) ((typeof(p))ALIGN_UP((unsigned long)(p), ptr_align_to)) 132 133 134static void *mmap_large_buffer(size_t need, size_t *allocated) 135{ 136 void *buffer; 137 size_t sz; 138 139 /* Attempt to use huge pages if possible. */ 140 sz = ALIGN_UP(need, map_align); 141 buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE, 142 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); 143 144 if (buffer == (void *)-1) { 145 sz = need; 146 buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE, 147 MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE, 148 -1, 0); 149 if (buffer != (void *)-1) 150 fprintf(stderr, "MAP_HUGETLB attempt failed, look at /sys/kernel/mm/hugepages for optimal performance\n"); 151 } 152 *allocated = sz; 153 return buffer; 154} 155 156static uint32_t tcp_info_get_rcv_mss(int fd) 157{ 158 socklen_t sz = sizeof(struct tcp_info); 159 struct tcp_info info; 160 161 if (getsockopt(fd, IPPROTO_TCP, TCP_INFO, &info, &sz)) { 162 fprintf(stderr, "Error fetching TCP_INFO\n"); 163 return 0; 164 } 165 166 return info.tcpi_rcv_mss; 167} 168 169void *child_thread(void *arg) 170{ 171 unsigned char digest[SHA256_DIGEST_LENGTH]; 172 unsigned long total_mmap = 0, total = 0; 173 struct tcp_zerocopy_receive zc; 174 unsigned char *buffer = NULL; 175 unsigned long delta_usec; 176 EVP_MD_CTX *ctx = NULL; 177 int flags = MAP_SHARED; 178 struct timeval t0, t1; 179 void *raddr = NULL; 180 void *addr = NULL; 181 double throughput; 182 struct rusage ru; 183 size_t buffer_sz; 184 int lu, fd; 185 186 fd = (int)(unsigned long)arg; 187 188 gettimeofday(&t0, NULL); 189 190 fcntl(fd, F_SETFL, O_NDELAY); 191 buffer = mmap_large_buffer(chunk_size, &buffer_sz); 192 if (buffer == (void *)-1) { 193 perror("mmap"); 194 goto error; 195 } 196 if (zflg) { 197 raddr = mmap(NULL, chunk_size + map_align, PROT_READ, flags, fd, 0); 198 if (raddr == (void *)-1) { 199 perror("mmap"); 200 zflg = 0; 201 } else { 202 addr = ALIGN_PTR_UP(raddr, map_align); 203 } 204 } 205 if (integrity) { 206 ctx = EVP_MD_CTX_new(); 207 if (!ctx) { 208 perror("cannot enable SHA computing"); 209 goto error; 210 } 211 EVP_DigestInit_ex(ctx, EVP_sha256(), NULL); 212 } 213 while (1) { 214 struct pollfd pfd = { .fd = fd, .events = POLLIN, }; 215 int sub; 216 217 poll(&pfd, 1, 10000); 218 if (zflg) { 219 socklen_t zc_len = sizeof(zc); 220 int res; 221 222 memset(&zc, 0, sizeof(zc)); 223 zc.address = (__u64)((unsigned long)addr); 224 zc.length = min(chunk_size, FILE_SZ - total); 225 226 res = getsockopt(fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE, 227 &zc, &zc_len); 228 if (res == -1) 229 break; 230 231 if (zc.length) { 232 assert(zc.length <= chunk_size); 233 if (integrity) 234 EVP_DigestUpdate(ctx, addr, zc.length); 235 total_mmap += zc.length; 236 if (xflg) 237 hash_zone(addr, zc.length); 238 /* It is more efficient to unmap the pages right now, 239 * instead of doing this in next TCP_ZEROCOPY_RECEIVE. 240 */ 241 madvise(addr, zc.length, MADV_DONTNEED); 242 total += zc.length; 243 } 244 if (zc.recv_skip_hint) { 245 assert(zc.recv_skip_hint <= chunk_size); 246 lu = read(fd, buffer, min(zc.recv_skip_hint, 247 FILE_SZ - total)); 248 if (lu > 0) { 249 if (integrity) 250 EVP_DigestUpdate(ctx, buffer, lu); 251 if (xflg) 252 hash_zone(buffer, lu); 253 total += lu; 254 } 255 if (lu == 0) 256 goto end; 257 } 258 continue; 259 } 260 sub = 0; 261 while (sub < chunk_size) { 262 lu = read(fd, buffer + sub, min(chunk_size - sub, 263 FILE_SZ - total)); 264 if (lu == 0) 265 goto end; 266 if (lu < 0) 267 break; 268 if (integrity) 269 EVP_DigestUpdate(ctx, buffer + sub, lu); 270 if (xflg) 271 hash_zone(buffer + sub, lu); 272 total += lu; 273 sub += lu; 274 } 275 } 276end: 277 gettimeofday(&t1, NULL); 278 delta_usec = (t1.tv_sec - t0.tv_sec) * 1000000 + t1.tv_usec - t0.tv_usec; 279 280 if (integrity) { 281 fcntl(fd, F_SETFL, 0); 282 EVP_DigestFinal_ex(ctx, digest, &digest_len); 283 lu = read(fd, buffer, SHA256_DIGEST_LENGTH); 284 if (lu != SHA256_DIGEST_LENGTH) 285 perror("Error: Cannot read SHA256\n"); 286 287 if (memcmp(digest, buffer, 288 SHA256_DIGEST_LENGTH)) 289 fprintf(stderr, "Error: SHA256 of the data is not right\n"); 290 else 291 printf("\nSHA256 is correct\n"); 292 } 293 294 throughput = 0; 295 if (delta_usec) 296 throughput = total * 8.0 / (double)delta_usec / 1000.0; 297 getrusage(RUSAGE_THREAD, &ru); 298 if (total > 1024*1024) { 299 unsigned long total_usec; 300 unsigned long mb = total >> 20; 301 total_usec = 1000000*ru.ru_utime.tv_sec + ru.ru_utime.tv_usec + 302 1000000*ru.ru_stime.tv_sec + ru.ru_stime.tv_usec; 303 printf("received %lg MB (%lg %% mmap'ed) in %lg s, %lg Gbit\n" 304 " cpu usage user:%lg sys:%lg, %lg usec per MB, %lu c-switches, rcv_mss %u\n", 305 total / (1024.0 * 1024.0), 306 100.0*total_mmap/total, 307 (double)delta_usec / 1000000.0, 308 throughput, 309 (double)ru.ru_utime.tv_sec + (double)ru.ru_utime.tv_usec / 1000000.0, 310 (double)ru.ru_stime.tv_sec + (double)ru.ru_stime.tv_usec / 1000000.0, 311 (double)total_usec/mb, 312 ru.ru_nvcsw, 313 tcp_info_get_rcv_mss(fd)); 314 } 315error: 316 munmap(buffer, buffer_sz); 317 close(fd); 318 if (zflg) 319 munmap(raddr, chunk_size + map_align); 320 pthread_exit(0); 321} 322 323static void apply_rcvsnd_buf(int fd) 324{ 325 if (rcvbuf && setsockopt(fd, SOL_SOCKET, 326 SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)) == -1) { 327 perror("setsockopt SO_RCVBUF"); 328 } 329 330 if (sndbuf && setsockopt(fd, SOL_SOCKET, 331 SO_SNDBUF, &sndbuf, sizeof(sndbuf)) == -1) { 332 perror("setsockopt SO_SNDBUF"); 333 } 334} 335 336 337static void setup_sockaddr(int domain, const char *str_addr, 338 struct sockaddr_storage *sockaddr) 339{ 340 struct sockaddr_in6 *addr6 = (void *) sockaddr; 341 struct sockaddr_in *addr4 = (void *) sockaddr; 342 343 switch (domain) { 344 case PF_INET: 345 memset(addr4, 0, sizeof(*addr4)); 346 addr4->sin_family = AF_INET; 347 addr4->sin_port = htons(cfg_port); 348 if (str_addr && 349 inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1) 350 error(1, 0, "ipv4 parse error: %s", str_addr); 351 break; 352 case PF_INET6: 353 memset(addr6, 0, sizeof(*addr6)); 354 addr6->sin6_family = AF_INET6; 355 addr6->sin6_port = htons(cfg_port); 356 if (str_addr && 357 inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1) 358 error(1, 0, "ipv6 parse error: %s", str_addr); 359 break; 360 default: 361 error(1, 0, "illegal domain"); 362 } 363} 364 365static void do_accept(int fdlisten) 366{ 367 pthread_attr_t attr; 368 int rcvlowat; 369 370 pthread_attr_init(&attr); 371 pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); 372 373 rcvlowat = chunk_size; 374 if (setsockopt(fdlisten, SOL_SOCKET, SO_RCVLOWAT, 375 &rcvlowat, sizeof(rcvlowat)) == -1) { 376 perror("setsockopt SO_RCVLOWAT"); 377 } 378 379 apply_rcvsnd_buf(fdlisten); 380 381 while (1) { 382 struct sockaddr_in addr; 383 socklen_t addrlen = sizeof(addr); 384 pthread_t th; 385 int fd, res; 386 387 fd = accept(fdlisten, (struct sockaddr *)&addr, &addrlen); 388 if (fd == -1) { 389 perror("accept"); 390 continue; 391 } 392 res = pthread_create(&th, &attr, child_thread, 393 (void *)(unsigned long)fd); 394 if (res) { 395 errno = res; 396 perror("pthread_create"); 397 close(fd); 398 } 399 } 400} 401 402/* Each thread should reserve a big enough vma to avoid 403 * spinlock collisions in ptl locks. 404 * This size is 2MB on x86_64, and is exported in /proc/meminfo. 405 */ 406static unsigned long default_huge_page_size(void) 407{ 408 FILE *f = fopen("/proc/meminfo", "r"); 409 unsigned long hps = 0; 410 size_t linelen = 0; 411 char *line = NULL; 412 413 if (!f) 414 return 0; 415 while (getline(&line, &linelen, f) > 0) { 416 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) { 417 hps <<= 10; 418 break; 419 } 420 } 421 free(line); 422 fclose(f); 423 return hps; 424} 425 426static void randomize(void *target, size_t count) 427{ 428 static int urandom = -1; 429 ssize_t got; 430 431 urandom = open("/dev/urandom", O_RDONLY); 432 if (urandom < 0) { 433 perror("open /dev/urandom"); 434 exit(1); 435 } 436 got = read(urandom, target, count); 437 if (got != count) { 438 perror("read /dev/urandom"); 439 exit(1); 440 } 441} 442 443int main(int argc, char *argv[]) 444{ 445 unsigned char digest[SHA256_DIGEST_LENGTH]; 446 struct sockaddr_storage listenaddr, addr; 447 unsigned int max_pacing_rate = 0; 448 EVP_MD_CTX *ctx = NULL; 449 unsigned char *buffer; 450 uint64_t total = 0; 451 char *host = NULL; 452 int fd, c, on = 1; 453 size_t buffer_sz; 454 int sflg = 0; 455 int mss = 0; 456 457 while ((c = getopt(argc, argv, "46p:svr:w:H:zxkP:M:C:a:i")) != -1) { 458 switch (c) { 459 case '4': 460 cfg_family = PF_INET; 461 cfg_alen = sizeof(struct sockaddr_in); 462 break; 463 case '6': 464 cfg_family = PF_INET6; 465 cfg_alen = sizeof(struct sockaddr_in6); 466 break; 467 case 'p': 468 cfg_port = atoi(optarg); 469 break; 470 case 'H': 471 host = optarg; 472 break; 473 case 's': /* server : listen for incoming connections */ 474 sflg++; 475 break; 476 case 'r': 477 rcvbuf = atoi(optarg); 478 break; 479 case 'w': 480 sndbuf = atoi(optarg); 481 break; 482 case 'z': 483 zflg = 1; 484 break; 485 case 'M': 486 mss = atoi(optarg); 487 break; 488 case 'x': 489 xflg = 1; 490 break; 491 case 'k': 492 keepflag = 1; 493 break; 494 case 'P': 495 max_pacing_rate = atoi(optarg) ; 496 break; 497 case 'C': 498 chunk_size = atol(optarg); 499 break; 500 case 'a': 501 map_align = atol(optarg); 502 break; 503 case 'i': 504 integrity = 1; 505 break; 506 default: 507 exit(1); 508 } 509 } 510 if (!map_align) { 511 map_align = default_huge_page_size(); 512 /* if really /proc/meminfo is not helping, 513 * we use the default x86_64 hugepagesize. 514 */ 515 if (!map_align) 516 map_align = 2*1024*1024; 517 } 518 if (sflg) { 519 int fdlisten = socket(cfg_family, SOCK_STREAM, 0); 520 521 if (fdlisten == -1) { 522 perror("socket"); 523 exit(1); 524 } 525 apply_rcvsnd_buf(fdlisten); 526 setsockopt(fdlisten, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)); 527 528 setup_sockaddr(cfg_family, host, &listenaddr); 529 530 if (mss && 531 setsockopt(fdlisten, IPPROTO_TCP, TCP_MAXSEG, 532 &mss, sizeof(mss)) == -1) { 533 perror("setsockopt TCP_MAXSEG"); 534 exit(1); 535 } 536 if (bind(fdlisten, (const struct sockaddr *)&listenaddr, cfg_alen) == -1) { 537 perror("bind"); 538 exit(1); 539 } 540 if (listen(fdlisten, 128) == -1) { 541 perror("listen"); 542 exit(1); 543 } 544 do_accept(fdlisten); 545 } 546 547 buffer = mmap_large_buffer(chunk_size, &buffer_sz); 548 if (buffer == (unsigned char *)-1) { 549 perror("mmap"); 550 exit(1); 551 } 552 553 fd = socket(cfg_family, SOCK_STREAM, 0); 554 if (fd == -1) { 555 perror("socket"); 556 exit(1); 557 } 558 apply_rcvsnd_buf(fd); 559 560 setup_sockaddr(cfg_family, host, &addr); 561 562 if (mss && 563 setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss)) == -1) { 564 perror("setsockopt TCP_MAXSEG"); 565 exit(1); 566 } 567 if (connect(fd, (const struct sockaddr *)&addr, cfg_alen) == -1) { 568 perror("connect"); 569 exit(1); 570 } 571 if (max_pacing_rate && 572 setsockopt(fd, SOL_SOCKET, SO_MAX_PACING_RATE, 573 &max_pacing_rate, sizeof(max_pacing_rate)) == -1) 574 perror("setsockopt SO_MAX_PACING_RATE"); 575 576 if (zflg && setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY, 577 &on, sizeof(on)) == -1) { 578 perror("setsockopt SO_ZEROCOPY, (-z option disabled)"); 579 zflg = 0; 580 } 581 if (integrity) { 582 randomize(buffer, buffer_sz); 583 ctx = EVP_MD_CTX_new(); 584 if (!ctx) { 585 perror("cannot enable SHA computing"); 586 exit(1); 587 } 588 EVP_DigestInit_ex(ctx, EVP_sha256(), NULL); 589 } 590 while (total < FILE_SZ) { 591 size_t offset = total % chunk_size; 592 int64_t wr = FILE_SZ - total; 593 594 if (wr > chunk_size - offset) 595 wr = chunk_size - offset; 596 /* Note : we just want to fill the pipe with random bytes */ 597 wr = send(fd, buffer + offset, 598 (size_t)wr, zflg ? MSG_ZEROCOPY : 0); 599 if (wr <= 0) 600 break; 601 if (integrity) 602 EVP_DigestUpdate(ctx, buffer + offset, wr); 603 total += wr; 604 } 605 if (integrity && total == FILE_SZ) { 606 EVP_DigestFinal_ex(ctx, digest, &digest_len); 607 send(fd, digest, (size_t)SHA256_DIGEST_LENGTH, 0); 608 } 609 close(fd); 610 munmap(buffer, buffer_sz); 611 return 0; 612} 613