1189623Srwatson/*- 2189623Srwatson * Copyright (c) 2008-2009 Robert N. M. Watson 3208873Srwatson * Copyright (c) 2010 Juniper Networks, Inc. 4189623Srwatson * All rights reserved. 5189623Srwatson * 6208873Srwatson * This software was developed by Robert N. M. Watson under contract 7208873Srwatson * to Juniper Networks, Inc. 8208873Srwatson * 9189623Srwatson * Redistribution and use in source and binary forms, with or without 10189623Srwatson * modification, are permitted provided that the following conditions 11189623Srwatson * are met: 12189623Srwatson * 1. Redistributions of source code must retain the above copyright 13189623Srwatson * notice, this list of conditions and the following disclaimer. 14189623Srwatson * 2. Redistributions in binary form must reproduce the above copyright 15189623Srwatson * notice, this list of conditions and the following disclaimer in the 16189623Srwatson * documentation and/or other materials provided with the distribution. 17189623Srwatson * 18189623Srwatson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19189623Srwatson * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20189623Srwatson * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21189623Srwatson * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22189623Srwatson * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23189623Srwatson * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24189623Srwatson * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25189623Srwatson * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26189623Srwatson * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27189623Srwatson * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28189623Srwatson * SUCH DAMAGE. 29189623Srwatson * 30189623Srwatson * $FreeBSD$ 31189623Srwatson */ 32189623Srwatson 33189623Srwatson#include <sys/types.h> 34189623Srwatson#include <sys/event.h> 35189623Srwatson#include <sys/resource.h> 36189623Srwatson#include <sys/sched.h> 37189623Srwatson#include <sys/socket.h> 38189623Srwatson#include <sys/sysctl.h> 39189623Srwatson#include <sys/time.h> 40206972Srwatson#include <sys/uio.h> 41189623Srwatson#include <sys/wait.h> 42189623Srwatson 43189623Srwatson#include <netinet/in.h> 44206972Srwatson#include <netinet/tcp.h> 45189623Srwatson 46189623Srwatson#include <err.h> 47189623Srwatson#include <errno.h> 48189623Srwatson#include <fcntl.h> 49189623Srwatson#include <inttypes.h> 50189623Srwatson#include <signal.h> 51189623Srwatson#include <stdio.h> 52189623Srwatson#include <stdlib.h> 53189623Srwatson#include <string.h> 54189623Srwatson#include <unistd.h> 55189623Srwatson 56189623Srwatson#include "tcpp.h" 57189623Srwatson 58189623Srwatson#define min(x, y) (x < y ? x : y) 59189623Srwatson 60189623Srwatson#define timespecsub(vvp, uvp) \ 61189623Srwatson do { \ 62189623Srwatson (vvp)->tv_sec -= (uvp)->tv_sec; \ 63189623Srwatson (vvp)->tv_nsec -= (uvp)->tv_nsec; \ 64189623Srwatson if ((vvp)->tv_nsec < 0) { \ 65189623Srwatson (vvp)->tv_sec--; \ 66189623Srwatson (vvp)->tv_nsec += 1000000000; \ 67189623Srwatson } \ 68189623Srwatson } while (0) 69189623Srwatson 70189623Srwatson 71189623Srwatson/* 72189623Srwatson * Gist of each client worker: build up to mflag connections at a time, and 73189623Srwatson * pump data in to them somewhat fairly until tflag connections have been 74189623Srwatson * completed. 75189623Srwatson */ 76189623Srwatson#define CONNECTION_MAGIC 0x87a3f56e 77189623Srwatsonstruct connection { 78189623Srwatson uint32_t conn_magic; /* Just magic. */ 79189623Srwatson int conn_fd; 80189623Srwatson struct tcpp_header conn_header; /* Header buffer. */ 81189623Srwatson u_int conn_header_sent; /* Header bytes sent. */ 82189623Srwatson u_int64_t conn_data_sent; /* Data bytes sent.*/ 83189623Srwatson}; 84189623Srwatson 85189623Srwatsonstatic u_char buffer[256 * 1024]; /* Buffer to send. */ 86189623Srwatsonstatic pid_t *pid_list; 87189623Srwatsonstatic int kq; 88189623Srwatsonstatic int started; /* Number started so far. */ 89189623Srwatsonstatic int finished; /* Number finished so far. */ 90189623Srwatsonstatic int counter; /* IP number offset. */ 91206972Srwatsonstatic uint64_t payload_len; 92189623Srwatson 93189623Srwatsonstatic struct connection * 94189623Srwatsontcpp_client_newconn(void) 95189623Srwatson{ 96189623Srwatson struct sockaddr_in sin; 97189623Srwatson struct connection *conn; 98189623Srwatson struct kevent kev; 99189623Srwatson int fd, i; 100189623Srwatson 101189623Srwatson /* 102189623Srwatson * Spread load over available IPs, roating through them as we go. No 103189623Srwatson * attempt to localize IPs to particular workers. 104189623Srwatson */ 105189623Srwatson sin = localipbase; 106189623Srwatson sin.sin_addr.s_addr = htonl(ntohl(localipbase.sin_addr.s_addr) + 107189623Srwatson (counter++ % Mflag)); 108189623Srwatson 109189623Srwatson fd = socket(PF_INET, SOCK_STREAM, 0); 110189623Srwatson if (fd < 0) 111189623Srwatson err(-1, "socket"); 112189623Srwatson 113189623Srwatson if (fcntl(fd, F_SETFL, O_NONBLOCK) < 0) 114189623Srwatson err(-1, "fcntl"); 115189623Srwatson 116189623Srwatson i = 1; 117189623Srwatson if (setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &i, sizeof(i)) < 0) 118189623Srwatson err(-1, "setsockopt"); 119206972Srwatson i = 1; 120206972Srwatson if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &i, sizeof(i)) < 0) 121206972Srwatson err(-1, "setsockopt"); 122189623Srwatson#if 0 123189623Srwatson i = 1; 124189623Srwatson if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &i, sizeof(i)) < 0) 125189623Srwatson err(-1, "setsockopt"); 126189623Srwatson#endif 127189623Srwatson 128189623Srwatson if (lflag) { 129189623Srwatson if (bind(fd, (struct sockaddr *)&sin, sizeof(sin)) < 0) 130189623Srwatson err(-1, "bind"); 131189623Srwatson } 132189623Srwatson 133189623Srwatson if (connect(fd, (struct sockaddr *)&remoteip, sizeof(remoteip)) < 0 && 134189623Srwatson errno != EINPROGRESS) 135189623Srwatson err(-1, "connect"); 136189623Srwatson 137189623Srwatson conn = malloc(sizeof(*conn)); 138189623Srwatson if (conn == NULL) 139189623Srwatson return (NULL); 140189623Srwatson bzero(conn, sizeof(*conn)); 141189623Srwatson conn->conn_magic = CONNECTION_MAGIC; 142189623Srwatson conn->conn_fd = fd; 143189623Srwatson conn->conn_header.th_magic = TCPP_MAGIC; 144206972Srwatson conn->conn_header.th_len = payload_len; 145189623Srwatson tcpp_header_encode(&conn->conn_header); 146189623Srwatson 147189623Srwatson EV_SET(&kev, fd, EVFILT_WRITE, EV_ADD, 0, 0, conn); 148189623Srwatson if (kevent(kq, &kev, 1, NULL, 0, NULL) < 0) 149189623Srwatson err(-1, "newconn kevent"); 150189623Srwatson 151189623Srwatson started++; 152189623Srwatson return (conn); 153189623Srwatson} 154189623Srwatson 155189623Srwatsonstatic void 156189623Srwatsontcpp_client_closeconn(struct connection *conn) 157189623Srwatson{ 158189623Srwatson 159189623Srwatson close(conn->conn_fd); 160189623Srwatson bzero(conn, sizeof(*conn)); 161189623Srwatson free(conn); 162189623Srwatson finished++; 163189623Srwatson} 164189623Srwatson 165189623Srwatsonstatic void 166189623Srwatsontcpp_client_handleconn(struct kevent *kev) 167189623Srwatson{ 168189623Srwatson struct connection *conn; 169206972Srwatson struct iovec iov[2]; 170206972Srwatson ssize_t len, header_left; 171189623Srwatson 172189623Srwatson conn = kev->udata; 173189623Srwatson if (conn->conn_magic != CONNECTION_MAGIC) 174189623Srwatson errx(-1, "tcpp_client_handleconn: magic"); 175189623Srwatson 176189623Srwatson if (conn->conn_header_sent < sizeof(conn->conn_header)) { 177206972Srwatson header_left = sizeof(conn->conn_header) - 178206972Srwatson conn->conn_header_sent; 179206972Srwatson iov[0].iov_base = ((u_char *)&conn->conn_header) + 180206972Srwatson conn->conn_header_sent; 181206972Srwatson iov[0].iov_len = header_left; 182206972Srwatson iov[1].iov_base = buffer; 183206972Srwatson iov[1].iov_len = min(sizeof(buffer), payload_len); 184206972Srwatson len = writev(conn->conn_fd, iov, 2); 185189623Srwatson if (len < 0) { 186189623Srwatson tcpp_client_closeconn(conn); 187189623Srwatson err(-1, "tcpp_client_handleconn: header write"); 188189623Srwatson } 189189623Srwatson if (len == 0) { 190189623Srwatson tcpp_client_closeconn(conn); 191189623Srwatson errx(-1, "tcpp_client_handleconn: header write " 192189623Srwatson "premature EOF"); 193189623Srwatson } 194206972Srwatson if (len > header_left) { 195206972Srwatson conn->conn_data_sent += (len - header_left); 196206972Srwatson conn->conn_header_sent += header_left; 197206972Srwatson } else 198206972Srwatson conn->conn_header_sent += len; 199189623Srwatson } else { 200189623Srwatson len = write(conn->conn_fd, buffer, min(sizeof(buffer), 201206972Srwatson payload_len - conn->conn_data_sent)); 202189623Srwatson if (len < 0) { 203189623Srwatson tcpp_client_closeconn(conn); 204189623Srwatson err(-1, "tcpp_client_handleconn: data write"); 205189623Srwatson } 206189623Srwatson if (len == 0) { 207189623Srwatson tcpp_client_closeconn(conn); 208189623Srwatson errx(-1, "tcpp_client_handleconn: data write: " 209189623Srwatson "premature EOF"); 210189623Srwatson } 211189623Srwatson conn->conn_data_sent += len; 212189623Srwatson } 213206972Srwatson if (conn->conn_data_sent >= payload_len) { 214206972Srwatson /* 215206972Srwatson * All is well. 216206972Srwatson */ 217206972Srwatson tcpp_client_closeconn(conn); 218206972Srwatson } 219189623Srwatson} 220189623Srwatson 221189623Srwatsonstatic void 222189623Srwatsontcpp_client_worker(int workernum) 223189623Srwatson{ 224189623Srwatson struct kevent *kev_array; 225189623Srwatson int i, numevents, kev_bytes; 226189623Srwatson#if defined(CPU_SETSIZE) && 0 227189623Srwatson cpu_set_t mask; 228189623Srwatson int ncpus; 229189623Srwatson size_t len; 230189623Srwatson 231208859Srwatson if (Pflag) { 232208859Srwatson len = sizeof(ncpus); 233208859Srwatson if (sysctlbyname(SYSCTLNAME_CPUS, &ncpus, &len, NULL, 0) < 0) 234208859Srwatson err(-1, "sysctlbyname: %s", SYSCTLNAME_CPUS); 235208859Srwatson if (len != sizeof(ncpus)) 236208859Srwatson errx(-1, "sysctlbyname: %s: len %jd", SYSCTLNAME_CPUS, 237208859Srwatson (intmax_t)len); 238189623Srwatson 239208859Srwatson CPU_ZERO(&mask); 240208859Srwatson CPU_SET(workernum % ncpus, &mask); 241208859Srwatson if (sched_setaffinity(0, CPU_SETSIZE, &mask) < 0) 242208859Srwatson err(-1, "sched_setaffinity"); 243208859Srwatson } 244189623Srwatson#endif 245189623Srwatson setproctitle("tcpp_client %d", workernum); 246189623Srwatson 247189623Srwatson /* 248189623Srwatson * Add the worker number to the remote port. 249189623Srwatson */ 250189623Srwatson remoteip.sin_port = htons(rflag + workernum); 251189623Srwatson 252189623Srwatson kev_bytes = sizeof(*kev_array) * mflag; 253189623Srwatson kev_array = malloc(kev_bytes); 254189623Srwatson if (kev_array == NULL) 255189623Srwatson err(-1, "malloc"); 256189623Srwatson bzero(kev_array, kev_bytes); 257189623Srwatson 258189623Srwatson kq = kqueue(); 259189623Srwatson if (kq < 0) 260189623Srwatson err(-1, "kqueue"); 261189623Srwatson 262189623Srwatson while (finished < tflag) { 263189623Srwatson while ((started - finished < mflag) && (started < tflag)) 264189623Srwatson (void)tcpp_client_newconn(); 265189623Srwatson numevents = kevent(kq, NULL, 0, kev_array, mflag, NULL); 266189623Srwatson if (numevents < 0) 267189623Srwatson err(-1, "kevent"); 268189623Srwatson if (numevents > mflag) 269189623Srwatson errx(-1, "kevent: %d", numevents); 270189623Srwatson for (i = 0; i < numevents; i++) 271189623Srwatson tcpp_client_handleconn(&kev_array[i]); 272189623Srwatson } 273189623Srwatson /* printf("Worker %d done - %d finished\n", workernum, finished); */ 274189623Srwatson} 275189623Srwatson 276189623Srwatsonvoid 277189623Srwatsontcpp_client(void) 278189623Srwatson{ 279189623Srwatson struct timespec ts_start, ts_finish; 280189623Srwatson long cp_time_start[CPUSTATES], cp_time_finish[CPUSTATES]; 281189623Srwatson long ticks; 282189623Srwatson size_t size; 283189623Srwatson pid_t pid; 284189623Srwatson int i, failed, status; 285189623Srwatson 286206972Srwatson if (bflag < sizeof(struct tcpp_header)) 287206972Srwatson errx(-1, "Can't use -b less than %zu\n", 288206972Srwatson sizeof(struct tcpp_header)); 289206972Srwatson payload_len = bflag - sizeof(struct tcpp_header); 290206972Srwatson 291189623Srwatson pid_list = malloc(sizeof(*pid_list) * pflag); 292189623Srwatson if (pid_list == NULL) 293189623Srwatson err(-1, "malloc pid_list"); 294189623Srwatson bzero(pid_list, sizeof(*pid_list) * pflag); 295189623Srwatson 296189623Srwatson /* 297189623Srwatson * Start workers. 298189623Srwatson */ 299189623Srwatson size = sizeof(cp_time_start); 300189623Srwatson if (sysctlbyname(SYSCTLNAME_CPTIME, &cp_time_start, &size, NULL, 0) 301189623Srwatson < 0) 302189623Srwatson err(-1, "sysctlbyname: %s", SYSCTLNAME_CPTIME); 303189623Srwatson if (clock_gettime(CLOCK_REALTIME, &ts_start) < 0) 304189623Srwatson err(-1, "clock_gettime"); 305189623Srwatson for (i = 0; i < pflag; i++) { 306189623Srwatson pid = fork(); 307189623Srwatson if (pid < 0) { 308189623Srwatson warn("fork"); 309189623Srwatson for (i = 0; i < pflag; i++) { 310189623Srwatson if (pid_list[i] != 0) 311189623Srwatson (void)kill(pid_list[i], SIGKILL); 312189623Srwatson } 313189623Srwatson exit(-1); 314189623Srwatson } 315189623Srwatson if (pid == 0) { 316189623Srwatson tcpp_client_worker(i); 317189623Srwatson exit(0); 318189623Srwatson } 319189623Srwatson pid_list[i] = pid; 320189623Srwatson } 321189623Srwatson 322189623Srwatson /* 323189623Srwatson * GC workers. 324189623Srwatson */ 325189623Srwatson failed = 0; 326189623Srwatson for (i = 0; i < pflag; i++) { 327189623Srwatson if (pid_list[i] != 0) { 328189623Srwatson while (waitpid(pid_list[i], &status, 0) != pid_list[i]); 329189623Srwatson if (WEXITSTATUS(status) != 0) 330189623Srwatson failed = 1; 331189623Srwatson } 332189623Srwatson } 333189623Srwatson if (clock_gettime(CLOCK_REALTIME, &ts_finish) < 0) 334189623Srwatson err(-1, "clock_gettime"); 335189623Srwatson size = sizeof(cp_time_finish); 336189623Srwatson if (sysctlbyname(SYSCTLNAME_CPTIME, &cp_time_finish, &size, NULL, 0) 337189623Srwatson < 0) 338189623Srwatson err(-1, "sysctlbyname: %s", SYSCTLNAME_CPTIME); 339189623Srwatson timespecsub(&ts_finish, &ts_start); 340189623Srwatson 341189623Srwatson if (failed) 342189623Srwatson errx(-1, "Too many errors"); 343189623Srwatson 344208873Srwatson if (hflag) 345208873Srwatson printf("bytes,seconds,conn/s,Gb/s,user%%,nice%%,sys%%," 346208873Srwatson "intr%%,idle%%\n"); 347208873Srwatson 348208873Srwatson /* 349208873Srwatson * Configuration parameters. 350208873Srwatson */ 351208873Srwatson printf("%jd,", bflag * tflag * pflag); 352208873Srwatson printf("%jd.%09jd,", (intmax_t)ts_finish.tv_sec, 353189623Srwatson (intmax_t)(ts_finish.tv_nsec)); 354189623Srwatson 355208873Srwatson /* 356208873Srwatson * Effective transmit rates. 357208873Srwatson */ 358208873Srwatson printf("%f,", (double)(pflag * tflag)/ 359208873Srwatson (ts_finish.tv_sec + ts_finish.tv_nsec * 1e-9)); 360208873Srwatson printf("%f,", (double)(bflag * tflag * pflag * 8) / 361208873Srwatson (ts_finish.tv_sec + ts_finish.tv_nsec * 1e-9) * 1e-9); 362208873Srwatson 363208873Srwatson /* 364208873Srwatson * CPU time (est). 365208873Srwatson */ 366208873Srwatson ticks = 0; 367208873Srwatson for (i = 0; i < CPUSTATES; i++) { 368208873Srwatson cp_time_finish[i] -= cp_time_start[i]; 369208873Srwatson ticks += cp_time_finish[i]; 370189623Srwatson } 371208873Srwatson printf("%0.02f,", (float)(100 * cp_time_finish[CP_USER]) / ticks); 372208873Srwatson printf("%0.02f,", (float)(100 * cp_time_finish[CP_NICE]) / ticks); 373208873Srwatson printf("%0.02f,", (float)(100 * cp_time_finish[CP_SYS]) / ticks); 374208873Srwatson printf("%0.02f,", (float)(100 * cp_time_finish[CP_INTR]) / ticks); 375208873Srwatson printf("%0.02f", (float)(100 * cp_time_finish[CP_IDLE]) / ticks); 376208873Srwatson printf("\n"); 377189623Srwatson} 378