1/* 2 * Authors: 3 * Copyright 2001, 2002 by Robert Olsson <robert.olsson@its.uu.se> 4 * Uppsala University and 5 * Swedish University of Agricultural Sciences 6 * 7 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 8 * Ben Greear <greearb@candelatech.com> 9 * Jens L����s <jens.laas@data.slu.se> 10 * 11 * This program is free software; you can redistribute it and/or 12 * modify it under the terms of the GNU General Public License 13 * as published by the Free Software Foundation; either version 14 * 2 of the License, or (at your option) any later version. 15 * 16 * 17 * A tool for loading the network with preconfigurated packets. 18 * The tool is implemented as a linux module. Parameters are output 19 * device, delay (to hard_xmit), number of packets, and whether 20 * to use multiple SKBs or just the same one. 21 * pktgen uses the installed interface's output routine. 22 * 23 * Additional hacking by: 24 * 25 * Jens.Laas@data.slu.se 26 * Improved by ANK. 010120. 27 * Improved by ANK even more. 010212. 28 * MAC address typo fixed. 010417 --ro 29 * Integrated. 020301 --DaveM 30 * Added multiskb option 020301 --DaveM 31 * Scaling of results. 020417--sigurdur@linpro.no 32 * Significant re-work of the module: 33 * * Convert to threaded model to more efficiently be able to transmit 34 * and receive on multiple interfaces at once. 35 * * Converted many counters to __u64 to allow longer runs. 36 * * Allow configuration of ranges, like min/max IP address, MACs, 37 * and UDP-ports, for both source and destination, and can 38 * set to use a random distribution or sequentially walk the range. 39 * * Can now change most values after starting. 40 * * Place 12-byte packet in UDP payload with magic number, 41 * sequence number, and timestamp. 42 * * Add receiver code that detects dropped pkts, re-ordered pkts, and 43 * latencies (with micro-second) precision. 44 * * Add IOCTL interface to easily get counters & configuration. 45 * --Ben Greear <greearb@candelatech.com> 46 * 47 * Renamed multiskb to clone_skb and cleaned up sending core for two distinct 48 * skb modes. A clone_skb=0 mode for Ben "ranges" work and a clone_skb != 0 49 * as a "fastpath" with a configurable number of clones after alloc's. 50 * clone_skb=0 means all packets are allocated this also means ranges time 51 * stamps etc can be used. clone_skb=100 means 1 malloc is followed by 100 52 * clones. 53 * 54 * Also moved to /proc/net/pktgen/ 55 * --ro 56 * 57 * Sept 10: Fixed threading/locking. Lots of bone-headed and more clever 58 * mistakes. Also merged in DaveM's patch in the -pre6 patch. 59 * --Ben Greear <greearb@candelatech.com> 60 * 61 * Integrated to 2.5.x 021029 --Lucio Maciel (luciomaciel@zipmail.com.br) 62 * 63 * 64 * 021124 Finished major redesign and rewrite for new functionality. 65 * See Documentation/networking/pktgen.txt for how to use this. 66 * 67 * The new operation: 68 * For each CPU one thread/process is created at start. This process checks 69 * for running devices in the if_list and sends packets until count is 0 it 70 * also the thread checks the thread->control which is used for inter-process 71 * communication. controlling process "posts" operations to the threads this 72 * way. The if_lock should be possible to remove when add/rem_device is merged 73 * into this too. 74 * 75 * By design there should only be *one* "controlling" process. In practice 76 * multiple write accesses gives unpredictable result. Understood by "write" 77 * to /proc gives result code thats should be read be the "writer". 78 * For practical use this should be no problem. 79 * 80 * Note when adding devices to a specific CPU there good idea to also assign 81 * /proc/irq/XX/smp_affinity so TX-interrupts gets bound to the same CPU. 82 * --ro 83 * 84 * Fix refcount off by one if first packet fails, potential null deref, 85 * memleak 030710- KJP 86 * 87 * First "ranges" functionality for ipv6 030726 --ro 88 * 89 * Included flow support. 030802 ANK. 90 * 91 * Fixed unaligned access on IA-64 Grant Grundler <grundler@parisc-linux.org> 92 * 93 * Remove if fix from added Harald Welte <laforge@netfilter.org> 040419 94 * ia64 compilation fix from Aron Griffis <aron@hp.com> 040604 95 * 96 * New xmit() return, do_div and misc clean up by Stephen Hemminger 97 * <shemminger@osdl.org> 040923 98 * 99 * Randy Dunlap fixed u64 printk compiler waring 100 * 101 * Remove FCS from BW calculation. Lennert Buytenhek <buytenh@wantstofly.org> 102 * New time handling. Lennert Buytenhek <buytenh@wantstofly.org> 041213 103 * 104 * Corrections from Nikolai Malykh (nmalykh@bilim.com) 105 * Removed unused flags F_SET_SRCMAC & F_SET_SRCIP 041230 106 * 107 * interruptible_sleep_on_timeout() replaced Nishanth Aravamudan <nacc@us.ibm.com> 108 * 050103 109 * 110 * MPLS support by Steven Whitehouse <steve@chygwyn.com> 111 * 112 * 802.1Q/Q-in-Q support by Francesco Fondelli (FF) <francesco.fondelli@gmail.com> 113 * 114 * Fixed src_mac command to set source mac of packet to value specified in 115 * command by Adit Ranadive <adit.262@gmail.com> 116 * 117 */ 118 119#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 120 121#include <linux/sys.h> 122#include <linux/types.h> 123#include <linux/module.h> 124#include <linux/moduleparam.h> 125#include <linux/kernel.h> 126#include <linux/mutex.h> 127#include <linux/sched.h> 128#include <linux/slab.h> 129#include <linux/vmalloc.h> 130#include <linux/unistd.h> 131#include <linux/string.h> 132#include <linux/ptrace.h> 133#include <linux/errno.h> 134#include <linux/ioport.h> 135#include <linux/interrupt.h> 136#include <linux/capability.h> 137#include <linux/hrtimer.h> 138#include <linux/freezer.h> 139#include <linux/delay.h> 140#include <linux/timer.h> 141#include <linux/list.h> 142#include <linux/init.h> 143#include <linux/skbuff.h> 144#include <linux/netdevice.h> 145#include <linux/inet.h> 146#include <linux/inetdevice.h> 147#include <linux/rtnetlink.h> 148#include <linux/if_arp.h> 149#include <linux/if_vlan.h> 150#include <linux/in.h> 151#include <linux/ip.h> 152#include <linux/ipv6.h> 153#include <linux/udp.h> 154#include <linux/proc_fs.h> 155#include <linux/seq_file.h> 156#include <linux/wait.h> 157#include <linux/etherdevice.h> 158#include <linux/kthread.h> 159#include <net/net_namespace.h> 160#include <net/checksum.h> 161#include <net/ipv6.h> 162#include <net/addrconf.h> 163#ifdef CONFIG_XFRM 164#include <net/xfrm.h> 165#endif 166#include <asm/byteorder.h> 167#include <linux/rcupdate.h> 168#include <linux/bitops.h> 169#include <linux/io.h> 170#include <linux/timex.h> 171#include <linux/uaccess.h> 172#include <asm/dma.h> 173#include <asm/div64.h> /* do_div */ 174 175#define VERSION "2.74" 176#define IP_NAME_SZ 32 177#define MAX_MPLS_LABELS 16 /* This is the max label stack depth */ 178#define MPLS_STACK_BOTTOM htonl(0x00000100) 179 180#define func_enter() pr_debug("entering %s\n", __func__); 181 182/* Device flag bits */ 183#define F_IPSRC_RND (1<<0) /* IP-Src Random */ 184#define F_IPDST_RND (1<<1) /* IP-Dst Random */ 185#define F_UDPSRC_RND (1<<2) /* UDP-Src Random */ 186#define F_UDPDST_RND (1<<3) /* UDP-Dst Random */ 187#define F_MACSRC_RND (1<<4) /* MAC-Src Random */ 188#define F_MACDST_RND (1<<5) /* MAC-Dst Random */ 189#define F_TXSIZE_RND (1<<6) /* Transmit size is random */ 190#define F_IPV6 (1<<7) /* Interface in IPV6 Mode */ 191#define F_MPLS_RND (1<<8) /* Random MPLS labels */ 192#define F_VID_RND (1<<9) /* Random VLAN ID */ 193#define F_SVID_RND (1<<10) /* Random SVLAN ID */ 194#define F_FLOW_SEQ (1<<11) /* Sequential flows */ 195#define F_IPSEC_ON (1<<12) /* ipsec on for flows */ 196#define F_QUEUE_MAP_RND (1<<13) /* queue map Random */ 197#define F_QUEUE_MAP_CPU (1<<14) /* queue map mirrors smp_processor_id() */ 198#define F_NODE (1<<15) /* Node memory alloc*/ 199 200/* Thread control flag bits */ 201#define T_STOP (1<<0) /* Stop run */ 202#define T_RUN (1<<1) /* Start run */ 203#define T_REMDEVALL (1<<2) /* Remove all devs */ 204#define T_REMDEV (1<<3) /* Remove one dev */ 205 206/* If lock -- can be removed after some work */ 207#define if_lock(t) spin_lock(&(t->if_lock)); 208#define if_unlock(t) spin_unlock(&(t->if_lock)); 209 210/* Used to help with determining the pkts on receive */ 211#define PKTGEN_MAGIC 0xbe9be955 212#define PG_PROC_DIR "pktgen" 213#define PGCTRL "pgctrl" 214static struct proc_dir_entry *pg_proc_dir; 215 216#define MAX_CFLOWS 65536 217 218#define VLAN_TAG_SIZE(x) ((x)->vlan_id == 0xffff ? 0 : 4) 219#define SVLAN_TAG_SIZE(x) ((x)->svlan_id == 0xffff ? 0 : 4) 220 221struct flow_state { 222 __be32 cur_daddr; 223 int count; 224#ifdef CONFIG_XFRM 225 struct xfrm_state *x; 226#endif 227 __u32 flags; 228}; 229 230/* flow flag bits */ 231#define F_INIT (1<<0) /* flow has been initialized */ 232 233struct pktgen_dev { 234 /* 235 * Try to keep frequent/infrequent used vars. separated. 236 */ 237 struct proc_dir_entry *entry; /* proc file */ 238 struct pktgen_thread *pg_thread;/* the owner */ 239 struct list_head list; /* chaining in the thread's run-queue */ 240 241 int running; /* if false, the test will stop */ 242 243 /* If min != max, then we will either do a linear iteration, or 244 * we will do a random selection from within the range. 245 */ 246 __u32 flags; 247 int removal_mark; /* non-zero => the device is marked for 248 * removal by worker thread */ 249 250 int min_pkt_size; /* = ETH_ZLEN; */ 251 int max_pkt_size; /* = ETH_ZLEN; */ 252 int pkt_overhead; /* overhead for MPLS, VLANs, IPSEC etc */ 253 int nfrags; 254 u64 delay; /* nano-seconds */ 255 256 __u64 count; /* Default No packets to send */ 257 __u64 sofar; /* How many pkts we've sent so far */ 258 __u64 tx_bytes; /* How many bytes we've transmitted */ 259 __u64 errors; /* Errors when trying to transmit, */ 260 261 /* runtime counters relating to clone_skb */ 262 263 __u64 allocated_skbs; 264 __u32 clone_count; 265 int last_ok; /* Was last skb sent? 266 * Or a failed transmit of some sort? 267 * This will keep sequence numbers in order 268 */ 269 ktime_t next_tx; 270 ktime_t started_at; 271 ktime_t stopped_at; 272 u64 idle_acc; /* nano-seconds */ 273 274 __u32 seq_num; 275 276 int clone_skb; /* 277 * Use multiple SKBs during packet gen. 278 * If this number is greater than 1, then 279 * that many copies of the same packet will be 280 * sent before a new packet is allocated. 281 * If you want to send 1024 identical packets 282 * before creating a new packet, 283 * set clone_skb to 1024. 284 */ 285 286 char dst_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 287 char dst_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 288 char src_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 289 char src_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 290 291 struct in6_addr in6_saddr; 292 struct in6_addr in6_daddr; 293 struct in6_addr cur_in6_daddr; 294 struct in6_addr cur_in6_saddr; 295 /* For ranges */ 296 struct in6_addr min_in6_daddr; 297 struct in6_addr max_in6_daddr; 298 struct in6_addr min_in6_saddr; 299 struct in6_addr max_in6_saddr; 300 301 /* If we're doing ranges, random or incremental, then this 302 * defines the min/max for those ranges. 303 */ 304 __be32 saddr_min; /* inclusive, source IP address */ 305 __be32 saddr_max; /* exclusive, source IP address */ 306 __be32 daddr_min; /* inclusive, dest IP address */ 307 __be32 daddr_max; /* exclusive, dest IP address */ 308 309 __u16 udp_src_min; /* inclusive, source UDP port */ 310 __u16 udp_src_max; /* exclusive, source UDP port */ 311 __u16 udp_dst_min; /* inclusive, dest UDP port */ 312 __u16 udp_dst_max; /* exclusive, dest UDP port */ 313 314 /* DSCP + ECN */ 315 __u8 tos; /* six MSB of (former) IPv4 TOS 316 are for dscp codepoint */ 317 __u8 traffic_class; /* ditto for the (former) Traffic Class in IPv6 318 (see RFC 3260, sec. 4) */ 319 320 /* MPLS */ 321 unsigned nr_labels; /* Depth of stack, 0 = no MPLS */ 322 __be32 labels[MAX_MPLS_LABELS]; 323 324 /* VLAN/SVLAN (802.1Q/Q-in-Q) */ 325 __u8 vlan_p; 326 __u8 vlan_cfi; 327 __u16 vlan_id; /* 0xffff means no vlan tag */ 328 329 __u8 svlan_p; 330 __u8 svlan_cfi; 331 __u16 svlan_id; /* 0xffff means no svlan tag */ 332 333 __u32 src_mac_count; /* How many MACs to iterate through */ 334 __u32 dst_mac_count; /* How many MACs to iterate through */ 335 336 unsigned char dst_mac[ETH_ALEN]; 337 unsigned char src_mac[ETH_ALEN]; 338 339 __u32 cur_dst_mac_offset; 340 __u32 cur_src_mac_offset; 341 __be32 cur_saddr; 342 __be32 cur_daddr; 343 __u16 ip_id; 344 __u16 cur_udp_dst; 345 __u16 cur_udp_src; 346 __u16 cur_queue_map; 347 __u32 cur_pkt_size; 348 __u32 last_pkt_size; 349 350 __u8 hh[14]; 351 /* = { 352 0x00, 0x80, 0xC8, 0x79, 0xB3, 0xCB, 353 354 We fill in SRC address later 355 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 356 0x08, 0x00 357 }; 358 */ 359 __u16 pad; /* pad out the hh struct to an even 16 bytes */ 360 361 struct sk_buff *skb; /* skb we are to transmit next, used for when we 362 * are transmitting the same one multiple times 363 */ 364 struct net_device *odev; /* The out-going device. 365 * Note that the device should have it's 366 * pg_info pointer pointing back to this 367 * device. 368 * Set when the user specifies the out-going 369 * device name (not when the inject is 370 * started as it used to do.) 371 */ 372 char odevname[32]; 373 struct flow_state *flows; 374 unsigned cflows; /* Concurrent flows (config) */ 375 unsigned lflow; /* Flow length (config) */ 376 unsigned nflows; /* accumulated flows (stats) */ 377 unsigned curfl; /* current sequenced flow (state)*/ 378 379 u16 queue_map_min; 380 u16 queue_map_max; 381 int node; /* Memory node */ 382 383#ifdef CONFIG_XFRM 384 __u8 ipsmode; /* IPSEC mode (config) */ 385 __u8 ipsproto; /* IPSEC type (config) */ 386#endif 387 char result[512]; 388}; 389 390struct pktgen_hdr { 391 __be32 pgh_magic; 392 __be32 seq_num; 393 __be32 tv_sec; 394 __be32 tv_usec; 395}; 396 397struct pktgen_thread { 398 spinlock_t if_lock; /* for list of devices */ 399 struct list_head if_list; /* All device here */ 400 struct list_head th_list; 401 struct task_struct *tsk; 402 char result[512]; 403 404 /* Field for thread to receive "posted" events terminate, 405 stop ifs etc. */ 406 407 u32 control; 408 int cpu; 409 410 wait_queue_head_t queue; 411 struct completion start_done; 412}; 413 414#define REMOVE 1 415#define FIND 0 416 417static inline ktime_t ktime_now(void) 418{ 419 struct timespec ts; 420 ktime_get_ts(&ts); 421 422 return timespec_to_ktime(ts); 423} 424 425/* This works even if 32 bit because of careful byte order choice */ 426static inline int ktime_lt(const ktime_t cmp1, const ktime_t cmp2) 427{ 428 return cmp1.tv64 < cmp2.tv64; 429} 430 431static const char version[] = 432 "Packet Generator for packet performance testing. " 433 "Version: " VERSION "\n"; 434 435static int pktgen_remove_device(struct pktgen_thread *t, struct pktgen_dev *i); 436static int pktgen_add_device(struct pktgen_thread *t, const char *ifname); 437static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t, 438 const char *ifname, bool exact); 439static int pktgen_device_event(struct notifier_block *, unsigned long, void *); 440static void pktgen_run_all_threads(void); 441static void pktgen_reset_all_threads(void); 442static void pktgen_stop_all_threads_ifs(void); 443 444static void pktgen_stop(struct pktgen_thread *t); 445static void pktgen_clear_counters(struct pktgen_dev *pkt_dev); 446 447static unsigned int scan_ip6(const char *s, char ip[16]); 448static unsigned int fmt_ip6(char *s, const char ip[16]); 449 450/* Module parameters, defaults. */ 451static int pg_count_d __read_mostly = 1000; 452static int pg_delay_d __read_mostly; 453static int pg_clone_skb_d __read_mostly; 454static int debug __read_mostly; 455 456static DEFINE_MUTEX(pktgen_thread_lock); 457static LIST_HEAD(pktgen_threads); 458 459static struct notifier_block pktgen_notifier_block = { 460 .notifier_call = pktgen_device_event, 461}; 462 463/* 464 * /proc handling functions 465 * 466 */ 467 468static int pgctrl_show(struct seq_file *seq, void *v) 469{ 470 seq_puts(seq, version); 471 return 0; 472} 473 474static ssize_t pgctrl_write(struct file *file, const char __user *buf, 475 size_t count, loff_t *ppos) 476{ 477 int err = 0; 478 char data[128]; 479 480 if (!capable(CAP_NET_ADMIN)) { 481 err = -EPERM; 482 goto out; 483 } 484 485 if (count > sizeof(data)) 486 count = sizeof(data); 487 488 if (copy_from_user(data, buf, count)) { 489 err = -EFAULT; 490 goto out; 491 } 492 data[count - 1] = 0; /* Make string */ 493 494 if (!strcmp(data, "stop")) 495 pktgen_stop_all_threads_ifs(); 496 497 else if (!strcmp(data, "start")) 498 pktgen_run_all_threads(); 499 500 else if (!strcmp(data, "reset")) 501 pktgen_reset_all_threads(); 502 503 else 504 pr_warning("Unknown command: %s\n", data); 505 506 err = count; 507 508out: 509 return err; 510} 511 512static int pgctrl_open(struct inode *inode, struct file *file) 513{ 514 return single_open(file, pgctrl_show, PDE(inode)->data); 515} 516 517static const struct file_operations pktgen_fops = { 518 .owner = THIS_MODULE, 519 .open = pgctrl_open, 520 .read = seq_read, 521 .llseek = seq_lseek, 522 .write = pgctrl_write, 523 .release = single_release, 524}; 525 526static int pktgen_if_show(struct seq_file *seq, void *v) 527{ 528 const struct pktgen_dev *pkt_dev = seq->private; 529 ktime_t stopped; 530 u64 idle; 531 532 seq_printf(seq, 533 "Params: count %llu min_pkt_size: %u max_pkt_size: %u\n", 534 (unsigned long long)pkt_dev->count, pkt_dev->min_pkt_size, 535 pkt_dev->max_pkt_size); 536 537 seq_printf(seq, 538 " frags: %d delay: %llu clone_skb: %d ifname: %s\n", 539 pkt_dev->nfrags, (unsigned long long) pkt_dev->delay, 540 pkt_dev->clone_skb, pkt_dev->odevname); 541 542 seq_printf(seq, " flows: %u flowlen: %u\n", pkt_dev->cflows, 543 pkt_dev->lflow); 544 545 seq_printf(seq, 546 " queue_map_min: %u queue_map_max: %u\n", 547 pkt_dev->queue_map_min, 548 pkt_dev->queue_map_max); 549 550 if (pkt_dev->flags & F_IPV6) { 551 char b1[128], b2[128], b3[128]; 552 fmt_ip6(b1, pkt_dev->in6_saddr.s6_addr); 553 fmt_ip6(b2, pkt_dev->min_in6_saddr.s6_addr); 554 fmt_ip6(b3, pkt_dev->max_in6_saddr.s6_addr); 555 seq_printf(seq, 556 " saddr: %s min_saddr: %s max_saddr: %s\n", b1, 557 b2, b3); 558 559 fmt_ip6(b1, pkt_dev->in6_daddr.s6_addr); 560 fmt_ip6(b2, pkt_dev->min_in6_daddr.s6_addr); 561 fmt_ip6(b3, pkt_dev->max_in6_daddr.s6_addr); 562 seq_printf(seq, 563 " daddr: %s min_daddr: %s max_daddr: %s\n", b1, 564 b2, b3); 565 566 } else { 567 seq_printf(seq, 568 " dst_min: %s dst_max: %s\n", 569 pkt_dev->dst_min, pkt_dev->dst_max); 570 seq_printf(seq, 571 " src_min: %s src_max: %s\n", 572 pkt_dev->src_min, pkt_dev->src_max); 573 } 574 575 seq_puts(seq, " src_mac: "); 576 577 seq_printf(seq, "%pM ", 578 is_zero_ether_addr(pkt_dev->src_mac) ? 579 pkt_dev->odev->dev_addr : pkt_dev->src_mac); 580 581 seq_printf(seq, "dst_mac: "); 582 seq_printf(seq, "%pM\n", pkt_dev->dst_mac); 583 584 seq_printf(seq, 585 " udp_src_min: %d udp_src_max: %d" 586 " udp_dst_min: %d udp_dst_max: %d\n", 587 pkt_dev->udp_src_min, pkt_dev->udp_src_max, 588 pkt_dev->udp_dst_min, pkt_dev->udp_dst_max); 589 590 seq_printf(seq, 591 " src_mac_count: %d dst_mac_count: %d\n", 592 pkt_dev->src_mac_count, pkt_dev->dst_mac_count); 593 594 if (pkt_dev->nr_labels) { 595 unsigned i; 596 seq_printf(seq, " mpls: "); 597 for (i = 0; i < pkt_dev->nr_labels; i++) 598 seq_printf(seq, "%08x%s", ntohl(pkt_dev->labels[i]), 599 i == pkt_dev->nr_labels-1 ? "\n" : ", "); 600 } 601 602 if (pkt_dev->vlan_id != 0xffff) 603 seq_printf(seq, " vlan_id: %u vlan_p: %u vlan_cfi: %u\n", 604 pkt_dev->vlan_id, pkt_dev->vlan_p, 605 pkt_dev->vlan_cfi); 606 607 if (pkt_dev->svlan_id != 0xffff) 608 seq_printf(seq, " svlan_id: %u vlan_p: %u vlan_cfi: %u\n", 609 pkt_dev->svlan_id, pkt_dev->svlan_p, 610 pkt_dev->svlan_cfi); 611 612 if (pkt_dev->tos) 613 seq_printf(seq, " tos: 0x%02x\n", pkt_dev->tos); 614 615 if (pkt_dev->traffic_class) 616 seq_printf(seq, " traffic_class: 0x%02x\n", pkt_dev->traffic_class); 617 618 if (pkt_dev->node >= 0) 619 seq_printf(seq, " node: %d\n", pkt_dev->node); 620 621 seq_printf(seq, " Flags: "); 622 623 if (pkt_dev->flags & F_IPV6) 624 seq_printf(seq, "IPV6 "); 625 626 if (pkt_dev->flags & F_IPSRC_RND) 627 seq_printf(seq, "IPSRC_RND "); 628 629 if (pkt_dev->flags & F_IPDST_RND) 630 seq_printf(seq, "IPDST_RND "); 631 632 if (pkt_dev->flags & F_TXSIZE_RND) 633 seq_printf(seq, "TXSIZE_RND "); 634 635 if (pkt_dev->flags & F_UDPSRC_RND) 636 seq_printf(seq, "UDPSRC_RND "); 637 638 if (pkt_dev->flags & F_UDPDST_RND) 639 seq_printf(seq, "UDPDST_RND "); 640 641 if (pkt_dev->flags & F_MPLS_RND) 642 seq_printf(seq, "MPLS_RND "); 643 644 if (pkt_dev->flags & F_QUEUE_MAP_RND) 645 seq_printf(seq, "QUEUE_MAP_RND "); 646 647 if (pkt_dev->flags & F_QUEUE_MAP_CPU) 648 seq_printf(seq, "QUEUE_MAP_CPU "); 649 650 if (pkt_dev->cflows) { 651 if (pkt_dev->flags & F_FLOW_SEQ) 652 seq_printf(seq, "FLOW_SEQ "); /*in sequence flows*/ 653 else 654 seq_printf(seq, "FLOW_RND "); 655 } 656 657#ifdef CONFIG_XFRM 658 if (pkt_dev->flags & F_IPSEC_ON) 659 seq_printf(seq, "IPSEC "); 660#endif 661 662 if (pkt_dev->flags & F_MACSRC_RND) 663 seq_printf(seq, "MACSRC_RND "); 664 665 if (pkt_dev->flags & F_MACDST_RND) 666 seq_printf(seq, "MACDST_RND "); 667 668 if (pkt_dev->flags & F_VID_RND) 669 seq_printf(seq, "VID_RND "); 670 671 if (pkt_dev->flags & F_SVID_RND) 672 seq_printf(seq, "SVID_RND "); 673 674 if (pkt_dev->flags & F_NODE) 675 seq_printf(seq, "NODE_ALLOC "); 676 677 seq_puts(seq, "\n"); 678 679 /* not really stopped, more like last-running-at */ 680 stopped = pkt_dev->running ? ktime_now() : pkt_dev->stopped_at; 681 idle = pkt_dev->idle_acc; 682 do_div(idle, NSEC_PER_USEC); 683 684 seq_printf(seq, 685 "Current:\n pkts-sofar: %llu errors: %llu\n", 686 (unsigned long long)pkt_dev->sofar, 687 (unsigned long long)pkt_dev->errors); 688 689 seq_printf(seq, 690 " started: %lluus stopped: %lluus idle: %lluus\n", 691 (unsigned long long) ktime_to_us(pkt_dev->started_at), 692 (unsigned long long) ktime_to_us(stopped), 693 (unsigned long long) idle); 694 695 seq_printf(seq, 696 " seq_num: %d cur_dst_mac_offset: %d cur_src_mac_offset: %d\n", 697 pkt_dev->seq_num, pkt_dev->cur_dst_mac_offset, 698 pkt_dev->cur_src_mac_offset); 699 700 if (pkt_dev->flags & F_IPV6) { 701 char b1[128], b2[128]; 702 fmt_ip6(b1, pkt_dev->cur_in6_daddr.s6_addr); 703 fmt_ip6(b2, pkt_dev->cur_in6_saddr.s6_addr); 704 seq_printf(seq, " cur_saddr: %s cur_daddr: %s\n", b2, b1); 705 } else 706 seq_printf(seq, " cur_saddr: 0x%x cur_daddr: 0x%x\n", 707 pkt_dev->cur_saddr, pkt_dev->cur_daddr); 708 709 seq_printf(seq, " cur_udp_dst: %d cur_udp_src: %d\n", 710 pkt_dev->cur_udp_dst, pkt_dev->cur_udp_src); 711 712 seq_printf(seq, " cur_queue_map: %u\n", pkt_dev->cur_queue_map); 713 714 seq_printf(seq, " flows: %u\n", pkt_dev->nflows); 715 716 if (pkt_dev->result[0]) 717 seq_printf(seq, "Result: %s\n", pkt_dev->result); 718 else 719 seq_printf(seq, "Result: Idle\n"); 720 721 return 0; 722} 723 724 725static int hex32_arg(const char __user *user_buffer, unsigned long maxlen, 726 __u32 *num) 727{ 728 int i = 0; 729 *num = 0; 730 731 for (; i < maxlen; i++) { 732 char c; 733 *num <<= 4; 734 if (get_user(c, &user_buffer[i])) 735 return -EFAULT; 736 if ((c >= '0') && (c <= '9')) 737 *num |= c - '0'; 738 else if ((c >= 'a') && (c <= 'f')) 739 *num |= c - 'a' + 10; 740 else if ((c >= 'A') && (c <= 'F')) 741 *num |= c - 'A' + 10; 742 else 743 break; 744 } 745 return i; 746} 747 748static int count_trail_chars(const char __user * user_buffer, 749 unsigned int maxlen) 750{ 751 int i; 752 753 for (i = 0; i < maxlen; i++) { 754 char c; 755 if (get_user(c, &user_buffer[i])) 756 return -EFAULT; 757 switch (c) { 758 case '\"': 759 case '\n': 760 case '\r': 761 case '\t': 762 case ' ': 763 case '=': 764 break; 765 default: 766 goto done; 767 } 768 } 769done: 770 return i; 771} 772 773static unsigned long num_arg(const char __user * user_buffer, 774 unsigned long maxlen, unsigned long *num) 775{ 776 int i = 0; 777 *num = 0; 778 779 for (; i < maxlen; i++) { 780 char c; 781 if (get_user(c, &user_buffer[i])) 782 return -EFAULT; 783 if ((c >= '0') && (c <= '9')) { 784 *num *= 10; 785 *num += c - '0'; 786 } else 787 break; 788 } 789 return i; 790} 791 792static int strn_len(const char __user * user_buffer, unsigned int maxlen) 793{ 794 int i = 0; 795 796 for (; i < maxlen; i++) { 797 char c; 798 if (get_user(c, &user_buffer[i])) 799 return -EFAULT; 800 switch (c) { 801 case '\"': 802 case '\n': 803 case '\r': 804 case '\t': 805 case ' ': 806 goto done_str; 807 break; 808 default: 809 break; 810 } 811 } 812done_str: 813 return i; 814} 815 816static ssize_t get_labels(const char __user *buffer, struct pktgen_dev *pkt_dev) 817{ 818 unsigned n = 0; 819 char c; 820 ssize_t i = 0; 821 int len; 822 823 pkt_dev->nr_labels = 0; 824 do { 825 __u32 tmp; 826 len = hex32_arg(&buffer[i], 8, &tmp); 827 if (len <= 0) 828 return len; 829 pkt_dev->labels[n] = htonl(tmp); 830 if (pkt_dev->labels[n] & MPLS_STACK_BOTTOM) 831 pkt_dev->flags |= F_MPLS_RND; 832 i += len; 833 if (get_user(c, &buffer[i])) 834 return -EFAULT; 835 i++; 836 n++; 837 if (n >= MAX_MPLS_LABELS) 838 return -E2BIG; 839 } while (c == ','); 840 841 pkt_dev->nr_labels = n; 842 return i; 843} 844 845static ssize_t pktgen_if_write(struct file *file, 846 const char __user * user_buffer, size_t count, 847 loff_t * offset) 848{ 849 struct seq_file *seq = file->private_data; 850 struct pktgen_dev *pkt_dev = seq->private; 851 int i = 0, max, len; 852 char name[16], valstr[32]; 853 unsigned long value = 0; 854 char *pg_result = NULL; 855 int tmp = 0; 856 char buf[128]; 857 858 pg_result = &(pkt_dev->result[0]); 859 860 if (count < 1) { 861 pr_warning("wrong command format\n"); 862 return -EINVAL; 863 } 864 865 max = count - i; 866 tmp = count_trail_chars(&user_buffer[i], max); 867 if (tmp < 0) { 868 pr_warning("illegal format\n"); 869 return tmp; 870 } 871 i += tmp; 872 873 /* Read variable name */ 874 875 len = strn_len(&user_buffer[i], sizeof(name) - 1); 876 if (len < 0) 877 return len; 878 879 memset(name, 0, sizeof(name)); 880 if (copy_from_user(name, &user_buffer[i], len)) 881 return -EFAULT; 882 i += len; 883 884 max = count - i; 885 len = count_trail_chars(&user_buffer[i], max); 886 if (len < 0) 887 return len; 888 889 i += len; 890 891 if (debug) { 892 char tb[count + 1]; 893 if (copy_from_user(tb, user_buffer, count)) 894 return -EFAULT; 895 tb[count] = 0; 896 printk(KERN_DEBUG "pktgen: %s,%lu buffer -:%s:-\n", name, 897 (unsigned long)count, tb); 898 } 899 900 if (!strcmp(name, "min_pkt_size")) { 901 len = num_arg(&user_buffer[i], 10, &value); 902 if (len < 0) 903 return len; 904 905 i += len; 906 if (value < 14 + 20 + 8) 907 value = 14 + 20 + 8; 908 if (value != pkt_dev->min_pkt_size) { 909 pkt_dev->min_pkt_size = value; 910 pkt_dev->cur_pkt_size = value; 911 } 912 sprintf(pg_result, "OK: min_pkt_size=%u", 913 pkt_dev->min_pkt_size); 914 return count; 915 } 916 917 if (!strcmp(name, "max_pkt_size")) { 918 len = num_arg(&user_buffer[i], 10, &value); 919 if (len < 0) 920 return len; 921 922 i += len; 923 if (value < 14 + 20 + 8) 924 value = 14 + 20 + 8; 925 if (value != pkt_dev->max_pkt_size) { 926 pkt_dev->max_pkt_size = value; 927 pkt_dev->cur_pkt_size = value; 928 } 929 sprintf(pg_result, "OK: max_pkt_size=%u", 930 pkt_dev->max_pkt_size); 931 return count; 932 } 933 934 /* Shortcut for min = max */ 935 936 if (!strcmp(name, "pkt_size")) { 937 len = num_arg(&user_buffer[i], 10, &value); 938 if (len < 0) 939 return len; 940 941 i += len; 942 if (value < 14 + 20 + 8) 943 value = 14 + 20 + 8; 944 if (value != pkt_dev->min_pkt_size) { 945 pkt_dev->min_pkt_size = value; 946 pkt_dev->max_pkt_size = value; 947 pkt_dev->cur_pkt_size = value; 948 } 949 sprintf(pg_result, "OK: pkt_size=%u", pkt_dev->min_pkt_size); 950 return count; 951 } 952 953 if (!strcmp(name, "debug")) { 954 len = num_arg(&user_buffer[i], 10, &value); 955 if (len < 0) 956 return len; 957 958 i += len; 959 debug = value; 960 sprintf(pg_result, "OK: debug=%u", debug); 961 return count; 962 } 963 964 if (!strcmp(name, "frags")) { 965 len = num_arg(&user_buffer[i], 10, &value); 966 if (len < 0) 967 return len; 968 969 i += len; 970 pkt_dev->nfrags = value; 971 sprintf(pg_result, "OK: frags=%u", pkt_dev->nfrags); 972 return count; 973 } 974 if (!strcmp(name, "delay")) { 975 len = num_arg(&user_buffer[i], 10, &value); 976 if (len < 0) 977 return len; 978 979 i += len; 980 if (value == 0x7FFFFFFF) 981 pkt_dev->delay = ULLONG_MAX; 982 else 983 pkt_dev->delay = (u64)value; 984 985 sprintf(pg_result, "OK: delay=%llu", 986 (unsigned long long) pkt_dev->delay); 987 return count; 988 } 989 if (!strcmp(name, "rate")) { 990 len = num_arg(&user_buffer[i], 10, &value); 991 if (len < 0) 992 return len; 993 994 i += len; 995 if (!value) 996 return len; 997 pkt_dev->delay = pkt_dev->min_pkt_size*8*NSEC_PER_USEC/value; 998 if (debug) 999 pr_info("Delay set at: %llu ns\n", pkt_dev->delay); 1000 1001 sprintf(pg_result, "OK: rate=%lu", value); 1002 return count; 1003 } 1004 if (!strcmp(name, "ratep")) { 1005 len = num_arg(&user_buffer[i], 10, &value); 1006 if (len < 0) 1007 return len; 1008 1009 i += len; 1010 if (!value) 1011 return len; 1012 pkt_dev->delay = NSEC_PER_SEC/value; 1013 if (debug) 1014 pr_info("Delay set at: %llu ns\n", pkt_dev->delay); 1015 1016 sprintf(pg_result, "OK: rate=%lu", value); 1017 return count; 1018 } 1019 if (!strcmp(name, "udp_src_min")) { 1020 len = num_arg(&user_buffer[i], 10, &value); 1021 if (len < 0) 1022 return len; 1023 1024 i += len; 1025 if (value != pkt_dev->udp_src_min) { 1026 pkt_dev->udp_src_min = value; 1027 pkt_dev->cur_udp_src = value; 1028 } 1029 sprintf(pg_result, "OK: udp_src_min=%u", pkt_dev->udp_src_min); 1030 return count; 1031 } 1032 if (!strcmp(name, "udp_dst_min")) { 1033 len = num_arg(&user_buffer[i], 10, &value); 1034 if (len < 0) 1035 return len; 1036 1037 i += len; 1038 if (value != pkt_dev->udp_dst_min) { 1039 pkt_dev->udp_dst_min = value; 1040 pkt_dev->cur_udp_dst = value; 1041 } 1042 sprintf(pg_result, "OK: udp_dst_min=%u", pkt_dev->udp_dst_min); 1043 return count; 1044 } 1045 if (!strcmp(name, "udp_src_max")) { 1046 len = num_arg(&user_buffer[i], 10, &value); 1047 if (len < 0) 1048 return len; 1049 1050 i += len; 1051 if (value != pkt_dev->udp_src_max) { 1052 pkt_dev->udp_src_max = value; 1053 pkt_dev->cur_udp_src = value; 1054 } 1055 sprintf(pg_result, "OK: udp_src_max=%u", pkt_dev->udp_src_max); 1056 return count; 1057 } 1058 if (!strcmp(name, "udp_dst_max")) { 1059 len = num_arg(&user_buffer[i], 10, &value); 1060 if (len < 0) 1061 return len; 1062 1063 i += len; 1064 if (value != pkt_dev->udp_dst_max) { 1065 pkt_dev->udp_dst_max = value; 1066 pkt_dev->cur_udp_dst = value; 1067 } 1068 sprintf(pg_result, "OK: udp_dst_max=%u", pkt_dev->udp_dst_max); 1069 return count; 1070 } 1071 if (!strcmp(name, "clone_skb")) { 1072 len = num_arg(&user_buffer[i], 10, &value); 1073 if (len < 0) 1074 return len; 1075 1076 i += len; 1077 pkt_dev->clone_skb = value; 1078 1079 sprintf(pg_result, "OK: clone_skb=%d", pkt_dev->clone_skb); 1080 return count; 1081 } 1082 if (!strcmp(name, "count")) { 1083 len = num_arg(&user_buffer[i], 10, &value); 1084 if (len < 0) 1085 return len; 1086 1087 i += len; 1088 pkt_dev->count = value; 1089 sprintf(pg_result, "OK: count=%llu", 1090 (unsigned long long)pkt_dev->count); 1091 return count; 1092 } 1093 if (!strcmp(name, "src_mac_count")) { 1094 len = num_arg(&user_buffer[i], 10, &value); 1095 if (len < 0) 1096 return len; 1097 1098 i += len; 1099 if (pkt_dev->src_mac_count != value) { 1100 pkt_dev->src_mac_count = value; 1101 pkt_dev->cur_src_mac_offset = 0; 1102 } 1103 sprintf(pg_result, "OK: src_mac_count=%d", 1104 pkt_dev->src_mac_count); 1105 return count; 1106 } 1107 if (!strcmp(name, "dst_mac_count")) { 1108 len = num_arg(&user_buffer[i], 10, &value); 1109 if (len < 0) 1110 return len; 1111 1112 i += len; 1113 if (pkt_dev->dst_mac_count != value) { 1114 pkt_dev->dst_mac_count = value; 1115 pkt_dev->cur_dst_mac_offset = 0; 1116 } 1117 sprintf(pg_result, "OK: dst_mac_count=%d", 1118 pkt_dev->dst_mac_count); 1119 return count; 1120 } 1121 if (!strcmp(name, "node")) { 1122 len = num_arg(&user_buffer[i], 10, &value); 1123 if (len < 0) 1124 return len; 1125 1126 i += len; 1127 1128 if (node_possible(value)) { 1129 pkt_dev->node = value; 1130 sprintf(pg_result, "OK: node=%d", pkt_dev->node); 1131 } 1132 else 1133 sprintf(pg_result, "ERROR: node not possible"); 1134 return count; 1135 } 1136 if (!strcmp(name, "flag")) { 1137 char f[32]; 1138 memset(f, 0, 32); 1139 len = strn_len(&user_buffer[i], sizeof(f) - 1); 1140 if (len < 0) 1141 return len; 1142 1143 if (copy_from_user(f, &user_buffer[i], len)) 1144 return -EFAULT; 1145 i += len; 1146 if (strcmp(f, "IPSRC_RND") == 0) 1147 pkt_dev->flags |= F_IPSRC_RND; 1148 1149 else if (strcmp(f, "!IPSRC_RND") == 0) 1150 pkt_dev->flags &= ~F_IPSRC_RND; 1151 1152 else if (strcmp(f, "TXSIZE_RND") == 0) 1153 pkt_dev->flags |= F_TXSIZE_RND; 1154 1155 else if (strcmp(f, "!TXSIZE_RND") == 0) 1156 pkt_dev->flags &= ~F_TXSIZE_RND; 1157 1158 else if (strcmp(f, "IPDST_RND") == 0) 1159 pkt_dev->flags |= F_IPDST_RND; 1160 1161 else if (strcmp(f, "!IPDST_RND") == 0) 1162 pkt_dev->flags &= ~F_IPDST_RND; 1163 1164 else if (strcmp(f, "UDPSRC_RND") == 0) 1165 pkt_dev->flags |= F_UDPSRC_RND; 1166 1167 else if (strcmp(f, "!UDPSRC_RND") == 0) 1168 pkt_dev->flags &= ~F_UDPSRC_RND; 1169 1170 else if (strcmp(f, "UDPDST_RND") == 0) 1171 pkt_dev->flags |= F_UDPDST_RND; 1172 1173 else if (strcmp(f, "!UDPDST_RND") == 0) 1174 pkt_dev->flags &= ~F_UDPDST_RND; 1175 1176 else if (strcmp(f, "MACSRC_RND") == 0) 1177 pkt_dev->flags |= F_MACSRC_RND; 1178 1179 else if (strcmp(f, "!MACSRC_RND") == 0) 1180 pkt_dev->flags &= ~F_MACSRC_RND; 1181 1182 else if (strcmp(f, "MACDST_RND") == 0) 1183 pkt_dev->flags |= F_MACDST_RND; 1184 1185 else if (strcmp(f, "!MACDST_RND") == 0) 1186 pkt_dev->flags &= ~F_MACDST_RND; 1187 1188 else if (strcmp(f, "MPLS_RND") == 0) 1189 pkt_dev->flags |= F_MPLS_RND; 1190 1191 else if (strcmp(f, "!MPLS_RND") == 0) 1192 pkt_dev->flags &= ~F_MPLS_RND; 1193 1194 else if (strcmp(f, "VID_RND") == 0) 1195 pkt_dev->flags |= F_VID_RND; 1196 1197 else if (strcmp(f, "!VID_RND") == 0) 1198 pkt_dev->flags &= ~F_VID_RND; 1199 1200 else if (strcmp(f, "SVID_RND") == 0) 1201 pkt_dev->flags |= F_SVID_RND; 1202 1203 else if (strcmp(f, "!SVID_RND") == 0) 1204 pkt_dev->flags &= ~F_SVID_RND; 1205 1206 else if (strcmp(f, "FLOW_SEQ") == 0) 1207 pkt_dev->flags |= F_FLOW_SEQ; 1208 1209 else if (strcmp(f, "QUEUE_MAP_RND") == 0) 1210 pkt_dev->flags |= F_QUEUE_MAP_RND; 1211 1212 else if (strcmp(f, "!QUEUE_MAP_RND") == 0) 1213 pkt_dev->flags &= ~F_QUEUE_MAP_RND; 1214 1215 else if (strcmp(f, "QUEUE_MAP_CPU") == 0) 1216 pkt_dev->flags |= F_QUEUE_MAP_CPU; 1217 1218 else if (strcmp(f, "!QUEUE_MAP_CPU") == 0) 1219 pkt_dev->flags &= ~F_QUEUE_MAP_CPU; 1220#ifdef CONFIG_XFRM 1221 else if (strcmp(f, "IPSEC") == 0) 1222 pkt_dev->flags |= F_IPSEC_ON; 1223#endif 1224 1225 else if (strcmp(f, "!IPV6") == 0) 1226 pkt_dev->flags &= ~F_IPV6; 1227 1228 else if (strcmp(f, "NODE_ALLOC") == 0) 1229 pkt_dev->flags |= F_NODE; 1230 1231 else if (strcmp(f, "!NODE_ALLOC") == 0) 1232 pkt_dev->flags &= ~F_NODE; 1233 1234 else { 1235 sprintf(pg_result, 1236 "Flag -:%s:- unknown\nAvailable flags, (prepend ! to un-set flag):\n%s", 1237 f, 1238 "IPSRC_RND, IPDST_RND, UDPSRC_RND, UDPDST_RND, " 1239 "MACSRC_RND, MACDST_RND, TXSIZE_RND, IPV6, MPLS_RND, VID_RND, SVID_RND, FLOW_SEQ, IPSEC, NODE_ALLOC\n"); 1240 return count; 1241 } 1242 sprintf(pg_result, "OK: flags=0x%x", pkt_dev->flags); 1243 return count; 1244 } 1245 if (!strcmp(name, "dst_min") || !strcmp(name, "dst")) { 1246 len = strn_len(&user_buffer[i], sizeof(pkt_dev->dst_min) - 1); 1247 if (len < 0) 1248 return len; 1249 1250 if (copy_from_user(buf, &user_buffer[i], len)) 1251 return -EFAULT; 1252 buf[len] = 0; 1253 if (strcmp(buf, pkt_dev->dst_min) != 0) { 1254 memset(pkt_dev->dst_min, 0, sizeof(pkt_dev->dst_min)); 1255 strncpy(pkt_dev->dst_min, buf, len); 1256 pkt_dev->daddr_min = in_aton(pkt_dev->dst_min); 1257 pkt_dev->cur_daddr = pkt_dev->daddr_min; 1258 } 1259 if (debug) 1260 printk(KERN_DEBUG "pktgen: dst_min set to: %s\n", 1261 pkt_dev->dst_min); 1262 i += len; 1263 sprintf(pg_result, "OK: dst_min=%s", pkt_dev->dst_min); 1264 return count; 1265 } 1266 if (!strcmp(name, "dst_max")) { 1267 len = strn_len(&user_buffer[i], sizeof(pkt_dev->dst_max) - 1); 1268 if (len < 0) 1269 return len; 1270 1271 1272 if (copy_from_user(buf, &user_buffer[i], len)) 1273 return -EFAULT; 1274 1275 buf[len] = 0; 1276 if (strcmp(buf, pkt_dev->dst_max) != 0) { 1277 memset(pkt_dev->dst_max, 0, sizeof(pkt_dev->dst_max)); 1278 strncpy(pkt_dev->dst_max, buf, len); 1279 pkt_dev->daddr_max = in_aton(pkt_dev->dst_max); 1280 pkt_dev->cur_daddr = pkt_dev->daddr_max; 1281 } 1282 if (debug) 1283 printk(KERN_DEBUG "pktgen: dst_max set to: %s\n", 1284 pkt_dev->dst_max); 1285 i += len; 1286 sprintf(pg_result, "OK: dst_max=%s", pkt_dev->dst_max); 1287 return count; 1288 } 1289 if (!strcmp(name, "dst6")) { 1290 len = strn_len(&user_buffer[i], sizeof(buf) - 1); 1291 if (len < 0) 1292 return len; 1293 1294 pkt_dev->flags |= F_IPV6; 1295 1296 if (copy_from_user(buf, &user_buffer[i], len)) 1297 return -EFAULT; 1298 buf[len] = 0; 1299 1300 scan_ip6(buf, pkt_dev->in6_daddr.s6_addr); 1301 fmt_ip6(buf, pkt_dev->in6_daddr.s6_addr); 1302 1303 ipv6_addr_copy(&pkt_dev->cur_in6_daddr, &pkt_dev->in6_daddr); 1304 1305 if (debug) 1306 printk(KERN_DEBUG "pktgen: dst6 set to: %s\n", buf); 1307 1308 i += len; 1309 sprintf(pg_result, "OK: dst6=%s", buf); 1310 return count; 1311 } 1312 if (!strcmp(name, "dst6_min")) { 1313 len = strn_len(&user_buffer[i], sizeof(buf) - 1); 1314 if (len < 0) 1315 return len; 1316 1317 pkt_dev->flags |= F_IPV6; 1318 1319 if (copy_from_user(buf, &user_buffer[i], len)) 1320 return -EFAULT; 1321 buf[len] = 0; 1322 1323 scan_ip6(buf, pkt_dev->min_in6_daddr.s6_addr); 1324 fmt_ip6(buf, pkt_dev->min_in6_daddr.s6_addr); 1325 1326 ipv6_addr_copy(&pkt_dev->cur_in6_daddr, 1327 &pkt_dev->min_in6_daddr); 1328 if (debug) 1329 printk(KERN_DEBUG "pktgen: dst6_min set to: %s\n", buf); 1330 1331 i += len; 1332 sprintf(pg_result, "OK: dst6_min=%s", buf); 1333 return count; 1334 } 1335 if (!strcmp(name, "dst6_max")) { 1336 len = strn_len(&user_buffer[i], sizeof(buf) - 1); 1337 if (len < 0) 1338 return len; 1339 1340 pkt_dev->flags |= F_IPV6; 1341 1342 if (copy_from_user(buf, &user_buffer[i], len)) 1343 return -EFAULT; 1344 buf[len] = 0; 1345 1346 scan_ip6(buf, pkt_dev->max_in6_daddr.s6_addr); 1347 fmt_ip6(buf, pkt_dev->max_in6_daddr.s6_addr); 1348 1349 if (debug) 1350 printk(KERN_DEBUG "pktgen: dst6_max set to: %s\n", buf); 1351 1352 i += len; 1353 sprintf(pg_result, "OK: dst6_max=%s", buf); 1354 return count; 1355 } 1356 if (!strcmp(name, "src6")) { 1357 len = strn_len(&user_buffer[i], sizeof(buf) - 1); 1358 if (len < 0) 1359 return len; 1360 1361 pkt_dev->flags |= F_IPV6; 1362 1363 if (copy_from_user(buf, &user_buffer[i], len)) 1364 return -EFAULT; 1365 buf[len] = 0; 1366 1367 scan_ip6(buf, pkt_dev->in6_saddr.s6_addr); 1368 fmt_ip6(buf, pkt_dev->in6_saddr.s6_addr); 1369 1370 ipv6_addr_copy(&pkt_dev->cur_in6_saddr, &pkt_dev->in6_saddr); 1371 1372 if (debug) 1373 printk(KERN_DEBUG "pktgen: src6 set to: %s\n", buf); 1374 1375 i += len; 1376 sprintf(pg_result, "OK: src6=%s", buf); 1377 return count; 1378 } 1379 if (!strcmp(name, "src_min")) { 1380 len = strn_len(&user_buffer[i], sizeof(pkt_dev->src_min) - 1); 1381 if (len < 0) 1382 return len; 1383 1384 if (copy_from_user(buf, &user_buffer[i], len)) 1385 return -EFAULT; 1386 buf[len] = 0; 1387 if (strcmp(buf, pkt_dev->src_min) != 0) { 1388 memset(pkt_dev->src_min, 0, sizeof(pkt_dev->src_min)); 1389 strncpy(pkt_dev->src_min, buf, len); 1390 pkt_dev->saddr_min = in_aton(pkt_dev->src_min); 1391 pkt_dev->cur_saddr = pkt_dev->saddr_min; 1392 } 1393 if (debug) 1394 printk(KERN_DEBUG "pktgen: src_min set to: %s\n", 1395 pkt_dev->src_min); 1396 i += len; 1397 sprintf(pg_result, "OK: src_min=%s", pkt_dev->src_min); 1398 return count; 1399 } 1400 if (!strcmp(name, "src_max")) { 1401 len = strn_len(&user_buffer[i], sizeof(pkt_dev->src_max) - 1); 1402 if (len < 0) 1403 return len; 1404 1405 if (copy_from_user(buf, &user_buffer[i], len)) 1406 return -EFAULT; 1407 buf[len] = 0; 1408 if (strcmp(buf, pkt_dev->src_max) != 0) { 1409 memset(pkt_dev->src_max, 0, sizeof(pkt_dev->src_max)); 1410 strncpy(pkt_dev->src_max, buf, len); 1411 pkt_dev->saddr_max = in_aton(pkt_dev->src_max); 1412 pkt_dev->cur_saddr = pkt_dev->saddr_max; 1413 } 1414 if (debug) 1415 printk(KERN_DEBUG "pktgen: src_max set to: %s\n", 1416 pkt_dev->src_max); 1417 i += len; 1418 sprintf(pg_result, "OK: src_max=%s", pkt_dev->src_max); 1419 return count; 1420 } 1421 if (!strcmp(name, "dst_mac")) { 1422 char *v = valstr; 1423 unsigned char old_dmac[ETH_ALEN]; 1424 unsigned char *m = pkt_dev->dst_mac; 1425 memcpy(old_dmac, pkt_dev->dst_mac, ETH_ALEN); 1426 1427 len = strn_len(&user_buffer[i], sizeof(valstr) - 1); 1428 if (len < 0) 1429 return len; 1430 1431 memset(valstr, 0, sizeof(valstr)); 1432 if (copy_from_user(valstr, &user_buffer[i], len)) 1433 return -EFAULT; 1434 i += len; 1435 1436 for (*m = 0; *v && m < pkt_dev->dst_mac + 6; v++) { 1437 int value; 1438 1439 value = hex_to_bin(*v); 1440 if (value >= 0) 1441 *m = *m * 16 + value; 1442 1443 if (*v == ':') { 1444 m++; 1445 *m = 0; 1446 } 1447 } 1448 1449 /* Set up Dest MAC */ 1450 if (compare_ether_addr(old_dmac, pkt_dev->dst_mac)) 1451 memcpy(&(pkt_dev->hh[0]), pkt_dev->dst_mac, ETH_ALEN); 1452 1453 sprintf(pg_result, "OK: dstmac"); 1454 return count; 1455 } 1456 if (!strcmp(name, "src_mac")) { 1457 char *v = valstr; 1458 unsigned char old_smac[ETH_ALEN]; 1459 unsigned char *m = pkt_dev->src_mac; 1460 1461 memcpy(old_smac, pkt_dev->src_mac, ETH_ALEN); 1462 1463 len = strn_len(&user_buffer[i], sizeof(valstr) - 1); 1464 if (len < 0) 1465 return len; 1466 1467 memset(valstr, 0, sizeof(valstr)); 1468 if (copy_from_user(valstr, &user_buffer[i], len)) 1469 return -EFAULT; 1470 i += len; 1471 1472 for (*m = 0; *v && m < pkt_dev->src_mac + 6; v++) { 1473 int value; 1474 1475 value = hex_to_bin(*v); 1476 if (value >= 0) 1477 *m = *m * 16 + value; 1478 1479 if (*v == ':') { 1480 m++; 1481 *m = 0; 1482 } 1483 } 1484 1485 /* Set up Src MAC */ 1486 if (compare_ether_addr(old_smac, pkt_dev->src_mac)) 1487 memcpy(&(pkt_dev->hh[6]), pkt_dev->src_mac, ETH_ALEN); 1488 1489 sprintf(pg_result, "OK: srcmac"); 1490 return count; 1491 } 1492 1493 if (!strcmp(name, "clear_counters")) { 1494 pktgen_clear_counters(pkt_dev); 1495 sprintf(pg_result, "OK: Clearing counters.\n"); 1496 return count; 1497 } 1498 1499 if (!strcmp(name, "flows")) { 1500 len = num_arg(&user_buffer[i], 10, &value); 1501 if (len < 0) 1502 return len; 1503 1504 i += len; 1505 if (value > MAX_CFLOWS) 1506 value = MAX_CFLOWS; 1507 1508 pkt_dev->cflows = value; 1509 sprintf(pg_result, "OK: flows=%u", pkt_dev->cflows); 1510 return count; 1511 } 1512 1513 if (!strcmp(name, "flowlen")) { 1514 len = num_arg(&user_buffer[i], 10, &value); 1515 if (len < 0) 1516 return len; 1517 1518 i += len; 1519 pkt_dev->lflow = value; 1520 sprintf(pg_result, "OK: flowlen=%u", pkt_dev->lflow); 1521 return count; 1522 } 1523 1524 if (!strcmp(name, "queue_map_min")) { 1525 len = num_arg(&user_buffer[i], 5, &value); 1526 if (len < 0) 1527 return len; 1528 1529 i += len; 1530 pkt_dev->queue_map_min = value; 1531 sprintf(pg_result, "OK: queue_map_min=%u", pkt_dev->queue_map_min); 1532 return count; 1533 } 1534 1535 if (!strcmp(name, "queue_map_max")) { 1536 len = num_arg(&user_buffer[i], 5, &value); 1537 if (len < 0) 1538 return len; 1539 1540 i += len; 1541 pkt_dev->queue_map_max = value; 1542 sprintf(pg_result, "OK: queue_map_max=%u", pkt_dev->queue_map_max); 1543 return count; 1544 } 1545 1546 if (!strcmp(name, "mpls")) { 1547 unsigned n, cnt; 1548 1549 len = get_labels(&user_buffer[i], pkt_dev); 1550 if (len < 0) 1551 return len; 1552 i += len; 1553 cnt = sprintf(pg_result, "OK: mpls="); 1554 for (n = 0; n < pkt_dev->nr_labels; n++) 1555 cnt += sprintf(pg_result + cnt, 1556 "%08x%s", ntohl(pkt_dev->labels[n]), 1557 n == pkt_dev->nr_labels-1 ? "" : ","); 1558 1559 if (pkt_dev->nr_labels && pkt_dev->vlan_id != 0xffff) { 1560 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */ 1561 pkt_dev->svlan_id = 0xffff; 1562 1563 if (debug) 1564 printk(KERN_DEBUG "pktgen: VLAN/SVLAN auto turned off\n"); 1565 } 1566 return count; 1567 } 1568 1569 if (!strcmp(name, "vlan_id")) { 1570 len = num_arg(&user_buffer[i], 4, &value); 1571 if (len < 0) 1572 return len; 1573 1574 i += len; 1575 if (value <= 4095) { 1576 pkt_dev->vlan_id = value; /* turn on VLAN */ 1577 1578 if (debug) 1579 printk(KERN_DEBUG "pktgen: VLAN turned on\n"); 1580 1581 if (debug && pkt_dev->nr_labels) 1582 printk(KERN_DEBUG "pktgen: MPLS auto turned off\n"); 1583 1584 pkt_dev->nr_labels = 0; /* turn off MPLS */ 1585 sprintf(pg_result, "OK: vlan_id=%u", pkt_dev->vlan_id); 1586 } else { 1587 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */ 1588 pkt_dev->svlan_id = 0xffff; 1589 1590 if (debug) 1591 printk(KERN_DEBUG "pktgen: VLAN/SVLAN turned off\n"); 1592 } 1593 return count; 1594 } 1595 1596 if (!strcmp(name, "vlan_p")) { 1597 len = num_arg(&user_buffer[i], 1, &value); 1598 if (len < 0) 1599 return len; 1600 1601 i += len; 1602 if ((value <= 7) && (pkt_dev->vlan_id != 0xffff)) { 1603 pkt_dev->vlan_p = value; 1604 sprintf(pg_result, "OK: vlan_p=%u", pkt_dev->vlan_p); 1605 } else { 1606 sprintf(pg_result, "ERROR: vlan_p must be 0-7"); 1607 } 1608 return count; 1609 } 1610 1611 if (!strcmp(name, "vlan_cfi")) { 1612 len = num_arg(&user_buffer[i], 1, &value); 1613 if (len < 0) 1614 return len; 1615 1616 i += len; 1617 if ((value <= 1) && (pkt_dev->vlan_id != 0xffff)) { 1618 pkt_dev->vlan_cfi = value; 1619 sprintf(pg_result, "OK: vlan_cfi=%u", pkt_dev->vlan_cfi); 1620 } else { 1621 sprintf(pg_result, "ERROR: vlan_cfi must be 0-1"); 1622 } 1623 return count; 1624 } 1625 1626 if (!strcmp(name, "svlan_id")) { 1627 len = num_arg(&user_buffer[i], 4, &value); 1628 if (len < 0) 1629 return len; 1630 1631 i += len; 1632 if ((value <= 4095) && ((pkt_dev->vlan_id != 0xffff))) { 1633 pkt_dev->svlan_id = value; /* turn on SVLAN */ 1634 1635 if (debug) 1636 printk(KERN_DEBUG "pktgen: SVLAN turned on\n"); 1637 1638 if (debug && pkt_dev->nr_labels) 1639 printk(KERN_DEBUG "pktgen: MPLS auto turned off\n"); 1640 1641 pkt_dev->nr_labels = 0; /* turn off MPLS */ 1642 sprintf(pg_result, "OK: svlan_id=%u", pkt_dev->svlan_id); 1643 } else { 1644 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */ 1645 pkt_dev->svlan_id = 0xffff; 1646 1647 if (debug) 1648 printk(KERN_DEBUG "pktgen: VLAN/SVLAN turned off\n"); 1649 } 1650 return count; 1651 } 1652 1653 if (!strcmp(name, "svlan_p")) { 1654 len = num_arg(&user_buffer[i], 1, &value); 1655 if (len < 0) 1656 return len; 1657 1658 i += len; 1659 if ((value <= 7) && (pkt_dev->svlan_id != 0xffff)) { 1660 pkt_dev->svlan_p = value; 1661 sprintf(pg_result, "OK: svlan_p=%u", pkt_dev->svlan_p); 1662 } else { 1663 sprintf(pg_result, "ERROR: svlan_p must be 0-7"); 1664 } 1665 return count; 1666 } 1667 1668 if (!strcmp(name, "svlan_cfi")) { 1669 len = num_arg(&user_buffer[i], 1, &value); 1670 if (len < 0) 1671 return len; 1672 1673 i += len; 1674 if ((value <= 1) && (pkt_dev->svlan_id != 0xffff)) { 1675 pkt_dev->svlan_cfi = value; 1676 sprintf(pg_result, "OK: svlan_cfi=%u", pkt_dev->svlan_cfi); 1677 } else { 1678 sprintf(pg_result, "ERROR: svlan_cfi must be 0-1"); 1679 } 1680 return count; 1681 } 1682 1683 if (!strcmp(name, "tos")) { 1684 __u32 tmp_value = 0; 1685 len = hex32_arg(&user_buffer[i], 2, &tmp_value); 1686 if (len < 0) 1687 return len; 1688 1689 i += len; 1690 if (len == 2) { 1691 pkt_dev->tos = tmp_value; 1692 sprintf(pg_result, "OK: tos=0x%02x", pkt_dev->tos); 1693 } else { 1694 sprintf(pg_result, "ERROR: tos must be 00-ff"); 1695 } 1696 return count; 1697 } 1698 1699 if (!strcmp(name, "traffic_class")) { 1700 __u32 tmp_value = 0; 1701 len = hex32_arg(&user_buffer[i], 2, &tmp_value); 1702 if (len < 0) 1703 return len; 1704 1705 i += len; 1706 if (len == 2) { 1707 pkt_dev->traffic_class = tmp_value; 1708 sprintf(pg_result, "OK: traffic_class=0x%02x", pkt_dev->traffic_class); 1709 } else { 1710 sprintf(pg_result, "ERROR: traffic_class must be 00-ff"); 1711 } 1712 return count; 1713 } 1714 1715 sprintf(pkt_dev->result, "No such parameter \"%s\"", name); 1716 return -EINVAL; 1717} 1718 1719static int pktgen_if_open(struct inode *inode, struct file *file) 1720{ 1721 return single_open(file, pktgen_if_show, PDE(inode)->data); 1722} 1723 1724static const struct file_operations pktgen_if_fops = { 1725 .owner = THIS_MODULE, 1726 .open = pktgen_if_open, 1727 .read = seq_read, 1728 .llseek = seq_lseek, 1729 .write = pktgen_if_write, 1730 .release = single_release, 1731}; 1732 1733static int pktgen_thread_show(struct seq_file *seq, void *v) 1734{ 1735 struct pktgen_thread *t = seq->private; 1736 const struct pktgen_dev *pkt_dev; 1737 1738 BUG_ON(!t); 1739 1740 seq_printf(seq, "Running: "); 1741 1742 if_lock(t); 1743 list_for_each_entry(pkt_dev, &t->if_list, list) 1744 if (pkt_dev->running) 1745 seq_printf(seq, "%s ", pkt_dev->odevname); 1746 1747 seq_printf(seq, "\nStopped: "); 1748 1749 list_for_each_entry(pkt_dev, &t->if_list, list) 1750 if (!pkt_dev->running) 1751 seq_printf(seq, "%s ", pkt_dev->odevname); 1752 1753 if (t->result[0]) 1754 seq_printf(seq, "\nResult: %s\n", t->result); 1755 else 1756 seq_printf(seq, "\nResult: NA\n"); 1757 1758 if_unlock(t); 1759 1760 return 0; 1761} 1762 1763static ssize_t pktgen_thread_write(struct file *file, 1764 const char __user * user_buffer, 1765 size_t count, loff_t * offset) 1766{ 1767 struct seq_file *seq = file->private_data; 1768 struct pktgen_thread *t = seq->private; 1769 int i = 0, max, len, ret; 1770 char name[40]; 1771 char *pg_result; 1772 1773 if (count < 1) { 1774 // sprintf(pg_result, "Wrong command format"); 1775 return -EINVAL; 1776 } 1777 1778 max = count - i; 1779 len = count_trail_chars(&user_buffer[i], max); 1780 if (len < 0) 1781 return len; 1782 1783 i += len; 1784 1785 /* Read variable name */ 1786 1787 len = strn_len(&user_buffer[i], sizeof(name) - 1); 1788 if (len < 0) 1789 return len; 1790 1791 memset(name, 0, sizeof(name)); 1792 if (copy_from_user(name, &user_buffer[i], len)) 1793 return -EFAULT; 1794 i += len; 1795 1796 max = count - i; 1797 len = count_trail_chars(&user_buffer[i], max); 1798 if (len < 0) 1799 return len; 1800 1801 i += len; 1802 1803 if (debug) 1804 printk(KERN_DEBUG "pktgen: t=%s, count=%lu\n", 1805 name, (unsigned long)count); 1806 1807 if (!t) { 1808 pr_err("ERROR: No thread\n"); 1809 ret = -EINVAL; 1810 goto out; 1811 } 1812 1813 pg_result = &(t->result[0]); 1814 1815 if (!strcmp(name, "add_device")) { 1816 char f[32]; 1817 memset(f, 0, 32); 1818 len = strn_len(&user_buffer[i], sizeof(f) - 1); 1819 if (len < 0) { 1820 ret = len; 1821 goto out; 1822 } 1823 if (copy_from_user(f, &user_buffer[i], len)) 1824 return -EFAULT; 1825 i += len; 1826 mutex_lock(&pktgen_thread_lock); 1827 pktgen_add_device(t, f); 1828 mutex_unlock(&pktgen_thread_lock); 1829 ret = count; 1830 sprintf(pg_result, "OK: add_device=%s", f); 1831 goto out; 1832 } 1833 1834 if (!strcmp(name, "rem_device_all")) { 1835 mutex_lock(&pktgen_thread_lock); 1836 t->control |= T_REMDEVALL; 1837 mutex_unlock(&pktgen_thread_lock); 1838 schedule_timeout_interruptible(msecs_to_jiffies(125)); /* Propagate thread->control */ 1839 ret = count; 1840 sprintf(pg_result, "OK: rem_device_all"); 1841 goto out; 1842 } 1843 1844 if (!strcmp(name, "max_before_softirq")) { 1845 sprintf(pg_result, "OK: Note! max_before_softirq is obsoleted -- Do not use"); 1846 ret = count; 1847 goto out; 1848 } 1849 1850 ret = -EINVAL; 1851out: 1852 return ret; 1853} 1854 1855static int pktgen_thread_open(struct inode *inode, struct file *file) 1856{ 1857 return single_open(file, pktgen_thread_show, PDE(inode)->data); 1858} 1859 1860static const struct file_operations pktgen_thread_fops = { 1861 .owner = THIS_MODULE, 1862 .open = pktgen_thread_open, 1863 .read = seq_read, 1864 .llseek = seq_lseek, 1865 .write = pktgen_thread_write, 1866 .release = single_release, 1867}; 1868 1869/* Think find or remove for NN */ 1870static struct pktgen_dev *__pktgen_NN_threads(const char *ifname, int remove) 1871{ 1872 struct pktgen_thread *t; 1873 struct pktgen_dev *pkt_dev = NULL; 1874 bool exact = (remove == FIND); 1875 1876 list_for_each_entry(t, &pktgen_threads, th_list) { 1877 pkt_dev = pktgen_find_dev(t, ifname, exact); 1878 if (pkt_dev) { 1879 if (remove) { 1880 if_lock(t); 1881 pkt_dev->removal_mark = 1; 1882 t->control |= T_REMDEV; 1883 if_unlock(t); 1884 } 1885 break; 1886 } 1887 } 1888 return pkt_dev; 1889} 1890 1891/* 1892 * mark a device for removal 1893 */ 1894static void pktgen_mark_device(const char *ifname) 1895{ 1896 struct pktgen_dev *pkt_dev = NULL; 1897 const int max_tries = 10, msec_per_try = 125; 1898 int i = 0; 1899 1900 mutex_lock(&pktgen_thread_lock); 1901 pr_debug("%s: marking %s for removal\n", __func__, ifname); 1902 1903 while (1) { 1904 1905 pkt_dev = __pktgen_NN_threads(ifname, REMOVE); 1906 if (pkt_dev == NULL) 1907 break; /* success */ 1908 1909 mutex_unlock(&pktgen_thread_lock); 1910 pr_debug("%s: waiting for %s to disappear....\n", 1911 __func__, ifname); 1912 schedule_timeout_interruptible(msecs_to_jiffies(msec_per_try)); 1913 mutex_lock(&pktgen_thread_lock); 1914 1915 if (++i >= max_tries) { 1916 pr_err("%s: timed out after waiting %d msec for device %s to be removed\n", 1917 __func__, msec_per_try * i, ifname); 1918 break; 1919 } 1920 1921 } 1922 1923 mutex_unlock(&pktgen_thread_lock); 1924} 1925 1926static void pktgen_change_name(struct net_device *dev) 1927{ 1928 struct pktgen_thread *t; 1929 1930 list_for_each_entry(t, &pktgen_threads, th_list) { 1931 struct pktgen_dev *pkt_dev; 1932 1933 list_for_each_entry(pkt_dev, &t->if_list, list) { 1934 if (pkt_dev->odev != dev) 1935 continue; 1936 1937 remove_proc_entry(pkt_dev->entry->name, pg_proc_dir); 1938 1939 pkt_dev->entry = proc_create_data(dev->name, 0600, 1940 pg_proc_dir, 1941 &pktgen_if_fops, 1942 pkt_dev); 1943 if (!pkt_dev->entry) 1944 pr_err("can't move proc entry for '%s'\n", 1945 dev->name); 1946 break; 1947 } 1948 } 1949} 1950 1951static int pktgen_device_event(struct notifier_block *unused, 1952 unsigned long event, void *ptr) 1953{ 1954 struct net_device *dev = ptr; 1955 1956 if (!net_eq(dev_net(dev), &init_net)) 1957 return NOTIFY_DONE; 1958 1959 /* It is OK that we do not hold the group lock right now, 1960 * as we run under the RTNL lock. 1961 */ 1962 1963 switch (event) { 1964 case NETDEV_CHANGENAME: 1965 pktgen_change_name(dev); 1966 break; 1967 1968 case NETDEV_UNREGISTER: 1969 pktgen_mark_device(dev->name); 1970 break; 1971 } 1972 1973 return NOTIFY_DONE; 1974} 1975 1976static struct net_device *pktgen_dev_get_by_name(struct pktgen_dev *pkt_dev, 1977 const char *ifname) 1978{ 1979 char b[IFNAMSIZ+5]; 1980 int i = 0; 1981 1982 for (i = 0; ifname[i] != '@'; i++) { 1983 if (i == IFNAMSIZ) 1984 break; 1985 1986 b[i] = ifname[i]; 1987 } 1988 b[i] = 0; 1989 1990 return dev_get_by_name(&init_net, b); 1991} 1992 1993 1994/* Associate pktgen_dev with a device. */ 1995 1996static int pktgen_setup_dev(struct pktgen_dev *pkt_dev, const char *ifname) 1997{ 1998 struct net_device *odev; 1999 int err; 2000 2001 /* Clean old setups */ 2002 if (pkt_dev->odev) { 2003 dev_put(pkt_dev->odev); 2004 pkt_dev->odev = NULL; 2005 } 2006 2007 odev = pktgen_dev_get_by_name(pkt_dev, ifname); 2008 if (!odev) { 2009 pr_err("no such netdevice: \"%s\"\n", ifname); 2010 return -ENODEV; 2011 } 2012 2013 if (odev->type != ARPHRD_ETHER) { 2014 pr_err("not an ethernet device: \"%s\"\n", ifname); 2015 err = -EINVAL; 2016 } else if (!netif_running(odev)) { 2017 pr_err("device is down: \"%s\"\n", ifname); 2018 err = -ENETDOWN; 2019 } else { 2020 pkt_dev->odev = odev; 2021 return 0; 2022 } 2023 2024 dev_put(odev); 2025 return err; 2026} 2027 2028/* Read pkt_dev from the interface and set up internal pktgen_dev 2029 * structure to have the right information to create/send packets 2030 */ 2031static void pktgen_setup_inject(struct pktgen_dev *pkt_dev) 2032{ 2033 int ntxq; 2034 2035 if (!pkt_dev->odev) { 2036 pr_err("ERROR: pkt_dev->odev == NULL in setup_inject\n"); 2037 sprintf(pkt_dev->result, 2038 "ERROR: pkt_dev->odev == NULL in setup_inject.\n"); 2039 return; 2040 } 2041 2042 /* make sure that we don't pick a non-existing transmit queue */ 2043 ntxq = pkt_dev->odev->real_num_tx_queues; 2044 2045 if (ntxq <= pkt_dev->queue_map_min) { 2046 pr_warning("WARNING: Requested queue_map_min (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n", 2047 pkt_dev->queue_map_min, (ntxq ?: 1) - 1, ntxq, 2048 pkt_dev->odevname); 2049 pkt_dev->queue_map_min = ntxq - 1; 2050 } 2051 if (pkt_dev->queue_map_max >= ntxq) { 2052 pr_warning("WARNING: Requested queue_map_max (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n", 2053 pkt_dev->queue_map_max, (ntxq ?: 1) - 1, ntxq, 2054 pkt_dev->odevname); 2055 pkt_dev->queue_map_max = ntxq - 1; 2056 } 2057 2058 /* Default to the interface's mac if not explicitly set. */ 2059 2060 if (is_zero_ether_addr(pkt_dev->src_mac)) 2061 memcpy(&(pkt_dev->hh[6]), pkt_dev->odev->dev_addr, ETH_ALEN); 2062 2063 /* Set up Dest MAC */ 2064 memcpy(&(pkt_dev->hh[0]), pkt_dev->dst_mac, ETH_ALEN); 2065 2066 /* Set up pkt size */ 2067 pkt_dev->cur_pkt_size = pkt_dev->min_pkt_size; 2068 2069 if (pkt_dev->flags & F_IPV6) { 2070 /* 2071 * Skip this automatic address setting until locks or functions 2072 * gets exported 2073 */ 2074 2075#ifdef NOTNOW 2076 int i, set = 0, err = 1; 2077 struct inet6_dev *idev; 2078 2079 for (i = 0; i < IN6_ADDR_HSIZE; i++) 2080 if (pkt_dev->cur_in6_saddr.s6_addr[i]) { 2081 set = 1; 2082 break; 2083 } 2084 2085 if (!set) { 2086 2087 /* 2088 * Use linklevel address if unconfigured. 2089 * 2090 * use ipv6_get_lladdr if/when it's get exported 2091 */ 2092 2093 rcu_read_lock(); 2094 idev = __in6_dev_get(pkt_dev->odev); 2095 if (idev) { 2096 struct inet6_ifaddr *ifp; 2097 2098 read_lock_bh(&idev->lock); 2099 for (ifp = idev->addr_list; ifp; 2100 ifp = ifp->if_next) { 2101 if (ifp->scope == IFA_LINK && 2102 !(ifp->flags & IFA_F_TENTATIVE)) { 2103 ipv6_addr_copy(&pkt_dev-> 2104 cur_in6_saddr, 2105 &ifp->addr); 2106 err = 0; 2107 break; 2108 } 2109 } 2110 read_unlock_bh(&idev->lock); 2111 } 2112 rcu_read_unlock(); 2113 if (err) 2114 pr_err("ERROR: IPv6 link address not available\n"); 2115 } 2116#endif 2117 } else { 2118 pkt_dev->saddr_min = 0; 2119 pkt_dev->saddr_max = 0; 2120 if (strlen(pkt_dev->src_min) == 0) { 2121 2122 struct in_device *in_dev; 2123 2124 rcu_read_lock(); 2125 in_dev = __in_dev_get_rcu(pkt_dev->odev); 2126 if (in_dev) { 2127 if (in_dev->ifa_list) { 2128 pkt_dev->saddr_min = 2129 in_dev->ifa_list->ifa_address; 2130 pkt_dev->saddr_max = pkt_dev->saddr_min; 2131 } 2132 } 2133 rcu_read_unlock(); 2134 } else { 2135 pkt_dev->saddr_min = in_aton(pkt_dev->src_min); 2136 pkt_dev->saddr_max = in_aton(pkt_dev->src_max); 2137 } 2138 2139 pkt_dev->daddr_min = in_aton(pkt_dev->dst_min); 2140 pkt_dev->daddr_max = in_aton(pkt_dev->dst_max); 2141 } 2142 /* Initialize current values. */ 2143 pkt_dev->cur_dst_mac_offset = 0; 2144 pkt_dev->cur_src_mac_offset = 0; 2145 pkt_dev->cur_saddr = pkt_dev->saddr_min; 2146 pkt_dev->cur_daddr = pkt_dev->daddr_min; 2147 pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min; 2148 pkt_dev->cur_udp_src = pkt_dev->udp_src_min; 2149 pkt_dev->nflows = 0; 2150} 2151 2152 2153static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until) 2154{ 2155 ktime_t start_time, end_time; 2156 s64 remaining; 2157 struct hrtimer_sleeper t; 2158 2159 hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 2160 hrtimer_set_expires(&t.timer, spin_until); 2161 2162 remaining = ktime_to_ns(hrtimer_expires_remaining(&t.timer)); 2163 if (remaining <= 0) { 2164 pkt_dev->next_tx = ktime_add_ns(spin_until, pkt_dev->delay); 2165 return; 2166 } 2167 2168 start_time = ktime_now(); 2169 if (remaining < 100000) 2170 ndelay(remaining); /* really small just spin */ 2171 else { 2172 /* see do_nanosleep */ 2173 hrtimer_init_sleeper(&t, current); 2174 do { 2175 set_current_state(TASK_INTERRUPTIBLE); 2176 hrtimer_start_expires(&t.timer, HRTIMER_MODE_ABS); 2177 if (!hrtimer_active(&t.timer)) 2178 t.task = NULL; 2179 2180 if (likely(t.task)) 2181 schedule(); 2182 2183 hrtimer_cancel(&t.timer); 2184 } while (t.task && pkt_dev->running && !signal_pending(current)); 2185 __set_current_state(TASK_RUNNING); 2186 } 2187 end_time = ktime_now(); 2188 2189 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(end_time, start_time)); 2190 pkt_dev->next_tx = ktime_add_ns(spin_until, pkt_dev->delay); 2191} 2192 2193static inline void set_pkt_overhead(struct pktgen_dev *pkt_dev) 2194{ 2195 pkt_dev->pkt_overhead = 0; 2196 pkt_dev->pkt_overhead += pkt_dev->nr_labels*sizeof(u32); 2197 pkt_dev->pkt_overhead += VLAN_TAG_SIZE(pkt_dev); 2198 pkt_dev->pkt_overhead += SVLAN_TAG_SIZE(pkt_dev); 2199} 2200 2201static inline int f_seen(const struct pktgen_dev *pkt_dev, int flow) 2202{ 2203 return !!(pkt_dev->flows[flow].flags & F_INIT); 2204} 2205 2206static inline int f_pick(struct pktgen_dev *pkt_dev) 2207{ 2208 int flow = pkt_dev->curfl; 2209 2210 if (pkt_dev->flags & F_FLOW_SEQ) { 2211 if (pkt_dev->flows[flow].count >= pkt_dev->lflow) { 2212 /* reset time */ 2213 pkt_dev->flows[flow].count = 0; 2214 pkt_dev->flows[flow].flags = 0; 2215 pkt_dev->curfl += 1; 2216 if (pkt_dev->curfl >= pkt_dev->cflows) 2217 pkt_dev->curfl = 0; /*reset */ 2218 } 2219 } else { 2220 flow = random32() % pkt_dev->cflows; 2221 pkt_dev->curfl = flow; 2222 2223 if (pkt_dev->flows[flow].count > pkt_dev->lflow) { 2224 pkt_dev->flows[flow].count = 0; 2225 pkt_dev->flows[flow].flags = 0; 2226 } 2227 } 2228 2229 return pkt_dev->curfl; 2230} 2231 2232 2233#ifdef CONFIG_XFRM 2234/* If there was already an IPSEC SA, we keep it as is, else 2235 * we go look for it ... 2236*/ 2237#define DUMMY_MARK 0 2238static void get_ipsec_sa(struct pktgen_dev *pkt_dev, int flow) 2239{ 2240 struct xfrm_state *x = pkt_dev->flows[flow].x; 2241 if (!x) { 2242 /*slow path: we dont already have xfrm_state*/ 2243 x = xfrm_stateonly_find(&init_net, DUMMY_MARK, 2244 (xfrm_address_t *)&pkt_dev->cur_daddr, 2245 (xfrm_address_t *)&pkt_dev->cur_saddr, 2246 AF_INET, 2247 pkt_dev->ipsmode, 2248 pkt_dev->ipsproto, 0); 2249 if (x) { 2250 pkt_dev->flows[flow].x = x; 2251 set_pkt_overhead(pkt_dev); 2252 pkt_dev->pkt_overhead += x->props.header_len; 2253 } 2254 2255 } 2256} 2257#endif 2258static void set_cur_queue_map(struct pktgen_dev *pkt_dev) 2259{ 2260 2261 if (pkt_dev->flags & F_QUEUE_MAP_CPU) 2262 pkt_dev->cur_queue_map = smp_processor_id(); 2263 2264 else if (pkt_dev->queue_map_min <= pkt_dev->queue_map_max) { 2265 __u16 t; 2266 if (pkt_dev->flags & F_QUEUE_MAP_RND) { 2267 t = random32() % 2268 (pkt_dev->queue_map_max - 2269 pkt_dev->queue_map_min + 1) 2270 + pkt_dev->queue_map_min; 2271 } else { 2272 t = pkt_dev->cur_queue_map + 1; 2273 if (t > pkt_dev->queue_map_max) 2274 t = pkt_dev->queue_map_min; 2275 } 2276 pkt_dev->cur_queue_map = t; 2277 } 2278 pkt_dev->cur_queue_map = pkt_dev->cur_queue_map % pkt_dev->odev->real_num_tx_queues; 2279} 2280 2281/* Increment/randomize headers according to flags and current values 2282 * for IP src/dest, UDP src/dst port, MAC-Addr src/dst 2283 */ 2284static void mod_cur_headers(struct pktgen_dev *pkt_dev) 2285{ 2286 __u32 imn; 2287 __u32 imx; 2288 int flow = 0; 2289 2290 if (pkt_dev->cflows) 2291 flow = f_pick(pkt_dev); 2292 2293 /* Deal with source MAC */ 2294 if (pkt_dev->src_mac_count > 1) { 2295 __u32 mc; 2296 __u32 tmp; 2297 2298 if (pkt_dev->flags & F_MACSRC_RND) 2299 mc = random32() % pkt_dev->src_mac_count; 2300 else { 2301 mc = pkt_dev->cur_src_mac_offset++; 2302 if (pkt_dev->cur_src_mac_offset >= 2303 pkt_dev->src_mac_count) 2304 pkt_dev->cur_src_mac_offset = 0; 2305 } 2306 2307 tmp = pkt_dev->src_mac[5] + (mc & 0xFF); 2308 pkt_dev->hh[11] = tmp; 2309 tmp = (pkt_dev->src_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8)); 2310 pkt_dev->hh[10] = tmp; 2311 tmp = (pkt_dev->src_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8)); 2312 pkt_dev->hh[9] = tmp; 2313 tmp = (pkt_dev->src_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8)); 2314 pkt_dev->hh[8] = tmp; 2315 tmp = (pkt_dev->src_mac[1] + (tmp >> 8)); 2316 pkt_dev->hh[7] = tmp; 2317 } 2318 2319 /* Deal with Destination MAC */ 2320 if (pkt_dev->dst_mac_count > 1) { 2321 __u32 mc; 2322 __u32 tmp; 2323 2324 if (pkt_dev->flags & F_MACDST_RND) 2325 mc = random32() % pkt_dev->dst_mac_count; 2326 2327 else { 2328 mc = pkt_dev->cur_dst_mac_offset++; 2329 if (pkt_dev->cur_dst_mac_offset >= 2330 pkt_dev->dst_mac_count) { 2331 pkt_dev->cur_dst_mac_offset = 0; 2332 } 2333 } 2334 2335 tmp = pkt_dev->dst_mac[5] + (mc & 0xFF); 2336 pkt_dev->hh[5] = tmp; 2337 tmp = (pkt_dev->dst_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8)); 2338 pkt_dev->hh[4] = tmp; 2339 tmp = (pkt_dev->dst_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8)); 2340 pkt_dev->hh[3] = tmp; 2341 tmp = (pkt_dev->dst_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8)); 2342 pkt_dev->hh[2] = tmp; 2343 tmp = (pkt_dev->dst_mac[1] + (tmp >> 8)); 2344 pkt_dev->hh[1] = tmp; 2345 } 2346 2347 if (pkt_dev->flags & F_MPLS_RND) { 2348 unsigned i; 2349 for (i = 0; i < pkt_dev->nr_labels; i++) 2350 if (pkt_dev->labels[i] & MPLS_STACK_BOTTOM) 2351 pkt_dev->labels[i] = MPLS_STACK_BOTTOM | 2352 ((__force __be32)random32() & 2353 htonl(0x000fffff)); 2354 } 2355 2356 if ((pkt_dev->flags & F_VID_RND) && (pkt_dev->vlan_id != 0xffff)) { 2357 pkt_dev->vlan_id = random32() & (4096-1); 2358 } 2359 2360 if ((pkt_dev->flags & F_SVID_RND) && (pkt_dev->svlan_id != 0xffff)) { 2361 pkt_dev->svlan_id = random32() & (4096 - 1); 2362 } 2363 2364 if (pkt_dev->udp_src_min < pkt_dev->udp_src_max) { 2365 if (pkt_dev->flags & F_UDPSRC_RND) 2366 pkt_dev->cur_udp_src = random32() % 2367 (pkt_dev->udp_src_max - pkt_dev->udp_src_min) 2368 + pkt_dev->udp_src_min; 2369 2370 else { 2371 pkt_dev->cur_udp_src++; 2372 if (pkt_dev->cur_udp_src >= pkt_dev->udp_src_max) 2373 pkt_dev->cur_udp_src = pkt_dev->udp_src_min; 2374 } 2375 } 2376 2377 if (pkt_dev->udp_dst_min < pkt_dev->udp_dst_max) { 2378 if (pkt_dev->flags & F_UDPDST_RND) { 2379 pkt_dev->cur_udp_dst = random32() % 2380 (pkt_dev->udp_dst_max - pkt_dev->udp_dst_min) 2381 + pkt_dev->udp_dst_min; 2382 } else { 2383 pkt_dev->cur_udp_dst++; 2384 if (pkt_dev->cur_udp_dst >= pkt_dev->udp_dst_max) 2385 pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min; 2386 } 2387 } 2388 2389 if (!(pkt_dev->flags & F_IPV6)) { 2390 2391 imn = ntohl(pkt_dev->saddr_min); 2392 imx = ntohl(pkt_dev->saddr_max); 2393 if (imn < imx) { 2394 __u32 t; 2395 if (pkt_dev->flags & F_IPSRC_RND) 2396 t = random32() % (imx - imn) + imn; 2397 else { 2398 t = ntohl(pkt_dev->cur_saddr); 2399 t++; 2400 if (t > imx) 2401 t = imn; 2402 2403 } 2404 pkt_dev->cur_saddr = htonl(t); 2405 } 2406 2407 if (pkt_dev->cflows && f_seen(pkt_dev, flow)) { 2408 pkt_dev->cur_daddr = pkt_dev->flows[flow].cur_daddr; 2409 } else { 2410 imn = ntohl(pkt_dev->daddr_min); 2411 imx = ntohl(pkt_dev->daddr_max); 2412 if (imn < imx) { 2413 __u32 t; 2414 __be32 s; 2415 if (pkt_dev->flags & F_IPDST_RND) { 2416 2417 t = random32() % (imx - imn) + imn; 2418 s = htonl(t); 2419 2420 while (ipv4_is_loopback(s) || 2421 ipv4_is_multicast(s) || 2422 ipv4_is_lbcast(s) || 2423 ipv4_is_zeronet(s) || 2424 ipv4_is_local_multicast(s)) { 2425 t = random32() % (imx - imn) + imn; 2426 s = htonl(t); 2427 } 2428 pkt_dev->cur_daddr = s; 2429 } else { 2430 t = ntohl(pkt_dev->cur_daddr); 2431 t++; 2432 if (t > imx) { 2433 t = imn; 2434 } 2435 pkt_dev->cur_daddr = htonl(t); 2436 } 2437 } 2438 if (pkt_dev->cflows) { 2439 pkt_dev->flows[flow].flags |= F_INIT; 2440 pkt_dev->flows[flow].cur_daddr = 2441 pkt_dev->cur_daddr; 2442#ifdef CONFIG_XFRM 2443 if (pkt_dev->flags & F_IPSEC_ON) 2444 get_ipsec_sa(pkt_dev, flow); 2445#endif 2446 pkt_dev->nflows++; 2447 } 2448 } 2449 } else { /* IPV6 * */ 2450 2451 if (pkt_dev->min_in6_daddr.s6_addr32[0] == 0 && 2452 pkt_dev->min_in6_daddr.s6_addr32[1] == 0 && 2453 pkt_dev->min_in6_daddr.s6_addr32[2] == 0 && 2454 pkt_dev->min_in6_daddr.s6_addr32[3] == 0) ; 2455 else { 2456 int i; 2457 2458 /* Only random destinations yet */ 2459 2460 for (i = 0; i < 4; i++) { 2461 pkt_dev->cur_in6_daddr.s6_addr32[i] = 2462 (((__force __be32)random32() | 2463 pkt_dev->min_in6_daddr.s6_addr32[i]) & 2464 pkt_dev->max_in6_daddr.s6_addr32[i]); 2465 } 2466 } 2467 } 2468 2469 if (pkt_dev->min_pkt_size < pkt_dev->max_pkt_size) { 2470 __u32 t; 2471 if (pkt_dev->flags & F_TXSIZE_RND) { 2472 t = random32() % 2473 (pkt_dev->max_pkt_size - pkt_dev->min_pkt_size) 2474 + pkt_dev->min_pkt_size; 2475 } else { 2476 t = pkt_dev->cur_pkt_size + 1; 2477 if (t > pkt_dev->max_pkt_size) 2478 t = pkt_dev->min_pkt_size; 2479 } 2480 pkt_dev->cur_pkt_size = t; 2481 } 2482 2483 set_cur_queue_map(pkt_dev); 2484 2485 pkt_dev->flows[flow].count++; 2486} 2487 2488 2489#ifdef CONFIG_XFRM 2490static int pktgen_output_ipsec(struct sk_buff *skb, struct pktgen_dev *pkt_dev) 2491{ 2492 struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x; 2493 int err = 0; 2494 struct iphdr *iph; 2495 2496 if (!x) 2497 return 0; 2498 if (x->props.mode != XFRM_MODE_TRANSPORT) 2499 return 0; 2500 2501 spin_lock(&x->lock); 2502 iph = ip_hdr(skb); 2503 2504 err = x->outer_mode->output(x, skb); 2505 if (err) 2506 goto error; 2507 err = x->type->output(x, skb); 2508 if (err) 2509 goto error; 2510 2511 x->curlft.bytes += skb->len; 2512 x->curlft.packets++; 2513error: 2514 spin_unlock(&x->lock); 2515 return err; 2516} 2517 2518static void free_SAs(struct pktgen_dev *pkt_dev) 2519{ 2520 if (pkt_dev->cflows) { 2521 /* let go of the SAs if we have them */ 2522 int i = 0; 2523 for (; i < pkt_dev->cflows; i++) { 2524 struct xfrm_state *x = pkt_dev->flows[i].x; 2525 if (x) { 2526 xfrm_state_put(x); 2527 pkt_dev->flows[i].x = NULL; 2528 } 2529 } 2530 } 2531} 2532 2533static int process_ipsec(struct pktgen_dev *pkt_dev, 2534 struct sk_buff *skb, __be16 protocol) 2535{ 2536 if (pkt_dev->flags & F_IPSEC_ON) { 2537 struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x; 2538 int nhead = 0; 2539 if (x) { 2540 int ret; 2541 __u8 *eth; 2542 nhead = x->props.header_len - skb_headroom(skb); 2543 if (nhead > 0) { 2544 ret = pskb_expand_head(skb, nhead, 0, GFP_ATOMIC); 2545 if (ret < 0) { 2546 pr_err("Error expanding ipsec packet %d\n", 2547 ret); 2548 goto err; 2549 } 2550 } 2551 2552 /* ipsec is not expecting ll header */ 2553 skb_pull(skb, ETH_HLEN); 2554 ret = pktgen_output_ipsec(skb, pkt_dev); 2555 if (ret) { 2556 pr_err("Error creating ipsec packet %d\n", ret); 2557 goto err; 2558 } 2559 /* restore ll */ 2560 eth = (__u8 *) skb_push(skb, ETH_HLEN); 2561 memcpy(eth, pkt_dev->hh, 12); 2562 *(u16 *) ð[12] = protocol; 2563 } 2564 } 2565 return 1; 2566err: 2567 kfree_skb(skb); 2568 return 0; 2569} 2570#endif 2571 2572static void mpls_push(__be32 *mpls, struct pktgen_dev *pkt_dev) 2573{ 2574 unsigned i; 2575 for (i = 0; i < pkt_dev->nr_labels; i++) 2576 *mpls++ = pkt_dev->labels[i] & ~MPLS_STACK_BOTTOM; 2577 2578 mpls--; 2579 *mpls |= MPLS_STACK_BOTTOM; 2580} 2581 2582static inline __be16 build_tci(unsigned int id, unsigned int cfi, 2583 unsigned int prio) 2584{ 2585 return htons(id | (cfi << 12) | (prio << 13)); 2586} 2587 2588static struct sk_buff *fill_packet_ipv4(struct net_device *odev, 2589 struct pktgen_dev *pkt_dev) 2590{ 2591 struct sk_buff *skb = NULL; 2592 __u8 *eth; 2593 struct udphdr *udph; 2594 int datalen, iplen; 2595 struct iphdr *iph; 2596 struct pktgen_hdr *pgh = NULL; 2597 __be16 protocol = htons(ETH_P_IP); 2598 __be32 *mpls; 2599 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */ 2600 __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */ 2601 __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */ 2602 __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */ 2603 u16 queue_map; 2604 2605 if (pkt_dev->nr_labels) 2606 protocol = htons(ETH_P_MPLS_UC); 2607 2608 if (pkt_dev->vlan_id != 0xffff) 2609 protocol = htons(ETH_P_8021Q); 2610 2611 /* Update any of the values, used when we're incrementing various 2612 * fields. 2613 */ 2614 queue_map = pkt_dev->cur_queue_map; 2615 mod_cur_headers(pkt_dev); 2616 2617 datalen = (odev->hard_header_len + 16) & ~0xf; 2618 2619 if (pkt_dev->flags & F_NODE) { 2620 int node; 2621 2622 if (pkt_dev->node >= 0) 2623 node = pkt_dev->node; 2624 else 2625 node = numa_node_id(); 2626 2627 skb = __alloc_skb(NET_SKB_PAD + pkt_dev->cur_pkt_size + 64 2628 + datalen + pkt_dev->pkt_overhead, GFP_NOWAIT, 0, node); 2629 if (likely(skb)) { 2630 skb_reserve(skb, NET_SKB_PAD); 2631 skb->dev = odev; 2632 } 2633 } 2634 else 2635 skb = __netdev_alloc_skb(odev, 2636 pkt_dev->cur_pkt_size + 64 2637 + datalen + pkt_dev->pkt_overhead, GFP_NOWAIT); 2638 2639 if (!skb) { 2640 sprintf(pkt_dev->result, "No memory"); 2641 return NULL; 2642 } 2643 2644 skb_reserve(skb, datalen); 2645 2646 /* Reserve for ethernet and IP header */ 2647 eth = (__u8 *) skb_push(skb, 14); 2648 mpls = (__be32 *)skb_put(skb, pkt_dev->nr_labels*sizeof(__u32)); 2649 if (pkt_dev->nr_labels) 2650 mpls_push(mpls, pkt_dev); 2651 2652 if (pkt_dev->vlan_id != 0xffff) { 2653 if (pkt_dev->svlan_id != 0xffff) { 2654 svlan_tci = (__be16 *)skb_put(skb, sizeof(__be16)); 2655 *svlan_tci = build_tci(pkt_dev->svlan_id, 2656 pkt_dev->svlan_cfi, 2657 pkt_dev->svlan_p); 2658 svlan_encapsulated_proto = (__be16 *)skb_put(skb, sizeof(__be16)); 2659 *svlan_encapsulated_proto = htons(ETH_P_8021Q); 2660 } 2661 vlan_tci = (__be16 *)skb_put(skb, sizeof(__be16)); 2662 *vlan_tci = build_tci(pkt_dev->vlan_id, 2663 pkt_dev->vlan_cfi, 2664 pkt_dev->vlan_p); 2665 vlan_encapsulated_proto = (__be16 *)skb_put(skb, sizeof(__be16)); 2666 *vlan_encapsulated_proto = htons(ETH_P_IP); 2667 } 2668 2669 skb->network_header = skb->tail; 2670 skb->transport_header = skb->network_header + sizeof(struct iphdr); 2671 skb_put(skb, sizeof(struct iphdr) + sizeof(struct udphdr)); 2672 skb_set_queue_mapping(skb, queue_map); 2673 iph = ip_hdr(skb); 2674 udph = udp_hdr(skb); 2675 2676 memcpy(eth, pkt_dev->hh, 12); 2677 *(__be16 *) & eth[12] = protocol; 2678 2679 /* Eth + IPh + UDPh + mpls */ 2680 datalen = pkt_dev->cur_pkt_size - 14 - 20 - 8 - 2681 pkt_dev->pkt_overhead; 2682 if (datalen < sizeof(struct pktgen_hdr)) 2683 datalen = sizeof(struct pktgen_hdr); 2684 2685 udph->source = htons(pkt_dev->cur_udp_src); 2686 udph->dest = htons(pkt_dev->cur_udp_dst); 2687 udph->len = htons(datalen + 8); /* DATA + udphdr */ 2688 udph->check = 0; /* No checksum */ 2689 2690 iph->ihl = 5; 2691 iph->version = 4; 2692 iph->ttl = 32; 2693 iph->tos = pkt_dev->tos; 2694 iph->protocol = IPPROTO_UDP; /* UDP */ 2695 iph->saddr = pkt_dev->cur_saddr; 2696 iph->daddr = pkt_dev->cur_daddr; 2697 iph->id = htons(pkt_dev->ip_id); 2698 pkt_dev->ip_id++; 2699 iph->frag_off = 0; 2700 iplen = 20 + 8 + datalen; 2701 iph->tot_len = htons(iplen); 2702 iph->check = 0; 2703 iph->check = ip_fast_csum((void *)iph, iph->ihl); 2704 skb->protocol = protocol; 2705 skb->mac_header = (skb->network_header - ETH_HLEN - 2706 pkt_dev->pkt_overhead); 2707 skb->dev = odev; 2708 skb->pkt_type = PACKET_HOST; 2709 2710 if (pkt_dev->nfrags <= 0) { 2711 pgh = (struct pktgen_hdr *)skb_put(skb, datalen); 2712 memset(pgh + 1, 0, datalen - sizeof(struct pktgen_hdr)); 2713 } else { 2714 int frags = pkt_dev->nfrags; 2715 int i, len; 2716 2717 pgh = (struct pktgen_hdr *)(((char *)(udph)) + 8); 2718 2719 if (frags > MAX_SKB_FRAGS) 2720 frags = MAX_SKB_FRAGS; 2721 if (datalen > frags * PAGE_SIZE) { 2722 len = datalen - frags * PAGE_SIZE; 2723 memset(skb_put(skb, len), 0, len); 2724 datalen = frags * PAGE_SIZE; 2725 } 2726 2727 i = 0; 2728 while (datalen > 0) { 2729 struct page *page = alloc_pages(GFP_KERNEL | __GFP_ZERO, 0); 2730 skb_shinfo(skb)->frags[i].page = page; 2731 skb_shinfo(skb)->frags[i].page_offset = 0; 2732 skb_shinfo(skb)->frags[i].size = 2733 (datalen < PAGE_SIZE ? datalen : PAGE_SIZE); 2734 datalen -= skb_shinfo(skb)->frags[i].size; 2735 skb->len += skb_shinfo(skb)->frags[i].size; 2736 skb->data_len += skb_shinfo(skb)->frags[i].size; 2737 i++; 2738 skb_shinfo(skb)->nr_frags = i; 2739 } 2740 2741 while (i < frags) { 2742 int rem; 2743 2744 if (i == 0) 2745 break; 2746 2747 rem = skb_shinfo(skb)->frags[i - 1].size / 2; 2748 if (rem == 0) 2749 break; 2750 2751 skb_shinfo(skb)->frags[i - 1].size -= rem; 2752 2753 skb_shinfo(skb)->frags[i] = 2754 skb_shinfo(skb)->frags[i - 1]; 2755 get_page(skb_shinfo(skb)->frags[i].page); 2756 skb_shinfo(skb)->frags[i].page = 2757 skb_shinfo(skb)->frags[i - 1].page; 2758 skb_shinfo(skb)->frags[i].page_offset += 2759 skb_shinfo(skb)->frags[i - 1].size; 2760 skb_shinfo(skb)->frags[i].size = rem; 2761 i++; 2762 skb_shinfo(skb)->nr_frags = i; 2763 } 2764 } 2765 2766 /* Stamp the time, and sequence number, 2767 * convert them to network byte order 2768 */ 2769 if (pgh) { 2770 struct timeval timestamp; 2771 2772 pgh->pgh_magic = htonl(PKTGEN_MAGIC); 2773 pgh->seq_num = htonl(pkt_dev->seq_num); 2774 2775 do_gettimeofday(×tamp); 2776 pgh->tv_sec = htonl(timestamp.tv_sec); 2777 pgh->tv_usec = htonl(timestamp.tv_usec); 2778 } 2779 2780#ifdef CONFIG_XFRM 2781 if (!process_ipsec(pkt_dev, skb, protocol)) 2782 return NULL; 2783#endif 2784 2785 return skb; 2786} 2787 2788/* 2789 * scan_ip6, fmt_ip taken from dietlibc-0.21 2790 * Author Felix von Leitner <felix-dietlibc@fefe.de> 2791 * 2792 * Slightly modified for kernel. 2793 * Should be candidate for net/ipv4/utils.c 2794 * --ro 2795 */ 2796 2797static unsigned int scan_ip6(const char *s, char ip[16]) 2798{ 2799 unsigned int i; 2800 unsigned int len = 0; 2801 unsigned long u; 2802 char suffix[16]; 2803 unsigned int prefixlen = 0; 2804 unsigned int suffixlen = 0; 2805 __be32 tmp; 2806 char *pos; 2807 2808 for (i = 0; i < 16; i++) 2809 ip[i] = 0; 2810 2811 for (;;) { 2812 if (*s == ':') { 2813 len++; 2814 if (s[1] == ':') { /* Found "::", skip to part 2 */ 2815 s += 2; 2816 len++; 2817 break; 2818 } 2819 s++; 2820 } 2821 2822 u = simple_strtoul(s, &pos, 16); 2823 i = pos - s; 2824 if (!i) 2825 return 0; 2826 if (prefixlen == 12 && s[i] == '.') { 2827 2828 /* the last 4 bytes may be written as IPv4 address */ 2829 2830 tmp = in_aton(s); 2831 memcpy((struct in_addr *)(ip + 12), &tmp, sizeof(tmp)); 2832 return i + len; 2833 } 2834 ip[prefixlen++] = (u >> 8); 2835 ip[prefixlen++] = (u & 255); 2836 s += i; 2837 len += i; 2838 if (prefixlen == 16) 2839 return len; 2840 } 2841 2842/* part 2, after "::" */ 2843 for (;;) { 2844 if (*s == ':') { 2845 if (suffixlen == 0) 2846 break; 2847 s++; 2848 len++; 2849 } else if (suffixlen != 0) 2850 break; 2851 2852 u = simple_strtol(s, &pos, 16); 2853 i = pos - s; 2854 if (!i) { 2855 if (*s) 2856 len--; 2857 break; 2858 } 2859 if (suffixlen + prefixlen <= 12 && s[i] == '.') { 2860 tmp = in_aton(s); 2861 memcpy((struct in_addr *)(suffix + suffixlen), &tmp, 2862 sizeof(tmp)); 2863 suffixlen += 4; 2864 len += strlen(s); 2865 break; 2866 } 2867 suffix[suffixlen++] = (u >> 8); 2868 suffix[suffixlen++] = (u & 255); 2869 s += i; 2870 len += i; 2871 if (prefixlen + suffixlen == 16) 2872 break; 2873 } 2874 for (i = 0; i < suffixlen; i++) 2875 ip[16 - suffixlen + i] = suffix[i]; 2876 return len; 2877} 2878 2879static char tohex(char hexdigit) 2880{ 2881 return hexdigit > 9 ? hexdigit + 'a' - 10 : hexdigit + '0'; 2882} 2883 2884static int fmt_xlong(char *s, unsigned int i) 2885{ 2886 char *bak = s; 2887 *s = tohex((i >> 12) & 0xf); 2888 if (s != bak || *s != '0') 2889 ++s; 2890 *s = tohex((i >> 8) & 0xf); 2891 if (s != bak || *s != '0') 2892 ++s; 2893 *s = tohex((i >> 4) & 0xf); 2894 if (s != bak || *s != '0') 2895 ++s; 2896 *s = tohex(i & 0xf); 2897 return s - bak + 1; 2898} 2899 2900static unsigned int fmt_ip6(char *s, const char ip[16]) 2901{ 2902 unsigned int len; 2903 unsigned int i; 2904 unsigned int temp; 2905 unsigned int compressing; 2906 int j; 2907 2908 len = 0; 2909 compressing = 0; 2910 for (j = 0; j < 16; j += 2) { 2911 2912#ifdef V4MAPPEDPREFIX 2913 if (j == 12 && !memcmp(ip, V4mappedprefix, 12)) { 2914 inet_ntoa_r(*(struct in_addr *)(ip + 12), s); 2915 temp = strlen(s); 2916 return len + temp; 2917 } 2918#endif 2919 temp = ((unsigned long)(unsigned char)ip[j] << 8) + 2920 (unsigned long)(unsigned char)ip[j + 1]; 2921 if (temp == 0) { 2922 if (!compressing) { 2923 compressing = 1; 2924 if (j == 0) { 2925 *s++ = ':'; 2926 ++len; 2927 } 2928 } 2929 } else { 2930 if (compressing) { 2931 compressing = 0; 2932 *s++ = ':'; 2933 ++len; 2934 } 2935 i = fmt_xlong(s, temp); 2936 len += i; 2937 s += i; 2938 if (j < 14) { 2939 *s++ = ':'; 2940 ++len; 2941 } 2942 } 2943 } 2944 if (compressing) { 2945 *s++ = ':'; 2946 ++len; 2947 } 2948 *s = 0; 2949 return len; 2950} 2951 2952static struct sk_buff *fill_packet_ipv6(struct net_device *odev, 2953 struct pktgen_dev *pkt_dev) 2954{ 2955 struct sk_buff *skb = NULL; 2956 __u8 *eth; 2957 struct udphdr *udph; 2958 int datalen; 2959 struct ipv6hdr *iph; 2960 struct pktgen_hdr *pgh = NULL; 2961 __be16 protocol = htons(ETH_P_IPV6); 2962 __be32 *mpls; 2963 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */ 2964 __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */ 2965 __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */ 2966 __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */ 2967 u16 queue_map; 2968 2969 if (pkt_dev->nr_labels) 2970 protocol = htons(ETH_P_MPLS_UC); 2971 2972 if (pkt_dev->vlan_id != 0xffff) 2973 protocol = htons(ETH_P_8021Q); 2974 2975 /* Update any of the values, used when we're incrementing various 2976 * fields. 2977 */ 2978 queue_map = pkt_dev->cur_queue_map; 2979 mod_cur_headers(pkt_dev); 2980 2981 skb = __netdev_alloc_skb(odev, 2982 pkt_dev->cur_pkt_size + 64 2983 + 16 + pkt_dev->pkt_overhead, GFP_NOWAIT); 2984 if (!skb) { 2985 sprintf(pkt_dev->result, "No memory"); 2986 return NULL; 2987 } 2988 2989 skb_reserve(skb, 16); 2990 2991 /* Reserve for ethernet and IP header */ 2992 eth = (__u8 *) skb_push(skb, 14); 2993 mpls = (__be32 *)skb_put(skb, pkt_dev->nr_labels*sizeof(__u32)); 2994 if (pkt_dev->nr_labels) 2995 mpls_push(mpls, pkt_dev); 2996 2997 if (pkt_dev->vlan_id != 0xffff) { 2998 if (pkt_dev->svlan_id != 0xffff) { 2999 svlan_tci = (__be16 *)skb_put(skb, sizeof(__be16)); 3000 *svlan_tci = build_tci(pkt_dev->svlan_id, 3001 pkt_dev->svlan_cfi, 3002 pkt_dev->svlan_p); 3003 svlan_encapsulated_proto = (__be16 *)skb_put(skb, sizeof(__be16)); 3004 *svlan_encapsulated_proto = htons(ETH_P_8021Q); 3005 } 3006 vlan_tci = (__be16 *)skb_put(skb, sizeof(__be16)); 3007 *vlan_tci = build_tci(pkt_dev->vlan_id, 3008 pkt_dev->vlan_cfi, 3009 pkt_dev->vlan_p); 3010 vlan_encapsulated_proto = (__be16 *)skb_put(skb, sizeof(__be16)); 3011 *vlan_encapsulated_proto = htons(ETH_P_IPV6); 3012 } 3013 3014 skb->network_header = skb->tail; 3015 skb->transport_header = skb->network_header + sizeof(struct ipv6hdr); 3016 skb_put(skb, sizeof(struct ipv6hdr) + sizeof(struct udphdr)); 3017 skb_set_queue_mapping(skb, queue_map); 3018 iph = ipv6_hdr(skb); 3019 udph = udp_hdr(skb); 3020 3021 memcpy(eth, pkt_dev->hh, 12); 3022 *(__be16 *) ð[12] = protocol; 3023 3024 /* Eth + IPh + UDPh + mpls */ 3025 datalen = pkt_dev->cur_pkt_size - 14 - 3026 sizeof(struct ipv6hdr) - sizeof(struct udphdr) - 3027 pkt_dev->pkt_overhead; 3028 3029 if (datalen < sizeof(struct pktgen_hdr)) { 3030 datalen = sizeof(struct pktgen_hdr); 3031 if (net_ratelimit()) 3032 pr_info("increased datalen to %d\n", datalen); 3033 } 3034 3035 udph->source = htons(pkt_dev->cur_udp_src); 3036 udph->dest = htons(pkt_dev->cur_udp_dst); 3037 udph->len = htons(datalen + sizeof(struct udphdr)); 3038 udph->check = 0; /* No checksum */ 3039 3040 *(__be32 *) iph = htonl(0x60000000); /* Version + flow */ 3041 3042 if (pkt_dev->traffic_class) { 3043 /* Version + traffic class + flow (0) */ 3044 *(__be32 *)iph |= htonl(0x60000000 | (pkt_dev->traffic_class << 20)); 3045 } 3046 3047 iph->hop_limit = 32; 3048 3049 iph->payload_len = htons(sizeof(struct udphdr) + datalen); 3050 iph->nexthdr = IPPROTO_UDP; 3051 3052 ipv6_addr_copy(&iph->daddr, &pkt_dev->cur_in6_daddr); 3053 ipv6_addr_copy(&iph->saddr, &pkt_dev->cur_in6_saddr); 3054 3055 skb->mac_header = (skb->network_header - ETH_HLEN - 3056 pkt_dev->pkt_overhead); 3057 skb->protocol = protocol; 3058 skb->dev = odev; 3059 skb->pkt_type = PACKET_HOST; 3060 3061 if (pkt_dev->nfrags <= 0) 3062 pgh = (struct pktgen_hdr *)skb_put(skb, datalen); 3063 else { 3064 int frags = pkt_dev->nfrags; 3065 int i; 3066 3067 pgh = (struct pktgen_hdr *)(((char *)(udph)) + 8); 3068 3069 if (frags > MAX_SKB_FRAGS) 3070 frags = MAX_SKB_FRAGS; 3071 if (datalen > frags * PAGE_SIZE) { 3072 skb_put(skb, datalen - frags * PAGE_SIZE); 3073 datalen = frags * PAGE_SIZE; 3074 } 3075 3076 i = 0; 3077 while (datalen > 0) { 3078 struct page *page = alloc_pages(GFP_KERNEL, 0); 3079 skb_shinfo(skb)->frags[i].page = page; 3080 skb_shinfo(skb)->frags[i].page_offset = 0; 3081 skb_shinfo(skb)->frags[i].size = 3082 (datalen < PAGE_SIZE ? datalen : PAGE_SIZE); 3083 datalen -= skb_shinfo(skb)->frags[i].size; 3084 skb->len += skb_shinfo(skb)->frags[i].size; 3085 skb->data_len += skb_shinfo(skb)->frags[i].size; 3086 i++; 3087 skb_shinfo(skb)->nr_frags = i; 3088 } 3089 3090 while (i < frags) { 3091 int rem; 3092 3093 if (i == 0) 3094 break; 3095 3096 rem = skb_shinfo(skb)->frags[i - 1].size / 2; 3097 if (rem == 0) 3098 break; 3099 3100 skb_shinfo(skb)->frags[i - 1].size -= rem; 3101 3102 skb_shinfo(skb)->frags[i] = 3103 skb_shinfo(skb)->frags[i - 1]; 3104 get_page(skb_shinfo(skb)->frags[i].page); 3105 skb_shinfo(skb)->frags[i].page = 3106 skb_shinfo(skb)->frags[i - 1].page; 3107 skb_shinfo(skb)->frags[i].page_offset += 3108 skb_shinfo(skb)->frags[i - 1].size; 3109 skb_shinfo(skb)->frags[i].size = rem; 3110 i++; 3111 skb_shinfo(skb)->nr_frags = i; 3112 } 3113 } 3114 3115 /* Stamp the time, and sequence number, 3116 * convert them to network byte order 3117 * should we update cloned packets too ? 3118 */ 3119 if (pgh) { 3120 struct timeval timestamp; 3121 3122 pgh->pgh_magic = htonl(PKTGEN_MAGIC); 3123 pgh->seq_num = htonl(pkt_dev->seq_num); 3124 3125 do_gettimeofday(×tamp); 3126 pgh->tv_sec = htonl(timestamp.tv_sec); 3127 pgh->tv_usec = htonl(timestamp.tv_usec); 3128 } 3129 /* pkt_dev->seq_num++; FF: you really mean this? */ 3130 3131 return skb; 3132} 3133 3134static struct sk_buff *fill_packet(struct net_device *odev, 3135 struct pktgen_dev *pkt_dev) 3136{ 3137 if (pkt_dev->flags & F_IPV6) 3138 return fill_packet_ipv6(odev, pkt_dev); 3139 else 3140 return fill_packet_ipv4(odev, pkt_dev); 3141} 3142 3143static void pktgen_clear_counters(struct pktgen_dev *pkt_dev) 3144{ 3145 pkt_dev->seq_num = 1; 3146 pkt_dev->idle_acc = 0; 3147 pkt_dev->sofar = 0; 3148 pkt_dev->tx_bytes = 0; 3149 pkt_dev->errors = 0; 3150} 3151 3152/* Set up structure for sending pkts, clear counters */ 3153 3154static void pktgen_run(struct pktgen_thread *t) 3155{ 3156 struct pktgen_dev *pkt_dev; 3157 int started = 0; 3158 3159 func_enter(); 3160 3161 if_lock(t); 3162 list_for_each_entry(pkt_dev, &t->if_list, list) { 3163 3164 /* 3165 * setup odev and create initial packet. 3166 */ 3167 pktgen_setup_inject(pkt_dev); 3168 3169 if (pkt_dev->odev) { 3170 pktgen_clear_counters(pkt_dev); 3171 pkt_dev->running = 1; /* Cranke yeself! */ 3172 pkt_dev->skb = NULL; 3173 pkt_dev->started_at = 3174 pkt_dev->next_tx = ktime_now(); 3175 3176 set_pkt_overhead(pkt_dev); 3177 3178 strcpy(pkt_dev->result, "Starting"); 3179 started++; 3180 } else 3181 strcpy(pkt_dev->result, "Error starting"); 3182 } 3183 if_unlock(t); 3184 if (started) 3185 t->control &= ~(T_STOP); 3186} 3187 3188static void pktgen_stop_all_threads_ifs(void) 3189{ 3190 struct pktgen_thread *t; 3191 3192 func_enter(); 3193 3194 mutex_lock(&pktgen_thread_lock); 3195 3196 list_for_each_entry(t, &pktgen_threads, th_list) 3197 t->control |= T_STOP; 3198 3199 mutex_unlock(&pktgen_thread_lock); 3200} 3201 3202static int thread_is_running(const struct pktgen_thread *t) 3203{ 3204 const struct pktgen_dev *pkt_dev; 3205 3206 list_for_each_entry(pkt_dev, &t->if_list, list) 3207 if (pkt_dev->running) 3208 return 1; 3209 return 0; 3210} 3211 3212static int pktgen_wait_thread_run(struct pktgen_thread *t) 3213{ 3214 if_lock(t); 3215 3216 while (thread_is_running(t)) { 3217 3218 if_unlock(t); 3219 3220 msleep_interruptible(100); 3221 3222 if (signal_pending(current)) 3223 goto signal; 3224 if_lock(t); 3225 } 3226 if_unlock(t); 3227 return 1; 3228signal: 3229 return 0; 3230} 3231 3232static int pktgen_wait_all_threads_run(void) 3233{ 3234 struct pktgen_thread *t; 3235 int sig = 1; 3236 3237 mutex_lock(&pktgen_thread_lock); 3238 3239 list_for_each_entry(t, &pktgen_threads, th_list) { 3240 sig = pktgen_wait_thread_run(t); 3241 if (sig == 0) 3242 break; 3243 } 3244 3245 if (sig == 0) 3246 list_for_each_entry(t, &pktgen_threads, th_list) 3247 t->control |= (T_STOP); 3248 3249 mutex_unlock(&pktgen_thread_lock); 3250 return sig; 3251} 3252 3253static void pktgen_run_all_threads(void) 3254{ 3255 struct pktgen_thread *t; 3256 3257 func_enter(); 3258 3259 mutex_lock(&pktgen_thread_lock); 3260 3261 list_for_each_entry(t, &pktgen_threads, th_list) 3262 t->control |= (T_RUN); 3263 3264 mutex_unlock(&pktgen_thread_lock); 3265 3266 /* Propagate thread->control */ 3267 schedule_timeout_interruptible(msecs_to_jiffies(125)); 3268 3269 pktgen_wait_all_threads_run(); 3270} 3271 3272static void pktgen_reset_all_threads(void) 3273{ 3274 struct pktgen_thread *t; 3275 3276 func_enter(); 3277 3278 mutex_lock(&pktgen_thread_lock); 3279 3280 list_for_each_entry(t, &pktgen_threads, th_list) 3281 t->control |= (T_REMDEVALL); 3282 3283 mutex_unlock(&pktgen_thread_lock); 3284 3285 /* Propagate thread->control */ 3286 schedule_timeout_interruptible(msecs_to_jiffies(125)); 3287 3288 pktgen_wait_all_threads_run(); 3289} 3290 3291static void show_results(struct pktgen_dev *pkt_dev, int nr_frags) 3292{ 3293 __u64 bps, mbps, pps; 3294 char *p = pkt_dev->result; 3295 ktime_t elapsed = ktime_sub(pkt_dev->stopped_at, 3296 pkt_dev->started_at); 3297 ktime_t idle = ns_to_ktime(pkt_dev->idle_acc); 3298 3299 p += sprintf(p, "OK: %llu(c%llu+d%llu) nsec, %llu (%dbyte,%dfrags)\n", 3300 (unsigned long long)ktime_to_us(elapsed), 3301 (unsigned long long)ktime_to_us(ktime_sub(elapsed, idle)), 3302 (unsigned long long)ktime_to_us(idle), 3303 (unsigned long long)pkt_dev->sofar, 3304 pkt_dev->cur_pkt_size, nr_frags); 3305 3306 pps = div64_u64(pkt_dev->sofar * NSEC_PER_SEC, 3307 ktime_to_ns(elapsed)); 3308 3309 bps = pps * 8 * pkt_dev->cur_pkt_size; 3310 3311 mbps = bps; 3312 do_div(mbps, 1000000); 3313 p += sprintf(p, " %llupps %lluMb/sec (%llubps) errors: %llu", 3314 (unsigned long long)pps, 3315 (unsigned long long)mbps, 3316 (unsigned long long)bps, 3317 (unsigned long long)pkt_dev->errors); 3318} 3319 3320/* Set stopped-at timer, remove from running list, do counters & statistics */ 3321static int pktgen_stop_device(struct pktgen_dev *pkt_dev) 3322{ 3323 int nr_frags = pkt_dev->skb ? skb_shinfo(pkt_dev->skb)->nr_frags : -1; 3324 3325 if (!pkt_dev->running) { 3326 pr_warning("interface: %s is already stopped\n", 3327 pkt_dev->odevname); 3328 return -EINVAL; 3329 } 3330 3331 kfree_skb(pkt_dev->skb); 3332 pkt_dev->skb = NULL; 3333 pkt_dev->stopped_at = ktime_now(); 3334 pkt_dev->running = 0; 3335 3336 show_results(pkt_dev, nr_frags); 3337 3338 return 0; 3339} 3340 3341static struct pktgen_dev *next_to_run(struct pktgen_thread *t) 3342{ 3343 struct pktgen_dev *pkt_dev, *best = NULL; 3344 3345 if_lock(t); 3346 3347 list_for_each_entry(pkt_dev, &t->if_list, list) { 3348 if (!pkt_dev->running) 3349 continue; 3350 if (best == NULL) 3351 best = pkt_dev; 3352 else if (ktime_lt(pkt_dev->next_tx, best->next_tx)) 3353 best = pkt_dev; 3354 } 3355 if_unlock(t); 3356 return best; 3357} 3358 3359static void pktgen_stop(struct pktgen_thread *t) 3360{ 3361 struct pktgen_dev *pkt_dev; 3362 3363 func_enter(); 3364 3365 if_lock(t); 3366 3367 list_for_each_entry(pkt_dev, &t->if_list, list) { 3368 pktgen_stop_device(pkt_dev); 3369 } 3370 3371 if_unlock(t); 3372} 3373 3374/* 3375 * one of our devices needs to be removed - find it 3376 * and remove it 3377 */ 3378static void pktgen_rem_one_if(struct pktgen_thread *t) 3379{ 3380 struct list_head *q, *n; 3381 struct pktgen_dev *cur; 3382 3383 func_enter(); 3384 3385 if_lock(t); 3386 3387 list_for_each_safe(q, n, &t->if_list) { 3388 cur = list_entry(q, struct pktgen_dev, list); 3389 3390 if (!cur->removal_mark) 3391 continue; 3392 3393 kfree_skb(cur->skb); 3394 cur->skb = NULL; 3395 3396 pktgen_remove_device(t, cur); 3397 3398 break; 3399 } 3400 3401 if_unlock(t); 3402} 3403 3404static void pktgen_rem_all_ifs(struct pktgen_thread *t) 3405{ 3406 struct list_head *q, *n; 3407 struct pktgen_dev *cur; 3408 3409 func_enter(); 3410 3411 /* Remove all devices, free mem */ 3412 3413 if_lock(t); 3414 3415 list_for_each_safe(q, n, &t->if_list) { 3416 cur = list_entry(q, struct pktgen_dev, list); 3417 3418 kfree_skb(cur->skb); 3419 cur->skb = NULL; 3420 3421 pktgen_remove_device(t, cur); 3422 } 3423 3424 if_unlock(t); 3425} 3426 3427static void pktgen_rem_thread(struct pktgen_thread *t) 3428{ 3429 /* Remove from the thread list */ 3430 3431 remove_proc_entry(t->tsk->comm, pg_proc_dir); 3432 3433 mutex_lock(&pktgen_thread_lock); 3434 3435 list_del(&t->th_list); 3436 3437 mutex_unlock(&pktgen_thread_lock); 3438} 3439 3440static void pktgen_resched(struct pktgen_dev *pkt_dev) 3441{ 3442 ktime_t idle_start = ktime_now(); 3443 schedule(); 3444 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_now(), idle_start)); 3445} 3446 3447static void pktgen_wait_for_skb(struct pktgen_dev *pkt_dev) 3448{ 3449 ktime_t idle_start = ktime_now(); 3450 3451 while (atomic_read(&(pkt_dev->skb->users)) != 1) { 3452 if (signal_pending(current)) 3453 break; 3454 3455 if (need_resched()) 3456 pktgen_resched(pkt_dev); 3457 else 3458 cpu_relax(); 3459 } 3460 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_now(), idle_start)); 3461} 3462 3463static void pktgen_xmit(struct pktgen_dev *pkt_dev) 3464{ 3465 struct net_device *odev = pkt_dev->odev; 3466 netdev_tx_t (*xmit)(struct sk_buff *, struct net_device *) 3467 = odev->netdev_ops->ndo_start_xmit; 3468 struct netdev_queue *txq; 3469 u16 queue_map; 3470 int ret; 3471 3472 /* If device is offline, then don't send */ 3473 if (unlikely(!netif_running(odev) || !netif_carrier_ok(odev))) { 3474 pktgen_stop_device(pkt_dev); 3475 return; 3476 } 3477 3478 /* This is max DELAY, this has special meaning of 3479 * "never transmit" 3480 */ 3481 if (unlikely(pkt_dev->delay == ULLONG_MAX)) { 3482 pkt_dev->next_tx = ktime_add_ns(ktime_now(), ULONG_MAX); 3483 return; 3484 } 3485 3486 /* If no skb or clone count exhausted then get new one */ 3487 if (!pkt_dev->skb || (pkt_dev->last_ok && 3488 ++pkt_dev->clone_count >= pkt_dev->clone_skb)) { 3489 /* build a new pkt */ 3490 kfree_skb(pkt_dev->skb); 3491 3492 pkt_dev->skb = fill_packet(odev, pkt_dev); 3493 if (pkt_dev->skb == NULL) { 3494 pr_err("ERROR: couldn't allocate skb in fill_packet\n"); 3495 schedule(); 3496 pkt_dev->clone_count--; /* back out increment, OOM */ 3497 return; 3498 } 3499 pkt_dev->last_pkt_size = pkt_dev->skb->len; 3500 pkt_dev->allocated_skbs++; 3501 pkt_dev->clone_count = 0; /* reset counter */ 3502 } 3503 3504 if (pkt_dev->delay && pkt_dev->last_ok) 3505 spin(pkt_dev, pkt_dev->next_tx); 3506 3507 queue_map = skb_get_queue_mapping(pkt_dev->skb); 3508 txq = netdev_get_tx_queue(odev, queue_map); 3509 3510 __netif_tx_lock_bh(txq); 3511 3512 if (unlikely(netif_tx_queue_stopped(txq) || netif_tx_queue_frozen(txq))) { 3513 ret = NETDEV_TX_BUSY; 3514 pkt_dev->last_ok = 0; 3515 goto unlock; 3516 } 3517 atomic_inc(&(pkt_dev->skb->users)); 3518 ret = (*xmit)(pkt_dev->skb, odev); 3519 3520 switch (ret) { 3521 case NETDEV_TX_OK: 3522 txq_trans_update(txq); 3523 pkt_dev->last_ok = 1; 3524 pkt_dev->sofar++; 3525 pkt_dev->seq_num++; 3526 pkt_dev->tx_bytes += pkt_dev->last_pkt_size; 3527 break; 3528 case NET_XMIT_DROP: 3529 case NET_XMIT_CN: 3530 case NET_XMIT_POLICED: 3531 /* skb has been consumed */ 3532 pkt_dev->errors++; 3533 break; 3534 default: /* Drivers are not supposed to return other values! */ 3535 if (net_ratelimit()) 3536 pr_info("pktgen: %s xmit error: %d\n", 3537 pkt_dev->odevname, ret); 3538 pkt_dev->errors++; 3539 /* fallthru */ 3540 case NETDEV_TX_LOCKED: 3541 case NETDEV_TX_BUSY: 3542 /* Retry it next time */ 3543 atomic_dec(&(pkt_dev->skb->users)); 3544 pkt_dev->last_ok = 0; 3545 } 3546unlock: 3547 __netif_tx_unlock_bh(txq); 3548 3549 /* If pkt_dev->count is zero, then run forever */ 3550 if ((pkt_dev->count != 0) && (pkt_dev->sofar >= pkt_dev->count)) { 3551 pktgen_wait_for_skb(pkt_dev); 3552 3553 /* Done with this */ 3554 pktgen_stop_device(pkt_dev); 3555 } 3556} 3557 3558/* 3559 * Main loop of the thread goes here 3560 */ 3561 3562static int pktgen_thread_worker(void *arg) 3563{ 3564 DEFINE_WAIT(wait); 3565 struct pktgen_thread *t = arg; 3566 struct pktgen_dev *pkt_dev = NULL; 3567 int cpu = t->cpu; 3568 3569 BUG_ON(smp_processor_id() != cpu); 3570 3571 init_waitqueue_head(&t->queue); 3572 complete(&t->start_done); 3573 3574 pr_debug("starting pktgen/%d: pid=%d\n", cpu, task_pid_nr(current)); 3575 3576 set_current_state(TASK_INTERRUPTIBLE); 3577 3578 set_freezable(); 3579 3580 while (!kthread_should_stop()) { 3581 pkt_dev = next_to_run(t); 3582 3583 if (unlikely(!pkt_dev && t->control == 0)) { 3584 wait_event_interruptible_timeout(t->queue, 3585 t->control != 0, 3586 HZ/10); 3587 try_to_freeze(); 3588 continue; 3589 } 3590 3591 __set_current_state(TASK_RUNNING); 3592 3593 if (likely(pkt_dev)) { 3594 pktgen_xmit(pkt_dev); 3595 3596 if (need_resched()) 3597 pktgen_resched(pkt_dev); 3598 else 3599 cpu_relax(); 3600 } 3601 3602 if (t->control & T_STOP) { 3603 pktgen_stop(t); 3604 t->control &= ~(T_STOP); 3605 } 3606 3607 if (t->control & T_RUN) { 3608 pktgen_run(t); 3609 t->control &= ~(T_RUN); 3610 } 3611 3612 if (t->control & T_REMDEVALL) { 3613 pktgen_rem_all_ifs(t); 3614 t->control &= ~(T_REMDEVALL); 3615 } 3616 3617 if (t->control & T_REMDEV) { 3618 pktgen_rem_one_if(t); 3619 t->control &= ~(T_REMDEV); 3620 } 3621 3622 try_to_freeze(); 3623 3624 set_current_state(TASK_INTERRUPTIBLE); 3625 } 3626 3627 pr_debug("%s stopping all device\n", t->tsk->comm); 3628 pktgen_stop(t); 3629 3630 pr_debug("%s removing all device\n", t->tsk->comm); 3631 pktgen_rem_all_ifs(t); 3632 3633 pr_debug("%s removing thread\n", t->tsk->comm); 3634 pktgen_rem_thread(t); 3635 3636 return 0; 3637} 3638 3639static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t, 3640 const char *ifname, bool exact) 3641{ 3642 struct pktgen_dev *p, *pkt_dev = NULL; 3643 size_t len = strlen(ifname); 3644 3645 if_lock(t); 3646 list_for_each_entry(p, &t->if_list, list) 3647 if (strncmp(p->odevname, ifname, len) == 0) { 3648 if (p->odevname[len]) { 3649 if (exact || p->odevname[len] != '@') 3650 continue; 3651 } 3652 pkt_dev = p; 3653 break; 3654 } 3655 3656 if_unlock(t); 3657 pr_debug("find_dev(%s) returning %p\n", ifname, pkt_dev); 3658 return pkt_dev; 3659} 3660 3661/* 3662 * Adds a dev at front of if_list. 3663 */ 3664 3665static int add_dev_to_thread(struct pktgen_thread *t, 3666 struct pktgen_dev *pkt_dev) 3667{ 3668 int rv = 0; 3669 3670 if_lock(t); 3671 3672 if (pkt_dev->pg_thread) { 3673 pr_err("ERROR: already assigned to a thread\n"); 3674 rv = -EBUSY; 3675 goto out; 3676 } 3677 3678 list_add(&pkt_dev->list, &t->if_list); 3679 pkt_dev->pg_thread = t; 3680 pkt_dev->running = 0; 3681 3682out: 3683 if_unlock(t); 3684 return rv; 3685} 3686 3687/* Called under thread lock */ 3688 3689static int pktgen_add_device(struct pktgen_thread *t, const char *ifname) 3690{ 3691 struct pktgen_dev *pkt_dev; 3692 int err; 3693 int node = cpu_to_node(t->cpu); 3694 3695 /* We don't allow a device to be on several threads */ 3696 3697 pkt_dev = __pktgen_NN_threads(ifname, FIND); 3698 if (pkt_dev) { 3699 pr_err("ERROR: interface already used\n"); 3700 return -EBUSY; 3701 } 3702 3703 pkt_dev = kzalloc_node(sizeof(struct pktgen_dev), GFP_KERNEL, node); 3704 if (!pkt_dev) 3705 return -ENOMEM; 3706 3707 strcpy(pkt_dev->odevname, ifname); 3708 pkt_dev->flows = vmalloc_node(MAX_CFLOWS * sizeof(struct flow_state), 3709 node); 3710 if (pkt_dev->flows == NULL) { 3711 kfree(pkt_dev); 3712 return -ENOMEM; 3713 } 3714 memset(pkt_dev->flows, 0, MAX_CFLOWS * sizeof(struct flow_state)); 3715 3716 pkt_dev->removal_mark = 0; 3717 pkt_dev->min_pkt_size = ETH_ZLEN; 3718 pkt_dev->max_pkt_size = ETH_ZLEN; 3719 pkt_dev->nfrags = 0; 3720 pkt_dev->clone_skb = pg_clone_skb_d; 3721 pkt_dev->delay = pg_delay_d; 3722 pkt_dev->count = pg_count_d; 3723 pkt_dev->sofar = 0; 3724 pkt_dev->udp_src_min = 9; /* sink port */ 3725 pkt_dev->udp_src_max = 9; 3726 pkt_dev->udp_dst_min = 9; 3727 pkt_dev->udp_dst_max = 9; 3728 3729 pkt_dev->vlan_p = 0; 3730 pkt_dev->vlan_cfi = 0; 3731 pkt_dev->vlan_id = 0xffff; 3732 pkt_dev->svlan_p = 0; 3733 pkt_dev->svlan_cfi = 0; 3734 pkt_dev->svlan_id = 0xffff; 3735 pkt_dev->node = -1; 3736 3737 err = pktgen_setup_dev(pkt_dev, ifname); 3738 if (err) 3739 goto out1; 3740 3741 pkt_dev->entry = proc_create_data(ifname, 0600, pg_proc_dir, 3742 &pktgen_if_fops, pkt_dev); 3743 if (!pkt_dev->entry) { 3744 pr_err("cannot create %s/%s procfs entry\n", 3745 PG_PROC_DIR, ifname); 3746 err = -EINVAL; 3747 goto out2; 3748 } 3749#ifdef CONFIG_XFRM 3750 pkt_dev->ipsmode = XFRM_MODE_TRANSPORT; 3751 pkt_dev->ipsproto = IPPROTO_ESP; 3752#endif 3753 3754 return add_dev_to_thread(t, pkt_dev); 3755out2: 3756 dev_put(pkt_dev->odev); 3757out1: 3758#ifdef CONFIG_XFRM 3759 free_SAs(pkt_dev); 3760#endif 3761 vfree(pkt_dev->flows); 3762 kfree(pkt_dev); 3763 return err; 3764} 3765 3766static int __init pktgen_create_thread(int cpu) 3767{ 3768 struct pktgen_thread *t; 3769 struct proc_dir_entry *pe; 3770 struct task_struct *p; 3771 3772 t = kzalloc_node(sizeof(struct pktgen_thread), GFP_KERNEL, 3773 cpu_to_node(cpu)); 3774 if (!t) { 3775 pr_err("ERROR: out of memory, can't create new thread\n"); 3776 return -ENOMEM; 3777 } 3778 3779 spin_lock_init(&t->if_lock); 3780 t->cpu = cpu; 3781 3782 INIT_LIST_HEAD(&t->if_list); 3783 3784 list_add_tail(&t->th_list, &pktgen_threads); 3785 init_completion(&t->start_done); 3786 3787 p = kthread_create(pktgen_thread_worker, t, "kpktgend_%d", cpu); 3788 if (IS_ERR(p)) { 3789 pr_err("kernel_thread() failed for cpu %d\n", t->cpu); 3790 list_del(&t->th_list); 3791 kfree(t); 3792 return PTR_ERR(p); 3793 } 3794 kthread_bind(p, cpu); 3795 t->tsk = p; 3796 3797 pe = proc_create_data(t->tsk->comm, 0600, pg_proc_dir, 3798 &pktgen_thread_fops, t); 3799 if (!pe) { 3800 pr_err("cannot create %s/%s procfs entry\n", 3801 PG_PROC_DIR, t->tsk->comm); 3802 kthread_stop(p); 3803 list_del(&t->th_list); 3804 kfree(t); 3805 return -EINVAL; 3806 } 3807 3808 wake_up_process(p); 3809 wait_for_completion(&t->start_done); 3810 3811 return 0; 3812} 3813 3814/* 3815 * Removes a device from the thread if_list. 3816 */ 3817static void _rem_dev_from_if_list(struct pktgen_thread *t, 3818 struct pktgen_dev *pkt_dev) 3819{ 3820 struct list_head *q, *n; 3821 struct pktgen_dev *p; 3822 3823 list_for_each_safe(q, n, &t->if_list) { 3824 p = list_entry(q, struct pktgen_dev, list); 3825 if (p == pkt_dev) 3826 list_del(&p->list); 3827 } 3828} 3829 3830static int pktgen_remove_device(struct pktgen_thread *t, 3831 struct pktgen_dev *pkt_dev) 3832{ 3833 3834 pr_debug("remove_device pkt_dev=%p\n", pkt_dev); 3835 3836 if (pkt_dev->running) { 3837 pr_warning("WARNING: trying to remove a running interface, stopping it now\n"); 3838 pktgen_stop_device(pkt_dev); 3839 } 3840 3841 /* Dis-associate from the interface */ 3842 3843 if (pkt_dev->odev) { 3844 dev_put(pkt_dev->odev); 3845 pkt_dev->odev = NULL; 3846 } 3847 3848 /* And update the thread if_list */ 3849 3850 _rem_dev_from_if_list(t, pkt_dev); 3851 3852 if (pkt_dev->entry) 3853 remove_proc_entry(pkt_dev->entry->name, pg_proc_dir); 3854 3855#ifdef CONFIG_XFRM 3856 free_SAs(pkt_dev); 3857#endif 3858 vfree(pkt_dev->flows); 3859 kfree(pkt_dev); 3860 return 0; 3861} 3862 3863static int __init pg_init(void) 3864{ 3865 int cpu; 3866 struct proc_dir_entry *pe; 3867 3868 pr_info("%s", version); 3869 3870 pg_proc_dir = proc_mkdir(PG_PROC_DIR, init_net.proc_net); 3871 if (!pg_proc_dir) 3872 return -ENODEV; 3873 3874 pe = proc_create(PGCTRL, 0600, pg_proc_dir, &pktgen_fops); 3875 if (pe == NULL) { 3876 pr_err("ERROR: cannot create %s procfs entry\n", PGCTRL); 3877 proc_net_remove(&init_net, PG_PROC_DIR); 3878 return -EINVAL; 3879 } 3880 3881 /* Register us to receive netdevice events */ 3882 register_netdevice_notifier(&pktgen_notifier_block); 3883 3884 for_each_online_cpu(cpu) { 3885 int err; 3886 3887 err = pktgen_create_thread(cpu); 3888 if (err) 3889 pr_warning("WARNING: Cannot create thread for cpu %d (%d)\n", 3890 cpu, err); 3891 } 3892 3893 if (list_empty(&pktgen_threads)) { 3894 pr_err("ERROR: Initialization failed for all threads\n"); 3895 unregister_netdevice_notifier(&pktgen_notifier_block); 3896 remove_proc_entry(PGCTRL, pg_proc_dir); 3897 proc_net_remove(&init_net, PG_PROC_DIR); 3898 return -ENODEV; 3899 } 3900 3901 return 0; 3902} 3903 3904static void __exit pg_cleanup(void) 3905{ 3906 struct pktgen_thread *t; 3907 struct list_head *q, *n; 3908 wait_queue_head_t queue; 3909 init_waitqueue_head(&queue); 3910 3911 /* Stop all interfaces & threads */ 3912 3913 list_for_each_safe(q, n, &pktgen_threads) { 3914 t = list_entry(q, struct pktgen_thread, th_list); 3915 kthread_stop(t->tsk); 3916 kfree(t); 3917 } 3918 3919 /* Un-register us from receiving netdevice events */ 3920 unregister_netdevice_notifier(&pktgen_notifier_block); 3921 3922 /* Clean up proc file system */ 3923 remove_proc_entry(PGCTRL, pg_proc_dir); 3924 proc_net_remove(&init_net, PG_PROC_DIR); 3925} 3926 3927module_init(pg_init); 3928module_exit(pg_cleanup); 3929 3930MODULE_AUTHOR("Robert Olsson <robert.olsson@its.uu.se>"); 3931MODULE_DESCRIPTION("Packet Generator tool"); 3932MODULE_LICENSE("GPL"); 3933MODULE_VERSION(VERSION); 3934module_param(pg_count_d, int, 0); 3935MODULE_PARM_DESC(pg_count_d, "Default number of packets to inject"); 3936module_param(pg_delay_d, int, 0); 3937MODULE_PARM_DESC(pg_delay_d, "Default delay between packets (nanoseconds)"); 3938module_param(pg_clone_skb_d, int, 0); 3939MODULE_PARM_DESC(pg_clone_skb_d, "Default number of copies of the same packet"); 3940module_param(debug, int, 0); 3941MODULE_PARM_DESC(debug, "Enable debugging of pktgen module"); 3942