1139823Simp/*- 2301772Struckman * Codel/FQ_Codel and PIE/FQ-PIE Code: 3301772Struckman * Copyright (C) 2016 Centre for Advanced Internet Architectures, 4301772Struckman * Swinburne University of Technology, Melbourne, Australia. 5301772Struckman * Portions of this code were made possible in part by a gift from 6301772Struckman * The Comcast Innovation Fund. 7301772Struckman * Implemented by Rasool Al-Saadi <ralsaadi@swin.edu.au> 8301772Struckman * 9204591Sluigi * Copyright (c) 1998-2002,2010 Luigi Rizzo, Universita` di Pisa 1055597Sluigi * Portions Copyright (c) 2000 Akamba Corp. 1155597Sluigi * All rights reserved 1239119Sluigi * 1355597Sluigi * Redistribution and use in source and binary forms, with or without 1455597Sluigi * modification, are permitted provided that the following conditions 1555597Sluigi * are met: 1655597Sluigi * 1. Redistributions of source code must retain the above copyright 1755597Sluigi * notice, this list of conditions and the following disclaimer. 1855597Sluigi * 2. Redistributions in binary form must reproduce the above copyright 1955597Sluigi * notice, this list of conditions and the following disclaimer in the 2055597Sluigi * documentation and/or other materials provided with the distribution. 2139119Sluigi * 2255597Sluigi * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 2355597Sluigi * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2455597Sluigi * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2555597Sluigi * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2655597Sluigi * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2755597Sluigi * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2855597Sluigi * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2955597Sluigi * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 3055597Sluigi * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 3155597Sluigi * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3255597Sluigi * SUCH DAMAGE. 3339119Sluigi */ 3439119Sluigi 35172467Ssilby#include <sys/cdefs.h> 36172467Ssilby__FBSDID("$FreeBSD: stable/10/sys/netpfil/ipfw/ip_dummynet.c 318155 2017-05-10 20:46:59Z marius $"); 37172467Ssilby 38204591Sluigi/* 39204591Sluigi * Configuration and internal object management for dummynet. 40204591Sluigi */ 4155597Sluigi 42145268Sphk#include "opt_inet6.h" 43145268Sphk 4439119Sluigi#include <sys/param.h> 4539119Sluigi#include <sys/systm.h> 4639119Sluigi#include <sys/malloc.h> 4739119Sluigi#include <sys/mbuf.h> 4839119Sluigi#include <sys/kernel.h> 49185895Szec#include <sys/lock.h> 5045869Speter#include <sys/module.h> 51179201Srwatson#include <sys/priv.h> 5283994Sbrooks#include <sys/proc.h> 53185895Szec#include <sys/rwlock.h> 5439119Sluigi#include <sys/socket.h> 5541793Sluigi#include <sys/socketvar.h> 5639119Sluigi#include <sys/time.h> 57163720Soleg#include <sys/taskqueue.h> 58188580Sluigi#include <net/if.h> /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */ 5939119Sluigi#include <netinet/in.h> 60201732Sluigi#include <netinet/ip_var.h> /* ip_output(), IP_FORWARDING */ 6139119Sluigi#include <netinet/ip_fw.h> 6239119Sluigi#include <netinet/ip_dummynet.h> 6339119Sluigi 64240494Sglebius#include <netpfil/ipfw/ip_fw_private.h> 65240494Sglebius#include <netpfil/ipfw/dn_heap.h> 66240494Sglebius#include <netpfil/ipfw/ip_dn_private.h> 67301772Struckman#ifdef NEW_AQM 68301772Struckman#include <netpfil/ipfw/dn_aqm.h> 69301772Struckman#endif 70240494Sglebius#include <netpfil/ipfw/dn_sched.h> 71240494Sglebius 72204591Sluigi/* which objects to copy */ 73204591Sluigi#define DN_C_LINK 0x01 74204591Sluigi#define DN_C_SCH 0x02 75204591Sluigi#define DN_C_FLOW 0x04 76204591Sluigi#define DN_C_FS 0x08 77204591Sluigi#define DN_C_QUEUE 0x10 7839119Sluigi 79204591Sluigi/* we use this argument in case of a schk_new */ 80204591Sluigistruct schk_new_arg { 81204591Sluigi struct dn_alg *fp; 82204591Sluigi struct dn_sch *sch; 83204591Sluigi}; 84145246Sbrooks 85204591Sluigi/*---- callout hooks. ----*/ 86204591Sluigistatic struct callout dn_timeout; 87297228Shselaskystatic int dn_gone; 88204591Sluigistatic struct task dn_task; 89204591Sluigistatic struct taskqueue *dn_tq = NULL; 9055597Sluigi 91204591Sluigistatic void 92238988Sluigidummynet(void *arg) 93204591Sluigi{ 9455597Sluigi 95238988Sluigi (void)arg; /* UNUSED */ 96254781Smav taskqueue_enqueue_fast(dn_tq, &dn_task); 97204591Sluigi} 9855597Sluigi 99204591Sluigivoid 100204591Sluigidn_reschedule(void) 101204591Sluigi{ 102254781Smav 103297228Shselasky if (dn_gone != 0) 104297228Shselasky return; 105254781Smav callout_reset_sbt(&dn_timeout, tick_sbt, 0, dummynet, NULL, 106254781Smav C_HARDCLOCK | C_DIRECT_EXEC); 107204591Sluigi} 108204591Sluigi/*----- end of callout hooks -----*/ 109176626Sdwmalone 110301772Struckman#ifdef NEW_AQM 111301772Struckman/* Return AQM descriptor for given type or name. */ 112301772Struckmanstatic struct dn_aqm * 113301772Struckmanfind_aqm_type(int type, char *name) 114301772Struckman{ 115301772Struckman struct dn_aqm *d; 116301772Struckman 117301772Struckman SLIST_FOREACH(d, &dn_cfg.aqmlist, next) { 118301772Struckman if (d->type == type || (name && !strcasecmp(d->name, name))) 119301772Struckman return d; 120301772Struckman } 121301772Struckman return NULL; /* not found */ 122301772Struckman} 123301772Struckman#endif 124301772Struckman 125204591Sluigi/* Return a scheduler descriptor given the type or name. */ 126204591Sluigistatic struct dn_alg * 127204591Sluigifind_sched_type(int type, char *name) 128204591Sluigi{ 129204591Sluigi struct dn_alg *d; 13061413Sluigi 131204591Sluigi SLIST_FOREACH(d, &dn_cfg.schedlist, next) { 132238063Sissyl0 if (d->type == type || (name && !strcasecmp(d->name, name))) 133204591Sluigi return d; 134204591Sluigi } 135204591Sluigi return NULL; /* not found */ 136204591Sluigi} 137163721Soleg 138204591Sluigiint 139204591Sluigiipdn_bound_var(int *v, int dflt, int lo, int hi, const char *msg) 140204591Sluigi{ 141204591Sluigi int oldv = *v; 142204591Sluigi const char *op = NULL; 143220831Sae if (dflt < lo) 144220831Sae dflt = lo; 145220831Sae if (dflt > hi) 146220831Sae dflt = hi; 147204591Sluigi if (oldv < lo) { 148204591Sluigi *v = dflt; 149204591Sluigi op = "Bump"; 150204591Sluigi } else if (oldv > hi) { 151204591Sluigi *v = hi; 152204591Sluigi op = "Clamp"; 153204591Sluigi } else 154204591Sluigi return *v; 155204591Sluigi if (op && msg) 156204591Sluigi printf("%s %s to %d (was %d)\n", op, msg, *v, oldv); 157204591Sluigi return *v; 158204591Sluigi} 159173399Soleg 160204591Sluigi/*---- flow_id mask, hash and compare functions ---*/ 161205173Sluigi/* 162205173Sluigi * The flow_id includes the 5-tuple, the queue/pipe number 163205173Sluigi * which we store in the extra area in host order, 164205173Sluigi * and for ipv6 also the flow_id6. 165205173Sluigi * XXX see if we want the tos byte (can store in 'flags') 166205173Sluigi */ 167204591Sluigistatic struct ipfw_flow_id * 168204591Sluigiflow_id_mask(struct ipfw_flow_id *mask, struct ipfw_flow_id *id) 169204591Sluigi{ 170204591Sluigi int is_v6 = IS_IP6_FLOW_ID(id); 17184516Sps 172204591Sluigi id->dst_port &= mask->dst_port; 173204591Sluigi id->src_port &= mask->src_port; 174204591Sluigi id->proto &= mask->proto; 175205173Sluigi id->extra &= mask->extra; 176204591Sluigi if (is_v6) { 177204591Sluigi APPLY_MASK(&id->dst_ip6, &mask->dst_ip6); 178204591Sluigi APPLY_MASK(&id->src_ip6, &mask->src_ip6); 179204591Sluigi id->flow_id6 &= mask->flow_id6; 180204591Sluigi } else { 181204591Sluigi id->dst_ip &= mask->dst_ip; 182204591Sluigi id->src_ip &= mask->src_ip; 183204591Sluigi } 184204591Sluigi return id; 185204591Sluigi} 18684516Sps 187204591Sluigi/* computes an OR of two masks, result in dst and also returned */ 188204591Sluigistatic struct ipfw_flow_id * 189204591Sluigiflow_id_or(struct ipfw_flow_id *src, struct ipfw_flow_id *dst) 190204591Sluigi{ 191204591Sluigi int is_v6 = IS_IP6_FLOW_ID(dst); 19271667Sluigi 193204591Sluigi dst->dst_port |= src->dst_port; 194204591Sluigi dst->src_port |= src->src_port; 195204591Sluigi dst->proto |= src->proto; 196205173Sluigi dst->extra |= src->extra; 197204591Sluigi if (is_v6) { 198204591Sluigi#define OR_MASK(_d, _s) \ 199204591Sluigi (_d)->__u6_addr.__u6_addr32[0] |= (_s)->__u6_addr.__u6_addr32[0]; \ 200204591Sluigi (_d)->__u6_addr.__u6_addr32[1] |= (_s)->__u6_addr.__u6_addr32[1]; \ 201204591Sluigi (_d)->__u6_addr.__u6_addr32[2] |= (_s)->__u6_addr.__u6_addr32[2]; \ 202204591Sluigi (_d)->__u6_addr.__u6_addr32[3] |= (_s)->__u6_addr.__u6_addr32[3]; 203204591Sluigi OR_MASK(&dst->dst_ip6, &src->dst_ip6); 204204591Sluigi OR_MASK(&dst->src_ip6, &src->src_ip6); 205204591Sluigi#undef OR_MASK 206204591Sluigi dst->flow_id6 |= src->flow_id6; 207204591Sluigi } else { 208204591Sluigi dst->dst_ip |= src->dst_ip; 209204591Sluigi dst->src_ip |= src->src_ip; 210204591Sluigi } 211204591Sluigi return dst; 212204591Sluigi} 21371667Sluigi 214204591Sluigistatic int 215204591Sluiginonzero_mask(struct ipfw_flow_id *m) 216204591Sluigi{ 217205173Sluigi if (m->dst_port || m->src_port || m->proto || m->extra) 218204591Sluigi return 1; 219204591Sluigi if (IS_IP6_FLOW_ID(m)) { 220204591Sluigi return 221204591Sluigi m->dst_ip6.__u6_addr.__u6_addr32[0] || 222204591Sluigi m->dst_ip6.__u6_addr.__u6_addr32[1] || 223204591Sluigi m->dst_ip6.__u6_addr.__u6_addr32[2] || 224204591Sluigi m->dst_ip6.__u6_addr.__u6_addr32[3] || 225204591Sluigi m->src_ip6.__u6_addr.__u6_addr32[0] || 226204591Sluigi m->src_ip6.__u6_addr.__u6_addr32[1] || 227204591Sluigi m->src_ip6.__u6_addr.__u6_addr32[2] || 228204591Sluigi m->src_ip6.__u6_addr.__u6_addr32[3] || 229204591Sluigi m->flow_id6; 230204591Sluigi } else { 231204591Sluigi return m->dst_ip || m->src_ip; 232204591Sluigi } 233204591Sluigi} 23439119Sluigi 235204591Sluigi/* XXX we may want a better hash function */ 236204591Sluigistatic uint32_t 237204591Sluigiflow_id_hash(struct ipfw_flow_id *id) 238204591Sluigi{ 239204591Sluigi uint32_t i; 24084516Sps 241204591Sluigi if (IS_IP6_FLOW_ID(id)) { 242204591Sluigi uint32_t *d = (uint32_t *)&id->dst_ip6; 243204591Sluigi uint32_t *s = (uint32_t *)&id->src_ip6; 244204591Sluigi i = (d[0] ) ^ (d[1]) ^ 245204591Sluigi (d[2] ) ^ (d[3]) ^ 246204591Sluigi (d[0] >> 15) ^ (d[1] >> 15) ^ 247204591Sluigi (d[2] >> 15) ^ (d[3] >> 15) ^ 248204591Sluigi (s[0] << 1) ^ (s[1] << 1) ^ 249204591Sluigi (s[2] << 1) ^ (s[3] << 1) ^ 250204591Sluigi (s[0] << 16) ^ (s[1] << 16) ^ 251204591Sluigi (s[2] << 16) ^ (s[3] << 16) ^ 252204591Sluigi (id->dst_port << 1) ^ (id->src_port) ^ 253205173Sluigi (id->extra) ^ 254204591Sluigi (id->proto ) ^ (id->flow_id6); 255204591Sluigi } else { 256204591Sluigi i = (id->dst_ip) ^ (id->dst_ip >> 15) ^ 257204591Sluigi (id->src_ip << 1) ^ (id->src_ip >> 16) ^ 258205173Sluigi (id->extra) ^ 259204591Sluigi (id->dst_port << 1) ^ (id->src_port) ^ (id->proto); 260204591Sluigi } 261204591Sluigi return i; 262204591Sluigi} 263147205Sthompsa 264204591Sluigi/* Like bcmp, returns 0 if ids match, 1 otherwise. */ 265204591Sluigistatic int 266204591Sluigiflow_id_cmp(struct ipfw_flow_id *id1, struct ipfw_flow_id *id2) 267204591Sluigi{ 268204591Sluigi int is_v6 = IS_IP6_FLOW_ID(id1); 269188580Sluigi 270205173Sluigi if (!is_v6) { 271205173Sluigi if (IS_IP6_FLOW_ID(id2)) 272205173Sluigi return 1; /* different address families */ 27339119Sluigi 274205173Sluigi return (id1->dst_ip == id2->dst_ip && 275205173Sluigi id1->src_ip == id2->src_ip && 276205173Sluigi id1->dst_port == id2->dst_port && 277205173Sluigi id1->src_port == id2->src_port && 278205173Sluigi id1->proto == id2->proto && 279205173Sluigi id1->extra == id2->extra) ? 0 : 1; 280205173Sluigi } 281205173Sluigi /* the ipv6 case */ 282205173Sluigi return ( 283204591Sluigi !bcmp(&id1->dst_ip6,&id2->dst_ip6, sizeof(id1->dst_ip6)) && 284204591Sluigi !bcmp(&id1->src_ip6,&id2->src_ip6, sizeof(id1->src_ip6)) && 285204591Sluigi id1->dst_port == id2->dst_port && 286204591Sluigi id1->src_port == id2->src_port && 287204591Sluigi id1->proto == id2->proto && 288205173Sluigi id1->extra == id2->extra && 289205173Sluigi id1->flow_id6 == id2->flow_id6) ? 0 : 1; 290204591Sluigi} 291204591Sluigi/*--------- end of flow-id mask, hash and compare ---------*/ 292120140Ssam 293204591Sluigi/*--- support functions for the qht hashtable ---- 294204591Sluigi * Entries are hashed by flow-id 295198845Soleg */ 296204591Sluigistatic uint32_t 297204591Sluigiq_hash(uintptr_t key, int flags, void *arg) 298107900Smaxim{ 299204591Sluigi /* compute the hash slot from the flow id */ 300204591Sluigi struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ? 301204591Sluigi &((struct dn_queue *)key)->ni.fid : 302204591Sluigi (struct ipfw_flow_id *)key; 30355597Sluigi 304204591Sluigi return flow_id_hash(id); 30555597Sluigi} 30655597Sluigi 30755597Sluigistatic int 308204591Sluigiq_match(void *obj, uintptr_t key, int flags, void *arg) 309107900Smaxim{ 310204591Sluigi struct dn_queue *o = (struct dn_queue *)obj; 311204591Sluigi struct ipfw_flow_id *id2; 31255597Sluigi 313204591Sluigi if (flags & DNHT_KEY_IS_OBJ) { 314204591Sluigi /* compare pointers */ 315204591Sluigi id2 = &((struct dn_queue *)key)->ni.fid; 316204591Sluigi } else { 317204591Sluigi id2 = (struct ipfw_flow_id *)key; 318204591Sluigi } 319204591Sluigi return (0 == flow_id_cmp(&o->ni.fid, id2)); 32055597Sluigi} 32155597Sluigi 32255597Sluigi/* 323204591Sluigi * create a new queue instance for the given 'key'. 32455597Sluigi */ 325204591Sluigistatic void * 326204591Sluigiq_new(uintptr_t key, int flags, void *arg) 327204591Sluigi{ 328204591Sluigi struct dn_queue *q, *template = arg; 329204591Sluigi struct dn_fsk *fs = template->fs; 330204591Sluigi int size = sizeof(*q) + fs->sched->fp->q_datalen; 33161413Sluigi 332204591Sluigi q = malloc(size, M_DUMMYNET, M_NOWAIT | M_ZERO); 333204591Sluigi if (q == NULL) { 334204591Sluigi D("no memory for new queue"); 335204591Sluigi return NULL; 33671395Sluigi } 337204591Sluigi 338204591Sluigi set_oid(&q->ni.oid, DN_QUEUE, size); 339204591Sluigi if (fs->fs.flags & DN_QHT_HASH) 340204591Sluigi q->ni.fid = *(struct ipfw_flow_id *)key; 341204591Sluigi q->fs = fs; 342204591Sluigi q->_si = template->_si; 343204591Sluigi q->_si->q_count++; 344204591Sluigi 345204591Sluigi if (fs->sched->fp->new_queue) 346204591Sluigi fs->sched->fp->new_queue(q); 347301772Struckman 348301772Struckman#ifdef NEW_AQM 349301772Struckman /* call AQM init function after creating a queue*/ 350301772Struckman if (fs->aqmfp && fs->aqmfp->init) 351301772Struckman if(fs->aqmfp->init(q)) 352301772Struckman D("unable to init AQM for fs %d", fs->fs.fs_nr); 353301772Struckman#endif 354204591Sluigi dn_cfg.queue_count++; 355301772Struckman 356204591Sluigi return q; 357107900Smaxim} 35839119Sluigi 35955597Sluigi/* 360204591Sluigi * Notify schedulers that a queue is going away. 361204591Sluigi * If (flags & DN_DESTROY), also free the packets. 362204591Sluigi * The version for callbacks is called q_delete_cb(). 36361413Sluigi */ 36461413Sluigistatic void 365204591Sluigidn_delete_queue(struct dn_queue *q, int flags) 36661413Sluigi{ 367204591Sluigi struct dn_fsk *fs = q->fs; 36861413Sluigi 369301772Struckman#ifdef NEW_AQM 370301772Struckman /* clean up AQM status for queue 'q' 371301772Struckman * cleanup here is called just with MULTIQUEUE 372301772Struckman */ 373301772Struckman if (fs && fs->aqmfp && fs->aqmfp->cleanup) 374301772Struckman fs->aqmfp->cleanup(q); 375301772Struckman#endif 376204591Sluigi // D("fs %p si %p\n", fs, q->_si); 377204591Sluigi /* notify the parent scheduler that the queue is going away */ 378204591Sluigi if (fs && fs->sched->fp->free_queue) 379204591Sluigi fs->sched->fp->free_queue(q); 380204591Sluigi q->_si->q_count--; 381204591Sluigi q->_si = NULL; 382204591Sluigi if (flags & DN_DESTROY) { 383204591Sluigi if (q->mq.head) 384204591Sluigi dn_free_pkts(q->mq.head); 385204591Sluigi bzero(q, sizeof(*q)); // safety 386204591Sluigi free(q, M_DUMMYNET); 387204591Sluigi dn_cfg.queue_count--; 38861413Sluigi } 38961413Sluigi} 39061413Sluigi 391204591Sluigistatic int 392204591Sluigiq_delete_cb(void *q, void *arg) 39355597Sluigi{ 394204591Sluigi int flags = (int)(uintptr_t)arg; 395204591Sluigi dn_delete_queue(q, flags); 396204591Sluigi return (flags & DN_DESTROY) ? DNHT_SCAN_DEL : 0; 39771133Sluigi} 39861413Sluigi 39961413Sluigi/* 400204591Sluigi * calls dn_delete_queue/q_delete_cb on all queues, 401204591Sluigi * which notifies the parent scheduler and possibly drains packets. 402204591Sluigi * flags & DN_DESTROY: drains queues and destroy qht; 40361413Sluigi */ 40461413Sluigistatic void 405204591Sluigiqht_delete(struct dn_fsk *fs, int flags) 40661413Sluigi{ 407204591Sluigi ND("fs %d start flags %d qht %p", 408204591Sluigi fs->fs.fs_nr, flags, fs->qht); 409204591Sluigi if (!fs->qht) 410204591Sluigi return; 411204591Sluigi if (fs->fs.flags & DN_QHT_HASH) { 412204591Sluigi dn_ht_scan(fs->qht, q_delete_cb, (void *)(uintptr_t)flags); 413204591Sluigi if (flags & DN_DESTROY) { 414204591Sluigi dn_ht_free(fs->qht, 0); 415204591Sluigi fs->qht = NULL; 416204591Sluigi } 417204591Sluigi } else { 418204591Sluigi dn_delete_queue((struct dn_queue *)(fs->qht), flags); 419204591Sluigi if (flags & DN_DESTROY) 420204591Sluigi fs->qht = NULL; 421204591Sluigi } 42255597Sluigi} 42361413Sluigi 42455597Sluigi/* 425204591Sluigi * Find and possibly create the queue for a MULTIQUEUE scheduler. 426204591Sluigi * We never call it for !MULTIQUEUE (the queue is in the sch_inst). 42755597Sluigi */ 428204591Sluigistruct dn_queue * 429204591Sluigiipdn_q_find(struct dn_fsk *fs, struct dn_sch_inst *si, 430204591Sluigi struct ipfw_flow_id *id) 431204591Sluigi{ 432204591Sluigi struct dn_queue template; 43339119Sluigi 434204591Sluigi template._si = si; 435204591Sluigi template.fs = fs; 436200358Sluigi 437204591Sluigi if (fs->fs.flags & DN_QHT_HASH) { 438204591Sluigi struct ipfw_flow_id masked_id; 439204591Sluigi if (fs->qht == NULL) { 440204591Sluigi fs->qht = dn_ht_init(NULL, fs->fs.buckets, 441204591Sluigi offsetof(struct dn_queue, q_next), 442204591Sluigi q_hash, q_match, q_new); 443204591Sluigi if (fs->qht == NULL) 444204591Sluigi return NULL; 445204591Sluigi } 446204591Sluigi masked_id = *id; 447204591Sluigi flow_id_mask(&fs->fsk_mask, &masked_id); 448204591Sluigi return dn_ht_find(fs->qht, (uintptr_t)&masked_id, 449204591Sluigi DNHT_INSERT, &template); 450204591Sluigi } else { 451204591Sluigi if (fs->qht == NULL) 452204591Sluigi fs->qht = q_new(0, 0, &template); 453204591Sluigi return (struct dn_queue *)fs->qht; 454200358Sluigi } 455200358Sluigi} 456204591Sluigi/*--- end of queue hash table ---*/ 457200358Sluigi 458204591Sluigi/*--- support functions for the sch_inst hashtable ---- 459204591Sluigi * 460204591Sluigi * These are hashed by flow-id 461126239Smlaier */ 462204591Sluigistatic uint32_t 463204591Sluigisi_hash(uintptr_t key, int flags, void *arg) 464126239Smlaier{ 465204591Sluigi /* compute the hash slot from the flow id */ 466204591Sluigi struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ? 467204591Sluigi &((struct dn_sch_inst *)key)->ni.fid : 468204591Sluigi (struct ipfw_flow_id *)key; 469204591Sluigi 470204591Sluigi return flow_id_hash(id); 471126239Smlaier} 472126239Smlaier 473204591Sluigistatic int 474204591Sluigisi_match(void *obj, uintptr_t key, int flags, void *arg) 475204591Sluigi{ 476204591Sluigi struct dn_sch_inst *o = obj; 477204591Sluigi struct ipfw_flow_id *id2; 478204591Sluigi 479204591Sluigi id2 = (flags & DNHT_KEY_IS_OBJ) ? 480204591Sluigi &((struct dn_sch_inst *)key)->ni.fid : 481204591Sluigi (struct ipfw_flow_id *)key; 482204591Sluigi return flow_id_cmp(&o->ni.fid, id2) == 0; 483204591Sluigi} 484204591Sluigi 485126239Smlaier/* 486204591Sluigi * create a new instance for the given 'key' 487204591Sluigi * Allocate memory for instance, delay line and scheduler private data. 48855597Sluigi */ 489204591Sluigistatic void * 490204591Sluigisi_new(uintptr_t key, int flags, void *arg) 49155597Sluigi{ 492204591Sluigi struct dn_schk *s = arg; 493204591Sluigi struct dn_sch_inst *si; 494204591Sluigi int l = sizeof(*si) + s->fp->si_datalen; 49555597Sluigi 496204591Sluigi si = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO); 497204591Sluigi if (si == NULL) 498204591Sluigi goto error; 499213267Sluigi 500204591Sluigi /* Set length only for the part passed up to userland. */ 501204591Sluigi set_oid(&si->ni.oid, DN_SCH_I, sizeof(struct dn_flow)); 502204591Sluigi set_oid(&(si->dline.oid), DN_DELAY_LINE, 503204591Sluigi sizeof(struct delay_line)); 504204591Sluigi /* mark si and dline as outside the event queue */ 505204591Sluigi si->ni.oid.id = si->dline.oid.id = -1; 506121140Ssam 507204591Sluigi si->sched = s; 508204591Sluigi si->dline.si = si; 50939119Sluigi 510204591Sluigi if (s->fp->new_sched && s->fp->new_sched(si)) { 511204591Sluigi D("new_sched error"); 512204591Sluigi goto error; 513155248Sglebius } 514204591Sluigi if (s->sch.flags & DN_HAVE_MASK) 515204591Sluigi si->ni.fid = *(struct ipfw_flow_id *)key; 51639119Sluigi 517301772Struckman#ifdef NEW_AQM 518301772Struckman /* init AQM status for !DN_MULTIQUEUE sched*/ 519301772Struckman if (!(s->fp->flags & DN_MULTIQUEUE)) 520301772Struckman if (s->fs->aqmfp && s->fs->aqmfp->init) 521301772Struckman if(s->fs->aqmfp->init((struct dn_queue *)(si + 1))) { 522301772Struckman D("unable to init AQM for fs %d", s->fs->fs.fs_nr); 523301772Struckman goto error; 524301772Struckman } 525301772Struckman#endif 526301772Struckman 527204591Sluigi dn_cfg.si_count++; 528204591Sluigi return si; 529204591Sluigi 530204591Sluigierror: 531204591Sluigi if (si) { 532204591Sluigi bzero(si, sizeof(*si)); // safety 533204591Sluigi free(si, M_DUMMYNET); 53439119Sluigi } 535204591Sluigi return NULL; 53639119Sluigi} 53755597Sluigi 53839119Sluigi/* 539204591Sluigi * Callback from siht to delete all scheduler instances. Remove 540204591Sluigi * si and delay line from the system heap, destroy all queues. 541204591Sluigi * We assume that all flowset have been notified and do not 542204591Sluigi * point to us anymore. 54361413Sluigi */ 544204591Sluigistatic int 545204591Sluigisi_destroy(void *_si, void *arg) 546190865Sluigi{ 547204591Sluigi struct dn_sch_inst *si = _si; 548204591Sluigi struct dn_schk *s = si->sched; 549204591Sluigi struct delay_line *dl = &si->dline; 55061413Sluigi 551204591Sluigi if (dl->oid.subtype) /* remove delay line from event heap */ 552204591Sluigi heap_extract(&dn_cfg.evheap, dl); 553204591Sluigi dn_free_pkts(dl->mq.head); /* drain delay line */ 554204591Sluigi if (si->kflags & DN_ACTIVE) /* remove si from event heap */ 555204591Sluigi heap_extract(&dn_cfg.evheap, si); 556301772Struckman 557301772Struckman#ifdef NEW_AQM 558301772Struckman /* clean up AQM status for !DN_MULTIQUEUE sched 559301772Struckman * Note that all queues belong to fs were cleaned up in fsk_detach. 560301772Struckman * When drain_scheduler is called s->fs and q->fs are pointing 561301772Struckman * to a correct fs, so we can use fs in this case. 562301772Struckman */ 563301772Struckman if (!(s->fp->flags & DN_MULTIQUEUE)) { 564301772Struckman struct dn_queue *q = (struct dn_queue *)(si + 1); 565301772Struckman if (q->aqm_status && q->fs->aqmfp) 566301772Struckman if (q->fs->aqmfp->cleanup) 567301772Struckman q->fs->aqmfp->cleanup(q); 568301772Struckman } 569301772Struckman#endif 570204591Sluigi if (s->fp->free_sched) 571204591Sluigi s->fp->free_sched(si); 572204591Sluigi bzero(si, sizeof(*si)); /* safety */ 573204591Sluigi free(si, M_DUMMYNET); 574204591Sluigi dn_cfg.si_count--; 575204591Sluigi return DNHT_SCAN_DEL; 576190865Sluigi} 577190865Sluigi 57861413Sluigi/* 579204591Sluigi * Find the scheduler instance for this packet. If we need to apply 580204591Sluigi * a mask, do on a local copy of the flow_id to preserve the original. 581204591Sluigi * Assume siht is always initialized if we have a mask. 582190865Sluigi */ 583204591Sluigistruct dn_sch_inst * 584204591Sluigiipdn_si_find(struct dn_schk *s, struct ipfw_flow_id *id) 585190865Sluigi{ 586190865Sluigi 587204591Sluigi if (s->sch.flags & DN_HAVE_MASK) { 588204591Sluigi struct ipfw_flow_id id_t = *id; 589204591Sluigi flow_id_mask(&s->sch.sched_mask, &id_t); 590204591Sluigi return dn_ht_find(s->siht, (uintptr_t)&id_t, 591204591Sluigi DNHT_INSERT, s); 592190865Sluigi } 593204591Sluigi if (!s->siht) 594204591Sluigi s->siht = si_new(0, 0, s); 595204591Sluigi return (struct dn_sch_inst *)s->siht; 596190865Sluigi} 597190865Sluigi 598204591Sluigi/* callback to flush credit for the scheduler instance */ 599204591Sluigistatic int 600204591Sluigisi_reset_credit(void *_si, void *arg) 601204591Sluigi{ 602204591Sluigi struct dn_sch_inst *si = _si; 603204591Sluigi struct dn_link *p = &si->sched->link; 604204591Sluigi 605204591Sluigi si->credit = p->burst + (dn_cfg.io_fast ? p->bandwidth : 0); 606204591Sluigi return 0; 607204591Sluigi} 608204591Sluigi 609190865Sluigistatic void 610204591Sluigischk_reset_credit(struct dn_schk *s) 611190865Sluigi{ 612204591Sluigi if (s->sch.flags & DN_HAVE_MASK) 613204591Sluigi dn_ht_scan(s->siht, si_reset_credit, NULL); 614204591Sluigi else if (s->siht) 615204591Sluigi si_reset_credit(s->siht, NULL); 616190865Sluigi} 617204591Sluigi/*---- end of sch_inst hashtable ---------------------*/ 618190865Sluigi 619204591Sluigi/*------------------------------------------------------- 620204591Sluigi * flowset hash (fshash) support. Entries are hashed by fs_nr. 621204591Sluigi * New allocations are put in the fsunlinked list, from which 622204591Sluigi * they are removed when they point to a specific scheduler. 62361413Sluigi */ 624204591Sluigistatic uint32_t 625204591Sluigifsk_hash(uintptr_t key, int flags, void *arg) 62661413Sluigi{ 627204591Sluigi uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key : 628204591Sluigi ((struct dn_fsk *)key)->fs.fs_nr; 629126239Smlaier 630204591Sluigi return ( (i>>8)^(i>>4)^i ); 631204591Sluigi} 63261413Sluigi 633204591Sluigistatic int 634204591Sluigifsk_match(void *obj, uintptr_t key, int flags, void *arg) 635204591Sluigi{ 636204591Sluigi struct dn_fsk *fs = obj; 637204591Sluigi int i = !(flags & DNHT_KEY_IS_OBJ) ? key : 638204591Sluigi ((struct dn_fsk *)key)->fs.fs_nr; 63961413Sluigi 640204591Sluigi return (fs->fs.fs_nr == i); 64161413Sluigi} 64261413Sluigi 643204591Sluigistatic void * 644204591Sluigifsk_new(uintptr_t key, int flags, void *arg) 64555597Sluigi{ 646204591Sluigi struct dn_fsk *fs; 64755597Sluigi 648204591Sluigi fs = malloc(sizeof(*fs), M_DUMMYNET, M_NOWAIT | M_ZERO); 649204591Sluigi if (fs) { 650204591Sluigi set_oid(&fs->fs.oid, DN_FS, sizeof(fs->fs)); 651204591Sluigi dn_cfg.fsk_count++; 652204591Sluigi fs->drain_bucket = 0; 653204591Sluigi SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain); 654173398Soleg } 655204591Sluigi return fs; 65661413Sluigi} 65761413Sluigi 658301772Struckman#ifdef NEW_AQM 659301772Struckman/* callback function for cleaning up AQM queue status belongs to a flowset 660301772Struckman * connected to scheduler instance '_si' (for !DN_MULTIQUEUE only). 661301772Struckman */ 662301772Struckmanstatic int 663301772Struckmansi_cleanup_q(void *_si, void *arg) 664301772Struckman{ 665301772Struckman struct dn_sch_inst *si = _si; 666301772Struckman 667301772Struckman if (!(si->sched->fp->flags & DN_MULTIQUEUE)) { 668301772Struckman if (si->sched->fs->aqmfp && si->sched->fs->aqmfp->cleanup) 669301772Struckman si->sched->fs->aqmfp->cleanup((struct dn_queue *) (si+1)); 670301772Struckman } 671301772Struckman return 0; 672301772Struckman} 673301772Struckman 674301772Struckman/* callback to clean up queue AQM status.*/ 675301772Struckmanstatic int 676301772Struckmanq_cleanup_q(void *_q, void *arg) 677301772Struckman{ 678301772Struckman struct dn_queue *q = _q; 679301772Struckman q->fs->aqmfp->cleanup(q); 680301772Struckman return 0; 681301772Struckman} 682301772Struckman 683301772Struckman/* Clean up all AQM queues status belongs to flowset 'fs' and then 684301772Struckman * deconfig AQM for flowset 'fs' 685301772Struckman */ 686301772Struckmanstatic void 687301772Struckmanaqm_cleanup_deconfig_fs(struct dn_fsk *fs) 688301772Struckman{ 689301772Struckman struct dn_sch_inst *si; 690301772Struckman 691301772Struckman /* clean up AQM status for all queues for !DN_MULTIQUEUE sched*/ 692301772Struckman if (fs->fs.fs_nr > DN_MAX_ID) { 693301772Struckman if (fs->sched && !(fs->sched->fp->flags & DN_MULTIQUEUE)) { 694301772Struckman if (fs->sched->sch.flags & DN_HAVE_MASK) 695301772Struckman dn_ht_scan(fs->sched->siht, si_cleanup_q, NULL); 696301772Struckman else { 697301772Struckman /* single si i.e. no sched mask */ 698301772Struckman si = (struct dn_sch_inst *) fs->sched->siht; 699301772Struckman if (si && fs->aqmfp && fs->aqmfp->cleanup) 700301772Struckman fs->aqmfp->cleanup((struct dn_queue *) (si+1)); 701301772Struckman } 702301772Struckman } 703301772Struckman } 704301772Struckman 705301772Struckman /* clean up AQM status for all queues for DN_MULTIQUEUE sched*/ 706301772Struckman if (fs->sched && fs->sched->fp->flags & DN_MULTIQUEUE && fs->qht) { 707301772Struckman if (fs->fs.flags & DN_QHT_HASH) 708301772Struckman dn_ht_scan(fs->qht, q_cleanup_q, NULL); 709301772Struckman else 710301772Struckman fs->aqmfp->cleanup((struct dn_queue *)(fs->qht)); 711301772Struckman } 712301772Struckman 713301772Struckman /* deconfig AQM */ 714301772Struckman if(fs->aqmcfg && fs->aqmfp && fs->aqmfp->deconfig) 715301772Struckman fs->aqmfp->deconfig(fs); 716301772Struckman} 717301772Struckman#endif 718301772Struckman 71961413Sluigi/* 720204591Sluigi * detach flowset from its current scheduler. Flags as follows: 721204591Sluigi * DN_DETACH removes from the fsk_list 722204591Sluigi * DN_DESTROY deletes individual queues 723204591Sluigi * DN_DELETE_FS destroys the flowset (otherwise goes in unlinked). 72461413Sluigi */ 72561413Sluigistatic void 726204591Sluigifsk_detach(struct dn_fsk *fs, int flags) 72761413Sluigi{ 728204591Sluigi if (flags & DN_DELETE_FS) 729204591Sluigi flags |= DN_DESTROY; 730204591Sluigi ND("fs %d from sched %d flags %s %s %s", 731204591Sluigi fs->fs.fs_nr, fs->fs.sched_nr, 732204591Sluigi (flags & DN_DELETE_FS) ? "DEL_FS":"", 733204591Sluigi (flags & DN_DESTROY) ? "DEL":"", 734204591Sluigi (flags & DN_DETACH) ? "DET":""); 735204591Sluigi if (flags & DN_DETACH) { /* detach from the list */ 736204591Sluigi struct dn_fsk_head *h; 737204591Sluigi h = fs->sched ? &fs->sched->fsk_list : &dn_cfg.fsu; 738204591Sluigi SLIST_REMOVE(h, fs, dn_fsk, sch_chain); 73961413Sluigi } 740204754Sluigi /* Free the RED parameters, they will be recomputed on 741204754Sluigi * subsequent attach if needed. 742204754Sluigi */ 743204754Sluigi if (fs->w_q_lookup) 744204754Sluigi free(fs->w_q_lookup, M_DUMMYNET); 745204754Sluigi fs->w_q_lookup = NULL; 746204591Sluigi qht_delete(fs, flags); 747301772Struckman#ifdef NEW_AQM 748301772Struckman aqm_cleanup_deconfig_fs(fs); 749301772Struckman#endif 750301772Struckman 751204591Sluigi if (fs->sched && fs->sched->fp->free_fsk) 752204591Sluigi fs->sched->fp->free_fsk(fs); 753204591Sluigi fs->sched = NULL; 754204591Sluigi if (flags & DN_DELETE_FS) { 755250131Seadler bzero(fs, sizeof(*fs)); /* safety */ 756204591Sluigi free(fs, M_DUMMYNET); 757204591Sluigi dn_cfg.fsk_count--; 758204591Sluigi } else { 759204591Sluigi SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain); 76061413Sluigi } 76155597Sluigi} 76255597Sluigi 76355597Sluigi/* 764204591Sluigi * Detach or destroy all flowsets in a list. 765204591Sluigi * flags specifies what to do: 766204591Sluigi * DN_DESTROY: flush all queues 767204591Sluigi * DN_DELETE_FS: DN_DESTROY + destroy flowset 768204591Sluigi * DN_DELETE_FS implies DN_DESTROY 76955597Sluigi */ 77055597Sluigistatic void 771204591Sluigifsk_detach_list(struct dn_fsk_head *h, int flags) 77239119Sluigi{ 773204591Sluigi struct dn_fsk *fs; 774204591Sluigi int n = 0; /* only for stats */ 775169454Srwatson 776204591Sluigi ND("head %p flags %x", h, flags); 777204591Sluigi while ((fs = SLIST_FIRST(h))) { 778204591Sluigi SLIST_REMOVE_HEAD(h, sch_chain); 779204591Sluigi n++; 780204591Sluigi fsk_detach(fs, flags); 781204591Sluigi } 782204591Sluigi ND("done %d flowsets", n); 783163720Soleg} 784163720Soleg 785163720Soleg/* 786204591Sluigi * called on 'queue X delete' -- removes the flowset from fshash, 787204591Sluigi * deletes all queues for the flowset, and removes the flowset. 788163720Soleg */ 789204591Sluigistatic int 790204591Sluigidelete_fs(int i, int locked) 791163720Soleg{ 792204591Sluigi struct dn_fsk *fs; 793204591Sluigi int err = 0; 79439119Sluigi 795204591Sluigi if (!locked) 796204591Sluigi DN_BH_WLOCK(); 797204591Sluigi fs = dn_ht_find(dn_cfg.fshash, i, DNHT_REMOVE, NULL); 798204591Sluigi ND("fs %d found %p", i, fs); 799204591Sluigi if (fs) { 800204591Sluigi fsk_detach(fs, DN_DETACH | DN_DELETE_FS); 801204591Sluigi err = 0; 802204591Sluigi } else 803204591Sluigi err = EINVAL; 804204591Sluigi if (!locked) 805204591Sluigi DN_BH_WUNLOCK(); 806204591Sluigi return err; 807204591Sluigi} 808163720Soleg 809204591Sluigi/*----- end of flowset hashtable support -------------*/ 810120140Ssam 811204591Sluigi/*------------------------------------------------------------ 812204591Sluigi * Scheduler hash. When searching by index we pass sched_nr, 813204591Sluigi * otherwise we pass struct dn_sch * which is the first field in 814204591Sluigi * struct dn_schk so we can cast between the two. We use this trick 815204591Sluigi * because in the create phase (but it should be fixed). 816204591Sluigi */ 817204591Sluigistatic uint32_t 818204591Sluigischk_hash(uintptr_t key, int flags, void *_arg) 819204591Sluigi{ 820204591Sluigi uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key : 821204591Sluigi ((struct dn_schk *)key)->sch.sched_nr; 822204591Sluigi return ( (i>>8)^(i>>4)^i ); 823204591Sluigi} 82471133Sluigi 825204591Sluigistatic int 826204591Sluigischk_match(void *obj, uintptr_t key, int flags, void *_arg) 827204591Sluigi{ 828204591Sluigi struct dn_schk *s = (struct dn_schk *)obj; 829204591Sluigi int i = !(flags & DNHT_KEY_IS_OBJ) ? key : 830204591Sluigi ((struct dn_schk *)key)->sch.sched_nr; 831204591Sluigi return (s->sch.sched_nr == i); 832204591Sluigi} 833152910Sglebius 834204591Sluigi/* 835204591Sluigi * Create the entry and intialize with the sched hash if needed. 836204591Sluigi * Leave s->fp unset so we can tell whether a dn_ht_find() returns 837204591Sluigi * a new object or a previously existing one. 838204591Sluigi */ 839204591Sluigistatic void * 840204591Sluigischk_new(uintptr_t key, int flags, void *arg) 841204591Sluigi{ 842204591Sluigi struct schk_new_arg *a = arg; 843204591Sluigi struct dn_schk *s; 844204591Sluigi int l = sizeof(*s) +a->fp->schk_datalen; 845120140Ssam 846204591Sluigi s = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO); 847204591Sluigi if (s == NULL) 848204591Sluigi return NULL; 849204591Sluigi set_oid(&s->link.oid, DN_LINK, sizeof(s->link)); 850204591Sluigi s->sch = *a->sch; // copy initial values 851204591Sluigi s->link.link_nr = s->sch.sched_nr; 852204591Sluigi SLIST_INIT(&s->fsk_list); 853204591Sluigi /* initialize the hash table or create the single instance */ 854204591Sluigi s->fp = a->fp; /* si_new needs this */ 855204591Sluigi s->drain_bucket = 0; 856204591Sluigi if (s->sch.flags & DN_HAVE_MASK) { 857204591Sluigi s->siht = dn_ht_init(NULL, s->sch.buckets, 858204591Sluigi offsetof(struct dn_sch_inst, si_next), 859204591Sluigi si_hash, si_match, si_new); 860204591Sluigi if (s->siht == NULL) { 861204591Sluigi free(s, M_DUMMYNET); 862204591Sluigi return NULL; 863201121Sluigi } 864201121Sluigi } 865204591Sluigi s->fp = NULL; /* mark as a new scheduler */ 866204591Sluigi dn_cfg.schk_count++; 867204591Sluigi return s; 86839119Sluigi} 869107900Smaxim 870204591Sluigi/* 871204591Sluigi * Callback for sched delete. Notify all attached flowsets to 872204591Sluigi * detach from the scheduler, destroy the internal flowset, and 873204591Sluigi * all instances. The scheduler goes away too. 874204591Sluigi * arg is 0 (only detach flowsets and destroy instances) 875204591Sluigi * DN_DESTROY (detach & delete queues, delete schk) 876204591Sluigi * or DN_DELETE_FS (delete queues and flowsets, delete schk) 877204591Sluigi */ 878204591Sluigistatic int 879204591Sluigischk_delete_cb(void *obj, void *arg) 880155248Sglebius{ 881204591Sluigi struct dn_schk *s = obj; 882204591Sluigi#if 0 883204591Sluigi int a = (int)arg; 884204591Sluigi ND("sched %d arg %s%s", 885204591Sluigi s->sch.sched_nr, 886204591Sluigi a&DN_DESTROY ? "DEL ":"", 887204591Sluigi a&DN_DELETE_FS ? "DEL_FS":""); 888155248Sglebius#endif 889204591Sluigi fsk_detach_list(&s->fsk_list, arg ? DN_DESTROY : 0); 890204591Sluigi /* no more flowset pointing to us now */ 891220203Sae if (s->sch.flags & DN_HAVE_MASK) { 892204591Sluigi dn_ht_scan(s->siht, si_destroy, NULL); 893220203Sae dn_ht_free(s->siht, 0); 894220203Sae } else if (s->siht) 895204591Sluigi si_destroy(s->siht, NULL); 896204591Sluigi if (s->profile) { 897204591Sluigi free(s->profile, M_DUMMYNET); 898204591Sluigi s->profile = NULL; 899155248Sglebius } 900204591Sluigi s->siht = NULL; 901204591Sluigi if (s->fp->destroy) 902204591Sluigi s->fp->destroy(s); 903204591Sluigi bzero(s, sizeof(*s)); // safety 904204591Sluigi free(obj, M_DUMMYNET); 905204591Sluigi dn_cfg.schk_count--; 906204591Sluigi return DNHT_SCAN_DEL; 907155248Sglebius} 908155248Sglebius 90955597Sluigi/* 910204591Sluigi * called on a 'sched X delete' command. Deletes a single scheduler. 911204591Sluigi * This is done by removing from the schedhash, unlinking all 912204591Sluigi * flowsets and deleting their traffic. 91361413Sluigi */ 91461413Sluigistatic int 915204591Sluigidelete_schk(int i) 91661413Sluigi{ 917204591Sluigi struct dn_schk *s; 91861413Sluigi 919204591Sluigi s = dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL); 920204591Sluigi ND("%d %p", i, s); 921204591Sluigi if (!s) 922204591Sluigi return EINVAL; 923204591Sluigi delete_fs(i + DN_MAX_ID, 1); /* first delete internal fs */ 924204591Sluigi /* then detach flowsets, delete traffic */ 925204591Sluigi schk_delete_cb(s, (void*)(uintptr_t)DN_DESTROY); 926204591Sluigi return 0; 92761413Sluigi} 928204591Sluigi/*--- end of schk hashtable support ---*/ 92961413Sluigi 930204591Sluigistatic int 931204591Sluigicopy_obj(char **start, char *end, void *_o, const char *msg, int i) 93261413Sluigi{ 933318155Smarius struct dn_id o; 934318155Smarius union { 935318155Smarius struct dn_link l; 936318155Smarius struct dn_schk s; 937318155Smarius } dn; 938204591Sluigi int have = end - *start; 93961413Sluigi 940318155Smarius memcpy(&o, _o, sizeof(o)); 941318155Smarius if (have < o.len || o.len == 0 || o.type == 0) { 942205050Sluigi D("(WARN) type %d %s %d have %d need %d", 943318155Smarius o.type, msg, i, have, o.len); 944204591Sluigi return 1; 945173398Soleg } 946318155Smarius ND("type %d %s %d len %d", o.type, msg, i, o.len); 947318155Smarius if (o.type == DN_LINK) { 948318155Smarius memcpy(&dn.l, _o, sizeof(dn.l)); 949204591Sluigi /* Adjust burst parameter for link */ 950318155Smarius dn.l.burst = div64(dn.l.burst, 8 * hz); 951318155Smarius dn.l.delay = dn.l.delay * 1000 / hz; 952318155Smarius memcpy(*start, &dn.l, sizeof(dn.l)); 953318155Smarius } else if (o.type == DN_SCH) { 954318155Smarius /* Set dn.s.sch.oid.id to the number of instances */ 955318155Smarius memcpy(&dn.s, _o, sizeof(dn.s)); 956318155Smarius dn.s.sch.oid.id = (dn.s.sch.flags & DN_HAVE_MASK) ? 957318155Smarius dn_ht_entries(dn.s.siht) : (dn.s.siht ? 1 : 0); 958318155Smarius memcpy(*start, &dn.s, sizeof(dn.s)); 959318155Smarius } else 960318155Smarius memcpy(*start, _o, o.len); 961318155Smarius *start += o.len; 962204591Sluigi return 0; 96361413Sluigi} 96461413Sluigi 965204591Sluigi/* Specific function to copy a queue. 966205178Sluigi * Copies only the user-visible part of a queue (which is in 967205178Sluigi * a struct dn_flow), and sets len accordingly. 96855597Sluigi */ 969204591Sluigistatic int 970204591Sluigicopy_obj_q(char **start, char *end, void *_o, const char *msg, int i) 97155597Sluigi{ 972204591Sluigi struct dn_id *o = _o; 973204591Sluigi int have = end - *start; 974205178Sluigi int len = sizeof(struct dn_flow); /* see above comment */ 97539119Sluigi 976204591Sluigi if (have < len || o->len == 0 || o->type != DN_QUEUE) { 977204591Sluigi D("ERROR type %d %s %d have %d need %d", 978204591Sluigi o->type, msg, i, have, len); 979204591Sluigi return 1; 980145246Sbrooks } 981204591Sluigi ND("type %d %s %d len %d", o->type, msg, i, len); 982318155Smarius memcpy(*start, _o, len); 983204591Sluigi ((struct dn_id*)(*start))->len = len; 984204591Sluigi *start += len; 985204591Sluigi return 0; 986204591Sluigi} 987145246Sbrooks 988204591Sluigistatic int 989204591Sluigicopy_q_cb(void *obj, void *arg) 990204591Sluigi{ 991204591Sluigi struct dn_queue *q = obj; 992204591Sluigi struct copy_args *a = arg; 993204591Sluigi struct dn_flow *ni = (struct dn_flow *)(*a->start); 994204591Sluigi if (copy_obj_q(a->start, a->end, &q->ni, "queue", -1)) 995204591Sluigi return DNHT_SCAN_END; 996204591Sluigi ni->oid.type = DN_FLOW; /* override the DN_QUEUE */ 997204591Sluigi ni->oid.id = si_hash((uintptr_t)&ni->fid, 0, NULL); 998204591Sluigi return 0; 999204591Sluigi} 1000145246Sbrooks 1001204591Sluigistatic int 1002204591Sluigicopy_q(struct copy_args *a, struct dn_fsk *fs, int flags) 1003204591Sluigi{ 1004204591Sluigi if (!fs->qht) 1005204591Sluigi return 0; 1006204591Sluigi if (fs->fs.flags & DN_QHT_HASH) 1007204591Sluigi dn_ht_scan(fs->qht, copy_q_cb, a); 1008204591Sluigi else 1009204591Sluigi copy_q_cb(fs->qht, a); 1010204591Sluigi return 0; 101155597Sluigi} 101255597Sluigi 1013204591Sluigi/* 1014204591Sluigi * This routine only copies the initial part of a profile ? XXX 1015204591Sluigi */ 101661413Sluigistatic int 1017204591Sluigicopy_profile(struct copy_args *a, struct dn_profile *p) 101861413Sluigi{ 1019204591Sluigi int have = a->end - *a->start; 1020204591Sluigi /* XXX here we check for max length */ 1021204591Sluigi int profile_len = sizeof(struct dn_profile) - 1022204591Sluigi ED_MAX_SAMPLES_NO*sizeof(int); 102361413Sluigi 1024204591Sluigi if (p == NULL) 1025204591Sluigi return 0; 1026204591Sluigi if (have < profile_len) { 1027204591Sluigi D("error have %d need %d", have, profile_len); 1028204591Sluigi return 1; 102961413Sluigi } 1030318155Smarius memcpy(*a->start, p, profile_len); 1031204591Sluigi ((struct dn_id *)(*a->start))->len = profile_len; 1032204591Sluigi *a->start += profile_len; 1033204591Sluigi return 0; 1034204591Sluigi} 103561413Sluigi 1036204591Sluigistatic int 1037204591Sluigicopy_flowset(struct copy_args *a, struct dn_fsk *fs, int flags) 1038204591Sluigi{ 1039204591Sluigi struct dn_fs *ufs = (struct dn_fs *)(*a->start); 1040204591Sluigi if (!fs) 1041204591Sluigi return 0; 1042204591Sluigi ND("flowset %d", fs->fs.fs_nr); 1043204591Sluigi if (copy_obj(a->start, a->end, &fs->fs, "flowset", fs->fs.fs_nr)) 1044204591Sluigi return DNHT_SCAN_END; 1045204591Sluigi ufs->oid.id = (fs->fs.flags & DN_QHT_HASH) ? 1046204591Sluigi dn_ht_entries(fs->qht) : (fs->qht ? 1 : 0); 1047204591Sluigi if (flags) { /* copy queues */ 1048204591Sluigi copy_q(a, fs, 0); 1049163717Soleg } 1050204591Sluigi return 0; 105161413Sluigi} 105261413Sluigi 1053204591Sluigistatic int 1054204591Sluigicopy_si_cb(void *obj, void *arg) 105561413Sluigi{ 1056204591Sluigi struct dn_sch_inst *si = obj; 1057204591Sluigi struct copy_args *a = arg; 1058204591Sluigi struct dn_flow *ni = (struct dn_flow *)(*a->start); 1059204591Sluigi if (copy_obj(a->start, a->end, &si->ni, "inst", 1060204591Sluigi si->sched->sch.sched_nr)) 1061204591Sluigi return DNHT_SCAN_END; 1062204591Sluigi ni->oid.type = DN_FLOW; /* override the DN_SCH_I */ 1063204591Sluigi ni->oid.id = si_hash((uintptr_t)si, DNHT_KEY_IS_OBJ, NULL); 1064204591Sluigi return 0; 1065152910Sglebius} 106698943Sluigi 1067204591Sluigistatic int 1068204591Sluigicopy_si(struct copy_args *a, struct dn_schk *s, int flags) 1069152910Sglebius{ 1070204591Sluigi if (s->sch.flags & DN_HAVE_MASK) 1071204591Sluigi dn_ht_scan(s->siht, copy_si_cb, a); 1072204591Sluigi else if (s->siht) 1073204591Sluigi copy_si_cb(s->siht, a); 1074204591Sluigi return 0; 107561413Sluigi} 107661413Sluigi 107739119Sluigi/* 1078204591Sluigi * compute a list of children of a scheduler and copy up 107939119Sluigi */ 108098613Sluigistatic int 1081204591Sluigicopy_fsk_list(struct copy_args *a, struct dn_schk *s, int flags) 108239119Sluigi{ 1083204591Sluigi struct dn_fsk *fs; 1084204591Sluigi struct dn_id *o; 1085204591Sluigi uint32_t *p; 108639119Sluigi 1087204591Sluigi int n = 0, space = sizeof(*o); 1088204591Sluigi SLIST_FOREACH(fs, &s->fsk_list, sch_chain) { 1089204591Sluigi if (fs->fs.fs_nr < DN_MAX_ID) 1090204591Sluigi n++; 109171667Sluigi } 1092204591Sluigi space += n * sizeof(uint32_t); 1093204591Sluigi DX(3, "sched %d has %d flowsets", s->sch.sched_nr, n); 1094204591Sluigi if (a->end - *(a->start) < space) 1095204591Sluigi return DNHT_SCAN_END; 1096204591Sluigi o = (struct dn_id *)(*(a->start)); 1097204591Sluigi o->len = space; 1098204591Sluigi *a->start += o->len; 1099204591Sluigi o->type = DN_TEXT; 1100204591Sluigi p = (uint32_t *)(o+1); 1101204591Sluigi SLIST_FOREACH(fs, &s->fsk_list, sch_chain) 1102204591Sluigi if (fs->fs.fs_nr < DN_MAX_ID) 1103204591Sluigi *p++ = fs->fs.fs_nr; 1104204591Sluigi return 0; 1105204591Sluigi} 110655597Sluigi 1107204591Sluigistatic int 1108204591Sluigicopy_data_helper(void *_o, void *_arg) 1109204591Sluigi{ 1110204591Sluigi struct copy_args *a = _arg; 1111205050Sluigi uint32_t *r = a->extra->r; /* start of first range */ 1112205050Sluigi uint32_t *lim; /* first invalid pointer */ 1113205050Sluigi int n; 1114126239Smlaier 1115205050Sluigi lim = (uint32_t *)((char *)(a->extra) + a->extra->o.len); 1116205050Sluigi 1117205050Sluigi if (a->type == DN_LINK || a->type == DN_SCH) { 1118205050Sluigi /* pipe|sched show, we receive a dn_schk */ 1119205050Sluigi struct dn_schk *s = _o; 1120205050Sluigi 1121205050Sluigi n = s->sch.sched_nr; 1122205050Sluigi if (a->type == DN_SCH && n >= DN_MAX_ID) 1123205050Sluigi return 0; /* not a scheduler */ 1124205050Sluigi if (a->type == DN_LINK && n <= DN_MAX_ID) 1125205050Sluigi return 0; /* not a pipe */ 1126205050Sluigi 1127205050Sluigi /* see if the object is within one of our ranges */ 1128205050Sluigi for (;r < lim; r += 2) { 1129205050Sluigi if (n < r[0] || n > r[1]) 1130205050Sluigi continue; 1131205050Sluigi /* Found a valid entry, copy and we are done */ 1132205050Sluigi if (a->flags & DN_C_LINK) { 1133205050Sluigi if (copy_obj(a->start, a->end, 1134205050Sluigi &s->link, "link", n)) 1135205050Sluigi return DNHT_SCAN_END; 1136205050Sluigi if (copy_profile(a, s->profile)) 1137205050Sluigi return DNHT_SCAN_END; 1138205050Sluigi if (copy_flowset(a, s->fs, 0)) 1139205050Sluigi return DNHT_SCAN_END; 1140205050Sluigi } 1141205050Sluigi if (a->flags & DN_C_SCH) { 1142205050Sluigi if (copy_obj(a->start, a->end, 1143205050Sluigi &s->sch, "sched", n)) 1144205050Sluigi return DNHT_SCAN_END; 1145205050Sluigi /* list all attached flowsets */ 1146205050Sluigi if (copy_fsk_list(a, s, 0)) 1147205050Sluigi return DNHT_SCAN_END; 1148205050Sluigi } 1149205050Sluigi if (a->flags & DN_C_FLOW) 1150205050Sluigi copy_si(a, s, 0); 1151205050Sluigi break; 1152194930Soleg } 1153205050Sluigi } else if (a->type == DN_FS) { 1154205050Sluigi /* queue show, skip internal flowsets */ 1155205050Sluigi struct dn_fsk *fs = _o; 1156173399Soleg 1157205050Sluigi n = fs->fs.fs_nr; 1158205050Sluigi if (n >= DN_MAX_ID) 1159205050Sluigi return 0; 1160205050Sluigi /* see if the object is within one of our ranges */ 1161205050Sluigi for (;r < lim; r += 2) { 1162205050Sluigi if (n < r[0] || n > r[1]) 1163205050Sluigi continue; 1164205050Sluigi if (copy_flowset(a, fs, 0)) 1165204591Sluigi return DNHT_SCAN_END; 1166205050Sluigi copy_q(a, fs, 0); 1167205050Sluigi break; /* we are done */ 1168190865Sluigi } 116961413Sluigi } 1170204591Sluigi return 0; 117161413Sluigi} 117261413Sluigi 1173204591Sluigistatic inline struct dn_schk * 1174204591Sluigilocate_scheduler(int i) 117561413Sluigi{ 1176204591Sluigi return dn_ht_find(dn_cfg.schedhash, i, 0, NULL); 117739119Sluigi} 117839119Sluigi 117939119Sluigi/* 1180204591Sluigi * red parameters are in fixed point arithmetic. 118139119Sluigi */ 1182204591Sluigistatic int 1183204591Sluigiconfig_red(struct dn_fsk *fs) 118439119Sluigi{ 1185204591Sluigi int64_t s, idle, weight, w0; 1186204591Sluigi int t, i; 118739119Sluigi 1188204591Sluigi fs->w_q = fs->fs.w_q; 1189204591Sluigi fs->max_p = fs->fs.max_p; 1190222560Sae ND("called"); 1191204591Sluigi /* Doing stuff that was in userland */ 1192204591Sluigi i = fs->sched->link.bandwidth; 1193204591Sluigi s = (i <= 0) ? 0 : 1194204591Sluigi hz * dn_cfg.red_avg_pkt_size * 8 * SCALE(1) / i; 1195120140Ssam 1196204591Sluigi idle = div64((s * 3) , fs->w_q); /* s, fs->w_q scaled; idle not scaled */ 1197204591Sluigi fs->lookup_step = div64(idle , dn_cfg.red_lookup_depth); 1198204591Sluigi /* fs->lookup_step not scaled, */ 1199204591Sluigi if (!fs->lookup_step) 1200204591Sluigi fs->lookup_step = 1; 1201204591Sluigi w0 = weight = SCALE(1) - fs->w_q; //fs->w_q scaled 120239119Sluigi 1203204591Sluigi for (t = fs->lookup_step; t > 1; --t) 1204204591Sluigi weight = SCALE_MUL(weight, w0); 1205204591Sluigi fs->lookup_weight = (int)(weight); // scaled 120641793Sluigi 1207204591Sluigi /* Now doing stuff that was in kerneland */ 1208204591Sluigi fs->min_th = SCALE(fs->fs.min_th); 1209204591Sluigi fs->max_th = SCALE(fs->fs.max_th); 121041793Sluigi 1211301231Struckman if (fs->fs.max_th == fs->fs.min_th) 1212301231Struckman fs->c_1 = fs->max_p; 1213301231Struckman else 1214301231Struckman fs->c_1 = SCALE((int64_t)(fs->max_p)) / (fs->fs.max_th - fs->fs.min_th); 1215204591Sluigi fs->c_2 = SCALE_MUL(fs->c_1, SCALE(fs->fs.min_th)); 121641793Sluigi 1217204591Sluigi if (fs->fs.flags & DN_IS_GENTLE_RED) { 1218204591Sluigi fs->c_3 = (SCALE(1) - fs->max_p) / fs->fs.max_th; 1219204591Sluigi fs->c_4 = SCALE(1) - 2 * fs->max_p; 1220163717Soleg } 1221163717Soleg 1222163717Soleg /* If the lookup table already exist, free and create it again. */ 1223204591Sluigi if (fs->w_q_lookup) { 1224204591Sluigi free(fs->w_q_lookup, M_DUMMYNET); 1225204591Sluigi fs->w_q_lookup = NULL; 1226163717Soleg } 1227204591Sluigi if (dn_cfg.red_lookup_depth == 0) { 1228163717Soleg printf("\ndummynet: net.inet.ip.dummynet.red_lookup_depth" 1229163717Soleg "must be > 0\n"); 1230204591Sluigi fs->fs.flags &= ~DN_IS_RED; 1231204591Sluigi fs->fs.flags &= ~DN_IS_GENTLE_RED; 1232163717Soleg return (EINVAL); 1233163717Soleg } 1234204591Sluigi fs->lookup_depth = dn_cfg.red_lookup_depth; 1235204591Sluigi fs->w_q_lookup = (u_int *)malloc(fs->lookup_depth * sizeof(int), 1236107898Smaxim M_DUMMYNET, M_NOWAIT); 1237204591Sluigi if (fs->w_q_lookup == NULL) { 1238163717Soleg printf("dummynet: sorry, cannot allocate red lookup table\n"); 1239204591Sluigi fs->fs.flags &= ~DN_IS_RED; 1240204591Sluigi fs->fs.flags &= ~DN_IS_GENTLE_RED; 1241163717Soleg return(ENOSPC); 1242163717Soleg } 124341793Sluigi 1244163717Soleg /* Fill the lookup table with (1 - w_q)^x */ 1245204591Sluigi fs->w_q_lookup[0] = SCALE(1) - fs->w_q; 1246163717Soleg 1247204591Sluigi for (i = 1; i < fs->lookup_depth; i++) 1248204591Sluigi fs->w_q_lookup[i] = 1249204591Sluigi SCALE_MUL(fs->w_q_lookup[i - 1], fs->lookup_weight); 1250163717Soleg 1251204591Sluigi if (dn_cfg.red_avg_pkt_size < 1) 1252204591Sluigi dn_cfg.red_avg_pkt_size = 512; 1253204591Sluigi fs->avg_pkt_size = dn_cfg.red_avg_pkt_size; 1254204591Sluigi if (dn_cfg.red_max_pkt_size < 1) 1255204591Sluigi dn_cfg.red_max_pkt_size = 1500; 1256204591Sluigi fs->max_pkt_size = dn_cfg.red_max_pkt_size; 1257222560Sae ND("exit"); 1258204591Sluigi return 0; 125961413Sluigi} 126061413Sluigi 1261204591Sluigi/* Scan all flowset attached to this scheduler and update red */ 1262204591Sluigistatic void 1263204591Sluigiupdate_red(struct dn_schk *s) 126461413Sluigi{ 1265204591Sluigi struct dn_fsk *fs; 1266204591Sluigi SLIST_FOREACH(fs, &s->fsk_list, sch_chain) { 1267204591Sluigi if (fs && (fs->fs.flags & DN_IS_RED)) 1268204591Sluigi config_red(fs); 1269204591Sluigi } 1270204591Sluigi} 127161413Sluigi 1272204591Sluigi/* attach flowset to scheduler s, possibly requeue */ 1273204591Sluigistatic void 1274204591Sluigifsk_attach(struct dn_fsk *fs, struct dn_schk *s) 1275204591Sluigi{ 1276204591Sluigi ND("remove fs %d from fsunlinked, link to sched %d", 1277204591Sluigi fs->fs.fs_nr, s->sch.sched_nr); 1278204591Sluigi SLIST_REMOVE(&dn_cfg.fsu, fs, dn_fsk, sch_chain); 1279204591Sluigi fs->sched = s; 1280204591Sluigi SLIST_INSERT_HEAD(&s->fsk_list, fs, sch_chain); 1281204591Sluigi if (s->fp->new_fsk) 1282204591Sluigi s->fp->new_fsk(fs); 1283204591Sluigi /* XXX compute fsk_mask */ 1284204591Sluigi fs->fsk_mask = fs->fs.flow_mask; 1285204591Sluigi if (fs->sched->sch.flags & DN_HAVE_MASK) 1286204591Sluigi flow_id_or(&fs->sched->sch.sched_mask, &fs->fsk_mask); 1287204591Sluigi if (fs->qht) { 1288204591Sluigi /* 1289204591Sluigi * we must drain qht according to the old 1290204591Sluigi * type, and reinsert according to the new one. 1291204591Sluigi * The requeue is complex -- in general we need to 1292204591Sluigi * reclassify every single packet. 1293204591Sluigi * For the time being, let's hope qht is never set 1294204591Sluigi * when we reach this point. 1295204591Sluigi */ 1296204591Sluigi D("XXX TODO requeue from fs %d to sch %d", 1297204591Sluigi fs->fs.fs_nr, s->sch.sched_nr); 1298204591Sluigi fs->qht = NULL; 1299204591Sluigi } 1300204591Sluigi /* set the new type for qht */ 1301204591Sluigi if (nonzero_mask(&fs->fsk_mask)) 1302204591Sluigi fs->fs.flags |= DN_QHT_HASH; 1303204591Sluigi else 1304204591Sluigi fs->fs.flags &= ~DN_QHT_HASH; 1305204591Sluigi 1306204591Sluigi /* XXX config_red() can fail... */ 1307204591Sluigi if (fs->fs.flags & DN_IS_RED) 1308204591Sluigi config_red(fs); 130961413Sluigi} 131061413Sluigi 1311204591Sluigi/* update all flowsets which may refer to this scheduler */ 131261413Sluigistatic void 1313204591Sluigiupdate_fs(struct dn_schk *s) 131461413Sluigi{ 1315204591Sluigi struct dn_fsk *fs, *tmp; 1316204591Sluigi 1317204591Sluigi SLIST_FOREACH_SAFE(fs, &dn_cfg.fsu, sch_chain, tmp) { 1318204591Sluigi if (s->sch.sched_nr != fs->fs.sched_nr) { 1319204591Sluigi D("fs %d for sch %d not %d still unlinked", 1320204591Sluigi fs->fs.fs_nr, fs->fs.sched_nr, 1321204591Sluigi s->sch.sched_nr); 1322204591Sluigi continue; 1323204591Sluigi } 1324204591Sluigi fsk_attach(fs, s); 1325163717Soleg } 132671133Sluigi} 132741793Sluigi 1328301772Struckman#ifdef NEW_AQM 1329301772Struckman/* Retrieve AQM configurations to ipfw userland 1330301772Struckman */ 1331301772Struckmanstatic int 1332301772Struckmanget_aqm_parms(struct sockopt *sopt) 1333301772Struckman{ 1334301772Struckman struct dn_extra_parms *ep; 1335301772Struckman struct dn_fsk *fs; 1336301772Struckman size_t sopt_valsize; 1337301772Struckman int l, err = 0; 1338301772Struckman 1339301772Struckman sopt_valsize = sopt->sopt_valsize; 1340301772Struckman l = sizeof(*ep); 1341301772Struckman if (sopt->sopt_valsize < l) { 1342301772Struckman D("bad len sopt->sopt_valsize %d len %d", 1343301772Struckman (int) sopt->sopt_valsize , l); 1344301772Struckman err = EINVAL; 1345301772Struckman return err; 1346301772Struckman } 1347301772Struckman ep = malloc(l, M_DUMMYNET, M_WAITOK); 1348301772Struckman if(!ep) { 1349301772Struckman err = ENOMEM ; 1350301772Struckman return err; 1351301772Struckman } 1352301772Struckman do { 1353301772Struckman err = sooptcopyin(sopt, ep, l, l); 1354301772Struckman if(err) 1355301772Struckman break; 1356301772Struckman sopt->sopt_valsize = sopt_valsize; 1357301772Struckman if (ep->oid.len < l) { 1358301772Struckman err = EINVAL; 1359301772Struckman break; 1360301772Struckman } 1361301772Struckman 1362301772Struckman fs = dn_ht_find(dn_cfg.fshash, ep->nr, 0, NULL); 1363301772Struckman if (!fs) { 1364301772Struckman D("fs %d not found", ep->nr); 1365301772Struckman err = EINVAL; 1366301772Struckman break; 1367301772Struckman } 1368301772Struckman 1369301772Struckman if (fs->aqmfp && fs->aqmfp->getconfig) { 1370301772Struckman if(fs->aqmfp->getconfig(fs, ep)) { 1371301772Struckman D("Error while trying to get AQM params"); 1372301772Struckman err = EINVAL; 1373301772Struckman break; 1374301772Struckman } 1375301772Struckman ep->oid.len = l; 1376301772Struckman err = sooptcopyout(sopt, ep, l); 1377301772Struckman } 1378301772Struckman }while(0); 1379301772Struckman 1380301772Struckman free(ep, M_DUMMYNET); 1381301772Struckman return err; 1382301772Struckman} 1383301772Struckman 1384301772Struckman/* Retrieve AQM configurations to ipfw userland 1385301772Struckman */ 1386301772Struckmanstatic int 1387301772Struckmanget_sched_parms(struct sockopt *sopt) 1388301772Struckman{ 1389301772Struckman struct dn_extra_parms *ep; 1390301772Struckman struct dn_schk *schk; 1391301772Struckman size_t sopt_valsize; 1392301772Struckman int l, err = 0; 1393301772Struckman 1394301772Struckman sopt_valsize = sopt->sopt_valsize; 1395301772Struckman l = sizeof(*ep); 1396301772Struckman if (sopt->sopt_valsize < l) { 1397301772Struckman D("bad len sopt->sopt_valsize %d len %d", 1398301772Struckman (int) sopt->sopt_valsize , l); 1399301772Struckman err = EINVAL; 1400301772Struckman return err; 1401301772Struckman } 1402301772Struckman ep = malloc(l, M_DUMMYNET, M_WAITOK); 1403301772Struckman if(!ep) { 1404301772Struckman err = ENOMEM ; 1405301772Struckman return err; 1406301772Struckman } 1407301772Struckman do { 1408301772Struckman err = sooptcopyin(sopt, ep, l, l); 1409301772Struckman if(err) 1410301772Struckman break; 1411301772Struckman sopt->sopt_valsize = sopt_valsize; 1412301772Struckman if (ep->oid.len < l) { 1413301772Struckman err = EINVAL; 1414301772Struckman break; 1415301772Struckman } 1416301772Struckman 1417301772Struckman schk = locate_scheduler(ep->nr); 1418301772Struckman if (!schk) { 1419301772Struckman D("sched %d not found", ep->nr); 1420301772Struckman err = EINVAL; 1421301772Struckman break; 1422301772Struckman } 1423301772Struckman 1424301772Struckman if (schk->fp && schk->fp->getconfig) { 1425301772Struckman if(schk->fp->getconfig(schk, ep)) { 1426301772Struckman D("Error while trying to get sched params"); 1427301772Struckman err = EINVAL; 1428301772Struckman break; 1429301772Struckman } 1430301772Struckman ep->oid.len = l; 1431301772Struckman err = sooptcopyout(sopt, ep, l); 1432301772Struckman } 1433301772Struckman }while(0); 1434301772Struckman free(ep, M_DUMMYNET); 1435301772Struckman 1436301772Struckman return err; 1437301772Struckman} 1438301772Struckman 1439301772Struckman/* Configure AQM for flowset 'fs'. 1440301772Struckman * extra parameters are passed from userland. 1441301772Struckman */ 1442301772Struckmanstatic int 1443301772Struckmanconfig_aqm(struct dn_fsk *fs, struct dn_extra_parms *ep, int busy) 1444301772Struckman{ 1445301772Struckman int err = 0; 1446301772Struckman 1447301772Struckman do { 1448301772Struckman /* no configurations */ 1449301772Struckman if (!ep) { 1450301772Struckman err = 0; 1451301772Struckman break; 1452301772Struckman } 1453301772Struckman 1454301772Struckman /* no AQM for this flowset*/ 1455301772Struckman if (!strcmp(ep->name,"")) { 1456301772Struckman err = 0; 1457301772Struckman break; 1458301772Struckman } 1459301772Struckman if (ep->oid.len < sizeof(*ep)) { 1460301772Struckman D("short aqm len %d", ep->oid.len); 1461301772Struckman err = EINVAL; 1462301772Struckman break; 1463301772Struckman } 1464301772Struckman 1465301772Struckman if (busy) { 1466301772Struckman D("Unable to configure flowset, flowset busy!"); 1467301772Struckman err = EINVAL; 1468301772Struckman break; 1469301772Struckman } 1470301772Struckman 1471301772Struckman /* deconfigure old aqm if exist */ 1472301772Struckman if (fs->aqmcfg && fs->aqmfp && fs->aqmfp->deconfig) { 1473301772Struckman aqm_cleanup_deconfig_fs(fs); 1474301772Struckman } 1475301772Struckman 1476301772Struckman if (!(fs->aqmfp = find_aqm_type(0, ep->name))) { 1477301772Struckman D("AQM functions not found for type %s!", ep->name); 1478301772Struckman fs->fs.flags &= ~DN_IS_AQM; 1479301772Struckman err = EINVAL; 1480301772Struckman break; 1481301772Struckman } else 1482301772Struckman fs->fs.flags |= DN_IS_AQM; 1483301772Struckman 1484301772Struckman if (ep->oid.subtype != DN_AQM_PARAMS) { 1485301772Struckman D("Wrong subtype"); 1486301772Struckman err = EINVAL; 1487301772Struckman break; 1488301772Struckman } 1489301772Struckman 1490301772Struckman if (fs->aqmfp->config) { 1491301772Struckman err = fs->aqmfp->config(fs, ep, ep->oid.len); 1492301772Struckman if (err) { 1493301772Struckman D("Unable to configure AQM for FS %d", fs->fs.fs_nr ); 1494301772Struckman fs->fs.flags &= ~DN_IS_AQM; 1495301772Struckman fs->aqmfp = NULL; 1496301772Struckman break; 1497301772Struckman } 1498301772Struckman } 1499301772Struckman } while(0); 1500301772Struckman 1501301772Struckman return err; 1502301772Struckman} 1503301772Struckman#endif 1504301772Struckman 150571133Sluigi/* 1506204591Sluigi * Configuration -- to preserve backward compatibility we use 1507204591Sluigi * the following scheme (N is 65536) 1508204591Sluigi * NUMBER SCHED LINK FLOWSET 1509204591Sluigi * 1 .. N-1 (1)WFQ (2)WFQ (3)queue 1510204591Sluigi * N+1 .. 2N-1 (4)FIFO (5)FIFO (6)FIFO for sched 1..N-1 1511204591Sluigi * 2N+1 .. 3N-1 -- -- (7)FIFO for sched N+1..2N-1 1512204591Sluigi * 1513204591Sluigi * "pipe i config" configures #1, #2 and #3 1514204591Sluigi * "sched i config" configures #1 and possibly #6 1515204591Sluigi * "queue i config" configures #3 1516204591Sluigi * #1 is configured with 'pipe i config' or 'sched i config' 1517204591Sluigi * #2 is configured with 'pipe i config', and created if not 1518204591Sluigi * existing with 'sched i config' 1519204591Sluigi * #3 is configured with 'queue i config' 1520204591Sluigi * #4 is automatically configured after #1, can only be FIFO 1521204591Sluigi * #5 is automatically configured after #2 1522204591Sluigi * #6 is automatically created when #1 is !MULTIQUEUE, 1523204591Sluigi * and can be updated. 1524204591Sluigi * #7 is automatically configured after #2 152571133Sluigi */ 1526204591Sluigi 1527204591Sluigi/* 1528204591Sluigi * configure a link (and its FIFO instance) 1529204591Sluigi */ 1530107900Smaximstatic int 1531204591Sluigiconfig_link(struct dn_link *p, struct dn_id *arg) 153261413Sluigi{ 1533204591Sluigi int i; 153455597Sluigi 1535204591Sluigi if (p->oid.len != sizeof(*p)) { 1536204591Sluigi D("invalid pipe len %d", p->oid.len); 1537204591Sluigi return EINVAL; 1538204591Sluigi } 1539204591Sluigi i = p->link_nr; 1540204591Sluigi if (i <= 0 || i >= DN_MAX_ID) 1541204591Sluigi return EINVAL; 1542163717Soleg /* 1543163717Soleg * The config program passes parameters as follows: 1544163717Soleg * bw = bits/second (0 means no limits), 1545163717Soleg * delay = ms, must be translated into ticks. 1546163717Soleg * qsize = slots/bytes 1547204591Sluigi * burst ??? 1548163717Soleg */ 1549163717Soleg p->delay = (p->delay * hz) / 1000; 1550194930Soleg /* Scale burst size: bytes -> bits * hz */ 1551194930Soleg p->burst *= 8 * hz; 1552120140Ssam 1553204591Sluigi DN_BH_WLOCK(); 1554204591Sluigi /* do it twice, base link and FIFO link */ 1555204591Sluigi for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) { 1556204591Sluigi struct dn_schk *s = locate_scheduler(i); 1557204591Sluigi if (s == NULL) { 1558204591Sluigi DN_BH_WUNLOCK(); 1559204591Sluigi D("sched %d not found", i); 1560204591Sluigi return EINVAL; 1561204591Sluigi } 1562204591Sluigi /* remove profile if exists */ 1563204591Sluigi if (s->profile) { 1564204591Sluigi free(s->profile, M_DUMMYNET); 1565204591Sluigi s->profile = NULL; 1566204591Sluigi } 1567204591Sluigi /* copy all parameters */ 1568204591Sluigi s->link.oid = p->oid; 1569204591Sluigi s->link.link_nr = i; 1570204591Sluigi s->link.delay = p->delay; 1571204591Sluigi if (s->link.bandwidth != p->bandwidth) { 1572204591Sluigi /* XXX bandwidth changes, need to update red params */ 1573204591Sluigi s->link.bandwidth = p->bandwidth; 1574204591Sluigi update_red(s); 1575204591Sluigi } 1576204591Sluigi s->link.burst = p->burst; 1577204591Sluigi schk_reset_credit(s); 1578204591Sluigi } 1579204591Sluigi dn_cfg.id++; 1580204591Sluigi DN_BH_WUNLOCK(); 1581204591Sluigi return 0; 1582204591Sluigi} 158361413Sluigi 1584204591Sluigi/* 1585204591Sluigi * configure a flowset. Can be called from inside with locked=1, 1586204591Sluigi */ 1587204591Sluigistatic struct dn_fsk * 1588204591Sluigiconfig_fs(struct dn_fs *nfs, struct dn_id *arg, int locked) 1589204591Sluigi{ 1590204591Sluigi int i; 1591204591Sluigi struct dn_fsk *fs; 1592318155Smarius#ifdef NEW_AQM 1593318155Smarius struct dn_extra_parms *ep; 1594318155Smarius#endif 159561413Sluigi 1596204591Sluigi if (nfs->oid.len != sizeof(*nfs)) { 1597204591Sluigi D("invalid flowset len %d", nfs->oid.len); 1598204591Sluigi return NULL; 1599204591Sluigi } 1600204591Sluigi i = nfs->fs_nr; 1601204591Sluigi if (i <= 0 || i >= 3*DN_MAX_ID) 1602204591Sluigi return NULL; 1603318155Smarius#ifdef NEW_AQM 1604318155Smarius ep = NULL; 1605318155Smarius if (arg != NULL) { 1606318155Smarius ep = malloc(sizeof(*ep), M_TEMP, locked ? M_NOWAIT : M_WAITOK); 1607318155Smarius if (ep == NULL) 1608318155Smarius return (NULL); 1609318155Smarius memcpy(ep, arg, sizeof(*ep)); 1610318155Smarius } 1611318155Smarius#endif 1612204591Sluigi ND("flowset %d", i); 1613204591Sluigi /* XXX other sanity checks */ 1614204591Sluigi if (nfs->flags & DN_QSIZE_BYTES) { 1615204591Sluigi ipdn_bound_var(&nfs->qsize, 16384, 1616204591Sluigi 1500, dn_cfg.byte_limit, NULL); // "queue byte size"); 1617204591Sluigi } else { 1618204591Sluigi ipdn_bound_var(&nfs->qsize, 50, 1619204591Sluigi 1, dn_cfg.slot_limit, NULL); // "queue slot size"); 1620204591Sluigi } 1621204591Sluigi if (nfs->flags & DN_HAVE_MASK) { 1622204591Sluigi /* make sure we have some buckets */ 1623238988Sluigi ipdn_bound_var((int *)&nfs->buckets, dn_cfg.hash_size, 1624204591Sluigi 1, dn_cfg.max_hash_size, "flowset buckets"); 1625204591Sluigi } else { 1626204591Sluigi nfs->buckets = 1; /* we only need 1 */ 1627204591Sluigi } 1628204591Sluigi if (!locked) 1629204591Sluigi DN_BH_WLOCK(); 1630204591Sluigi do { /* exit with break when done */ 1631204591Sluigi struct dn_schk *s; 1632204591Sluigi int flags = nfs->sched_nr ? DNHT_INSERT : 0; 1633204591Sluigi int j; 1634204591Sluigi int oldc = dn_cfg.fsk_count; 1635204591Sluigi fs = dn_ht_find(dn_cfg.fshash, i, flags, NULL); 1636204591Sluigi if (fs == NULL) { 1637204591Sluigi D("missing sched for flowset %d", i); 1638204591Sluigi break; 1639204591Sluigi } 1640204591Sluigi /* grab some defaults from the existing one */ 1641204591Sluigi if (nfs->sched_nr == 0) /* reuse */ 1642204591Sluigi nfs->sched_nr = fs->fs.sched_nr; 1643204591Sluigi for (j = 0; j < sizeof(nfs->par)/sizeof(nfs->par[0]); j++) { 1644204591Sluigi if (nfs->par[j] == -1) /* reuse */ 1645204591Sluigi nfs->par[j] = fs->fs.par[j]; 1646204591Sluigi } 1647204591Sluigi if (bcmp(&fs->fs, nfs, sizeof(*nfs)) == 0) { 1648204591Sluigi ND("flowset %d unchanged", i); 1649301772Struckman#ifdef NEW_AQM 1650318155Smarius if (ep != NULL) { 1651318155Smarius /* 1652318155Smarius * Reconfigure AQM as the parameters can be changed. 1653318155Smarius * We consider the flowset as busy if it has scheduler 1654318155Smarius * instance(s). 1655318155Smarius */ 1656318155Smarius s = locate_scheduler(nfs->sched_nr); 1657318155Smarius config_aqm(fs, ep, s != NULL && s->siht != NULL); 1658318155Smarius } 1659301772Struckman#endif 1660204591Sluigi break; /* no change, nothing to do */ 1661204591Sluigi } 1662204591Sluigi if (oldc != dn_cfg.fsk_count) /* new item */ 1663204591Sluigi dn_cfg.id++; 1664204591Sluigi s = locate_scheduler(nfs->sched_nr); 1665204591Sluigi /* detach from old scheduler if needed, preserving 1666204591Sluigi * queues if we need to reattach. Then update the 1667204591Sluigi * configuration, and possibly attach to the new sched. 1668204591Sluigi */ 1669204591Sluigi DX(2, "fs %d changed sched %d@%p to %d@%p", 1670204591Sluigi fs->fs.fs_nr, 1671204591Sluigi fs->fs.sched_nr, fs->sched, nfs->sched_nr, s); 1672204591Sluigi if (fs->sched) { 1673204591Sluigi int flags = s ? DN_DETACH : (DN_DETACH | DN_DESTROY); 1674204591Sluigi flags |= DN_DESTROY; /* XXX temporary */ 1675204591Sluigi fsk_detach(fs, flags); 1676204591Sluigi } 1677204591Sluigi fs->fs = *nfs; /* copy configuration */ 1678301772Struckman#ifdef NEW_AQM 1679301772Struckman fs->aqmfp = NULL; 1680318155Smarius if (ep != NULL) 1681318155Smarius config_aqm(fs, ep, s != NULL && 1682318155Smarius s->siht != NULL); 1683301772Struckman#endif 1684204591Sluigi if (s != NULL) 1685204591Sluigi fsk_attach(fs, s); 1686204591Sluigi } while (0); 1687204591Sluigi if (!locked) 1688204591Sluigi DN_BH_WUNLOCK(); 1689318155Smarius#ifdef NEW_AQM 1690318155Smarius if (ep != NULL) 1691318155Smarius free(ep, M_TEMP); 1692318155Smarius#endif 1693204591Sluigi return fs; 1694204591Sluigi} 169561413Sluigi 1696204591Sluigi/* 1697204591Sluigi * config/reconfig a scheduler and its FIFO variant. 1698204591Sluigi * For !MULTIQUEUE schedulers, also set up the flowset. 1699204591Sluigi * 1700204591Sluigi * On reconfigurations (detected because s->fp is set), 1701204591Sluigi * detach existing flowsets preserving traffic, preserve link, 1702204591Sluigi * and delete the old scheduler creating a new one. 1703204591Sluigi */ 1704204591Sluigistatic int 1705204591Sluigiconfig_sched(struct dn_sch *_nsch, struct dn_id *arg) 1706204591Sluigi{ 1707204591Sluigi struct dn_schk *s; 1708204591Sluigi struct schk_new_arg a; /* argument for schk_new */ 1709204591Sluigi int i; 1710204591Sluigi struct dn_link p; /* copy of oldlink */ 1711204763Sluigi struct dn_profile *pf = NULL; /* copy of old link profile */ 1712204591Sluigi /* Used to preserv mask parameter */ 1713204591Sluigi struct ipfw_flow_id new_mask; 1714204591Sluigi int new_buckets = 0; 1715204591Sluigi int new_flags = 0; 1716204591Sluigi int pipe_cmd; 1717204763Sluigi int err = ENOMEM; 1718190865Sluigi 1719204591Sluigi a.sch = _nsch; 1720204591Sluigi if (a.sch->oid.len != sizeof(*a.sch)) { 1721204591Sluigi D("bad sched len %d", a.sch->oid.len); 1722204591Sluigi return EINVAL; 1723204591Sluigi } 1724204591Sluigi i = a.sch->sched_nr; 1725204591Sluigi if (i <= 0 || i >= DN_MAX_ID) 1726204591Sluigi return EINVAL; 1727204591Sluigi /* make sure we have some buckets */ 1728204591Sluigi if (a.sch->flags & DN_HAVE_MASK) 1729238988Sluigi ipdn_bound_var((int *)&a.sch->buckets, dn_cfg.hash_size, 1730204591Sluigi 1, dn_cfg.max_hash_size, "sched buckets"); 1731204591Sluigi /* XXX other sanity checks */ 1732204591Sluigi bzero(&p, sizeof(p)); 1733190865Sluigi 1734204591Sluigi pipe_cmd = a.sch->flags & DN_PIPE_CMD; 1735204591Sluigi a.sch->flags &= ~DN_PIPE_CMD; //XXX do it even if is not set? 1736204591Sluigi if (pipe_cmd) { 1737204591Sluigi /* Copy mask parameter */ 1738204591Sluigi new_mask = a.sch->sched_mask; 1739204591Sluigi new_buckets = a.sch->buckets; 1740204591Sluigi new_flags = a.sch->flags; 1741204591Sluigi } 1742204591Sluigi DN_BH_WLOCK(); 1743204591Sluigiagain: /* run twice, for wfq and fifo */ 1744204591Sluigi /* 1745204591Sluigi * lookup the type. If not supplied, use the previous one 1746204591Sluigi * or default to WF2Q+. Otherwise, return an error. 1747204591Sluigi */ 1748204591Sluigi dn_cfg.id++; 1749204591Sluigi a.fp = find_sched_type(a.sch->oid.subtype, a.sch->name); 1750204591Sluigi if (a.fp != NULL) { 1751204591Sluigi /* found. Lookup or create entry */ 1752204591Sluigi s = dn_ht_find(dn_cfg.schedhash, i, DNHT_INSERT, &a); 1753204591Sluigi } else if (a.sch->oid.subtype == 0 && !a.sch->name[0]) { 1754204591Sluigi /* No type. search existing s* or retry with WF2Q+ */ 1755204591Sluigi s = dn_ht_find(dn_cfg.schedhash, i, 0, &a); 1756204591Sluigi if (s != NULL) { 1757204591Sluigi a.fp = s->fp; 1758204591Sluigi /* Scheduler exists, skip to FIFO scheduler 1759204591Sluigi * if command was pipe config... 1760204591Sluigi */ 1761204591Sluigi if (pipe_cmd) 1762204591Sluigi goto next; 1763163717Soleg } else { 1764204591Sluigi /* New scheduler, create a wf2q+ with no mask 1765204591Sluigi * if command was pipe config... 1766163717Soleg */ 1767204591Sluigi if (pipe_cmd) { 1768204591Sluigi /* clear mask parameter */ 1769204591Sluigi bzero(&a.sch->sched_mask, sizeof(new_mask)); 1770204591Sluigi a.sch->buckets = 0; 1771204591Sluigi a.sch->flags &= ~DN_HAVE_MASK; 1772163717Soleg } 1773204591Sluigi a.sch->oid.subtype = DN_SCHED_WF2QP; 1774204591Sluigi goto again; 1775163717Soleg } 1776204591Sluigi } else { 1777204591Sluigi D("invalid scheduler type %d %s", 1778204591Sluigi a.sch->oid.subtype, a.sch->name); 1779204763Sluigi err = EINVAL; 1780204763Sluigi goto error; 1781204591Sluigi } 1782204591Sluigi /* normalize name and subtype */ 1783204591Sluigi a.sch->oid.subtype = a.fp->type; 1784204591Sluigi bzero(a.sch->name, sizeof(a.sch->name)); 1785204591Sluigi strlcpy(a.sch->name, a.fp->name, sizeof(a.sch->name)); 1786204591Sluigi if (s == NULL) { 1787204591Sluigi D("cannot allocate scheduler %d", i); 1788204763Sluigi goto error; 1789204591Sluigi } 1790204591Sluigi /* restore existing link if any */ 1791204591Sluigi if (p.link_nr) { 1792204591Sluigi s->link = p; 1793204763Sluigi if (!pf || pf->link_nr != p.link_nr) { /* no saved value */ 1794204591Sluigi s->profile = NULL; /* XXX maybe not needed */ 1795204763Sluigi } else { 1796204763Sluigi s->profile = malloc(sizeof(struct dn_profile), 1797204763Sluigi M_DUMMYNET, M_NOWAIT | M_ZERO); 1798204763Sluigi if (s->profile == NULL) { 1799204763Sluigi D("cannot allocate profile"); 1800204763Sluigi goto error; //XXX 1801204763Sluigi } 1802318155Smarius memcpy(s->profile, pf, sizeof(*pf)); 1803204763Sluigi } 1804204591Sluigi } 1805204591Sluigi p.link_nr = 0; 1806204591Sluigi if (s->fp == NULL) { 1807204591Sluigi DX(2, "sched %d new type %s", i, a.fp->name); 1808204591Sluigi } else if (s->fp != a.fp || 1809204591Sluigi bcmp(a.sch, &s->sch, sizeof(*a.sch)) ) { 1810204591Sluigi /* already existing. */ 1811204591Sluigi DX(2, "sched %d type changed from %s to %s", 1812204591Sluigi i, s->fp->name, a.fp->name); 1813204591Sluigi DX(4, " type/sub %d/%d -> %d/%d", 1814204591Sluigi s->sch.oid.type, s->sch.oid.subtype, 1815204591Sluigi a.sch->oid.type, a.sch->oid.subtype); 1816204591Sluigi if (s->link.link_nr == 0) 1817204591Sluigi D("XXX WARNING link 0 for sched %d", i); 1818204591Sluigi p = s->link; /* preserve link */ 1819204763Sluigi if (s->profile) {/* preserve profile */ 1820204763Sluigi if (!pf) 1821204763Sluigi pf = malloc(sizeof(*pf), 1822204763Sluigi M_DUMMYNET, M_NOWAIT | M_ZERO); 1823204763Sluigi if (pf) /* XXX should issue a warning otherwise */ 1824318155Smarius memcpy(pf, s->profile, sizeof(*pf)); 1825204763Sluigi } 1826204591Sluigi /* remove from the hash */ 1827204591Sluigi dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL); 1828204591Sluigi /* Detach flowsets, preserve queues. */ 1829204591Sluigi // schk_delete_cb(s, NULL); 1830204591Sluigi // XXX temporarily, kill queues 1831204591Sluigi schk_delete_cb(s, (void *)DN_DESTROY); 1832204591Sluigi goto again; 1833204591Sluigi } else { 1834204591Sluigi DX(4, "sched %d unchanged type %s", i, a.fp->name); 1835204591Sluigi } 1836204591Sluigi /* complete initialization */ 1837204591Sluigi s->sch = *a.sch; 1838204591Sluigi s->fp = a.fp; 1839204591Sluigi s->cfg = arg; 1840204591Sluigi // XXX schk_reset_credit(s); 1841204591Sluigi /* create the internal flowset if needed, 1842204591Sluigi * trying to reuse existing ones if available 1843204591Sluigi */ 1844204591Sluigi if (!(s->fp->flags & DN_MULTIQUEUE) && !s->fs) { 1845204591Sluigi s->fs = dn_ht_find(dn_cfg.fshash, i, 0, NULL); 1846204591Sluigi if (!s->fs) { 1847204591Sluigi struct dn_fs fs; 1848204591Sluigi bzero(&fs, sizeof(fs)); 1849204591Sluigi set_oid(&fs.oid, DN_FS, sizeof(fs)); 1850204591Sluigi fs.fs_nr = i + DN_MAX_ID; 1851204591Sluigi fs.sched_nr = i; 1852204591Sluigi s->fs = config_fs(&fs, NULL, 1 /* locked */); 1853204591Sluigi } 1854204591Sluigi if (!s->fs) { 1855204591Sluigi schk_delete_cb(s, (void *)DN_DESTROY); 1856204591Sluigi D("error creating internal fs for %d", i); 1857204763Sluigi goto error; 1858204591Sluigi } 1859204591Sluigi } 1860204591Sluigi /* call init function after the flowset is created */ 1861204591Sluigi if (s->fp->config) 1862204591Sluigi s->fp->config(s); 1863204591Sluigi update_fs(s); 1864204591Sluiginext: 1865204591Sluigi if (i < DN_MAX_ID) { /* now configure the FIFO instance */ 1866204591Sluigi i += DN_MAX_ID; 1867204591Sluigi if (pipe_cmd) { 1868204591Sluigi /* Restore mask parameter for FIFO */ 1869204591Sluigi a.sch->sched_mask = new_mask; 1870204591Sluigi a.sch->buckets = new_buckets; 1871204591Sluigi a.sch->flags = new_flags; 1872204591Sluigi } else { 1873204591Sluigi /* sched config shouldn't modify the FIFO scheduler */ 1874204591Sluigi if (dn_ht_find(dn_cfg.schedhash, i, 0, &a) != NULL) { 1875204591Sluigi /* FIFO already exist, don't touch it */ 1876204862Sluigi err = 0; /* and this is not an error */ 1877204763Sluigi goto error; 1878163717Soleg } 1879163717Soleg } 1880204591Sluigi a.sch->sched_nr = i; 1881204591Sluigi a.sch->oid.subtype = DN_SCHED_FIFO; 1882204591Sluigi bzero(a.sch->name, sizeof(a.sch->name)); 1883204591Sluigi goto again; 188461413Sluigi } 1885204763Sluigi err = 0; 1886204763Sluigierror: 1887204591Sluigi DN_BH_WUNLOCK(); 1888204763Sluigi if (pf) 1889204763Sluigi free(pf, M_DUMMYNET); 1890204763Sluigi return err; 189161413Sluigi} 189261413Sluigi 189371667Sluigi/* 1894204591Sluigi * attach a profile to a link 189571909Sluigi */ 1896204591Sluigistatic int 1897204591Sluigiconfig_profile(struct dn_profile *pf, struct dn_id *arg) 189861413Sluigi{ 1899204591Sluigi struct dn_schk *s; 1900204591Sluigi int i, olen, err = 0; 1901201121Sluigi 1902204591Sluigi if (pf->oid.len < sizeof(*pf)) { 1903204591Sluigi D("short profile len %d", pf->oid.len); 1904204591Sluigi return EINVAL; 1905204591Sluigi } 1906204591Sluigi i = pf->link_nr; 1907204591Sluigi if (i <= 0 || i >= DN_MAX_ID) 1908204591Sluigi return EINVAL; 1909204591Sluigi /* XXX other sanity checks */ 1910204591Sluigi DN_BH_WLOCK(); 1911204591Sluigi for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) { 1912206845Sluigi s = locate_scheduler(i); 1913204591Sluigi 1914206845Sluigi if (s == NULL) { 1915204591Sluigi err = EINVAL; 1916204591Sluigi break; 1917206845Sluigi } 1918206845Sluigi dn_cfg.id++; 1919206845Sluigi /* 1920206845Sluigi * If we had a profile and the new one does not fit, 1921206845Sluigi * or it is deleted, then we need to free memory. 1922206845Sluigi */ 1923206845Sluigi if (s->profile && (pf->samples_no == 0 || 1924206845Sluigi s->profile->oid.len < pf->oid.len)) { 1925206845Sluigi free(s->profile, M_DUMMYNET); 1926206845Sluigi s->profile = NULL; 1927206845Sluigi } 1928204591Sluigi if (pf->samples_no == 0) 1929204591Sluigi continue; 1930206845Sluigi /* 1931204591Sluigi * new profile, possibly allocate memory 1932206845Sluigi * and copy data. 1933206845Sluigi */ 1934204591Sluigi if (s->profile == NULL) 1935204591Sluigi s->profile = malloc(pf->oid.len, 1936213267Sluigi M_DUMMYNET, M_NOWAIT | M_ZERO); 1937204591Sluigi if (s->profile == NULL) { 1938204591Sluigi D("no memory for profile %d", i); 1939204591Sluigi err = ENOMEM; 1940204591Sluigi break; 1941204591Sluigi } 1942204591Sluigi /* preserve larger length XXX double check */ 1943204591Sluigi olen = s->profile->oid.len; 1944204591Sluigi if (olen < pf->oid.len) 1945204591Sluigi olen = pf->oid.len; 1946318155Smarius memcpy(s->profile, pf, pf->oid.len); 1947204591Sluigi s->profile->oid.len = olen; 1948204591Sluigi } 1949204591Sluigi DN_BH_WUNLOCK(); 1950204591Sluigi return err; 195171133Sluigi} 195261413Sluigi 195361413Sluigi/* 1954204591Sluigi * Delete all objects: 195561413Sluigi */ 195661413Sluigistatic void 1957204591Sluigidummynet_flush(void) 195861413Sluigi{ 1959201121Sluigi 1960204591Sluigi /* delete all schedulers and related links/queues/flowsets */ 1961204591Sluigi dn_ht_scan(dn_cfg.schedhash, schk_delete_cb, 1962204591Sluigi (void *)(uintptr_t)DN_DELETE_FS); 1963204591Sluigi /* delete all remaining (unlinked) flowsets */ 1964204591Sluigi DX(4, "still %d unlinked fs", dn_cfg.fsk_count); 1965204591Sluigi dn_ht_free(dn_cfg.fshash, DNHT_REMOVE); 1966204591Sluigi fsk_detach_list(&dn_cfg.fsu, DN_DELETE_FS); 1967204591Sluigi /* Reinitialize system heap... */ 1968204591Sluigi heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id)); 196971133Sluigi} 197061413Sluigi 197161413Sluigi/* 1972204591Sluigi * Main handler for configuration. We are guaranteed to be called 1973204591Sluigi * with an oid which is at least a dn_id. 1974204591Sluigi * - the first object is the command (config, delete, flush, ...) 1975204591Sluigi * - config_link must be issued after the corresponding config_sched 1976204591Sluigi * - parameters (DN_TXT) for an object must preceed the object 1977204591Sluigi * processed on a config_sched. 197861413Sluigi */ 1979204591Sluigiint 1980204591Sluigido_config(void *p, int l) 198161413Sluigi{ 1982318155Smarius struct dn_id o; 1983318155Smarius union { 1984318155Smarius struct dn_profile profile; 1985318155Smarius struct dn_fs fs; 1986318155Smarius struct dn_link link; 1987318155Smarius struct dn_sch sched; 1988318155Smarius } *dn; 1989318155Smarius struct dn_id *arg; 1990318155Smarius uintptr_t a; 1991318155Smarius int err, err2, off; 1992169454Srwatson 1993318155Smarius memcpy(&o, p, sizeof(o)); 1994318155Smarius if (o.id != DN_API_VERSION) { 1995318155Smarius D("invalid api version got %d need %d", o.id, DN_API_VERSION); 1996204591Sluigi return EINVAL; 1997204591Sluigi } 1998318155Smarius arg = NULL; 1999318155Smarius dn = NULL; 2000318155Smarius for (off = 0; l >= sizeof(o); memcpy(&o, (char *)p + off, sizeof(o))) { 2001318155Smarius if (o.len < sizeof(o) || l < o.len) { 2002318155Smarius D("bad len o.len %d len %d", o.len, l); 2003204591Sluigi err = EINVAL; 2004204591Sluigi break; 2005204591Sluigi } 2006318155Smarius l -= o.len; 2007204591Sluigi err = 0; 2008318155Smarius switch (o.type) { 2009204591Sluigi default: 2010318155Smarius D("cmd %d not implemented", o.type); 2011204591Sluigi break; 2012206845Sluigi 2013206845Sluigi#ifdef EMULATE_SYSCTL 2014204591Sluigi /* sysctl emulation. 2015204591Sluigi * if we recognize the command, jump to the correct 2016204591Sluigi * handler and return 2017204591Sluigi */ 2018204591Sluigi case DN_SYSCTL_SET: 2019204591Sluigi err = kesysctl_emu_set(p, l); 2020204591Sluigi return err; 2021204591Sluigi#endif 2022206845Sluigi 2023204591Sluigi case DN_CMD_CONFIG: /* simply a header */ 2024204591Sluigi break; 202561413Sluigi 2026204591Sluigi case DN_CMD_DELETE: 2027204591Sluigi /* the argument is in the first uintptr_t after o */ 2028318155Smarius if (o.len < sizeof(o) + sizeof(a)) { 2029204591Sluigi err = EINVAL; 2030204591Sluigi break; 2031204591Sluigi } 2032318155Smarius memcpy(&a, (char *)p + off + sizeof(o), sizeof(a)); 2033318155Smarius switch (o.subtype) { 2034204591Sluigi case DN_LINK: 2035204591Sluigi /* delete base and derived schedulers */ 2036204591Sluigi DN_BH_WLOCK(); 2037318155Smarius err = delete_schk(a); 2038318155Smarius err2 = delete_schk(a + DN_MAX_ID); 2039204591Sluigi DN_BH_WUNLOCK(); 2040204591Sluigi if (!err) 2041204591Sluigi err = err2; 2042204591Sluigi break; 2043152910Sglebius 2044204591Sluigi default: 2045318155Smarius D("invalid delete type %d", o.subtype); 2046204591Sluigi err = EINVAL; 2047204591Sluigi break; 204861413Sluigi 2049204591Sluigi case DN_FS: 2050318155Smarius err = (a < 1 || a >= DN_MAX_ID) ? 2051318155Smarius EINVAL : delete_fs(a, 0) ; 2052204591Sluigi break; 2053204591Sluigi } 2054204591Sluigi break; 205561413Sluigi 2056204591Sluigi case DN_CMD_FLUSH: 2057204591Sluigi DN_BH_WLOCK(); 2058204591Sluigi dummynet_flush(); 2059204591Sluigi DN_BH_WUNLOCK(); 2060204591Sluigi break; 2061318155Smarius case DN_TEXT: /* store argument of next block */ 2062318155Smarius if (arg != NULL) 2063318155Smarius free(arg, M_TEMP); 2064318155Smarius arg = malloc(o.len, M_TEMP, M_WAITOK); 2065318155Smarius memcpy(arg, (char *)p + off, o.len); 2066204591Sluigi break; 2067204591Sluigi case DN_LINK: 2068318155Smarius if (dn == NULL) 2069318155Smarius dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK); 2070318155Smarius memcpy(&dn->link, (char *)p + off, sizeof(dn->link)); 2071318155Smarius err = config_link(&dn->link, arg); 2072204591Sluigi break; 2073204591Sluigi case DN_PROFILE: 2074318155Smarius if (dn == NULL) 2075318155Smarius dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK); 2076318155Smarius memcpy(&dn->profile, (char *)p + off, 2077318155Smarius sizeof(dn->profile)); 2078318155Smarius err = config_profile(&dn->profile, arg); 2079204591Sluigi break; 2080204591Sluigi case DN_SCH: 2081318155Smarius if (dn == NULL) 2082318155Smarius dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK); 2083318155Smarius memcpy(&dn->sched, (char *)p + off, 2084318155Smarius sizeof(dn->sched)); 2085318155Smarius err = config_sched(&dn->sched, arg); 2086204591Sluigi break; 2087204591Sluigi case DN_FS: 2088318155Smarius if (dn == NULL) 2089318155Smarius dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK); 2090318155Smarius memcpy(&dn->fs, (char *)p + off, sizeof(dn->fs)); 2091318155Smarius err = (NULL == config_fs(&dn->fs, arg, 0)); 2092204591Sluigi break; 2093152910Sglebius } 2094204591Sluigi if (err != 0) 2095204591Sluigi break; 2096318155Smarius off += o.len; 2097201121Sluigi } 2098318155Smarius if (arg != NULL) 2099318155Smarius free(arg, M_TEMP); 2100318155Smarius if (dn != NULL) 2101318155Smarius free(dn, M_TEMP); 2102204591Sluigi return err; 2103204591Sluigi} 2104120140Ssam 2105204591Sluigistatic int 2106205050Sluigicompute_space(struct dn_id *cmd, struct copy_args *a) 2107204591Sluigi{ 2108204591Sluigi int x = 0, need = 0; 2109204591Sluigi int profile_size = sizeof(struct dn_profile) - 2110204591Sluigi ED_MAX_SAMPLES_NO*sizeof(int); 211161413Sluigi 2112204591Sluigi /* NOTE about compute space: 2113204591Sluigi * NP = dn_cfg.schk_count 2114204591Sluigi * NSI = dn_cfg.si_count 2115204591Sluigi * NF = dn_cfg.fsk_count 2116204591Sluigi * NQ = dn_cfg.queue_count 2117204591Sluigi * - ipfw pipe show 2118204591Sluigi * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler 2119204591Sluigi * link, scheduler template, flowset 2120204591Sluigi * integrated in scheduler and header 2121204591Sluigi * for flowset list 2122204714Sluigi * (NSI)*(dn_flow) all scheduler instance (includes 2123204714Sluigi * the queue instance) 2124204591Sluigi * - ipfw sched show 2125204591Sluigi * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler 2126204591Sluigi * link, scheduler template, flowset 2127204591Sluigi * integrated in scheduler and header 2128204591Sluigi * for flowset list 2129204591Sluigi * (NSI * dn_flow) all scheduler instances 2130204591Sluigi * (NF * sizeof(uint_32)) space for flowset list linked to scheduler 2131204591Sluigi * (NQ * dn_queue) all queue [XXXfor now not listed] 2132204591Sluigi * - ipfw queue show 2133204591Sluigi * (NF * dn_fs) all flowset 2134204591Sluigi * (NQ * dn_queue) all queues 2135204591Sluigi */ 2136204591Sluigi switch (cmd->subtype) { 2137204591Sluigi default: 2138204591Sluigi return -1; 2139204591Sluigi /* XXX where do LINK and SCH differ ? */ 2140204714Sluigi /* 'ipfw sched show' could list all queues associated to 2141204714Sluigi * a scheduler. This feature for now is disabled 2142204714Sluigi */ 2143204591Sluigi case DN_LINK: /* pipe show */ 2144204591Sluigi x = DN_C_LINK | DN_C_SCH | DN_C_FLOW; 2145204591Sluigi need += dn_cfg.schk_count * 2146204591Sluigi (sizeof(struct dn_fs) + profile_size) / 2; 2147204591Sluigi need += dn_cfg.fsk_count * sizeof(uint32_t); 2148204591Sluigi break; 2149204591Sluigi case DN_SCH: /* sched show */ 2150204591Sluigi need += dn_cfg.schk_count * 2151204591Sluigi (sizeof(struct dn_fs) + profile_size) / 2; 2152204591Sluigi need += dn_cfg.fsk_count * sizeof(uint32_t); 2153204591Sluigi x = DN_C_SCH | DN_C_LINK | DN_C_FLOW; 2154204591Sluigi break; 2155204591Sluigi case DN_FS: /* queue show */ 2156204591Sluigi x = DN_C_FS | DN_C_QUEUE; 2157204591Sluigi break; 2158204591Sluigi case DN_GET_COMPAT: /* compatibility mode */ 2159206425Sluigi need = dn_compat_calc_size(); 2160204591Sluigi break; 2161120140Ssam } 2162205050Sluigi a->flags = x; 2163204591Sluigi if (x & DN_C_SCH) { 2164204591Sluigi need += dn_cfg.schk_count * sizeof(struct dn_sch) / 2; 2165204591Sluigi /* NOT also, each fs might be attached to a sched */ 2166204591Sluigi need += dn_cfg.schk_count * sizeof(struct dn_id) / 2; 2167204591Sluigi } 2168204591Sluigi if (x & DN_C_FS) 2169204591Sluigi need += dn_cfg.fsk_count * sizeof(struct dn_fs); 2170204591Sluigi if (x & DN_C_LINK) { 2171204591Sluigi need += dn_cfg.schk_count * sizeof(struct dn_link) / 2; 2172204591Sluigi } 2173205178Sluigi /* 2174205178Sluigi * When exporting a queue to userland, only pass up the 2175205178Sluigi * struct dn_flow, which is the only visible part. 2176205178Sluigi */ 2177205178Sluigi 2178204591Sluigi if (x & DN_C_QUEUE) 2179205178Sluigi need += dn_cfg.queue_count * sizeof(struct dn_flow); 2180204591Sluigi if (x & DN_C_FLOW) 2181204591Sluigi need += dn_cfg.si_count * (sizeof(struct dn_flow)); 2182204591Sluigi return need; 2183204591Sluigi} 218461413Sluigi 2185204591Sluigi/* 2186204591Sluigi * If compat != NULL dummynet_get is called in compatibility mode. 2187204591Sluigi * *compat will be the pointer to the buffer to pass to ipfw 2188204591Sluigi */ 2189204591Sluigiint 2190204591Sluigidummynet_get(struct sockopt *sopt, void **compat) 2191204591Sluigi{ 2192204591Sluigi int have, i, need, error; 2193204591Sluigi char *start = NULL, *buf; 2194204591Sluigi size_t sopt_valsize; 2195205050Sluigi struct dn_id *cmd; 2196204591Sluigi struct copy_args a; 2197205050Sluigi struct copy_range r; 2198205050Sluigi int l = sizeof(struct dn_id); 219961413Sluigi 2200205050Sluigi bzero(&a, sizeof(a)); 2201205050Sluigi bzero(&r, sizeof(r)); 2202205050Sluigi 2203204591Sluigi /* save and restore original sopt_valsize around copyin */ 2204204591Sluigi sopt_valsize = sopt->sopt_valsize; 2205205050Sluigi 2206205050Sluigi cmd = &r.o; 2207205050Sluigi 2208204591Sluigi if (!compat) { 2209205050Sluigi /* copy at least an oid, and possibly a full object */ 2210205050Sluigi error = sooptcopyin(sopt, cmd, sizeof(r), sizeof(*cmd)); 2211205050Sluigi sopt->sopt_valsize = sopt_valsize; 2212204591Sluigi if (error) 2213205050Sluigi goto done; 2214205050Sluigi l = cmd->len; 2215204591Sluigi#ifdef EMULATE_SYSCTL 2216204591Sluigi /* sysctl emulation. */ 2217205050Sluigi if (cmd->type == DN_SYSCTL_GET) 2218204591Sluigi return kesysctl_emu_get(sopt); 221971133Sluigi#endif 2220205050Sluigi if (l > sizeof(r)) { 2221205050Sluigi /* request larger than default, allocate buffer */ 2222220812Sae cmd = malloc(l, M_DUMMYNET, M_WAITOK); 2223205050Sluigi error = sooptcopyin(sopt, cmd, l, l); 2224205050Sluigi sopt->sopt_valsize = sopt_valsize; 2225205050Sluigi if (error) 2226205050Sluigi goto done; 2227205050Sluigi } 2228205050Sluigi } else { /* compatibility */ 2229204591Sluigi error = 0; 2230205050Sluigi cmd->type = DN_CMD_GET; 2231205050Sluigi cmd->len = sizeof(struct dn_id); 2232205050Sluigi cmd->subtype = DN_GET_COMPAT; 2233205050Sluigi // cmd->id = sopt_valsize; 2234204591Sluigi D("compatibility mode"); 223555597Sluigi } 2236301772Struckman 2237301772Struckman#ifdef NEW_AQM 2238301772Struckman /* get AQM params */ 2239301772Struckman if(cmd->subtype == DN_AQM_PARAMS) { 2240301772Struckman error = get_aqm_parms(sopt); 2241301772Struckman goto done; 2242301772Struckman /* get Scheduler params */ 2243301772Struckman } else if (cmd->subtype == DN_SCH_PARAMS) { 2244301772Struckman error = get_sched_parms(sopt); 2245301772Struckman goto done; 2246301772Struckman } 2247301772Struckman#endif 2248301772Struckman 2249205050Sluigi a.extra = (struct copy_range *)cmd; 2250205050Sluigi if (cmd->len == sizeof(*cmd)) { /* no range, create a default */ 2251205050Sluigi uint32_t *rp = (uint32_t *)(cmd + 1); 2252205050Sluigi cmd->len += 2* sizeof(uint32_t); 2253205050Sluigi rp[0] = 1; 2254205050Sluigi rp[1] = DN_MAX_ID - 1; 2255205050Sluigi if (cmd->subtype == DN_LINK) { 2256205050Sluigi rp[0] += DN_MAX_ID; 2257205050Sluigi rp[1] += DN_MAX_ID; 2258205050Sluigi } 2259205050Sluigi } 2260204591Sluigi /* Count space (under lock) and allocate (outside lock). 2261204591Sluigi * Exit with lock held if we manage to get enough buffer. 2262204591Sluigi * Try a few times then give up. 2263204591Sluigi */ 2264204591Sluigi for (have = 0, i = 0; i < 10; i++) { 2265204591Sluigi DN_BH_WLOCK(); 2266205050Sluigi need = compute_space(cmd, &a); 2267205050Sluigi 2268205050Sluigi /* if there is a range, ignore value from compute_space() */ 2269205050Sluigi if (l > sizeof(*cmd)) 2270205050Sluigi need = sopt_valsize - sizeof(*cmd); 2271205050Sluigi 2272204591Sluigi if (need < 0) { 2273204591Sluigi DN_BH_WUNLOCK(); 2274205050Sluigi error = EINVAL; 2275205050Sluigi goto done; 2276204591Sluigi } 2277205050Sluigi need += sizeof(*cmd); 2278205050Sluigi cmd->id = need; 2279204591Sluigi if (have >= need) 2280204591Sluigi break; 2281205050Sluigi 2282204591Sluigi DN_BH_WUNLOCK(); 2283204591Sluigi if (start) 2284204591Sluigi free(start, M_DUMMYNET); 2285204591Sluigi start = NULL; 2286204591Sluigi if (need > sopt_valsize) 2287204591Sluigi break; 2288205050Sluigi 2289204591Sluigi have = need; 2290204591Sluigi start = malloc(have, M_DUMMYNET, M_WAITOK | M_ZERO); 2291204591Sluigi } 2292205050Sluigi 2293204591Sluigi if (start == NULL) { 2294204591Sluigi if (compat) { 2295204591Sluigi *compat = NULL; 2296205050Sluigi error = 1; // XXX 2297205050Sluigi } else { 2298205050Sluigi error = sooptcopyout(sopt, cmd, sizeof(*cmd)); 2299204591Sluigi } 2300205050Sluigi goto done; 2301204591Sluigi } 2302204591Sluigi ND("have %d:%d sched %d, %d:%d links %d, %d:%d flowsets %d, " 2303204591Sluigi "%d:%d si %d, %d:%d queues %d", 2304204591Sluigi dn_cfg.schk_count, sizeof(struct dn_sch), DN_SCH, 2305204591Sluigi dn_cfg.schk_count, sizeof(struct dn_link), DN_LINK, 2306204591Sluigi dn_cfg.fsk_count, sizeof(struct dn_fs), DN_FS, 2307204591Sluigi dn_cfg.si_count, sizeof(struct dn_flow), DN_SCH_I, 2308204591Sluigi dn_cfg.queue_count, sizeof(struct dn_queue), DN_QUEUE); 2309204591Sluigi sopt->sopt_valsize = sopt_valsize; 2310205050Sluigi a.type = cmd->subtype; 2311205050Sluigi 2312204591Sluigi if (compat == NULL) { 2313318155Smarius memcpy(start, cmd, sizeof(*cmd)); 2314205050Sluigi ((struct dn_id*)(start))->len = sizeof(struct dn_id); 2315205050Sluigi buf = start + sizeof(*cmd); 2316204591Sluigi } else 2317204591Sluigi buf = start; 2318204591Sluigi a.start = &buf; 2319204591Sluigi a.end = start + have; 2320204591Sluigi /* start copying other objects */ 2321204591Sluigi if (compat) { 2322204591Sluigi a.type = DN_COMPAT_PIPE; 2323204591Sluigi dn_ht_scan(dn_cfg.schedhash, copy_data_helper_compat, &a); 2324204591Sluigi a.type = DN_COMPAT_QUEUE; 2325204591Sluigi dn_ht_scan(dn_cfg.fshash, copy_data_helper_compat, &a); 2326205050Sluigi } else if (a.type == DN_FS) { 2327204591Sluigi dn_ht_scan(dn_cfg.fshash, copy_data_helper, &a); 2328205050Sluigi } else { 2329204591Sluigi dn_ht_scan(dn_cfg.schedhash, copy_data_helper, &a); 2330205050Sluigi } 2331204591Sluigi DN_BH_WUNLOCK(); 2332205050Sluigi 2333204591Sluigi if (compat) { 2334204591Sluigi *compat = start; 2335204591Sluigi sopt->sopt_valsize = buf - start; 2336204591Sluigi /* free() is done by ip_dummynet_compat() */ 2337205050Sluigi start = NULL; //XXX hack 2338204591Sluigi } else { 2339204591Sluigi error = sooptcopyout(sopt, start, buf - start); 2340205050Sluigi } 2341205050Sluigidone: 2342205050Sluigi if (cmd && cmd != &r.o) 2343205050Sluigi free(cmd, M_DUMMYNET); 2344205050Sluigi if (start) 2345204591Sluigi free(start, M_DUMMYNET); 2346204591Sluigi return error; 234771133Sluigi} 234861413Sluigi 2349204591Sluigi/* Callback called on scheduler instance to delete it if idle */ 2350204591Sluigistatic int 2351204591Sluigidrain_scheduler_cb(void *_si, void *arg) 235261413Sluigi{ 2353204591Sluigi struct dn_sch_inst *si = _si; 235461413Sluigi 2355204591Sluigi if ((si->kflags & DN_ACTIVE) || si->dline.mq.head != NULL) 2356204591Sluigi return 0; 2357120140Ssam 2358204591Sluigi if (si->sched->fp->flags & DN_MULTIQUEUE) { 2359204591Sluigi if (si->q_count == 0) 2360204591Sluigi return si_destroy(si, NULL); 2361204591Sluigi else 2362204591Sluigi return 0; 2363204591Sluigi } else { /* !DN_MULTIQUEUE */ 2364204591Sluigi if ((si+1)->ni.length == 0) 2365204591Sluigi return si_destroy(si, NULL); 2366204591Sluigi else 2367204591Sluigi return 0; 236861413Sluigi } 2369204591Sluigi return 0; /* unreachable */ 237061413Sluigi} 237161413Sluigi 2372204591Sluigi/* Callback called on scheduler to check if it has instances */ 2373204591Sluigistatic int 2374204591Sluigidrain_scheduler_sch_cb(void *_s, void *arg) 237561413Sluigi{ 2376204591Sluigi struct dn_schk *s = _s; 237761413Sluigi 2378204591Sluigi if (s->sch.flags & DN_HAVE_MASK) { 2379204591Sluigi dn_ht_scan_bucket(s->siht, &s->drain_bucket, 2380204591Sluigi drain_scheduler_cb, NULL); 2381204591Sluigi s->drain_bucket++; 2382204591Sluigi } else { 2383204591Sluigi if (s->siht) { 2384204591Sluigi if (drain_scheduler_cb(s->siht, NULL) == DNHT_SCAN_DEL) 2385204591Sluigi s->siht = NULL; 2386204591Sluigi } 2387204591Sluigi } 2388204591Sluigi return 0; 2389134290Spjd} 2390134290Spjd 2391204591Sluigi/* Called every tick, try to delete a 'bucket' of scheduler */ 2392204591Sluigivoid 2393204591Sluigidn_drain_scheduler(void) 2394204591Sluigi{ 2395204591Sluigi dn_ht_scan_bucket(dn_cfg.schedhash, &dn_cfg.drain_sch, 2396204591Sluigi drain_scheduler_sch_cb, NULL); 2397204591Sluigi dn_cfg.drain_sch++; 2398204591Sluigi} 2399204591Sluigi 2400204591Sluigi/* Callback called on queue to delete if it is idle */ 2401134290Spjdstatic int 2402204591Sluigidrain_queue_cb(void *_q, void *arg) 2403134290Spjd{ 2404204591Sluigi struct dn_queue *q = _q; 2405134290Spjd 2406204591Sluigi if (q->ni.length == 0) { 2407204591Sluigi dn_delete_queue(q, DN_DESTROY); 2408204591Sluigi return DNHT_SCAN_DEL; /* queue is deleted */ 2409204591Sluigi } 241061413Sluigi 2411204591Sluigi return 0; /* queue isn't deleted */ 2412204591Sluigi} 241361413Sluigi 2414204591Sluigi/* Callback called on flowset used to check if it has queues */ 2415204591Sluigistatic int 2416204591Sluigidrain_queue_fs_cb(void *_fs, void *arg) 2417204591Sluigi{ 2418204591Sluigi struct dn_fsk *fs = _fs; 2419152910Sglebius 2420204591Sluigi if (fs->fs.flags & DN_QHT_HASH) { 2421204591Sluigi /* Flowset has a hash table for queues */ 2422204591Sluigi dn_ht_scan_bucket(fs->qht, &fs->drain_bucket, 2423204591Sluigi drain_queue_cb, NULL); 2424204591Sluigi fs->drain_bucket++; 2425205050Sluigi } else { 2426204591Sluigi /* No hash table for this flowset, null the pointer 2427204591Sluigi * if the queue is deleted 2428204591Sluigi */ 2429204591Sluigi if (fs->qht) { 2430204591Sluigi if (drain_queue_cb(fs->qht, NULL) == DNHT_SCAN_DEL) 2431204591Sluigi fs->qht = NULL; 2432204591Sluigi } 2433204591Sluigi } 2434204591Sluigi return 0; 2435204591Sluigi} 2436152910Sglebius 2437204591Sluigi/* Called every tick, try to delete a 'bucket' of queue */ 2438204591Sluigivoid 2439204591Sluigidn_drain_queue(void) 2440204591Sluigi{ 2441204591Sluigi /* scan a bucket of flowset */ 2442204591Sluigi dn_ht_scan_bucket(dn_cfg.fshash, &dn_cfg.drain_fs, 2443204591Sluigi drain_queue_fs_cb, NULL); 2444204591Sluigi dn_cfg.drain_fs++; 244561413Sluigi} 244661413Sluigi 244761413Sluigi/* 2448204591Sluigi * Handler for the various dummynet socket options 244961413Sluigi */ 245061413Sluigistatic int 245161413Sluigiip_dn_ctl(struct sockopt *sopt) 245261413Sluigi{ 2453204591Sluigi void *p = NULL; 2454204591Sluigi int error, l; 245561413Sluigi 2456204591Sluigi error = priv_check(sopt->sopt_td, PRIV_NETINET_DUMMYNET); 245783970Srwatson if (error) 2458204591Sluigi return (error); 245961413Sluigi 2460204591Sluigi /* Disallow sets in really-really secure mode. */ 2461204591Sluigi if (sopt->sopt_dir == SOPT_SET) { 2462204591Sluigi error = securelevel_ge(sopt->sopt_td->td_ucred, 3); 2463204591Sluigi if (error) 2464204591Sluigi return (error); 2465204591Sluigi } 246661413Sluigi 2467204591Sluigi switch (sopt->sopt_name) { 2468204591Sluigi default : 2469204591Sluigi D("dummynet: unknown option %d", sopt->sopt_name); 2470204591Sluigi error = EINVAL; 2471204591Sluigi break; 247261413Sluigi 2473204591Sluigi case IP_DUMMYNET_FLUSH: 2474204591Sluigi case IP_DUMMYNET_CONFIGURE: 2475204591Sluigi case IP_DUMMYNET_DEL: /* remove a pipe or queue */ 2476204591Sluigi case IP_DUMMYNET_GET: 2477204591Sluigi D("dummynet: compat option %d", sopt->sopt_name); 2478204591Sluigi error = ip_dummynet_compat(sopt); 2479204591Sluigi break; 248061413Sluigi 2481204591Sluigi case IP_DUMMYNET3 : 2482204591Sluigi if (sopt->sopt_dir == SOPT_GET) { 2483204591Sluigi error = dummynet_get(sopt, NULL); 2484204591Sluigi break; 2485204591Sluigi } 2486204591Sluigi l = sopt->sopt_valsize; 2487204591Sluigi if (l < sizeof(struct dn_id) || l > 12000) { 2488204591Sluigi D("argument len %d invalid", l); 2489204591Sluigi break; 2490204591Sluigi } 2491204591Sluigi p = malloc(l, M_TEMP, M_WAITOK); // XXX can it fail ? 2492204591Sluigi error = sooptcopyin(sopt, p, l, l); 2493204591Sluigi if (error) 2494204591Sluigi break ; 2495204591Sluigi error = do_config(p, l); 2496204591Sluigi break; 2497204591Sluigi } 2498190865Sluigi 2499204591Sluigi if (p != NULL) 2500204591Sluigi free(p, M_TEMP); 250161413Sluigi 2502204591Sluigi return error ; 2503204591Sluigi} 250461413Sluigi 2505200029Sluigi 250645869Speterstatic void 250739119Sluigiip_dn_init(void) 250839119Sluigi{ 2509206428Sluigi if (dn_cfg.init_done) 2510204591Sluigi return; 2511206428Sluigi printf("DUMMYNET %p with IPv6 initialized (100409)\n", curvnet); 2512206428Sluigi dn_cfg.init_done = 1; 2513205173Sluigi /* Set defaults here. MSVC does not accept initializers, 2514205173Sluigi * and this is also useful for vimages 2515205173Sluigi */ 2516204591Sluigi /* queue limits */ 2517204591Sluigi dn_cfg.slot_limit = 100; /* Foot shooting limit for queues. */ 2518204591Sluigi dn_cfg.byte_limit = 1024 * 1024; 2519205173Sluigi dn_cfg.expire = 1; 2520120140Ssam 2521204591Sluigi /* RED parameters */ 2522204591Sluigi dn_cfg.red_lookup_depth = 256; /* default lookup table depth */ 2523204591Sluigi dn_cfg.red_avg_pkt_size = 512; /* default medium packet size */ 2524204591Sluigi dn_cfg.red_max_pkt_size = 1500; /* default max packet size */ 252561413Sluigi 2526204591Sluigi /* hash tables */ 2527220831Sae dn_cfg.max_hash_size = 65536; /* max in the hash tables */ 2528204591Sluigi dn_cfg.hash_size = 64; /* default hash size */ 252961413Sluigi 2530204591Sluigi /* create hash tables for schedulers and flowsets. 2531204591Sluigi * In both we search by key and by pointer. 2532204591Sluigi */ 2533204591Sluigi dn_cfg.schedhash = dn_ht_init(NULL, dn_cfg.hash_size, 2534204591Sluigi offsetof(struct dn_schk, schk_next), 2535204591Sluigi schk_hash, schk_match, schk_new); 2536204591Sluigi dn_cfg.fshash = dn_ht_init(NULL, dn_cfg.hash_size, 2537204591Sluigi offsetof(struct dn_fsk, fsk_next), 2538204591Sluigi fsk_hash, fsk_match, fsk_new); 2539120140Ssam 2540204591Sluigi /* bucket index to drain object */ 2541204591Sluigi dn_cfg.drain_fs = 0; 2542204591Sluigi dn_cfg.drain_sch = 0; 2543204591Sluigi 2544204591Sluigi heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id)); 2545204591Sluigi SLIST_INIT(&dn_cfg.fsu); 2546204591Sluigi SLIST_INIT(&dn_cfg.schedlist); 2547204591Sluigi 2548204591Sluigi DN_LOCK_INIT(); 2549120140Ssam 2550206428Sluigi TASK_INIT(&dn_task, 0, dummynet_task, curvnet); 2551254781Smav dn_tq = taskqueue_create_fast("dummynet", M_WAITOK, 2552163720Soleg taskqueue_thread_enqueue, &dn_tq); 2553163720Soleg taskqueue_start_threads(&dn_tq, 1, PI_NET, "dummynet"); 2554163720Soleg 2555314667Savg callout_init(&dn_timeout, 1); 2556254781Smav dn_reschedule(); 2557163721Soleg 2558163721Soleg /* Initialize curr_time adjustment mechanics. */ 2559204591Sluigi getmicrouptime(&dn_cfg.prev_t); 256039119Sluigi} 256139119Sluigi 2562120140Ssamstatic void 2563206428Sluigiip_dn_destroy(int last) 2564120140Ssam{ 2565297228Shselasky DN_BH_WLOCK(); 2566297228Shselasky /* ensure no more callouts are started */ 2567297228Shselasky dn_gone = 1; 2568204714Sluigi 2569297228Shselasky /* check for last */ 2570206428Sluigi if (last) { 2571222560Sae ND("removing last instance\n"); 2572206428Sluigi ip_dn_ctl_ptr = NULL; 2573206428Sluigi ip_dn_io_ptr = NULL; 2574206428Sluigi } 2575120140Ssam 2576204591Sluigi dummynet_flush(); 2577204591Sluigi DN_BH_WUNLOCK(); 2578297228Shselasky 2579297228Shselasky callout_drain(&dn_timeout); 2580163720Soleg taskqueue_drain(dn_tq, &dn_task); 2581163720Soleg taskqueue_free(dn_tq); 2582163720Soleg 2583204591Sluigi dn_ht_free(dn_cfg.schedhash, 0); 2584204591Sluigi dn_ht_free(dn_cfg.fshash, 0); 2585204591Sluigi heap_free(&dn_cfg.evheap); 2586120140Ssam 2587204591Sluigi DN_LOCK_DESTROY(); 2588120140Ssam} 2589120140Ssam 259039119Sluigistatic int 259145743Speterdummynet_modevent(module_t mod, int type, void *data) 259239119Sluigi{ 2593169454Srwatson 2594204591Sluigi if (type == MOD_LOAD) { 2595193497Sluigi if (ip_dn_io_ptr) { 2596204591Sluigi printf("DUMMYNET already loaded\n"); 2597204591Sluigi return EEXIST ; 259886047Sluigi } 259945743Speter ip_dn_init(); 2600206428Sluigi ip_dn_ctl_ptr = ip_dn_ctl; 2601206428Sluigi ip_dn_io_ptr = dummynet_io; 2602204591Sluigi return 0; 2603204591Sluigi } else if (type == MOD_UNLOAD) { 2604206428Sluigi ip_dn_destroy(1 /* last */); 2605204591Sluigi return 0; 2606204591Sluigi } else 2607132199Sphk return EOPNOTSUPP; 2608204591Sluigi} 2609204591Sluigi 2610204591Sluigi/* modevent helpers for the modules */ 2611204591Sluigistatic int 2612204591Sluigiload_dn_sched(struct dn_alg *d) 2613204591Sluigi{ 2614204591Sluigi struct dn_alg *s; 2615204591Sluigi 2616204591Sluigi if (d == NULL) 2617204591Sluigi return 1; /* error */ 2618204591Sluigi ip_dn_init(); /* just in case, we need the lock */ 2619204591Sluigi 2620204591Sluigi /* Check that mandatory funcs exists */ 2621204591Sluigi if (d->enqueue == NULL || d->dequeue == NULL) { 2622204591Sluigi D("missing enqueue or dequeue for %s", d->name); 2623204591Sluigi return 1; 262445743Speter } 2625204591Sluigi 2626204591Sluigi /* Search if scheduler already exists */ 2627204591Sluigi DN_BH_WLOCK(); 2628204591Sluigi SLIST_FOREACH(s, &dn_cfg.schedlist, next) { 2629204591Sluigi if (strcmp(s->name, d->name) == 0) { 2630204591Sluigi D("%s already loaded", d->name); 2631204591Sluigi break; /* scheduler already exists */ 2632204591Sluigi } 2633204591Sluigi } 2634204591Sluigi if (s == NULL) 2635204591Sluigi SLIST_INSERT_HEAD(&dn_cfg.schedlist, d, next); 2636204591Sluigi DN_BH_WUNLOCK(); 2637204591Sluigi D("dn_sched %s %sloaded", d->name, s ? "not ":""); 2638204591Sluigi return s ? 1 : 0; 263939119Sluigi} 264039119Sluigi 2641204591Sluigistatic int 2642204591Sluigiunload_dn_sched(struct dn_alg *s) 2643204591Sluigi{ 2644204591Sluigi struct dn_alg *tmp, *r; 2645204591Sluigi int err = EINVAL; 2646204591Sluigi 2647222560Sae ND("called for %s", s->name); 2648204591Sluigi 2649204591Sluigi DN_BH_WLOCK(); 2650204591Sluigi SLIST_FOREACH_SAFE(r, &dn_cfg.schedlist, next, tmp) { 2651204591Sluigi if (strcmp(s->name, r->name) != 0) 2652204591Sluigi continue; 2653222560Sae ND("ref_count = %d", r->ref_count); 2654204591Sluigi err = (r->ref_count != 0) ? EBUSY : 0; 2655204591Sluigi if (err == 0) 2656204591Sluigi SLIST_REMOVE(&dn_cfg.schedlist, r, dn_alg, next); 2657204591Sluigi break; 2658204591Sluigi } 2659204591Sluigi DN_BH_WUNLOCK(); 2660204591Sluigi D("dn_sched %s %sunloaded", s->name, err ? "not ":""); 2661204591Sluigi return err; 2662204591Sluigi} 2663204591Sluigi 2664204591Sluigiint 2665204591Sluigidn_sched_modevent(module_t mod, int cmd, void *arg) 2666204591Sluigi{ 2667204591Sluigi struct dn_alg *sch = arg; 2668204591Sluigi 2669204591Sluigi if (cmd == MOD_LOAD) 2670204591Sluigi return load_dn_sched(sch); 2671204591Sluigi else if (cmd == MOD_UNLOAD) 2672204591Sluigi return unload_dn_sched(sch); 2673204591Sluigi else 2674204591Sluigi return EINVAL; 2675204591Sluigi} 2676204591Sluigi 267745743Speterstatic moduledata_t dummynet_mod = { 2678204591Sluigi "dummynet", dummynet_modevent, NULL 267984516Sps}; 2680204591Sluigi 2681206428Sluigi#define DN_SI_SUB SI_SUB_PROTO_IFATTACHDOMAIN 2682206428Sluigi#define DN_MODEV_ORD (SI_ORDER_ANY - 128) /* after ipfw */ 2683206428SluigiDECLARE_MODULE(dummynet, dummynet_mod, DN_SI_SUB, DN_MODEV_ORD); 2684133920SandreMODULE_DEPEND(dummynet, ipfw, 2, 2, 2); 2685218741SpluknetMODULE_VERSION(dummynet, 3); 2686206428Sluigi 2687206428Sluigi/* 2688206428Sluigi * Starting up. Done in order after dummynet_modevent() has been called. 2689206428Sluigi * VNET_SYSINIT is also called for each existing vnet and each new vnet. 2690206428Sluigi */ 2691206428Sluigi//VNET_SYSINIT(vnet_dn_init, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_init, NULL); 2692213253Sluigi 2693206428Sluigi/* 2694206428Sluigi * Shutdown handlers up shop. These are done in REVERSE ORDER, but still 2695206428Sluigi * after dummynet_modevent() has been called. Not called on reboot. 2696206428Sluigi * VNET_SYSUNINIT is also called for each exiting vnet as it exits. 2697206428Sluigi * or when the module is unloaded. 2698206428Sluigi */ 2699206428Sluigi//VNET_SYSUNINIT(vnet_dn_uninit, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_destroy, NULL); 2700206428Sluigi 2701301772Struckman#ifdef NEW_AQM 2702301772Struckman 2703301772Struckman/* modevent helpers for the AQM modules */ 2704301772Struckmanstatic int 2705301772Struckmanload_dn_aqm(struct dn_aqm *d) 2706301772Struckman{ 2707301772Struckman struct dn_aqm *aqm=NULL; 2708301772Struckman 2709301772Struckman if (d == NULL) 2710301772Struckman return 1; /* error */ 2711301772Struckman ip_dn_init(); /* just in case, we need the lock */ 2712301772Struckman 2713301772Struckman /* Check that mandatory funcs exists */ 2714301772Struckman if (d->enqueue == NULL || d->dequeue == NULL) { 2715301772Struckman D("missing enqueue or dequeue for %s", d->name); 2716301772Struckman return 1; 2717301772Struckman } 2718301772Struckman 2719301772Struckman /* Search if AQM already exists */ 2720301772Struckman DN_BH_WLOCK(); 2721301772Struckman SLIST_FOREACH(aqm, &dn_cfg.aqmlist, next) { 2722301772Struckman if (strcmp(aqm->name, d->name) == 0) { 2723301772Struckman D("%s already loaded", d->name); 2724301772Struckman break; /* AQM already exists */ 2725301772Struckman } 2726301772Struckman } 2727301772Struckman if (aqm == NULL) 2728301772Struckman SLIST_INSERT_HEAD(&dn_cfg.aqmlist, d, next); 2729301772Struckman DN_BH_WUNLOCK(); 2730301772Struckman D("dn_aqm %s %sloaded", d->name, aqm ? "not ":""); 2731301772Struckman return aqm ? 1 : 0; 2732301772Struckman} 2733301772Struckman 2734301772Struckman 2735301772Struckman/* Callback to clean up AQM status for queues connected to a flowset 2736301772Struckman * and then deconfigure the flowset. 2737301772Struckman * This function is called before an AQM module is unloaded 2738301772Struckman */ 2739301772Struckmanstatic int 2740301772Struckmanfs_cleanup(void *_fs, void *arg) 2741301772Struckman{ 2742301772Struckman struct dn_fsk *fs = _fs; 2743301772Struckman uint32_t type = *(uint32_t *)arg; 2744301772Struckman 2745301772Struckman if (fs->aqmfp && fs->aqmfp->type == type) 2746301772Struckman aqm_cleanup_deconfig_fs(fs); 2747301772Struckman 2748301772Struckman return 0; 2749301772Struckman} 2750301772Struckman 2751301772Struckmanstatic int 2752301772Struckmanunload_dn_aqm(struct dn_aqm *aqm) 2753301772Struckman{ 2754301772Struckman struct dn_aqm *tmp, *r; 2755301772Struckman int err = EINVAL; 2756301772Struckman err = 0; 2757301772Struckman ND("called for %s", aqm->name); 2758301772Struckman 2759301772Struckman DN_BH_WLOCK(); 2760301772Struckman 2761301772Struckman /* clean up AQM status and deconfig flowset */ 2762301772Struckman dn_ht_scan(dn_cfg.fshash, fs_cleanup, &aqm->type); 2763301772Struckman 2764301772Struckman SLIST_FOREACH_SAFE(r, &dn_cfg.aqmlist, next, tmp) { 2765301772Struckman if (strcmp(aqm->name, r->name) != 0) 2766301772Struckman continue; 2767301772Struckman ND("ref_count = %d", r->ref_count); 2768301772Struckman err = (r->ref_count != 0 || r->cfg_ref_count != 0) ? EBUSY : 0; 2769301772Struckman if (err == 0) 2770301772Struckman SLIST_REMOVE(&dn_cfg.aqmlist, r, dn_aqm, next); 2771301772Struckman break; 2772301772Struckman } 2773301772Struckman DN_BH_WUNLOCK(); 2774301772Struckman D("%s %sunloaded", aqm->name, err ? "not ":""); 2775301772Struckman if (err) 2776301772Struckman D("ref_count=%d, cfg_ref_count=%d", r->ref_count, r->cfg_ref_count); 2777301772Struckman return err; 2778301772Struckman} 2779301772Struckman 2780301772Struckmanint 2781301772Struckmandn_aqm_modevent(module_t mod, int cmd, void *arg) 2782301772Struckman{ 2783301772Struckman struct dn_aqm *aqm = arg; 2784301772Struckman 2785301772Struckman if (cmd == MOD_LOAD) 2786301772Struckman return load_dn_aqm(aqm); 2787301772Struckman else if (cmd == MOD_UNLOAD) 2788301772Struckman return unload_dn_aqm(aqm); 2789301772Struckman else 2790301772Struckman return EINVAL; 2791301772Struckman} 2792301772Struckman#endif 2793301772Struckman 2794200601Sluigi/* end of file */ 2795301772Struckman 2796