wqinput.c revision 1.3
1/* $NetBSD: wqinput.c,v 1.3 2017/06/02 19:10:19 para Exp $ */ 2 3/*- 4 * Copyright (c) 2017 Internet Initiative Japan Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29#include <sys/param.h> 30#include <sys/kmem.h> 31#include <sys/mbuf.h> 32#include <sys/protosw.h> 33#include <sys/socketvar.h> 34#include <sys/syslog.h> 35#include <sys/workqueue.h> 36#include <sys/atomic.h> 37#include <sys/queue.h> 38#include <sys/percpu.h> 39#include <sys/sysctl.h> 40 41#include <net/if.h> 42#include <netinet/wqinput.h> 43 44#define WQINPUT_LIST_MAXLEN IFQ_MAXLEN 45 46struct wqinput_work { 47 struct mbuf *ww_mbuf; 48 int ww_off; 49 int ww_proto; 50 struct wqinput_work *ww_next; 51}; 52 53struct wqinput_worklist { 54 /* 55 * XXX: TAILQ cannot be used because TAILQ_INIT memories the address 56 * of percpu data while percpu(9) may move percpu data during bootup. 57 */ 58 struct wqinput_work *wwl_head; 59 struct wqinput_work *wwl_tail; 60 unsigned int wwl_len; 61 unsigned long wwl_dropped; 62 struct work wwl_work; 63 bool wwl_wq_is_active; 64}; 65 66struct wqinput { 67 struct workqueue *wqi_wq; 68 struct pool wqi_work_pool; 69 struct percpu *wqi_worklists; /* struct wqinput_worklist */ 70 void (*wqi_input)(struct mbuf *, int, int); 71}; 72 73static void wqinput_work(struct work *, void *); 74static void wqinput_sysctl_setup(const char *, struct wqinput *); 75 76static void 77wqinput_drops(void *p, void *arg, struct cpu_info *ci __unused) 78{ 79 struct wqinput_worklist *const wwl = p; 80 int *sum = arg; 81 82 *sum += wwl->wwl_dropped; 83} 84 85static int 86wqinput_sysctl_drops_handler(SYSCTLFN_ARGS) 87{ 88 struct sysctlnode node; 89 struct wqinput *wqi; 90 int sum = 0; 91 int error; 92 93 node = *rnode; 94 wqi = node.sysctl_data; 95 96 percpu_foreach(wqi->wqi_worklists, wqinput_drops, &sum); 97 98 node.sysctl_data = ∑ 99 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 100 if (error != 0 || newp == NULL) 101 return error; 102 103 return 0; 104} 105 106static void 107wqinput_sysctl_setup(const char *name, struct wqinput *wqi) 108{ 109 const struct sysctlnode *cnode, *rnode; 110 int error; 111 112 error = sysctl_createv(NULL, 0, NULL, &rnode, 113 CTLFLAG_PERMANENT, CTLTYPE_NODE, "wqinput", 114 SYSCTL_DESCR("workqueue-based pr_input controls"), 115 NULL, 0, NULL, 0, CTL_NET, CTL_CREATE, CTL_EOL); 116 if (error != 0) 117 goto bad; 118 119 error = sysctl_createv(NULL, 0, &rnode, &rnode, 120 CTLFLAG_PERMANENT, CTLTYPE_NODE, name, 121 SYSCTL_DESCR("Protocol controls for workqueue-based pr_input"), 122 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 123 if (error != 0) 124 goto bad; 125 126 error = sysctl_createv(NULL, 0, &rnode, &rnode, 127 CTLFLAG_PERMANENT, CTLTYPE_NODE, "inputq", 128 SYSCTL_DESCR("wqinput input queue controls"), 129 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 130 if (error != 0) 131 goto bad; 132 133 error = sysctl_createv(NULL, 0, &rnode, &cnode, 134 CTLFLAG_PERMANENT, CTLTYPE_INT, "drops", 135 SYSCTL_DESCR("Total packets dropped due to full input queue"), 136 wqinput_sysctl_drops_handler, 0, (void *)wqi, 0, CTL_CREATE, CTL_EOL); 137 if (error != 0) 138 goto bad; 139 140 return; 141bad: 142 log(LOG_ERR, "%s: could not create a sysctl node for %s\n", 143 __func__, name); 144 return; 145} 146 147struct wqinput * 148wqinput_create(const char *name, void (*func)(struct mbuf *, int, int)) 149{ 150 struct wqinput *wqi; 151 int error; 152 char namebuf[32]; 153 154 snprintf(namebuf, sizeof(namebuf), "%s_wqinput", name); 155 156 wqi = kmem_alloc(sizeof(*wqi), KM_SLEEP); 157 158 error = workqueue_create(&wqi->wqi_wq, namebuf, wqinput_work, wqi, 159 PRI_SOFTNET, IPL_SOFTNET, WQ_MPSAFE|WQ_PERCPU); 160 if (error != 0) 161 panic("%s: workqueue_create failed (%d)\n", __func__, error); 162 pool_init(&wqi->wqi_work_pool, sizeof(struct wqinput_work), 0, 0, 0, 163 name, NULL, IPL_SOFTNET); 164 wqi->wqi_worklists = percpu_alloc(sizeof(struct wqinput_worklist)); 165 wqi->wqi_input = func; 166 167 wqinput_sysctl_setup(name, wqi); 168 169 return wqi; 170} 171 172static struct wqinput_work * 173wqinput_work_get(struct wqinput_worklist *wwl) 174{ 175 struct wqinput_work *work; 176 177 /* Must be called at IPL_SOFTNET */ 178 179 work = wwl->wwl_head; 180 if (work != NULL) { 181 KASSERTMSG(wwl->wwl_len > 0, "wwl->wwl_len=%d", wwl->wwl_len); 182 wwl->wwl_len--; 183 wwl->wwl_head = work->ww_next; 184 work->ww_next = NULL; 185 186 if (wwl->wwl_head == NULL) 187 wwl->wwl_tail = NULL; 188 } else { 189 KASSERT(wwl->wwl_len == 0); 190 } 191 192 return work; 193} 194 195static void 196wqinput_work(struct work *wk, void *arg) 197{ 198 struct wqinput *wqi = arg; 199 struct wqinput_work *work; 200 struct wqinput_worklist *wwl; 201 int s; 202 203 /* Users expect to run at IPL_SOFTNET */ 204 s = splsoftnet(); 205 /* This also prevents LWP migrations between CPUs */ 206 wwl = percpu_getref(wqi->wqi_worklists); 207 208 /* We can allow enqueuing another work at this point */ 209 wwl->wwl_wq_is_active = false; 210 211 while ((work = wqinput_work_get(wwl)) != NULL) { 212 mutex_enter(softnet_lock); 213 wqi->wqi_input(work->ww_mbuf, work->ww_off, work->ww_proto); 214 mutex_exit(softnet_lock); 215 216 pool_put(&wqi->wqi_work_pool, work); 217 } 218 219 percpu_putref(wqi->wqi_worklists); 220 splx(s); 221} 222 223static void 224wqinput_work_put(struct wqinput_worklist *wwl, struct wqinput_work *work) 225{ 226 227 if (wwl->wwl_tail != NULL) { 228 wwl->wwl_tail->ww_next = work; 229 } else { 230 wwl->wwl_head = work; 231 } 232 wwl->wwl_tail = work; 233 wwl->wwl_len++; 234} 235 236void 237wqinput_input(struct wqinput *wqi, struct mbuf *m, int off, int proto) 238{ 239 struct wqinput_work *work; 240 struct wqinput_worklist *wwl; 241 242 wwl = percpu_getref(wqi->wqi_worklists); 243 244 /* Prevent too much work and mbuf from being queued */ 245 if (wwl->wwl_len >= WQINPUT_LIST_MAXLEN) { 246 wwl->wwl_dropped++; 247 m_freem(m); 248 goto out; 249 } 250 251 work = pool_get(&wqi->wqi_work_pool, PR_NOWAIT); 252 if (work == NULL) { 253 wwl->wwl_dropped++; 254 m_freem(m); 255 goto out; 256 } 257 work->ww_mbuf = m; 258 work->ww_off = off; 259 work->ww_proto = proto; 260 work->ww_next = NULL; 261 262 wqinput_work_put(wwl, work); 263 264 /* Avoid enqueuing another work when one is already enqueued */ 265 if (wwl->wwl_wq_is_active) 266 goto out; 267 wwl->wwl_wq_is_active = true; 268 269 workqueue_enqueue(wqi->wqi_wq, &wwl->wwl_work, NULL); 270out: 271 percpu_putref(wqi->wqi_worklists); 272} 273