32
33#include "opt_bpf.h"
34
35#include <sys/param.h>
36#include <sys/lock.h>
37#include <sys/malloc.h>
38#include <sys/mbuf.h>
39#include <sys/mutex.h>
40#include <sys/proc.h>
41#include <sys/sf_buf.h>
42#include <sys/socket.h>
43#include <sys/uio.h>
44
45#include <machine/atomic.h>
46
47#include <net/if.h>
48#include <net/bpf.h>
49#include <net/bpf_zerocopy.h>
50#include <net/bpfdesc.h>
51
52#include <vm/vm.h>
53#include <vm/pmap.h>
54#include <vm/vm_extern.h>
55#include <vm/vm_map.h>
56#include <vm/vm_page.h>
57
58/*
59 * Zero-copy buffer scheme for BPF: user space "donates" two buffers, which
60 * are mapped into the kernel address space using sf_bufs and used directly
61 * by BPF. Memory is wired since page faults cannot be tolerated in the
62 * contexts where the buffers are copied to (locks held, interrupt context,
63 * etc). Access to shared memory buffers is synchronized using a header on
64 * each buffer, allowing the number of system calls to go to zero as BPF
65 * reaches saturation (buffers filled as fast as they can be drained by the
66 * user process). Full details of the protocol for communicating between the
67 * user process and BPF may be found in bpf(4).
68 */
69
70/*
71 * Maximum number of pages per buffer. Since all BPF devices use two, the
72 * maximum per device is 2*BPF_MAX_PAGES. Resource limits on the number of
73 * sf_bufs may be an issue, so do not set this too high. On older systems,
74 * kernel address space limits may also be an issue.
75 */
76#define BPF_MAX_PAGES 512
77
78/*
79 * struct zbuf describes a memory buffer loaned by a user process to the
80 * kernel. We represent this as a series of pages managed using an array of
81 * sf_bufs. Even though the memory is contiguous in user space, it may not
82 * be mapped contiguously in the kernel (i.e., a set of physically
83 * non-contiguous pages in the direct map region) so we must implement
84 * scatter-gather copying. One significant mitigating factor is that on
85 * systems with a direct memory map, we can avoid TLB misses.
86 *
87 * At the front of the shared memory region is a bpf_zbuf_header, which
88 * contains shared control data to allow user space and the kernel to
89 * synchronize; this is included in zb_size, but not bpf_bufsize, so that BPF
90 * knows that the space is not available.
91 */
92struct zbuf {
93 vm_offset_t zb_uaddr; /* User address at time of setup. */
94 size_t zb_size; /* Size of buffer, incl. header. */
95 u_int zb_numpages; /* Number of pages. */
96 int zb_flags; /* Flags on zbuf. */
97 struct sf_buf **zb_pages; /* Pages themselves. */
98 struct bpf_zbuf_header *zb_header; /* Shared header. */
99};
100
101/*
102 * When a buffer has been assigned to userspace, flag it as such, as the
103 * buffer may remain in the store position as a result of the user process
104 * not yet having acknowledged the buffer in the hold position yet.
105 */
106#define ZBUF_FLAG_ASSIGNED 0x00000001 /* Set when owned by user. */
107
108/*
109 * Release a page we've previously wired.
110 */
111static void
112zbuf_page_free(vm_page_t pp)
113{
114
115 vm_page_lock(pp);
116 vm_page_unwire(pp, 0);
117 if (pp->wire_count == 0 && pp->object == NULL)
118 vm_page_free(pp);
119 vm_page_unlock(pp);
120}
121
122/*
123 * Free an sf_buf with attached page.
124 */
125static void
126zbuf_sfbuf_free(struct sf_buf *sf)
127{
128 vm_page_t pp;
129
130 pp = sf_buf_page(sf);
131 sf_buf_free(sf);
132 zbuf_page_free(pp);
133}
134
135/*
136 * Free a zbuf, including its page array, sbufs, and pages. Allow partially
137 * allocated zbufs to be freed so that it may be used even during a zbuf
138 * setup.
139 */
140static void
141zbuf_free(struct zbuf *zb)
142{
143 int i;
144
145 for (i = 0; i < zb->zb_numpages; i++) {
146 if (zb->zb_pages[i] != NULL)
147 zbuf_sfbuf_free(zb->zb_pages[i]);
148 }
149 free(zb->zb_pages, M_BPF);
150 free(zb, M_BPF);
151}
152
153/*
154 * Given a user pointer to a page of user memory, return an sf_buf for the
155 * page. Because we may be requesting quite a few sf_bufs, prefer failure to
156 * deadlock and use SFB_NOWAIT.
157 */
158static struct sf_buf *
159zbuf_sfbuf_get(struct vm_map *map, vm_offset_t uaddr)
160{
161 struct sf_buf *sf;
162 vm_page_t pp;
163
164 if (vm_fault_quick_hold_pages(map, uaddr, PAGE_SIZE, VM_PROT_READ |
165 VM_PROT_WRITE, &pp, 1) < 0)
166 return (NULL);
167 vm_page_lock(pp);
168 vm_page_wire(pp);
169 vm_page_unhold(pp);
170 vm_page_unlock(pp);
171 sf = sf_buf_alloc(pp, SFB_NOWAIT);
172 if (sf == NULL) {
173 zbuf_page_free(pp);
174 return (NULL);
175 }
176 return (sf);
177}
178
179/*
180 * Create a zbuf describing a range of user address space memory. Validate
181 * page alignment, size requirements, etc.
182 */
183static int
184zbuf_setup(struct thread *td, vm_offset_t uaddr, size_t len,
185 struct zbuf **zbp)
186{
187 struct zbuf *zb;
188 struct vm_map *map;
189 int error, i;
190
191 *zbp = NULL;
192
193 /*
194 * User address must be page-aligned.
195 */
196 if (uaddr & PAGE_MASK)
197 return (EINVAL);
198
199 /*
200 * Length must be an integer number of full pages.
201 */
202 if (len & PAGE_MASK)
203 return (EINVAL);
204
205 /*
206 * Length must not exceed per-buffer resource limit.
207 */
208 if ((len / PAGE_SIZE) > BPF_MAX_PAGES)
209 return (EINVAL);
210
211 /*
212 * Allocate the buffer and set up each page with is own sf_buf.
213 */
214 error = 0;
215 zb = malloc(sizeof(*zb), M_BPF, M_ZERO | M_WAITOK);
216 zb->zb_uaddr = uaddr;
217 zb->zb_size = len;
218 zb->zb_numpages = len / PAGE_SIZE;
219 zb->zb_pages = malloc(sizeof(struct sf_buf *) *
220 zb->zb_numpages, M_BPF, M_ZERO | M_WAITOK);
221 map = &td->td_proc->p_vmspace->vm_map;
222 for (i = 0; i < zb->zb_numpages; i++) {
223 zb->zb_pages[i] = zbuf_sfbuf_get(map,
224 uaddr + (i * PAGE_SIZE));
225 if (zb->zb_pages[i] == NULL) {
226 error = EFAULT;
227 goto error;
228 }
229 }
230 zb->zb_header =
231 (struct bpf_zbuf_header *)sf_buf_kva(zb->zb_pages[0]);
232 bzero(zb->zb_header, sizeof(*zb->zb_header));
233 *zbp = zb;
234 return (0);
235
236error:
237 zbuf_free(zb);
238 return (error);
239}
240
241/*
242 * Copy bytes from a source into the specified zbuf. The caller is
243 * responsible for performing bounds checking, etc.
244 */
245void
246bpf_zerocopy_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset,
247 void *src, u_int len)
248{
249 u_int count, page, poffset;
250 u_char *src_bytes;
251 struct zbuf *zb;
252
253 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
254 ("bpf_zerocopy_append_bytes: not in zbuf mode"));
255 KASSERT(buf != NULL, ("bpf_zerocopy_append_bytes: NULL buf"));
256
257 src_bytes = (u_char *)src;
258 zb = (struct zbuf *)buf;
259
260 KASSERT((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0,
261 ("bpf_zerocopy_append_bytes: ZBUF_FLAG_ASSIGNED"));
262
263 /*
264 * Scatter-gather copy to user pages mapped into kernel address space
265 * using sf_bufs: copy up to a page at a time.
266 */
267 offset += sizeof(struct bpf_zbuf_header);
268 page = offset / PAGE_SIZE;
269 poffset = offset % PAGE_SIZE;
270 while (len > 0) {
271 KASSERT(page < zb->zb_numpages, ("bpf_zerocopy_append_bytes:"
272 " page overflow (%d p %d np)\n", page, zb->zb_numpages));
273
274 count = min(len, PAGE_SIZE - poffset);
275 bcopy(src_bytes, ((u_char *)sf_buf_kva(zb->zb_pages[page])) +
276 poffset, count);
277 poffset += count;
278 if (poffset == PAGE_SIZE) {
279 poffset = 0;
280 page++;
281 }
282 KASSERT(poffset < PAGE_SIZE,
283 ("bpf_zerocopy_append_bytes: page offset overflow (%d)",
284 poffset));
285 len -= count;
286 src_bytes += count;
287 }
288}
289
290/*
291 * Copy bytes from an mbuf chain to the specified zbuf: copying will be
292 * scatter-gather both from mbufs, which may be fragmented over memory, and
293 * to pages, which may not be contiguously mapped in kernel address space.
294 * As with bpf_zerocopy_append_bytes(), the caller is responsible for
295 * checking that this will not exceed the buffer limit.
296 */
297void
298bpf_zerocopy_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset,
299 void *src, u_int len)
300{
301 u_int count, moffset, page, poffset;
302 const struct mbuf *m;
303 struct zbuf *zb;
304
305 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
306 ("bpf_zerocopy_append_mbuf not in zbuf mode"));
307 KASSERT(buf != NULL, ("bpf_zerocopy_append_mbuf: NULL buf"));
308
309 m = (struct mbuf *)src;
310 zb = (struct zbuf *)buf;
311
312 KASSERT((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0,
313 ("bpf_zerocopy_append_mbuf: ZBUF_FLAG_ASSIGNED"));
314
315 /*
316 * Scatter gather both from an mbuf chain and to a user page set
317 * mapped into kernel address space using sf_bufs. If we're lucky,
318 * each mbuf requires one copy operation, but if page alignment and
319 * mbuf alignment work out less well, we'll be doing two copies per
320 * mbuf.
321 */
322 offset += sizeof(struct bpf_zbuf_header);
323 page = offset / PAGE_SIZE;
324 poffset = offset % PAGE_SIZE;
325 moffset = 0;
326 while (len > 0) {
327 KASSERT(page < zb->zb_numpages,
328 ("bpf_zerocopy_append_mbuf: page overflow (%d p %d "
329 "np)\n", page, zb->zb_numpages));
330 KASSERT(m != NULL,
331 ("bpf_zerocopy_append_mbuf: end of mbuf chain"));
332
333 count = min(m->m_len - moffset, len);
334 count = min(count, PAGE_SIZE - poffset);
335 bcopy(mtod(m, u_char *) + moffset,
336 ((u_char *)sf_buf_kva(zb->zb_pages[page])) + poffset,
337 count);
338 poffset += count;
339 if (poffset == PAGE_SIZE) {
340 poffset = 0;
341 page++;
342 }
343 KASSERT(poffset < PAGE_SIZE,
344 ("bpf_zerocopy_append_mbuf: page offset overflow (%d)",
345 poffset));
346 moffset += count;
347 if (moffset == m->m_len) {
348 m = m->m_next;
349 moffset = 0;
350 }
351 len -= count;
352 }
353}
354
355/*
356 * Notification from the BPF framework that a buffer in the store position is
357 * rejecting packets and may be considered full. We mark the buffer as
358 * immutable and assign to userspace so that it is immediately available for
359 * the user process to access.
360 */
361void
362bpf_zerocopy_buffull(struct bpf_d *d)
363{
364 struct zbuf *zb;
365
366 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
367 ("bpf_zerocopy_buffull: not in zbuf mode"));
368
369 zb = (struct zbuf *)d->bd_sbuf;
370 KASSERT(zb != NULL, ("bpf_zerocopy_buffull: zb == NULL"));
371
372 if ((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0) {
373 zb->zb_flags |= ZBUF_FLAG_ASSIGNED;
374 zb->zb_header->bzh_kernel_len = d->bd_slen;
375 atomic_add_rel_int(&zb->zb_header->bzh_kernel_gen, 1);
376 }
377}
378
379/*
380 * Notification from the BPF framework that a buffer has moved into the held
381 * slot on a descriptor. Zero-copy BPF will update the shared page to let
382 * the user process know and flag the buffer as assigned if it hasn't already
383 * been marked assigned due to filling while it was in the store position.
384 *
385 * Note: identical logic as in bpf_zerocopy_buffull(), except that we operate
386 * on bd_hbuf and bd_hlen.
387 */
388void
389bpf_zerocopy_bufheld(struct bpf_d *d)
390{
391 struct zbuf *zb;
392
393 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
394 ("bpf_zerocopy_bufheld: not in zbuf mode"));
395
396 zb = (struct zbuf *)d->bd_hbuf;
397 KASSERT(zb != NULL, ("bpf_zerocopy_bufheld: zb == NULL"));
398
399 if ((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0) {
400 zb->zb_flags |= ZBUF_FLAG_ASSIGNED;
401 zb->zb_header->bzh_kernel_len = d->bd_hlen;
402 atomic_add_rel_int(&zb->zb_header->bzh_kernel_gen, 1);
403 }
404}
405
406/*
407 * Notification from the BPF framework that the free buffer has been been
408 * rotated out of the held position to the free position. This happens when
409 * the user acknowledges the held buffer.
410 */
411void
412bpf_zerocopy_buf_reclaimed(struct bpf_d *d)
413{
414 struct zbuf *zb;
415
416 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
417 ("bpf_zerocopy_reclaim_buf: not in zbuf mode"));
418
419 KASSERT(d->bd_fbuf != NULL,
420 ("bpf_zerocopy_buf_reclaimed: NULL free buf"));
421 zb = (struct zbuf *)d->bd_fbuf;
422 zb->zb_flags &= ~ZBUF_FLAG_ASSIGNED;
423}
424
425/*
426 * Query from the BPF framework regarding whether the buffer currently in the
427 * held position can be moved to the free position, which can be indicated by
428 * the user process making their generation number equal to the kernel
429 * generation number.
430 */
431int
432bpf_zerocopy_canfreebuf(struct bpf_d *d)
433{
434 struct zbuf *zb;
435
436 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
437 ("bpf_zerocopy_canfreebuf: not in zbuf mode"));
438
439 zb = (struct zbuf *)d->bd_hbuf;
440 if (zb == NULL)
441 return (0);
442 if (zb->zb_header->bzh_kernel_gen ==
443 atomic_load_acq_int(&zb->zb_header->bzh_user_gen))
444 return (1);
445 return (0);
446}
447
448/*
449 * Query from the BPF framework as to whether or not the buffer current in
450 * the store position can actually be written to. This may return false if
451 * the store buffer is assigned to userspace before the hold buffer is
452 * acknowledged.
453 */
454int
455bpf_zerocopy_canwritebuf(struct bpf_d *d)
456{
457 struct zbuf *zb;
458
459 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
460 ("bpf_zerocopy_canwritebuf: not in zbuf mode"));
461
462 zb = (struct zbuf *)d->bd_sbuf;
463 KASSERT(zb != NULL, ("bpf_zerocopy_canwritebuf: bd_sbuf NULL"));
464
465 if (zb->zb_flags & ZBUF_FLAG_ASSIGNED)
466 return (0);
467 return (1);
468}
469
470/*
471 * Free zero copy buffers at request of descriptor.
472 */
473void
474bpf_zerocopy_free(struct bpf_d *d)
475{
476 struct zbuf *zb;
477
478 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
479 ("bpf_zerocopy_free: not in zbuf mode"));
480
481 zb = (struct zbuf *)d->bd_sbuf;
482 if (zb != NULL)
483 zbuf_free(zb);
484 zb = (struct zbuf *)d->bd_hbuf;
485 if (zb != NULL)
486 zbuf_free(zb);
487 zb = (struct zbuf *)d->bd_fbuf;
488 if (zb != NULL)
489 zbuf_free(zb);
490}
491
492/*
493 * Ioctl to return the maximum buffer size.
494 */
495int
496bpf_zerocopy_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i)
497{
498
499 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
500 ("bpf_zerocopy_ioctl_getzmax: not in zbuf mode"));
501
502 *i = BPF_MAX_PAGES * PAGE_SIZE;
503 return (0);
504}
505
506/*
507 * Ioctl to force rotation of the two buffers, if there's any data available.
508 * This can be used by user space to implement timeouts when waiting for a
509 * buffer to fill.
510 */
511int
512bpf_zerocopy_ioctl_rotzbuf(struct thread *td, struct bpf_d *d,
513 struct bpf_zbuf *bz)
514{
515 struct zbuf *bzh;
516
517 bzero(bz, sizeof(*bz));
518 BPFD_LOCK(d);
519 if (d->bd_hbuf == NULL && d->bd_slen != 0) {
520 ROTATE_BUFFERS(d);
521 bzh = (struct zbuf *)d->bd_hbuf;
522 bz->bz_bufa = (void *)bzh->zb_uaddr;
523 bz->bz_buflen = d->bd_hlen;
524 }
525 BPFD_UNLOCK(d);
526 return (0);
527}
528
529/*
530 * Ioctl to configure zero-copy buffers -- may be done only once.
531 */
532int
533bpf_zerocopy_ioctl_setzbuf(struct thread *td, struct bpf_d *d,
534 struct bpf_zbuf *bz)
535{
536 struct zbuf *zba, *zbb;
537 int error;
538
539 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
540 ("bpf_zerocopy_ioctl_setzbuf: not in zbuf mode"));
541
542 /*
543 * Must set both buffers. Cannot clear them.
544 */
545 if (bz->bz_bufa == NULL || bz->bz_bufb == NULL)
546 return (EINVAL);
547
548 /*
549 * Buffers must have a size greater than 0. Alignment and other size
550 * validity checking is done in zbuf_setup().
551 */
552 if (bz->bz_buflen == 0)
553 return (EINVAL);
554
555 /*
556 * Allocate new buffers.
557 */
558 error = zbuf_setup(td, (vm_offset_t)bz->bz_bufa, bz->bz_buflen,
559 &zba);
560 if (error)
561 return (error);
562 error = zbuf_setup(td, (vm_offset_t)bz->bz_bufb, bz->bz_buflen,
563 &zbb);
564 if (error) {
565 zbuf_free(zba);
566 return (error);
567 }
568
569 /*
570 * We only allow buffers to be installed once, so atomically check
571 * that no buffers are currently installed and install new buffers.
572 */
573 BPFD_LOCK(d);
574 if (d->bd_hbuf != NULL || d->bd_sbuf != NULL || d->bd_fbuf != NULL ||
575 d->bd_bif != NULL) {
576 BPFD_UNLOCK(d);
577 zbuf_free(zba);
578 zbuf_free(zbb);
579 return (EINVAL);
580 }
581
582 /*
583 * Point BPF descriptor at buffers; initialize sbuf as zba so that
584 * it is always filled first in the sequence, per bpf(4).
585 */
586 d->bd_fbuf = (caddr_t)zbb;
587 d->bd_sbuf = (caddr_t)zba;
588 d->bd_slen = 0;
589 d->bd_hlen = 0;
590
591 /*
592 * We expose only the space left in the buffer after the size of the
593 * shared management region.
594 */
595 d->bd_bufsize = bz->bz_buflen - sizeof(struct bpf_zbuf_header);
596 BPFD_UNLOCK(d);
597 return (0);
598}
|