1/*-
2 * Copyright (c) 2002 Poul-Henning Kamp
3 * Copyright (c) 2002 Networks Associates Technology, Inc.
4 * All rights reserved.
5 *
6 * This software was developed for the FreeBSD Project by Poul-Henning Kamp
7 * and NAI Labs, the Security Research Division of Network Associates, Inc.
8 * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the
9 * DARPA CHATS research program.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 *
32 * $FreeBSD: head/sys/geom/bde/g_bde_work.c 114152 2003-04-28 06:19:41Z phk $
33 *
34 * This source file contains the state-engine which makes things happen in the
35 * right order.
36 *
37 * Outline:
38 * 1) g_bde_start1()
39 * Break the struct bio into multiple work packets one per zone.
40 * 2) g_bde_start2()
41 * Setup the necessary sector buffers and start those read operations
42 * which we can start at this time and put the item on the work-list.
43 * 3) g_bde_worker()
44 * Scan the work-list for items which are ready for crypto processing
45 * and call the matching crypto function in g_bde_crypt.c and schedule
46 * any writes needed. Read operations finish here by releasing the
47 * sector buffers and delivering the original bio request.
48 * 4) g_bde_write_done()
49 * Release sector buffers and deliver the original bio request.
50 *
51 * Because of the C-scope rules, the functions are almost perfectly in the
52 * opposite order in this source file.
53 *
54 * XXX: A switch to the hardware assisted crypto in src/sys/opencrypto will add
55 * XXX: additional states to this state-engine. Since no hardware available
56 * XXX: at this time has AES support, implementing this has been postponed
57 * XXX: until such time as it would result in a benefit.
58 */
59
60#include <sys/param.h>
61#include <sys/bio.h>
62#include <sys/lock.h>
63#include <sys/mutex.h>
64#include <sys/queue.h>
65#include <sys/malloc.h>
66#include <sys/systm.h>
67#include <sys/kernel.h>
68#include <sys/sysctl.h>
69#include <sys/proc.h>
70#include <sys/kthread.h>
71
72#include <crypto/rijndael/rijndael.h>
73#include <crypto/sha2/sha2.h>
74#include <geom/geom.h>
75#include <geom/bde/g_bde.h>
76
77static void g_bde_delete_sector(struct g_bde_softc *wp, struct g_bde_sector *sp);
78static struct g_bde_sector * g_bde_new_sector(struct g_bde_work *wp, u_int len);
79static void g_bde_release_keysector(struct g_bde_work *wp);
80static struct g_bde_sector *g_bde_get_sector(struct g_bde_work *wp, off_t offset);
81static int g_bde_start_read(struct g_bde_sector *sp);
82static void g_bde_purge_sector(struct g_bde_softc *sc, int fraction);
83
84/*
85 * Work item allocation.
86 *
87 * C++ would call these constructors and destructors.
88 */
89static u_int g_bde_nwork;
90SYSCTL_UINT(_debug, OID_AUTO, gbde_nwork, CTLFLAG_RD, &g_bde_nwork, 0, "");
91
92static struct g_bde_work *
93g_bde_new_work(struct g_bde_softc *sc)
94{
95 struct g_bde_work *wp;
96
97 wp = g_malloc(sizeof *wp, M_NOWAIT | M_ZERO);
98 if (wp == NULL)
99 return (wp);
100 wp->state = SETUP;
101 wp->softc = sc;
102 g_bde_nwork++;
103 sc->nwork++;
104 TAILQ_INSERT_TAIL(&sc->worklist, wp, list);
105 return (wp);
106}
107
108static void
109g_bde_delete_work(struct g_bde_work *wp)
110{
111 struct g_bde_softc *sc;
112
113 sc = wp->softc;
114 g_bde_nwork--;
115 sc->nwork--;
116 TAILQ_REMOVE(&sc->worklist, wp, list);
117 g_free(wp);
118}
119
120/*
121 * Sector buffer allocation
122 *
123 * These two functions allocate and free back variable sized sector buffers
124 */
125
126static u_int g_bde_nsect;
127SYSCTL_UINT(_debug, OID_AUTO, gbde_nsect, CTLFLAG_RD, &g_bde_nsect, 0, "");
128
129static void
130g_bde_delete_sector(struct g_bde_softc *sc, struct g_bde_sector *sp)
131{
132
133 g_bde_nsect--;
134 sc->nsect--;
135 if (sp->malloc)
136 g_free(sp->data);
137 g_free(sp);
138}
139
140static struct g_bde_sector *
141g_bde_new_sector(struct g_bde_work *wp, u_int len)
142{
143 struct g_bde_sector *sp;
144
145 sp = g_malloc(sizeof *sp, M_NOWAIT | M_ZERO);
146 if (sp == NULL)
147 return (sp);
148 if (len > 0) {
149 sp->data = g_malloc(len, M_NOWAIT | M_ZERO);
150 if (sp->data == NULL) {
151 g_free(sp);
152 return (NULL);
153 }
154 sp->malloc = 1;
155 }
156 g_bde_nsect++;
157 wp->softc->nsect++;
158 sp->size = len;
159 sp->softc = wp->softc;
160 sp->ref = 1;
161 sp->owner = wp;
162 sp->offset = wp->so;
163 sp->state = JUNK;
164 return (sp);
165}
166
167/*
168 * Skey sector cache.
169 *
170 * Nothing prevents two separate I/O requests from addressing the same zone
171 * and thereby needing the same skey sector. We therefore need to sequence
172 * I/O operations to the skey sectors. A certain amount of caching is also
173 * desirable, although the extent of benefit from this is not at this point
174 * determined.
175 *
176 * XXX: GEOM may be able to grow a generic caching facility at some point
177 * XXX: to support such needs.
178 */
179
180static u_int g_bde_ncache;
181SYSCTL_UINT(_debug, OID_AUTO, gbde_ncache, CTLFLAG_RD, &g_bde_ncache, 0, "");
182
183static void
184g_bde_purge_one_sector(struct g_bde_softc *sc, struct g_bde_sector *sp)
185{
186
187 g_trace(G_T_TOPOLOGY, "g_bde_purge_one_sector(%p, %p)", sc, sp);
188 if (sp->ref != 0)
189 return;
190 TAILQ_REMOVE(&sc->freelist, sp, list);
191 g_bde_ncache--;
192 sc->ncache--;
193 bzero(sp->data, sp->size);
194 g_bde_delete_sector(sc, sp);
195}
196
197static struct g_bde_sector *
198g_bde_get_sector(struct g_bde_work *wp, off_t offset)
199{
200 struct g_bde_sector *sp;
201 struct g_bde_softc *sc;
202
203 g_trace(G_T_TOPOLOGY, "g_bde_get_sector(%p, %jd)", wp, (intmax_t)offset);
204 sc = wp->softc;
205
206 if (malloc_last_fail() < g_bde_ncache)
207 g_bde_purge_sector(sc, -1);
208
209 sp = TAILQ_FIRST(&sc->freelist);
210 if (sp != NULL && sp->ref == 0 && sp->used + 300 < time_uptime)
211 g_bde_purge_one_sector(sc, sp);
212
213 TAILQ_FOREACH(sp, &sc->freelist, list) {
214 if (sp->offset == offset)
215 break;
216 }
217 if (sp != NULL) {
218 sp->ref++;
219 KASSERT(sp->offset == offset, ("wrong offset"));
220 KASSERT(sp->softc == wp->softc, ("wrong softc"));
221 if (sp->ref == 1)
222 sp->owner = wp;
223 } else {
224 if (malloc_last_fail() < g_bde_ncache) {
225 TAILQ_FOREACH(sp, &sc->freelist, list)
226 if (sp->ref == 0)
227 break;
228 }
229 if (sp == NULL && !TAILQ_EMPTY(&sc->freelist))
230 sp = TAILQ_FIRST(&sc->freelist);
231 if (sp != NULL && sp->ref > 0)
232 sp = NULL;
233 if (sp == NULL) {
234 sp = g_bde_new_sector(wp, sc->sectorsize);
235 if (sp != NULL) {
236 g_bde_ncache++;
237 sc->ncache++;
238 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
239 sp->malloc = 2;
240 }
241 }
242 if (sp != NULL) {
243 sp->offset = offset;
244 sp->softc = wp->softc;
245 sp->ref = 1;
246 sp->owner = wp;
247 sp->state = JUNK;
248 sp->error = 0;
249 }
250 }
251 if (sp != NULL) {
252 TAILQ_REMOVE(&sc->freelist, sp, list);
253 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
254 sp->used = time_uptime;
255 }
256 wp->ksp = sp;
257 return(sp);
258}
259
260static void
261g_bde_release_keysector(struct g_bde_work *wp)
262{
263 struct g_bde_softc *sc;
264 struct g_bde_work *wp2;
265 struct g_bde_sector *sp;
266
267 sp = wp->ksp;
268 g_trace(G_T_TOPOLOGY, "g_bde_release_keysector(%p)", sp);
269 KASSERT(sp->malloc == 2, ("Wrong sector released"));
270 sc = sp->softc;
271 KASSERT(sc != NULL, ("NULL sp->softc"));
272 KASSERT(wp == sp->owner, ("Releasing, not owner"));
273 sp->owner = NULL;
274 wp->ksp = NULL;
275 sp->ref--;
276 if (sp->ref > 0) {
277 TAILQ_REMOVE(&sc->freelist, sp, list);
278 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
279 TAILQ_FOREACH(wp2, &sc->worklist, list) {
280 if (wp2->ksp == sp) {
281 KASSERT(wp2 != wp, ("Self-reowning"));
282 sp->owner = wp2;
283 wakeup(sp->softc);
284 break;
285 }
286 }
287 KASSERT(wp2 != NULL, ("Failed to pick up owner for %p\n", sp));
288 } else if (sp->error != 0) {
289 sp->offset = ~0;
290 sp->error = 0;
291 sp->state = JUNK;
292 }
293 TAILQ_REMOVE(&sc->freelist, sp, list);
294 TAILQ_INSERT_HEAD(&sc->freelist, sp, list);
295}
296
297static void
298g_bde_purge_sector(struct g_bde_softc *sc, int fraction)
299{
300 struct g_bde_sector *sp;
301 int n;
302
303 g_trace(G_T_TOPOLOGY, "g_bde_purge_sector(%p)", sc);
304 if (fraction > 0)
305 n = sc->ncache / fraction + 1;
306 else
307 n = g_bde_ncache - malloc_last_fail();
308 if (n < 0)
309 return;
310 if (n > sc->ncache)
311 n = sc->ncache;
312 while(n--) {
313 TAILQ_FOREACH(sp, &sc->freelist, list) {
314 if (sp->ref != 0)
315 continue;
316 TAILQ_REMOVE(&sc->freelist, sp, list);
317 g_bde_ncache--;
318 sc->ncache--;
319 bzero(sp->data, sp->size);
320 g_bde_delete_sector(sc, sp);
321 break;
322 }
323 }
324}
325
326static struct g_bde_sector *
327g_bde_read_keysector(struct g_bde_softc *sc, struct g_bde_work *wp)
328{
329 struct g_bde_sector *sp;
330
331 g_trace(G_T_TOPOLOGY, "g_bde_read_keysector(%p)", wp);
332 sp = g_bde_get_sector(wp, wp->kso);
333 if (sp == NULL) {
334 g_bde_purge_sector(sc, -1);
335 sp = g_bde_get_sector(wp, wp->kso);
336 }
337 if (sp == NULL)
338 return (sp);
339 if (sp->owner != wp)
340 return (sp);
341 if (sp->state == VALID)
342 return (sp);
343 if (g_bde_start_read(sp) == 0)
344 return (sp);
345 g_bde_release_keysector(wp);
346 return (NULL);
347}
348
349/*
350 * Contribute to the completion of the original bio request.
351 *
352 * We have no simple way to tell how many bits the original bio request has
353 * been segmented into, so the easiest way to determine when we can deliver
354 * it is to keep track of the number of bytes we have completed. We keep
355 * track of any errors underway and latch onto the first one.
356 *
357 * We always report "nothing done" in case of error, because random bits here
358 * and there may be completed and returning a number of completed bytes does
359 * not convey any useful information about which bytes they were. If some
360 * piece of broken code somewhere interprets this to mean that nothing has
361 * changed on the underlying media they deserve the lossage headed for them.
362 *
363 * A single mutex per g_bde instance is used to prevent contention.
364 */
365
366static void
367g_bde_contribute(struct bio *bp, off_t bytes, int error)
368{
369 struct g_bde_softc *sc;
370
371 g_trace(G_T_TOPOLOGY, "g_bde_contribute bp %p bytes %jd error %d",
372 bp, (intmax_t)bytes, error);
373 sc = bp->bio_driver1;
374 if (bp->bio_error == 0)
375 bp->bio_error = error;
376 bp->bio_completed += bytes;
377 KASSERT(bp->bio_completed <= bp->bio_length, ("Too large contribution"));
378 if (bp->bio_completed == bp->bio_length) {
379 if (bp->bio_error != 0)
380 bp->bio_completed = 0;
381 g_io_deliver(bp, bp->bio_error);
382 }
383}
384
385/*
386 * A write operation has finished. When we have all expected cows in the
387 * barn close the door and call it a day.
388 */
389
390static void
391g_bde_write_done(struct bio *bp)
392{
393 struct g_bde_sector *sp;
394 struct g_bde_work *wp;
395 struct g_bde_softc *sc;
396
397 sp = bp->bio_caller1;
398 sc = bp->bio_caller2;
399 mtx_lock(&sc->worklist_mutex);
400 KASSERT(sp != NULL, ("NULL sp"));
401 KASSERT(sc != NULL, ("NULL sc"));
402 KASSERT(sp->owner != NULL, ("NULL sp->owner"));
403 g_trace(G_T_TOPOLOGY, "g_bde_write_done(%p)", sp);
404 sp->error = bp->bio_error;
405 g_destroy_bio(bp);
406 wp = sp->owner;
407 if (wp->error == 0)
408 wp->error = sp->error;
409
410 if (wp->bp->bio_cmd == BIO_DELETE) {
411 KASSERT(sp == wp->sp, ("trashed delete op"));
412 g_bde_contribute(wp->bp, wp->length, wp->error);
413 g_bde_delete_sector(sc, sp);
414 g_bde_delete_work(wp);
415 mtx_unlock(&sc->worklist_mutex);
416 return;
417 }
418
419 KASSERT(wp->bp->bio_cmd == BIO_WRITE, ("Confused in g_bde_write_done()"));
420 KASSERT(sp == wp->sp || sp == wp->ksp, ("trashed write op"));
421 if (wp->sp == sp) {
422 g_bde_delete_sector(sc, wp->sp);
423 wp->sp = NULL;
424 } else {
425 sp->state = VALID;
426 }
427 if (wp->sp == NULL && wp->ksp != NULL && wp->ksp->state == VALID) {
428 g_bde_contribute(wp->bp, wp->length, wp->error);
429 g_bde_release_keysector(wp);
430 g_bde_delete_work(wp);
431 }
432 mtx_unlock(&sc->worklist_mutex);
433 return;
434}
435
436/*
437 * Send a write request for the given sector down the pipeline.
438 */
439
440static int
441g_bde_start_write(struct g_bde_sector *sp)
442{
443 struct bio *bp;
444 struct g_bde_softc *sc;
445
446 g_trace(G_T_TOPOLOGY, "g_bde_start_write(%p)", sp);
447 sc = sp->softc;
448 KASSERT(sc != NULL, ("NULL sc in g_bde_start_write"));
449 KASSERT(sp->owner != NULL, ("NULL sp->owner in g_bde_start_write"));
450 bp = g_new_bio();
451 if (bp == NULL)
452 return (ENOMEM);
453 bp->bio_cmd = BIO_WRITE;
454 bp->bio_offset = sp->offset;
455 bp->bio_data = sp->data;
456 bp->bio_length = sp->size;
457 bp->bio_done = g_bde_write_done;
458 bp->bio_caller1 = sp;
459 bp->bio_caller2 = sc;
460 sp->state = IO;
461 g_io_request(bp, sc->consumer);
462 return(0);
463}
464
465/*
466 * A read operation has finished. Mark the sector no longer iobusy and
467 * wake up the worker thread and let it do its thing.
468 */
469
470static void
471g_bde_read_done(struct bio *bp)
472{
473 struct g_bde_sector *sp;
474 struct g_bde_softc *sc;
475
476 sp = bp->bio_caller1;
477 g_trace(G_T_TOPOLOGY, "g_bde_read_done(%p)", sp);
478 sc = bp->bio_caller2;
479 mtx_lock(&sc->worklist_mutex);
480 sp->error = bp->bio_error;
481 if (sp->error == 0)
482 sp->state = VALID;
483 else
484 sp->state = JUNK;
485 wakeup(sc);
486 g_destroy_bio(bp);
487 mtx_unlock(&sc->worklist_mutex);
488}
489
490/*
491 * Send a read request for the given sector down the pipeline.
492 */
493
494static int
495g_bde_start_read(struct g_bde_sector *sp)
496{
497 struct bio *bp;
498 struct g_bde_softc *sc;
499
500 g_trace(G_T_TOPOLOGY, "g_bde_start_read(%p)", sp);
501 sc = sp->softc;
502 KASSERT(sc != NULL, ("Null softc in sp %p", sp));
503 bp = g_new_bio();
504 if (bp == NULL)
505 return (ENOMEM);
506 bp->bio_cmd = BIO_READ;
507 bp->bio_offset = sp->offset;
508 bp->bio_data = sp->data;
509 bp->bio_length = sp->size;
510 bp->bio_done = g_bde_read_done;
511 bp->bio_caller1 = sp;
512 bp->bio_caller2 = sc;
513 sp->state = IO;
514 g_io_request(bp, sc->consumer);
515 return(0);
516}
517
518/*
519 * The worker thread.
520 *
521 * The up/down path of GEOM is not allowed to sleep or do any major work
522 * so we use this thread to do the actual crypto operations and to push
523 * the state engine onwards.
524 *
525 * XXX: if we switch to the src/sys/opencrypt hardware assisted encryption
526 * XXX: using a thread here is probably not needed.
527 */
528
529void
530g_bde_worker(void *arg)
531{
532 struct g_bde_softc *sc;
533 struct g_bde_work *wp;
534 struct g_geom *gp;
535 int busy, error;
536
537 gp = arg;
538 sc = gp->softc;
539
540 mtx_lock(&sc->worklist_mutex);
541 for (;;) {
542 busy = 0;
543 g_trace(G_T_TOPOLOGY, "g_bde_worker scan");
544 TAILQ_FOREACH(wp, &sc->worklist, list) {
545 KASSERT(wp != NULL, ("NULL wp"));
546 KASSERT(wp->softc != NULL, ("NULL wp->softc"));
547 if (wp->state != WAIT)
548 continue; /* Not interesting here */
549
550 KASSERT(wp->bp != NULL, ("NULL wp->bp"));
551 KASSERT(wp->sp != NULL, ("NULL wp->sp"));
552
553 if (wp->ksp != NULL) {
554 if (wp->ksp->owner != wp)
555 continue;
556 if (wp->ksp->state == IO)
557 continue;
558 KASSERT(wp->ksp->state == VALID,
559 ("Illegal sector state (JUNK ?)"));
560 }
561
562 if (wp->bp->bio_cmd == BIO_READ &&
563 wp->sp->state == IO)
564 continue;
565
566 if (wp->ksp != NULL && wp->ksp->error != 0) {
567 g_bde_contribute(wp->bp, wp->length,
568 wp->ksp->error);
569 g_bde_delete_sector(sc, wp->sp);
570 g_bde_release_keysector(wp);
571 g_bde_delete_work(wp);
572 busy++;
573 break;
574 }
575 switch(wp->bp->bio_cmd) {
576 case BIO_READ:
577 if (wp->ksp == NULL) {
578 KASSERT(wp->error != 0,
579 ("BIO_READ, no ksp and no error"));
580 g_bde_contribute(wp->bp, wp->length,
581 wp->error);
582 } else {
583 if (wp->sp->error == 0) {
584 mtx_unlock(&sc->worklist_mutex);
585 g_bde_crypt_read(wp);
586 mtx_lock(&sc->worklist_mutex);
587 }
588 g_bde_contribute(wp->bp, wp->length,
589 wp->sp->error);
590 }
591 g_bde_delete_sector(sc, wp->sp);
592 if (wp->ksp != NULL)
593 g_bde_release_keysector(wp);
594 g_bde_delete_work(wp);
595 break;
596 case BIO_WRITE:
597 wp->state = FINISH;
598 KASSERT(wp->sp->owner == wp, ("Write not owner sp"));
599 KASSERT(wp->ksp->owner == wp, ("Write not owner ksp"));
600 mtx_unlock(&sc->worklist_mutex);
601 g_bde_crypt_write(wp);
602 mtx_lock(&sc->worklist_mutex);
603 error = g_bde_start_write(wp->sp);
604 if (error) {
605 g_bde_contribute(wp->bp, wp->length, error);
606 g_bde_release_keysector(wp);
607 g_bde_delete_sector(sc, wp->sp);
608 g_bde_delete_work(wp);
609 break;
610 }
611 error = g_bde_start_write(wp->ksp);
612 if (wp->error == 0)
613 wp->error = error;
614 break;
615 case BIO_DELETE:
616 wp->state = FINISH;
617 mtx_unlock(&sc->worklist_mutex);
618 g_bde_crypt_delete(wp);
619 mtx_lock(&sc->worklist_mutex);
620 g_bde_start_write(wp->sp);
621 break;
622 }
623 busy++;
624 break;
625 }
626 if (!busy) {
627 /*
628 * We don't look for our death-warrant until we are
629 * idle. Shouldn't make a difference in practice.
630 */
631 if (sc->dead)
632 break;
633 g_trace(G_T_TOPOLOGY, "g_bde_worker sleep");
634 error = msleep(sc, &sc->worklist_mutex,
635 PRIBIO, "g_bde", hz);
636 if (error == EWOULDBLOCK) {
637 /*
638 * Loose our skey cache in an orderly fashion.
639 * The exact rate can be tuned to be less
640 * aggressive if this is desirable. 10% per
641 * second means that the cache is gone in a
642 * few minutes.
643 */
644 g_bde_purge_sector(sc, 10);
645 }
646 }
647 }
648 g_trace(G_T_TOPOLOGY, "g_bde_worker die");
649 g_bde_purge_sector(sc, 1);
650 KASSERT(sc->nwork == 0, ("Dead but %d work remaining", sc->nwork));
651 KASSERT(sc->ncache == 0, ("Dead but %d cache remaining", sc->ncache));
652 KASSERT(sc->nsect == 0, ("Dead but %d sect remaining", sc->nsect));
653 mtx_unlock(&sc->worklist_mutex);
654 sc->dead = 2;
655 wakeup(sc);
656 mtx_lock(&Giant);
657 kthread_exit(0);
658}
659
660/*
661 * g_bde_start1 has chopped the incoming request up so all the requests
662 * we see here are inside a single zone. Map the data and key locations
663 * grab the buffers we need and fire off the first volley of read requests.
664 */
665
666static void
667g_bde_start2(struct g_bde_work *wp)
668{
669 struct g_bde_softc *sc;
670
671 KASSERT(wp != NULL, ("NULL wp in g_bde_start2"));
672 KASSERT(wp->softc != NULL, ("NULL wp->softc"));
673 g_trace(G_T_TOPOLOGY, "g_bde_start2(%p)", wp);
674 sc = wp->softc;
675 if (wp->bp->bio_cmd == BIO_READ) {
676 wp->sp = g_bde_new_sector(wp, 0);
677 if (wp->sp == NULL) {
678 g_bde_contribute(wp->bp, wp->length, ENOMEM);
679 g_bde_delete_work(wp);
680 return;
681 }
682 wp->sp->size = wp->length;
683 wp->sp->data = wp->data;
684 if (g_bde_start_read(wp->sp) != 0) {
685 g_bde_contribute(wp->bp, wp->length, ENOMEM);
686 g_bde_delete_sector(sc, wp->sp);
687 g_bde_delete_work(wp);
688 return;
689 }
690 g_bde_read_keysector(sc, wp);
691 if (wp->ksp == NULL)
692 wp->error = ENOMEM;
693 } else if (wp->bp->bio_cmd == BIO_DELETE) {
694 wp->sp = g_bde_new_sector(wp, wp->length);
695 if (wp->sp == NULL) {
696 g_bde_contribute(wp->bp, wp->length, ENOMEM);
697 g_bde_delete_work(wp);
698 return;
699 }
700 } else if (wp->bp->bio_cmd == BIO_WRITE) {
701 wp->sp = g_bde_new_sector(wp, wp->length);
702 if (wp->sp == NULL) {
703 g_bde_contribute(wp->bp, wp->length, ENOMEM);
704 g_bde_delete_work(wp);
705 return;
706 }
707 g_bde_read_keysector(sc, wp);
708 if (wp->ksp == NULL) {
709 g_bde_contribute(wp->bp, wp->length, ENOMEM);
710 g_bde_delete_sector(sc, wp->sp);
711 g_bde_delete_work(wp);
712 return;
713 }
714 } else {
715 KASSERT(0 == 1,
716 ("Wrong bio_cmd %d in g_bde_start2", wp->bp->bio_cmd));
717 }
718
719 wp->state = WAIT;
720 wakeup(sc);
721}
722
723/*
724 * Create a sequence of work structures, and have g_bde_map_sector() determine
725 * how long they each can be. Feed them to g_bde_start2().
726 */
727
728void
729g_bde_start1(struct bio *bp)
730{
731 struct g_bde_softc *sc;
732 struct g_bde_work *wp;
733 off_t done;
734
735 sc = bp->bio_to->geom->softc;
736 bp->bio_driver1 = sc;
737
738 mtx_lock(&sc->worklist_mutex);
739 for(done = 0; done < bp->bio_length; ) {
740 wp = g_bde_new_work(sc);
741 if (wp != NULL) {
742 wp->bp = bp;
743 wp->offset = bp->bio_offset + done;
744 wp->data = bp->bio_data + done;
745 wp->length = bp->bio_length - done;
746 g_bde_map_sector(wp);
747 done += wp->length;
748 g_bde_start2(wp);
749 }
750 if (wp == NULL || bp->bio_error != 0) {
751 g_bde_contribute(bp, bp->bio_length - done, ENOMEM);
752 break;
753 }
754 }
755 mtx_unlock(&sc->worklist_mutex);
756 return;
757}
2 * Copyright (c) 2002 Poul-Henning Kamp
3 * Copyright (c) 2002 Networks Associates Technology, Inc.
4 * All rights reserved.
5 *
6 * This software was developed for the FreeBSD Project by Poul-Henning Kamp
7 * and NAI Labs, the Security Research Division of Network Associates, Inc.
8 * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the
9 * DARPA CHATS research program.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 *
32 * $FreeBSD: head/sys/geom/bde/g_bde_work.c 114152 2003-04-28 06:19:41Z phk $
33 *
34 * This source file contains the state-engine which makes things happen in the
35 * right order.
36 *
37 * Outline:
38 * 1) g_bde_start1()
39 * Break the struct bio into multiple work packets one per zone.
40 * 2) g_bde_start2()
41 * Setup the necessary sector buffers and start those read operations
42 * which we can start at this time and put the item on the work-list.
43 * 3) g_bde_worker()
44 * Scan the work-list for items which are ready for crypto processing
45 * and call the matching crypto function in g_bde_crypt.c and schedule
46 * any writes needed. Read operations finish here by releasing the
47 * sector buffers and delivering the original bio request.
48 * 4) g_bde_write_done()
49 * Release sector buffers and deliver the original bio request.
50 *
51 * Because of the C-scope rules, the functions are almost perfectly in the
52 * opposite order in this source file.
53 *
54 * XXX: A switch to the hardware assisted crypto in src/sys/opencrypto will add
55 * XXX: additional states to this state-engine. Since no hardware available
56 * XXX: at this time has AES support, implementing this has been postponed
57 * XXX: until such time as it would result in a benefit.
58 */
59
60#include <sys/param.h>
61#include <sys/bio.h>
62#include <sys/lock.h>
63#include <sys/mutex.h>
64#include <sys/queue.h>
65#include <sys/malloc.h>
66#include <sys/systm.h>
67#include <sys/kernel.h>
68#include <sys/sysctl.h>
69#include <sys/proc.h>
70#include <sys/kthread.h>
71
72#include <crypto/rijndael/rijndael.h>
73#include <crypto/sha2/sha2.h>
74#include <geom/geom.h>
75#include <geom/bde/g_bde.h>
76
77static void g_bde_delete_sector(struct g_bde_softc *wp, struct g_bde_sector *sp);
78static struct g_bde_sector * g_bde_new_sector(struct g_bde_work *wp, u_int len);
79static void g_bde_release_keysector(struct g_bde_work *wp);
80static struct g_bde_sector *g_bde_get_sector(struct g_bde_work *wp, off_t offset);
81static int g_bde_start_read(struct g_bde_sector *sp);
82static void g_bde_purge_sector(struct g_bde_softc *sc, int fraction);
83
84/*
85 * Work item allocation.
86 *
87 * C++ would call these constructors and destructors.
88 */
89static u_int g_bde_nwork;
90SYSCTL_UINT(_debug, OID_AUTO, gbde_nwork, CTLFLAG_RD, &g_bde_nwork, 0, "");
91
92static struct g_bde_work *
93g_bde_new_work(struct g_bde_softc *sc)
94{
95 struct g_bde_work *wp;
96
97 wp = g_malloc(sizeof *wp, M_NOWAIT | M_ZERO);
98 if (wp == NULL)
99 return (wp);
100 wp->state = SETUP;
101 wp->softc = sc;
102 g_bde_nwork++;
103 sc->nwork++;
104 TAILQ_INSERT_TAIL(&sc->worklist, wp, list);
105 return (wp);
106}
107
108static void
109g_bde_delete_work(struct g_bde_work *wp)
110{
111 struct g_bde_softc *sc;
112
113 sc = wp->softc;
114 g_bde_nwork--;
115 sc->nwork--;
116 TAILQ_REMOVE(&sc->worklist, wp, list);
117 g_free(wp);
118}
119
120/*
121 * Sector buffer allocation
122 *
123 * These two functions allocate and free back variable sized sector buffers
124 */
125
126static u_int g_bde_nsect;
127SYSCTL_UINT(_debug, OID_AUTO, gbde_nsect, CTLFLAG_RD, &g_bde_nsect, 0, "");
128
129static void
130g_bde_delete_sector(struct g_bde_softc *sc, struct g_bde_sector *sp)
131{
132
133 g_bde_nsect--;
134 sc->nsect--;
135 if (sp->malloc)
136 g_free(sp->data);
137 g_free(sp);
138}
139
140static struct g_bde_sector *
141g_bde_new_sector(struct g_bde_work *wp, u_int len)
142{
143 struct g_bde_sector *sp;
144
145 sp = g_malloc(sizeof *sp, M_NOWAIT | M_ZERO);
146 if (sp == NULL)
147 return (sp);
148 if (len > 0) {
149 sp->data = g_malloc(len, M_NOWAIT | M_ZERO);
150 if (sp->data == NULL) {
151 g_free(sp);
152 return (NULL);
153 }
154 sp->malloc = 1;
155 }
156 g_bde_nsect++;
157 wp->softc->nsect++;
158 sp->size = len;
159 sp->softc = wp->softc;
160 sp->ref = 1;
161 sp->owner = wp;
162 sp->offset = wp->so;
163 sp->state = JUNK;
164 return (sp);
165}
166
167/*
168 * Skey sector cache.
169 *
170 * Nothing prevents two separate I/O requests from addressing the same zone
171 * and thereby needing the same skey sector. We therefore need to sequence
172 * I/O operations to the skey sectors. A certain amount of caching is also
173 * desirable, although the extent of benefit from this is not at this point
174 * determined.
175 *
176 * XXX: GEOM may be able to grow a generic caching facility at some point
177 * XXX: to support such needs.
178 */
179
180static u_int g_bde_ncache;
181SYSCTL_UINT(_debug, OID_AUTO, gbde_ncache, CTLFLAG_RD, &g_bde_ncache, 0, "");
182
183static void
184g_bde_purge_one_sector(struct g_bde_softc *sc, struct g_bde_sector *sp)
185{
186
187 g_trace(G_T_TOPOLOGY, "g_bde_purge_one_sector(%p, %p)", sc, sp);
188 if (sp->ref != 0)
189 return;
190 TAILQ_REMOVE(&sc->freelist, sp, list);
191 g_bde_ncache--;
192 sc->ncache--;
193 bzero(sp->data, sp->size);
194 g_bde_delete_sector(sc, sp);
195}
196
197static struct g_bde_sector *
198g_bde_get_sector(struct g_bde_work *wp, off_t offset)
199{
200 struct g_bde_sector *sp;
201 struct g_bde_softc *sc;
202
203 g_trace(G_T_TOPOLOGY, "g_bde_get_sector(%p, %jd)", wp, (intmax_t)offset);
204 sc = wp->softc;
205
206 if (malloc_last_fail() < g_bde_ncache)
207 g_bde_purge_sector(sc, -1);
208
209 sp = TAILQ_FIRST(&sc->freelist);
210 if (sp != NULL && sp->ref == 0 && sp->used + 300 < time_uptime)
211 g_bde_purge_one_sector(sc, sp);
212
213 TAILQ_FOREACH(sp, &sc->freelist, list) {
214 if (sp->offset == offset)
215 break;
216 }
217 if (sp != NULL) {
218 sp->ref++;
219 KASSERT(sp->offset == offset, ("wrong offset"));
220 KASSERT(sp->softc == wp->softc, ("wrong softc"));
221 if (sp->ref == 1)
222 sp->owner = wp;
223 } else {
224 if (malloc_last_fail() < g_bde_ncache) {
225 TAILQ_FOREACH(sp, &sc->freelist, list)
226 if (sp->ref == 0)
227 break;
228 }
229 if (sp == NULL && !TAILQ_EMPTY(&sc->freelist))
230 sp = TAILQ_FIRST(&sc->freelist);
231 if (sp != NULL && sp->ref > 0)
232 sp = NULL;
233 if (sp == NULL) {
234 sp = g_bde_new_sector(wp, sc->sectorsize);
235 if (sp != NULL) {
236 g_bde_ncache++;
237 sc->ncache++;
238 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
239 sp->malloc = 2;
240 }
241 }
242 if (sp != NULL) {
243 sp->offset = offset;
244 sp->softc = wp->softc;
245 sp->ref = 1;
246 sp->owner = wp;
247 sp->state = JUNK;
248 sp->error = 0;
249 }
250 }
251 if (sp != NULL) {
252 TAILQ_REMOVE(&sc->freelist, sp, list);
253 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
254 sp->used = time_uptime;
255 }
256 wp->ksp = sp;
257 return(sp);
258}
259
260static void
261g_bde_release_keysector(struct g_bde_work *wp)
262{
263 struct g_bde_softc *sc;
264 struct g_bde_work *wp2;
265 struct g_bde_sector *sp;
266
267 sp = wp->ksp;
268 g_trace(G_T_TOPOLOGY, "g_bde_release_keysector(%p)", sp);
269 KASSERT(sp->malloc == 2, ("Wrong sector released"));
270 sc = sp->softc;
271 KASSERT(sc != NULL, ("NULL sp->softc"));
272 KASSERT(wp == sp->owner, ("Releasing, not owner"));
273 sp->owner = NULL;
274 wp->ksp = NULL;
275 sp->ref--;
276 if (sp->ref > 0) {
277 TAILQ_REMOVE(&sc->freelist, sp, list);
278 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
279 TAILQ_FOREACH(wp2, &sc->worklist, list) {
280 if (wp2->ksp == sp) {
281 KASSERT(wp2 != wp, ("Self-reowning"));
282 sp->owner = wp2;
283 wakeup(sp->softc);
284 break;
285 }
286 }
287 KASSERT(wp2 != NULL, ("Failed to pick up owner for %p\n", sp));
288 } else if (sp->error != 0) {
289 sp->offset = ~0;
290 sp->error = 0;
291 sp->state = JUNK;
292 }
293 TAILQ_REMOVE(&sc->freelist, sp, list);
294 TAILQ_INSERT_HEAD(&sc->freelist, sp, list);
295}
296
297static void
298g_bde_purge_sector(struct g_bde_softc *sc, int fraction)
299{
300 struct g_bde_sector *sp;
301 int n;
302
303 g_trace(G_T_TOPOLOGY, "g_bde_purge_sector(%p)", sc);
304 if (fraction > 0)
305 n = sc->ncache / fraction + 1;
306 else
307 n = g_bde_ncache - malloc_last_fail();
308 if (n < 0)
309 return;
310 if (n > sc->ncache)
311 n = sc->ncache;
312 while(n--) {
313 TAILQ_FOREACH(sp, &sc->freelist, list) {
314 if (sp->ref != 0)
315 continue;
316 TAILQ_REMOVE(&sc->freelist, sp, list);
317 g_bde_ncache--;
318 sc->ncache--;
319 bzero(sp->data, sp->size);
320 g_bde_delete_sector(sc, sp);
321 break;
322 }
323 }
324}
325
326static struct g_bde_sector *
327g_bde_read_keysector(struct g_bde_softc *sc, struct g_bde_work *wp)
328{
329 struct g_bde_sector *sp;
330
331 g_trace(G_T_TOPOLOGY, "g_bde_read_keysector(%p)", wp);
332 sp = g_bde_get_sector(wp, wp->kso);
333 if (sp == NULL) {
334 g_bde_purge_sector(sc, -1);
335 sp = g_bde_get_sector(wp, wp->kso);
336 }
337 if (sp == NULL)
338 return (sp);
339 if (sp->owner != wp)
340 return (sp);
341 if (sp->state == VALID)
342 return (sp);
343 if (g_bde_start_read(sp) == 0)
344 return (sp);
345 g_bde_release_keysector(wp);
346 return (NULL);
347}
348
349/*
350 * Contribute to the completion of the original bio request.
351 *
352 * We have no simple way to tell how many bits the original bio request has
353 * been segmented into, so the easiest way to determine when we can deliver
354 * it is to keep track of the number of bytes we have completed. We keep
355 * track of any errors underway and latch onto the first one.
356 *
357 * We always report "nothing done" in case of error, because random bits here
358 * and there may be completed and returning a number of completed bytes does
359 * not convey any useful information about which bytes they were. If some
360 * piece of broken code somewhere interprets this to mean that nothing has
361 * changed on the underlying media they deserve the lossage headed for them.
362 *
363 * A single mutex per g_bde instance is used to prevent contention.
364 */
365
366static void
367g_bde_contribute(struct bio *bp, off_t bytes, int error)
368{
369 struct g_bde_softc *sc;
370
371 g_trace(G_T_TOPOLOGY, "g_bde_contribute bp %p bytes %jd error %d",
372 bp, (intmax_t)bytes, error);
373 sc = bp->bio_driver1;
374 if (bp->bio_error == 0)
375 bp->bio_error = error;
376 bp->bio_completed += bytes;
377 KASSERT(bp->bio_completed <= bp->bio_length, ("Too large contribution"));
378 if (bp->bio_completed == bp->bio_length) {
379 if (bp->bio_error != 0)
380 bp->bio_completed = 0;
381 g_io_deliver(bp, bp->bio_error);
382 }
383}
384
385/*
386 * A write operation has finished. When we have all expected cows in the
387 * barn close the door and call it a day.
388 */
389
390static void
391g_bde_write_done(struct bio *bp)
392{
393 struct g_bde_sector *sp;
394 struct g_bde_work *wp;
395 struct g_bde_softc *sc;
396
397 sp = bp->bio_caller1;
398 sc = bp->bio_caller2;
399 mtx_lock(&sc->worklist_mutex);
400 KASSERT(sp != NULL, ("NULL sp"));
401 KASSERT(sc != NULL, ("NULL sc"));
402 KASSERT(sp->owner != NULL, ("NULL sp->owner"));
403 g_trace(G_T_TOPOLOGY, "g_bde_write_done(%p)", sp);
404 sp->error = bp->bio_error;
405 g_destroy_bio(bp);
406 wp = sp->owner;
407 if (wp->error == 0)
408 wp->error = sp->error;
409
410 if (wp->bp->bio_cmd == BIO_DELETE) {
411 KASSERT(sp == wp->sp, ("trashed delete op"));
412 g_bde_contribute(wp->bp, wp->length, wp->error);
413 g_bde_delete_sector(sc, sp);
414 g_bde_delete_work(wp);
415 mtx_unlock(&sc->worklist_mutex);
416 return;
417 }
418
419 KASSERT(wp->bp->bio_cmd == BIO_WRITE, ("Confused in g_bde_write_done()"));
420 KASSERT(sp == wp->sp || sp == wp->ksp, ("trashed write op"));
421 if (wp->sp == sp) {
422 g_bde_delete_sector(sc, wp->sp);
423 wp->sp = NULL;
424 } else {
425 sp->state = VALID;
426 }
427 if (wp->sp == NULL && wp->ksp != NULL && wp->ksp->state == VALID) {
428 g_bde_contribute(wp->bp, wp->length, wp->error);
429 g_bde_release_keysector(wp);
430 g_bde_delete_work(wp);
431 }
432 mtx_unlock(&sc->worklist_mutex);
433 return;
434}
435
436/*
437 * Send a write request for the given sector down the pipeline.
438 */
439
440static int
441g_bde_start_write(struct g_bde_sector *sp)
442{
443 struct bio *bp;
444 struct g_bde_softc *sc;
445
446 g_trace(G_T_TOPOLOGY, "g_bde_start_write(%p)", sp);
447 sc = sp->softc;
448 KASSERT(sc != NULL, ("NULL sc in g_bde_start_write"));
449 KASSERT(sp->owner != NULL, ("NULL sp->owner in g_bde_start_write"));
450 bp = g_new_bio();
451 if (bp == NULL)
452 return (ENOMEM);
453 bp->bio_cmd = BIO_WRITE;
454 bp->bio_offset = sp->offset;
455 bp->bio_data = sp->data;
456 bp->bio_length = sp->size;
457 bp->bio_done = g_bde_write_done;
458 bp->bio_caller1 = sp;
459 bp->bio_caller2 = sc;
460 sp->state = IO;
461 g_io_request(bp, sc->consumer);
462 return(0);
463}
464
465/*
466 * A read operation has finished. Mark the sector no longer iobusy and
467 * wake up the worker thread and let it do its thing.
468 */
469
470static void
471g_bde_read_done(struct bio *bp)
472{
473 struct g_bde_sector *sp;
474 struct g_bde_softc *sc;
475
476 sp = bp->bio_caller1;
477 g_trace(G_T_TOPOLOGY, "g_bde_read_done(%p)", sp);
478 sc = bp->bio_caller2;
479 mtx_lock(&sc->worklist_mutex);
480 sp->error = bp->bio_error;
481 if (sp->error == 0)
482 sp->state = VALID;
483 else
484 sp->state = JUNK;
485 wakeup(sc);
486 g_destroy_bio(bp);
487 mtx_unlock(&sc->worklist_mutex);
488}
489
490/*
491 * Send a read request for the given sector down the pipeline.
492 */
493
494static int
495g_bde_start_read(struct g_bde_sector *sp)
496{
497 struct bio *bp;
498 struct g_bde_softc *sc;
499
500 g_trace(G_T_TOPOLOGY, "g_bde_start_read(%p)", sp);
501 sc = sp->softc;
502 KASSERT(sc != NULL, ("Null softc in sp %p", sp));
503 bp = g_new_bio();
504 if (bp == NULL)
505 return (ENOMEM);
506 bp->bio_cmd = BIO_READ;
507 bp->bio_offset = sp->offset;
508 bp->bio_data = sp->data;
509 bp->bio_length = sp->size;
510 bp->bio_done = g_bde_read_done;
511 bp->bio_caller1 = sp;
512 bp->bio_caller2 = sc;
513 sp->state = IO;
514 g_io_request(bp, sc->consumer);
515 return(0);
516}
517
518/*
519 * The worker thread.
520 *
521 * The up/down path of GEOM is not allowed to sleep or do any major work
522 * so we use this thread to do the actual crypto operations and to push
523 * the state engine onwards.
524 *
525 * XXX: if we switch to the src/sys/opencrypt hardware assisted encryption
526 * XXX: using a thread here is probably not needed.
527 */
528
529void
530g_bde_worker(void *arg)
531{
532 struct g_bde_softc *sc;
533 struct g_bde_work *wp;
534 struct g_geom *gp;
535 int busy, error;
536
537 gp = arg;
538 sc = gp->softc;
539
540 mtx_lock(&sc->worklist_mutex);
541 for (;;) {
542 busy = 0;
543 g_trace(G_T_TOPOLOGY, "g_bde_worker scan");
544 TAILQ_FOREACH(wp, &sc->worklist, list) {
545 KASSERT(wp != NULL, ("NULL wp"));
546 KASSERT(wp->softc != NULL, ("NULL wp->softc"));
547 if (wp->state != WAIT)
548 continue; /* Not interesting here */
549
550 KASSERT(wp->bp != NULL, ("NULL wp->bp"));
551 KASSERT(wp->sp != NULL, ("NULL wp->sp"));
552
553 if (wp->ksp != NULL) {
554 if (wp->ksp->owner != wp)
555 continue;
556 if (wp->ksp->state == IO)
557 continue;
558 KASSERT(wp->ksp->state == VALID,
559 ("Illegal sector state (JUNK ?)"));
560 }
561
562 if (wp->bp->bio_cmd == BIO_READ &&
563 wp->sp->state == IO)
564 continue;
565
566 if (wp->ksp != NULL && wp->ksp->error != 0) {
567 g_bde_contribute(wp->bp, wp->length,
568 wp->ksp->error);
569 g_bde_delete_sector(sc, wp->sp);
570 g_bde_release_keysector(wp);
571 g_bde_delete_work(wp);
572 busy++;
573 break;
574 }
575 switch(wp->bp->bio_cmd) {
576 case BIO_READ:
577 if (wp->ksp == NULL) {
578 KASSERT(wp->error != 0,
579 ("BIO_READ, no ksp and no error"));
580 g_bde_contribute(wp->bp, wp->length,
581 wp->error);
582 } else {
583 if (wp->sp->error == 0) {
584 mtx_unlock(&sc->worklist_mutex);
585 g_bde_crypt_read(wp);
586 mtx_lock(&sc->worklist_mutex);
587 }
588 g_bde_contribute(wp->bp, wp->length,
589 wp->sp->error);
590 }
591 g_bde_delete_sector(sc, wp->sp);
592 if (wp->ksp != NULL)
593 g_bde_release_keysector(wp);
594 g_bde_delete_work(wp);
595 break;
596 case BIO_WRITE:
597 wp->state = FINISH;
598 KASSERT(wp->sp->owner == wp, ("Write not owner sp"));
599 KASSERT(wp->ksp->owner == wp, ("Write not owner ksp"));
600 mtx_unlock(&sc->worklist_mutex);
601 g_bde_crypt_write(wp);
602 mtx_lock(&sc->worklist_mutex);
603 error = g_bde_start_write(wp->sp);
604 if (error) {
605 g_bde_contribute(wp->bp, wp->length, error);
606 g_bde_release_keysector(wp);
607 g_bde_delete_sector(sc, wp->sp);
608 g_bde_delete_work(wp);
609 break;
610 }
611 error = g_bde_start_write(wp->ksp);
612 if (wp->error == 0)
613 wp->error = error;
614 break;
615 case BIO_DELETE:
616 wp->state = FINISH;
617 mtx_unlock(&sc->worklist_mutex);
618 g_bde_crypt_delete(wp);
619 mtx_lock(&sc->worklist_mutex);
620 g_bde_start_write(wp->sp);
621 break;
622 }
623 busy++;
624 break;
625 }
626 if (!busy) {
627 /*
628 * We don't look for our death-warrant until we are
629 * idle. Shouldn't make a difference in practice.
630 */
631 if (sc->dead)
632 break;
633 g_trace(G_T_TOPOLOGY, "g_bde_worker sleep");
634 error = msleep(sc, &sc->worklist_mutex,
635 PRIBIO, "g_bde", hz);
636 if (error == EWOULDBLOCK) {
637 /*
638 * Loose our skey cache in an orderly fashion.
639 * The exact rate can be tuned to be less
640 * aggressive if this is desirable. 10% per
641 * second means that the cache is gone in a
642 * few minutes.
643 */
644 g_bde_purge_sector(sc, 10);
645 }
646 }
647 }
648 g_trace(G_T_TOPOLOGY, "g_bde_worker die");
649 g_bde_purge_sector(sc, 1);
650 KASSERT(sc->nwork == 0, ("Dead but %d work remaining", sc->nwork));
651 KASSERT(sc->ncache == 0, ("Dead but %d cache remaining", sc->ncache));
652 KASSERT(sc->nsect == 0, ("Dead but %d sect remaining", sc->nsect));
653 mtx_unlock(&sc->worklist_mutex);
654 sc->dead = 2;
655 wakeup(sc);
656 mtx_lock(&Giant);
657 kthread_exit(0);
658}
659
660/*
661 * g_bde_start1 has chopped the incoming request up so all the requests
662 * we see here are inside a single zone. Map the data and key locations
663 * grab the buffers we need and fire off the first volley of read requests.
664 */
665
666static void
667g_bde_start2(struct g_bde_work *wp)
668{
669 struct g_bde_softc *sc;
670
671 KASSERT(wp != NULL, ("NULL wp in g_bde_start2"));
672 KASSERT(wp->softc != NULL, ("NULL wp->softc"));
673 g_trace(G_T_TOPOLOGY, "g_bde_start2(%p)", wp);
674 sc = wp->softc;
675 if (wp->bp->bio_cmd == BIO_READ) {
676 wp->sp = g_bde_new_sector(wp, 0);
677 if (wp->sp == NULL) {
678 g_bde_contribute(wp->bp, wp->length, ENOMEM);
679 g_bde_delete_work(wp);
680 return;
681 }
682 wp->sp->size = wp->length;
683 wp->sp->data = wp->data;
684 if (g_bde_start_read(wp->sp) != 0) {
685 g_bde_contribute(wp->bp, wp->length, ENOMEM);
686 g_bde_delete_sector(sc, wp->sp);
687 g_bde_delete_work(wp);
688 return;
689 }
690 g_bde_read_keysector(sc, wp);
691 if (wp->ksp == NULL)
692 wp->error = ENOMEM;
693 } else if (wp->bp->bio_cmd == BIO_DELETE) {
694 wp->sp = g_bde_new_sector(wp, wp->length);
695 if (wp->sp == NULL) {
696 g_bde_contribute(wp->bp, wp->length, ENOMEM);
697 g_bde_delete_work(wp);
698 return;
699 }
700 } else if (wp->bp->bio_cmd == BIO_WRITE) {
701 wp->sp = g_bde_new_sector(wp, wp->length);
702 if (wp->sp == NULL) {
703 g_bde_contribute(wp->bp, wp->length, ENOMEM);
704 g_bde_delete_work(wp);
705 return;
706 }
707 g_bde_read_keysector(sc, wp);
708 if (wp->ksp == NULL) {
709 g_bde_contribute(wp->bp, wp->length, ENOMEM);
710 g_bde_delete_sector(sc, wp->sp);
711 g_bde_delete_work(wp);
712 return;
713 }
714 } else {
715 KASSERT(0 == 1,
716 ("Wrong bio_cmd %d in g_bde_start2", wp->bp->bio_cmd));
717 }
718
719 wp->state = WAIT;
720 wakeup(sc);
721}
722
723/*
724 * Create a sequence of work structures, and have g_bde_map_sector() determine
725 * how long they each can be. Feed them to g_bde_start2().
726 */
727
728void
729g_bde_start1(struct bio *bp)
730{
731 struct g_bde_softc *sc;
732 struct g_bde_work *wp;
733 off_t done;
734
735 sc = bp->bio_to->geom->softc;
736 bp->bio_driver1 = sc;
737
738 mtx_lock(&sc->worklist_mutex);
739 for(done = 0; done < bp->bio_length; ) {
740 wp = g_bde_new_work(sc);
741 if (wp != NULL) {
742 wp->bp = bp;
743 wp->offset = bp->bio_offset + done;
744 wp->data = bp->bio_data + done;
745 wp->length = bp->bio_length - done;
746 g_bde_map_sector(wp);
747 done += wp->length;
748 g_bde_start2(wp);
749 }
750 if (wp == NULL || bp->bio_error != 0) {
751 g_bde_contribute(bp, bp->bio_length - done, ENOMEM);
752 break;
753 }
754 }
755 mtx_unlock(&sc->worklist_mutex);
756 return;
757}