33 */
34/*
35 * This source file contains the state-engine which makes things happen in the
36 * right order.
37 *
38 * Outline:
39 * 1) g_bde_start1()
40 * Break the struct bio into multiple work packets one per zone.
41 * 2) g_bde_start2()
42 * Setup the necessary sector buffers and start those read operations
43 * which we can start at this time and put the item on the work-list.
44 * 3) g_bde_worker()
45 * Scan the work-list for items which are ready for crypto processing
46 * and call the matching crypto function in g_bde_crypt.c and schedule
47 * any writes needed. Read operations finish here by releasing the
48 * sector buffers and delivering the original bio request.
49 * 4) g_bde_write_done()
50 * Release sector buffers and deliver the original bio request.
51 *
52 * Because of the C-scope rules, the functions are almost perfectly in the
53 * opposite order in this source file.
54 *
55 * XXX: A switch to the hardware assisted crypto in src/sys/opencrypto will add
56 * XXX: additional states to this state-engine. Since no hardware available
57 * XXX: at this time has AES support, implementing this has been postponed
58 * XXX: until such time as it would result in a benefit.
59 */
60
61#include <sys/param.h>
62#include <sys/bio.h>
63#include <sys/lock.h>
64#include <sys/mutex.h>
65#include <sys/queue.h>
66#include <sys/malloc.h>
67#include <sys/systm.h>
68#include <sys/kernel.h>
69#include <sys/sysctl.h>
70#include <sys/proc.h>
71#include <sys/kthread.h>
72
73#include <crypto/rijndael/rijndael-api-fst.h>
74#include <crypto/sha2/sha2.h>
75#include <geom/geom.h>
76#include <geom/bde/g_bde.h>
77
78static void g_bde_delete_sector(struct g_bde_softc *wp, struct g_bde_sector *sp);
79static struct g_bde_sector * g_bde_new_sector(struct g_bde_work *wp, u_int len);
80static void g_bde_release_keysector(struct g_bde_work *wp);
81static struct g_bde_sector *g_bde_get_keysector(struct g_bde_work *wp);
82static int g_bde_start_read(struct g_bde_sector *sp);
83static void g_bde_purge_sector(struct g_bde_softc *sc, int fraction);
84
85/*
86 * Work item allocation.
87 *
88 * C++ would call these constructors and destructors.
89 */
90static u_int g_bde_nwork;
91SYSCTL_UINT(_debug, OID_AUTO, gbde_nwork, CTLFLAG_RD, &g_bde_nwork, 0, "");
92
93static MALLOC_DEFINE(M_GBDE, "gbde", "GBDE data structures");
94
95static struct g_bde_work *
96g_bde_new_work(struct g_bde_softc *sc)
97{
98 struct g_bde_work *wp;
99
100 wp = malloc(sizeof *wp, M_GBDE, M_NOWAIT | M_ZERO);
101 if (wp == NULL)
102 return (wp);
103 wp->state = SETUP;
104 wp->softc = sc;
105 g_bde_nwork++;
106 sc->nwork++;
107 TAILQ_INSERT_TAIL(&sc->worklist, wp, list);
108 return (wp);
109}
110
111static void
112g_bde_delete_work(struct g_bde_work *wp)
113{
114 struct g_bde_softc *sc;
115
116 sc = wp->softc;
117 g_bde_nwork--;
118 sc->nwork--;
119 TAILQ_REMOVE(&sc->worklist, wp, list);
120 free(wp, M_GBDE);
121}
122
123/*
124 * Sector buffer allocation
125 *
126 * These two functions allocate and free back variable sized sector buffers
127 */
128
129static u_int g_bde_nsect;
130SYSCTL_UINT(_debug, OID_AUTO, gbde_nsect, CTLFLAG_RD, &g_bde_nsect, 0, "");
131
132static void
133g_bde_delete_sector(struct g_bde_softc *sc, struct g_bde_sector *sp)
134{
135
136 g_bde_nsect--;
137 sc->nsect--;
138 if (sp->malloc)
139 free(sp->data, M_GBDE);
140 free(sp, M_GBDE);
141}
142
143static struct g_bde_sector *
144g_bde_new_sector(struct g_bde_work *wp, u_int len)
145{
146 struct g_bde_sector *sp;
147
148 sp = malloc(sizeof *sp, M_GBDE, M_NOWAIT | M_ZERO);
149 if (sp == NULL)
150 return (sp);
151 if (len > 0) {
152 sp->data = malloc(len, M_GBDE, M_NOWAIT | M_ZERO);
153 if (sp->data == NULL) {
154 free(sp, M_GBDE);
155 return (NULL);
156 }
157 sp->malloc = 1;
158 }
159 g_bde_nsect++;
160 wp->softc->nsect++;
161 sp->size = len;
162 sp->softc = wp->softc;
163 sp->ref = 1;
164 sp->owner = wp;
165 sp->offset = wp->so;
166 sp->state = JUNK;
167 return (sp);
168}
169
170/*
171 * Skey sector cache.
172 *
173 * Nothing prevents two separate I/O requests from addressing the same zone
174 * and thereby needing the same skey sector. We therefore need to sequence
175 * I/O operations to the skey sectors. A certain amount of caching is also
176 * desirable, although the extent of benefit from this is not at this point
177 * determined.
178 *
179 * XXX: GEOM may be able to grow a generic caching facility at some point
180 * XXX: to support such needs.
181 */
182
183static u_int g_bde_ncache;
184SYSCTL_UINT(_debug, OID_AUTO, gbde_ncache, CTLFLAG_RD, &g_bde_ncache, 0, "");
185
186static void
187g_bde_purge_one_sector(struct g_bde_softc *sc, struct g_bde_sector *sp)
188{
189
190 g_trace(G_T_TOPOLOGY, "g_bde_purge_one_sector(%p, %p)", sc, sp);
191 if (sp->ref != 0)
192 return;
193 TAILQ_REMOVE(&sc->freelist, sp, list);
194 g_bde_ncache--;
195 sc->ncache--;
196 bzero(sp->data, sp->size);
197 g_bde_delete_sector(sc, sp);
198}
199
200static struct g_bde_sector *
201g_bde_get_keysector(struct g_bde_work *wp)
202{
203 struct g_bde_sector *sp;
204 struct g_bde_softc *sc;
205 off_t offset;
206
207 offset = wp->kso;
208 g_trace(G_T_TOPOLOGY, "g_bde_get_keysector(%p, %jd)", wp, (intmax_t)offset);
209 sc = wp->softc;
210
211 if (malloc_last_fail() < g_bde_ncache)
212 g_bde_purge_sector(sc, -1);
213
214 sp = TAILQ_FIRST(&sc->freelist);
215 if (sp != NULL && sp->ref == 0 && sp->used + 300 < time_uptime)
216 g_bde_purge_one_sector(sc, sp);
217
218 TAILQ_FOREACH(sp, &sc->freelist, list) {
219 if (sp->offset == offset)
220 break;
221 }
222 if (sp != NULL) {
223 sp->ref++;
224 KASSERT(sp->offset == offset, ("wrong offset"));
225 KASSERT(sp->softc == wp->softc, ("wrong softc"));
226 if (sp->ref == 1)
227 sp->owner = wp;
228 } else {
229 if (malloc_last_fail() < g_bde_ncache) {
230 TAILQ_FOREACH(sp, &sc->freelist, list)
231 if (sp->ref == 0)
232 break;
233 }
234 if (sp == NULL && !TAILQ_EMPTY(&sc->freelist))
235 sp = TAILQ_FIRST(&sc->freelist);
236 if (sp != NULL && sp->ref > 0)
237 sp = NULL;
238 if (sp == NULL) {
239 sp = g_bde_new_sector(wp, sc->sectorsize);
240 if (sp != NULL) {
241 g_bde_ncache++;
242 sc->ncache++;
243 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
244 sp->malloc = 2;
245 }
246 }
247 if (sp != NULL) {
248 sp->offset = offset;
249 sp->softc = wp->softc;
250 sp->ref = 1;
251 sp->owner = wp;
252 sp->state = JUNK;
253 sp->error = 0;
254 }
255 }
256 if (sp != NULL) {
257 TAILQ_REMOVE(&sc->freelist, sp, list);
258 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
259 sp->used = time_uptime;
260 }
261 wp->ksp = sp;
262 return(sp);
263}
264
265static void
266g_bde_release_keysector(struct g_bde_work *wp)
267{
268 struct g_bde_softc *sc;
269 struct g_bde_work *wp2;
270 struct g_bde_sector *sp;
271
272 sp = wp->ksp;
273 g_trace(G_T_TOPOLOGY, "g_bde_release_keysector(%p)", sp);
274 KASSERT(sp->malloc == 2, ("Wrong sector released"));
275 sc = sp->softc;
276 KASSERT(sc != NULL, ("NULL sp->softc"));
277 KASSERT(wp == sp->owner, ("Releasing, not owner"));
278 sp->owner = NULL;
279 wp->ksp = NULL;
280 sp->ref--;
281 if (sp->ref > 0) {
282 TAILQ_REMOVE(&sc->freelist, sp, list);
283 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
284 TAILQ_FOREACH(wp2, &sc->worklist, list) {
285 if (wp2->ksp == sp) {
286 KASSERT(wp2 != wp, ("Self-reowning"));
287 sp->owner = wp2;
288 wakeup(sp->softc);
289 break;
290 }
291 }
292 KASSERT(wp2 != NULL, ("Failed to pick up owner for %p\n", sp));
293 } else if (sp->error != 0) {
294 sp->offset = ~0;
295 sp->error = 0;
296 sp->state = JUNK;
297 }
298 TAILQ_REMOVE(&sc->freelist, sp, list);
299 TAILQ_INSERT_HEAD(&sc->freelist, sp, list);
300}
301
302static void
303g_bde_purge_sector(struct g_bde_softc *sc, int fraction)
304{
305 struct g_bde_sector *sp;
306 int n;
307
308 g_trace(G_T_TOPOLOGY, "g_bde_purge_sector(%p)", sc);
309 if (fraction > 0)
310 n = sc->ncache / fraction + 1;
311 else
312 n = g_bde_ncache - malloc_last_fail();
313 if (n < 0)
314 return;
315 if (n > sc->ncache)
316 n = sc->ncache;
317 while(n--) {
318 TAILQ_FOREACH(sp, &sc->freelist, list) {
319 if (sp->ref != 0)
320 continue;
321 TAILQ_REMOVE(&sc->freelist, sp, list);
322 g_bde_ncache--;
323 sc->ncache--;
324 bzero(sp->data, sp->size);
325 g_bde_delete_sector(sc, sp);
326 break;
327 }
328 }
329}
330
331static struct g_bde_sector *
332g_bde_read_keysector(struct g_bde_softc *sc, struct g_bde_work *wp)
333{
334 struct g_bde_sector *sp;
335
336 g_trace(G_T_TOPOLOGY, "g_bde_read_keysector(%p)", wp);
337 sp = g_bde_get_keysector(wp);
338 if (sp == NULL) {
339 g_bde_purge_sector(sc, -1);
340 sp = g_bde_get_keysector(wp);
341 }
342 if (sp == NULL)
343 return (sp);
344 if (sp->owner != wp)
345 return (sp);
346 if (sp->state == VALID)
347 return (sp);
348 if (g_bde_start_read(sp) == 0)
349 return (sp);
350 g_bde_release_keysector(wp);
351 return (NULL);
352}
353
354/*
355 * Contribute to the completion of the original bio request.
356 *
357 * We have no simple way to tell how many bits the original bio request has
358 * been segmented into, so the easiest way to determine when we can deliver
359 * it is to keep track of the number of bytes we have completed. We keep
360 * track of any errors underway and latch onto the first one.
361 *
362 * We always report "nothing done" in case of error, because random bits here
363 * and there may be completed and returning a number of completed bytes does
364 * not convey any useful information about which bytes they were. If some
365 * piece of broken code somewhere interprets this to mean that nothing has
366 * changed on the underlying media they deserve the lossage headed for them.
367 *
368 * A single mutex per g_bde instance is used to prevent contention.
369 */
370
371static void
372g_bde_contribute(struct bio *bp, off_t bytes, int error)
373{
374
375 g_trace(G_T_TOPOLOGY, "g_bde_contribute bp %p bytes %jd error %d",
376 bp, (intmax_t)bytes, error);
377 if (bp->bio_error == 0)
378 bp->bio_error = error;
379 bp->bio_completed += bytes;
380 KASSERT(bp->bio_completed <= bp->bio_length, ("Too large contribution"));
381 if (bp->bio_completed == bp->bio_length) {
382 if (bp->bio_error != 0)
383 bp->bio_completed = 0;
384 g_io_deliver(bp, bp->bio_error);
385 }
386}
387
388/*
389 * This is the common case "we're done with this work package" function
390 */
391
392static void
393g_bde_work_done(struct g_bde_work *wp, int error)
394{
395
396 g_bde_contribute(wp->bp, wp->length, error);
397 if (wp->sp != NULL)
398 g_bde_delete_sector(wp->softc, wp->sp);
399 if (wp->ksp != NULL)
400 g_bde_release_keysector(wp);
401 g_bde_delete_work(wp);
402}
403
404/*
405 * A write operation has finished. When we have all expected cows in the
406 * barn close the door and call it a day.
407 */
408
409static void
410g_bde_write_done(struct bio *bp)
411{
412 struct g_bde_sector *sp;
413 struct g_bde_work *wp;
414 struct g_bde_softc *sc;
415
416 sp = bp->bio_caller1;
417 sc = bp->bio_caller2;
418 mtx_lock(&sc->worklist_mutex);
419 KASSERT(sp != NULL, ("NULL sp"));
420 KASSERT(sc != NULL, ("NULL sc"));
421 KASSERT(sp->owner != NULL, ("NULL sp->owner"));
422 g_trace(G_T_TOPOLOGY, "g_bde_write_done(%p)", sp);
423 if (bp->bio_error == 0 && bp->bio_completed != sp->size)
424 bp->bio_error = EIO;
425 sp->error = bp->bio_error;
426 g_destroy_bio(bp);
427 wp = sp->owner;
428 if (wp->error == 0)
429 wp->error = sp->error;
430
431 if (wp->bp->bio_cmd == BIO_DELETE) {
432 KASSERT(sp == wp->sp, ("trashed delete op"));
433 g_bde_work_done(wp, wp->error);
434 mtx_unlock(&sc->worklist_mutex);
435 return;
436 }
437
438 KASSERT(wp->bp->bio_cmd == BIO_WRITE, ("Confused in g_bde_write_done()"));
439 KASSERT(sp == wp->sp || sp == wp->ksp, ("trashed write op"));
440 if (wp->sp == sp) {
441 g_bde_delete_sector(sc, wp->sp);
442 wp->sp = NULL;
443 } else {
444 sp->state = VALID;
445 }
446 if (wp->sp == NULL && wp->ksp != NULL && wp->ksp->state == VALID)
447 g_bde_work_done(wp, wp->error);
448 mtx_unlock(&sc->worklist_mutex);
449 return;
450}
451
452/*
453 * Send a write request for the given sector down the pipeline.
454 */
455
456static int
457g_bde_start_write(struct g_bde_sector *sp)
458{
459 struct bio *bp;
460 struct g_bde_softc *sc;
461
462 g_trace(G_T_TOPOLOGY, "g_bde_start_write(%p)", sp);
463 sc = sp->softc;
464 KASSERT(sc != NULL, ("NULL sc in g_bde_start_write"));
465 KASSERT(sp->owner != NULL, ("NULL sp->owner in g_bde_start_write"));
466 bp = g_new_bio();
467 if (bp == NULL)
468 return (ENOMEM);
469 bp->bio_cmd = BIO_WRITE;
470 bp->bio_offset = sp->offset;
471 bp->bio_data = sp->data;
472 bp->bio_length = sp->size;
473 bp->bio_done = g_bde_write_done;
474 bp->bio_caller1 = sp;
475 bp->bio_caller2 = sc;
476 sp->state = IO;
477 g_io_request(bp, sc->consumer);
478 return(0);
479}
480
481/*
482 * A read operation has finished. Mark the sector no longer iobusy and
483 * wake up the worker thread and let it do its thing.
484 */
485
486static void
487g_bde_read_done(struct bio *bp)
488{
489 struct g_bde_sector *sp;
490 struct g_bde_softc *sc;
491
492 sp = bp->bio_caller1;
493 g_trace(G_T_TOPOLOGY, "g_bde_read_done(%p)", sp);
494 sc = bp->bio_caller2;
495 mtx_lock(&sc->worklist_mutex);
496 if (bp->bio_error == 0 && bp->bio_completed != sp->size)
497 bp->bio_error = EIO;
498 sp->error = bp->bio_error;
499 if (sp->error == 0)
500 sp->state = VALID;
501 else
502 sp->state = JUNK;
503 wakeup(sc);
504 g_destroy_bio(bp);
505 mtx_unlock(&sc->worklist_mutex);
506}
507
508/*
509 * Send a read request for the given sector down the pipeline.
510 */
511
512static int
513g_bde_start_read(struct g_bde_sector *sp)
514{
515 struct bio *bp;
516 struct g_bde_softc *sc;
517
518 g_trace(G_T_TOPOLOGY, "g_bde_start_read(%p)", sp);
519 sc = sp->softc;
520 KASSERT(sc != NULL, ("Null softc in sp %p", sp));
521 bp = g_new_bio();
522 if (bp == NULL)
523 return (ENOMEM);
524 bp->bio_cmd = BIO_READ;
525 bp->bio_offset = sp->offset;
526 bp->bio_data = sp->data;
527 bp->bio_length = sp->size;
528 bp->bio_done = g_bde_read_done;
529 bp->bio_caller1 = sp;
530 bp->bio_caller2 = sc;
531 sp->state = IO;
532 g_io_request(bp, sc->consumer);
533 return(0);
534}
535
536/*
537 * The worker thread.
538 *
539 * The up/down path of GEOM is not allowed to sleep or do any major work
540 * so we use this thread to do the actual crypto operations and to push
541 * the state engine onwards.
542 *
543 * XXX: if we switch to the src/sys/opencrypt hardware assisted encryption
544 * XXX: using a thread here is probably not needed.
545 */
546
547void
548g_bde_worker(void *arg)
549{
550 struct g_bde_softc *sc;
551 struct g_bde_work *wp, *twp;
552 struct g_geom *gp;
553 int restart, error;
554
555 gp = arg;
556 sc = gp->softc;
557
558 mtx_lock(&sc->worklist_mutex);
559 for (;;) {
560 restart = 0;
561 g_trace(G_T_TOPOLOGY, "g_bde_worker scan");
562 TAILQ_FOREACH_SAFE(wp, &sc->worklist, list, twp) {
563 KASSERT(wp != NULL, ("NULL wp"));
564 KASSERT(wp->softc != NULL, ("NULL wp->softc"));
565 if (wp->state != WAIT)
566 continue; /* Not interesting here */
567
568 KASSERT(wp->bp != NULL, ("NULL wp->bp"));
569 KASSERT(wp->sp != NULL, ("NULL wp->sp"));
570
571 if (wp->ksp != NULL) {
572 if (wp->ksp->owner != wp)
573 continue;
574 if (wp->ksp->state == IO)
575 continue;
576 KASSERT(wp->ksp->state == VALID,
577 ("Illegal sector state (%d)",
578 wp->ksp->state));
579 }
580
581 if (wp->bp->bio_cmd == BIO_READ && wp->sp->state == IO)
582 continue;
583
584 if (wp->ksp != NULL && wp->ksp->error != 0) {
585 g_bde_work_done(wp, wp->ksp->error);
586 continue;
587 }
588 switch(wp->bp->bio_cmd) {
589 case BIO_READ:
590 if (wp->ksp == NULL) {
591 KASSERT(wp->error != 0,
592 ("BIO_READ, no ksp and no error"));
593 g_bde_work_done(wp, wp->error);
594 break;
595 }
596 if (wp->sp->error != 0) {
597 g_bde_work_done(wp, wp->sp->error);
598 break;
599 }
600 mtx_unlock(&sc->worklist_mutex);
601 g_bde_crypt_read(wp);
602 mtx_lock(&sc->worklist_mutex);
603 restart++;
604 g_bde_work_done(wp, wp->sp->error);
605 break;
606 case BIO_WRITE:
607 wp->state = FINISH;
608 KASSERT(wp->sp->owner == wp,
609 ("Write not owner sp"));
610 KASSERT(wp->ksp->owner == wp,
611 ("Write not owner ksp"));
612 mtx_unlock(&sc->worklist_mutex);
613 g_bde_crypt_write(wp);
614 mtx_lock(&sc->worklist_mutex);
615 restart++;
616 error = g_bde_start_write(wp->sp);
617 if (error) {
618 g_bde_work_done(wp, error);
619 break;
620 }
621 error = g_bde_start_write(wp->ksp);
622 if (wp->error != 0)
623 wp->error = error;
624 break;
625 case BIO_DELETE:
626 wp->state = FINISH;
627 mtx_unlock(&sc->worklist_mutex);
628 g_bde_crypt_delete(wp);
629 mtx_lock(&sc->worklist_mutex);
630 restart++;
631 g_bde_start_write(wp->sp);
632 break;
633 }
634 if (restart)
635 break;
636 }
637 if (!restart) {
638 /*
639 * We don't look for our death-warrant until we are
640 * idle. Shouldn't make a difference in practice.
641 */
642 if (sc->dead)
643 break;
644 g_trace(G_T_TOPOLOGY, "g_bde_worker sleep");
645 error = msleep(sc, &sc->worklist_mutex,
646 PRIBIO, "-", hz);
647 if (error == EWOULDBLOCK) {
648 /*
| 33 */
34/*
35 * This source file contains the state-engine which makes things happen in the
36 * right order.
37 *
38 * Outline:
39 * 1) g_bde_start1()
40 * Break the struct bio into multiple work packets one per zone.
41 * 2) g_bde_start2()
42 * Setup the necessary sector buffers and start those read operations
43 * which we can start at this time and put the item on the work-list.
44 * 3) g_bde_worker()
45 * Scan the work-list for items which are ready for crypto processing
46 * and call the matching crypto function in g_bde_crypt.c and schedule
47 * any writes needed. Read operations finish here by releasing the
48 * sector buffers and delivering the original bio request.
49 * 4) g_bde_write_done()
50 * Release sector buffers and deliver the original bio request.
51 *
52 * Because of the C-scope rules, the functions are almost perfectly in the
53 * opposite order in this source file.
54 *
55 * XXX: A switch to the hardware assisted crypto in src/sys/opencrypto will add
56 * XXX: additional states to this state-engine. Since no hardware available
57 * XXX: at this time has AES support, implementing this has been postponed
58 * XXX: until such time as it would result in a benefit.
59 */
60
61#include <sys/param.h>
62#include <sys/bio.h>
63#include <sys/lock.h>
64#include <sys/mutex.h>
65#include <sys/queue.h>
66#include <sys/malloc.h>
67#include <sys/systm.h>
68#include <sys/kernel.h>
69#include <sys/sysctl.h>
70#include <sys/proc.h>
71#include <sys/kthread.h>
72
73#include <crypto/rijndael/rijndael-api-fst.h>
74#include <crypto/sha2/sha2.h>
75#include <geom/geom.h>
76#include <geom/bde/g_bde.h>
77
78static void g_bde_delete_sector(struct g_bde_softc *wp, struct g_bde_sector *sp);
79static struct g_bde_sector * g_bde_new_sector(struct g_bde_work *wp, u_int len);
80static void g_bde_release_keysector(struct g_bde_work *wp);
81static struct g_bde_sector *g_bde_get_keysector(struct g_bde_work *wp);
82static int g_bde_start_read(struct g_bde_sector *sp);
83static void g_bde_purge_sector(struct g_bde_softc *sc, int fraction);
84
85/*
86 * Work item allocation.
87 *
88 * C++ would call these constructors and destructors.
89 */
90static u_int g_bde_nwork;
91SYSCTL_UINT(_debug, OID_AUTO, gbde_nwork, CTLFLAG_RD, &g_bde_nwork, 0, "");
92
93static MALLOC_DEFINE(M_GBDE, "gbde", "GBDE data structures");
94
95static struct g_bde_work *
96g_bde_new_work(struct g_bde_softc *sc)
97{
98 struct g_bde_work *wp;
99
100 wp = malloc(sizeof *wp, M_GBDE, M_NOWAIT | M_ZERO);
101 if (wp == NULL)
102 return (wp);
103 wp->state = SETUP;
104 wp->softc = sc;
105 g_bde_nwork++;
106 sc->nwork++;
107 TAILQ_INSERT_TAIL(&sc->worklist, wp, list);
108 return (wp);
109}
110
111static void
112g_bde_delete_work(struct g_bde_work *wp)
113{
114 struct g_bde_softc *sc;
115
116 sc = wp->softc;
117 g_bde_nwork--;
118 sc->nwork--;
119 TAILQ_REMOVE(&sc->worklist, wp, list);
120 free(wp, M_GBDE);
121}
122
123/*
124 * Sector buffer allocation
125 *
126 * These two functions allocate and free back variable sized sector buffers
127 */
128
129static u_int g_bde_nsect;
130SYSCTL_UINT(_debug, OID_AUTO, gbde_nsect, CTLFLAG_RD, &g_bde_nsect, 0, "");
131
132static void
133g_bde_delete_sector(struct g_bde_softc *sc, struct g_bde_sector *sp)
134{
135
136 g_bde_nsect--;
137 sc->nsect--;
138 if (sp->malloc)
139 free(sp->data, M_GBDE);
140 free(sp, M_GBDE);
141}
142
143static struct g_bde_sector *
144g_bde_new_sector(struct g_bde_work *wp, u_int len)
145{
146 struct g_bde_sector *sp;
147
148 sp = malloc(sizeof *sp, M_GBDE, M_NOWAIT | M_ZERO);
149 if (sp == NULL)
150 return (sp);
151 if (len > 0) {
152 sp->data = malloc(len, M_GBDE, M_NOWAIT | M_ZERO);
153 if (sp->data == NULL) {
154 free(sp, M_GBDE);
155 return (NULL);
156 }
157 sp->malloc = 1;
158 }
159 g_bde_nsect++;
160 wp->softc->nsect++;
161 sp->size = len;
162 sp->softc = wp->softc;
163 sp->ref = 1;
164 sp->owner = wp;
165 sp->offset = wp->so;
166 sp->state = JUNK;
167 return (sp);
168}
169
170/*
171 * Skey sector cache.
172 *
173 * Nothing prevents two separate I/O requests from addressing the same zone
174 * and thereby needing the same skey sector. We therefore need to sequence
175 * I/O operations to the skey sectors. A certain amount of caching is also
176 * desirable, although the extent of benefit from this is not at this point
177 * determined.
178 *
179 * XXX: GEOM may be able to grow a generic caching facility at some point
180 * XXX: to support such needs.
181 */
182
183static u_int g_bde_ncache;
184SYSCTL_UINT(_debug, OID_AUTO, gbde_ncache, CTLFLAG_RD, &g_bde_ncache, 0, "");
185
186static void
187g_bde_purge_one_sector(struct g_bde_softc *sc, struct g_bde_sector *sp)
188{
189
190 g_trace(G_T_TOPOLOGY, "g_bde_purge_one_sector(%p, %p)", sc, sp);
191 if (sp->ref != 0)
192 return;
193 TAILQ_REMOVE(&sc->freelist, sp, list);
194 g_bde_ncache--;
195 sc->ncache--;
196 bzero(sp->data, sp->size);
197 g_bde_delete_sector(sc, sp);
198}
199
200static struct g_bde_sector *
201g_bde_get_keysector(struct g_bde_work *wp)
202{
203 struct g_bde_sector *sp;
204 struct g_bde_softc *sc;
205 off_t offset;
206
207 offset = wp->kso;
208 g_trace(G_T_TOPOLOGY, "g_bde_get_keysector(%p, %jd)", wp, (intmax_t)offset);
209 sc = wp->softc;
210
211 if (malloc_last_fail() < g_bde_ncache)
212 g_bde_purge_sector(sc, -1);
213
214 sp = TAILQ_FIRST(&sc->freelist);
215 if (sp != NULL && sp->ref == 0 && sp->used + 300 < time_uptime)
216 g_bde_purge_one_sector(sc, sp);
217
218 TAILQ_FOREACH(sp, &sc->freelist, list) {
219 if (sp->offset == offset)
220 break;
221 }
222 if (sp != NULL) {
223 sp->ref++;
224 KASSERT(sp->offset == offset, ("wrong offset"));
225 KASSERT(sp->softc == wp->softc, ("wrong softc"));
226 if (sp->ref == 1)
227 sp->owner = wp;
228 } else {
229 if (malloc_last_fail() < g_bde_ncache) {
230 TAILQ_FOREACH(sp, &sc->freelist, list)
231 if (sp->ref == 0)
232 break;
233 }
234 if (sp == NULL && !TAILQ_EMPTY(&sc->freelist))
235 sp = TAILQ_FIRST(&sc->freelist);
236 if (sp != NULL && sp->ref > 0)
237 sp = NULL;
238 if (sp == NULL) {
239 sp = g_bde_new_sector(wp, sc->sectorsize);
240 if (sp != NULL) {
241 g_bde_ncache++;
242 sc->ncache++;
243 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
244 sp->malloc = 2;
245 }
246 }
247 if (sp != NULL) {
248 sp->offset = offset;
249 sp->softc = wp->softc;
250 sp->ref = 1;
251 sp->owner = wp;
252 sp->state = JUNK;
253 sp->error = 0;
254 }
255 }
256 if (sp != NULL) {
257 TAILQ_REMOVE(&sc->freelist, sp, list);
258 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
259 sp->used = time_uptime;
260 }
261 wp->ksp = sp;
262 return(sp);
263}
264
265static void
266g_bde_release_keysector(struct g_bde_work *wp)
267{
268 struct g_bde_softc *sc;
269 struct g_bde_work *wp2;
270 struct g_bde_sector *sp;
271
272 sp = wp->ksp;
273 g_trace(G_T_TOPOLOGY, "g_bde_release_keysector(%p)", sp);
274 KASSERT(sp->malloc == 2, ("Wrong sector released"));
275 sc = sp->softc;
276 KASSERT(sc != NULL, ("NULL sp->softc"));
277 KASSERT(wp == sp->owner, ("Releasing, not owner"));
278 sp->owner = NULL;
279 wp->ksp = NULL;
280 sp->ref--;
281 if (sp->ref > 0) {
282 TAILQ_REMOVE(&sc->freelist, sp, list);
283 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
284 TAILQ_FOREACH(wp2, &sc->worklist, list) {
285 if (wp2->ksp == sp) {
286 KASSERT(wp2 != wp, ("Self-reowning"));
287 sp->owner = wp2;
288 wakeup(sp->softc);
289 break;
290 }
291 }
292 KASSERT(wp2 != NULL, ("Failed to pick up owner for %p\n", sp));
293 } else if (sp->error != 0) {
294 sp->offset = ~0;
295 sp->error = 0;
296 sp->state = JUNK;
297 }
298 TAILQ_REMOVE(&sc->freelist, sp, list);
299 TAILQ_INSERT_HEAD(&sc->freelist, sp, list);
300}
301
302static void
303g_bde_purge_sector(struct g_bde_softc *sc, int fraction)
304{
305 struct g_bde_sector *sp;
306 int n;
307
308 g_trace(G_T_TOPOLOGY, "g_bde_purge_sector(%p)", sc);
309 if (fraction > 0)
310 n = sc->ncache / fraction + 1;
311 else
312 n = g_bde_ncache - malloc_last_fail();
313 if (n < 0)
314 return;
315 if (n > sc->ncache)
316 n = sc->ncache;
317 while(n--) {
318 TAILQ_FOREACH(sp, &sc->freelist, list) {
319 if (sp->ref != 0)
320 continue;
321 TAILQ_REMOVE(&sc->freelist, sp, list);
322 g_bde_ncache--;
323 sc->ncache--;
324 bzero(sp->data, sp->size);
325 g_bde_delete_sector(sc, sp);
326 break;
327 }
328 }
329}
330
331static struct g_bde_sector *
332g_bde_read_keysector(struct g_bde_softc *sc, struct g_bde_work *wp)
333{
334 struct g_bde_sector *sp;
335
336 g_trace(G_T_TOPOLOGY, "g_bde_read_keysector(%p)", wp);
337 sp = g_bde_get_keysector(wp);
338 if (sp == NULL) {
339 g_bde_purge_sector(sc, -1);
340 sp = g_bde_get_keysector(wp);
341 }
342 if (sp == NULL)
343 return (sp);
344 if (sp->owner != wp)
345 return (sp);
346 if (sp->state == VALID)
347 return (sp);
348 if (g_bde_start_read(sp) == 0)
349 return (sp);
350 g_bde_release_keysector(wp);
351 return (NULL);
352}
353
354/*
355 * Contribute to the completion of the original bio request.
356 *
357 * We have no simple way to tell how many bits the original bio request has
358 * been segmented into, so the easiest way to determine when we can deliver
359 * it is to keep track of the number of bytes we have completed. We keep
360 * track of any errors underway and latch onto the first one.
361 *
362 * We always report "nothing done" in case of error, because random bits here
363 * and there may be completed and returning a number of completed bytes does
364 * not convey any useful information about which bytes they were. If some
365 * piece of broken code somewhere interprets this to mean that nothing has
366 * changed on the underlying media they deserve the lossage headed for them.
367 *
368 * A single mutex per g_bde instance is used to prevent contention.
369 */
370
371static void
372g_bde_contribute(struct bio *bp, off_t bytes, int error)
373{
374
375 g_trace(G_T_TOPOLOGY, "g_bde_contribute bp %p bytes %jd error %d",
376 bp, (intmax_t)bytes, error);
377 if (bp->bio_error == 0)
378 bp->bio_error = error;
379 bp->bio_completed += bytes;
380 KASSERT(bp->bio_completed <= bp->bio_length, ("Too large contribution"));
381 if (bp->bio_completed == bp->bio_length) {
382 if (bp->bio_error != 0)
383 bp->bio_completed = 0;
384 g_io_deliver(bp, bp->bio_error);
385 }
386}
387
388/*
389 * This is the common case "we're done with this work package" function
390 */
391
392static void
393g_bde_work_done(struct g_bde_work *wp, int error)
394{
395
396 g_bde_contribute(wp->bp, wp->length, error);
397 if (wp->sp != NULL)
398 g_bde_delete_sector(wp->softc, wp->sp);
399 if (wp->ksp != NULL)
400 g_bde_release_keysector(wp);
401 g_bde_delete_work(wp);
402}
403
404/*
405 * A write operation has finished. When we have all expected cows in the
406 * barn close the door and call it a day.
407 */
408
409static void
410g_bde_write_done(struct bio *bp)
411{
412 struct g_bde_sector *sp;
413 struct g_bde_work *wp;
414 struct g_bde_softc *sc;
415
416 sp = bp->bio_caller1;
417 sc = bp->bio_caller2;
418 mtx_lock(&sc->worklist_mutex);
419 KASSERT(sp != NULL, ("NULL sp"));
420 KASSERT(sc != NULL, ("NULL sc"));
421 KASSERT(sp->owner != NULL, ("NULL sp->owner"));
422 g_trace(G_T_TOPOLOGY, "g_bde_write_done(%p)", sp);
423 if (bp->bio_error == 0 && bp->bio_completed != sp->size)
424 bp->bio_error = EIO;
425 sp->error = bp->bio_error;
426 g_destroy_bio(bp);
427 wp = sp->owner;
428 if (wp->error == 0)
429 wp->error = sp->error;
430
431 if (wp->bp->bio_cmd == BIO_DELETE) {
432 KASSERT(sp == wp->sp, ("trashed delete op"));
433 g_bde_work_done(wp, wp->error);
434 mtx_unlock(&sc->worklist_mutex);
435 return;
436 }
437
438 KASSERT(wp->bp->bio_cmd == BIO_WRITE, ("Confused in g_bde_write_done()"));
439 KASSERT(sp == wp->sp || sp == wp->ksp, ("trashed write op"));
440 if (wp->sp == sp) {
441 g_bde_delete_sector(sc, wp->sp);
442 wp->sp = NULL;
443 } else {
444 sp->state = VALID;
445 }
446 if (wp->sp == NULL && wp->ksp != NULL && wp->ksp->state == VALID)
447 g_bde_work_done(wp, wp->error);
448 mtx_unlock(&sc->worklist_mutex);
449 return;
450}
451
452/*
453 * Send a write request for the given sector down the pipeline.
454 */
455
456static int
457g_bde_start_write(struct g_bde_sector *sp)
458{
459 struct bio *bp;
460 struct g_bde_softc *sc;
461
462 g_trace(G_T_TOPOLOGY, "g_bde_start_write(%p)", sp);
463 sc = sp->softc;
464 KASSERT(sc != NULL, ("NULL sc in g_bde_start_write"));
465 KASSERT(sp->owner != NULL, ("NULL sp->owner in g_bde_start_write"));
466 bp = g_new_bio();
467 if (bp == NULL)
468 return (ENOMEM);
469 bp->bio_cmd = BIO_WRITE;
470 bp->bio_offset = sp->offset;
471 bp->bio_data = sp->data;
472 bp->bio_length = sp->size;
473 bp->bio_done = g_bde_write_done;
474 bp->bio_caller1 = sp;
475 bp->bio_caller2 = sc;
476 sp->state = IO;
477 g_io_request(bp, sc->consumer);
478 return(0);
479}
480
481/*
482 * A read operation has finished. Mark the sector no longer iobusy and
483 * wake up the worker thread and let it do its thing.
484 */
485
486static void
487g_bde_read_done(struct bio *bp)
488{
489 struct g_bde_sector *sp;
490 struct g_bde_softc *sc;
491
492 sp = bp->bio_caller1;
493 g_trace(G_T_TOPOLOGY, "g_bde_read_done(%p)", sp);
494 sc = bp->bio_caller2;
495 mtx_lock(&sc->worklist_mutex);
496 if (bp->bio_error == 0 && bp->bio_completed != sp->size)
497 bp->bio_error = EIO;
498 sp->error = bp->bio_error;
499 if (sp->error == 0)
500 sp->state = VALID;
501 else
502 sp->state = JUNK;
503 wakeup(sc);
504 g_destroy_bio(bp);
505 mtx_unlock(&sc->worklist_mutex);
506}
507
508/*
509 * Send a read request for the given sector down the pipeline.
510 */
511
512static int
513g_bde_start_read(struct g_bde_sector *sp)
514{
515 struct bio *bp;
516 struct g_bde_softc *sc;
517
518 g_trace(G_T_TOPOLOGY, "g_bde_start_read(%p)", sp);
519 sc = sp->softc;
520 KASSERT(sc != NULL, ("Null softc in sp %p", sp));
521 bp = g_new_bio();
522 if (bp == NULL)
523 return (ENOMEM);
524 bp->bio_cmd = BIO_READ;
525 bp->bio_offset = sp->offset;
526 bp->bio_data = sp->data;
527 bp->bio_length = sp->size;
528 bp->bio_done = g_bde_read_done;
529 bp->bio_caller1 = sp;
530 bp->bio_caller2 = sc;
531 sp->state = IO;
532 g_io_request(bp, sc->consumer);
533 return(0);
534}
535
536/*
537 * The worker thread.
538 *
539 * The up/down path of GEOM is not allowed to sleep or do any major work
540 * so we use this thread to do the actual crypto operations and to push
541 * the state engine onwards.
542 *
543 * XXX: if we switch to the src/sys/opencrypt hardware assisted encryption
544 * XXX: using a thread here is probably not needed.
545 */
546
547void
548g_bde_worker(void *arg)
549{
550 struct g_bde_softc *sc;
551 struct g_bde_work *wp, *twp;
552 struct g_geom *gp;
553 int restart, error;
554
555 gp = arg;
556 sc = gp->softc;
557
558 mtx_lock(&sc->worklist_mutex);
559 for (;;) {
560 restart = 0;
561 g_trace(G_T_TOPOLOGY, "g_bde_worker scan");
562 TAILQ_FOREACH_SAFE(wp, &sc->worklist, list, twp) {
563 KASSERT(wp != NULL, ("NULL wp"));
564 KASSERT(wp->softc != NULL, ("NULL wp->softc"));
565 if (wp->state != WAIT)
566 continue; /* Not interesting here */
567
568 KASSERT(wp->bp != NULL, ("NULL wp->bp"));
569 KASSERT(wp->sp != NULL, ("NULL wp->sp"));
570
571 if (wp->ksp != NULL) {
572 if (wp->ksp->owner != wp)
573 continue;
574 if (wp->ksp->state == IO)
575 continue;
576 KASSERT(wp->ksp->state == VALID,
577 ("Illegal sector state (%d)",
578 wp->ksp->state));
579 }
580
581 if (wp->bp->bio_cmd == BIO_READ && wp->sp->state == IO)
582 continue;
583
584 if (wp->ksp != NULL && wp->ksp->error != 0) {
585 g_bde_work_done(wp, wp->ksp->error);
586 continue;
587 }
588 switch(wp->bp->bio_cmd) {
589 case BIO_READ:
590 if (wp->ksp == NULL) {
591 KASSERT(wp->error != 0,
592 ("BIO_READ, no ksp and no error"));
593 g_bde_work_done(wp, wp->error);
594 break;
595 }
596 if (wp->sp->error != 0) {
597 g_bde_work_done(wp, wp->sp->error);
598 break;
599 }
600 mtx_unlock(&sc->worklist_mutex);
601 g_bde_crypt_read(wp);
602 mtx_lock(&sc->worklist_mutex);
603 restart++;
604 g_bde_work_done(wp, wp->sp->error);
605 break;
606 case BIO_WRITE:
607 wp->state = FINISH;
608 KASSERT(wp->sp->owner == wp,
609 ("Write not owner sp"));
610 KASSERT(wp->ksp->owner == wp,
611 ("Write not owner ksp"));
612 mtx_unlock(&sc->worklist_mutex);
613 g_bde_crypt_write(wp);
614 mtx_lock(&sc->worklist_mutex);
615 restart++;
616 error = g_bde_start_write(wp->sp);
617 if (error) {
618 g_bde_work_done(wp, error);
619 break;
620 }
621 error = g_bde_start_write(wp->ksp);
622 if (wp->error != 0)
623 wp->error = error;
624 break;
625 case BIO_DELETE:
626 wp->state = FINISH;
627 mtx_unlock(&sc->worklist_mutex);
628 g_bde_crypt_delete(wp);
629 mtx_lock(&sc->worklist_mutex);
630 restart++;
631 g_bde_start_write(wp->sp);
632 break;
633 }
634 if (restart)
635 break;
636 }
637 if (!restart) {
638 /*
639 * We don't look for our death-warrant until we are
640 * idle. Shouldn't make a difference in practice.
641 */
642 if (sc->dead)
643 break;
644 g_trace(G_T_TOPOLOGY, "g_bde_worker sleep");
645 error = msleep(sc, &sc->worklist_mutex,
646 PRIBIO, "-", hz);
647 if (error == EWOULDBLOCK) {
648 /*
|