1/*-
2 * Copyright (c) 2005 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/geom/eli/g_eli.c 159306 2006-06-05 21:25:19Z pjd $");
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/kernel.h>
33#include <sys/linker.h>
34#include <sys/module.h>
35#include <sys/lock.h>
36#include <sys/mutex.h>
37#include <sys/bio.h>
38#include <sys/sysctl.h>
39#include <sys/malloc.h>
40#include <sys/kthread.h>
41#include <sys/proc.h>
42#include <sys/sched.h>
43#include <sys/smp.h>
44#include <sys/uio.h>
45#include <sys/vnode.h>
46
47#include <vm/uma.h>
48
49#include <geom/geom.h>
50#include <geom/eli/g_eli.h>
51#include <geom/eli/pkcs5v2.h>
52
53
54MALLOC_DEFINE(M_ELI, "eli data", "GEOM_ELI Data");
55
56SYSCTL_DECL(_kern_geom);
57SYSCTL_NODE(_kern_geom, OID_AUTO, eli, CTLFLAG_RW, 0, "GEOM_ELI stuff");
58u_int g_eli_debug = 0;
59TUNABLE_INT("kern.geom.eli.debug", &g_eli_debug);
60SYSCTL_UINT(_kern_geom_eli, OID_AUTO, debug, CTLFLAG_RW, &g_eli_debug, 0,
61 "Debug level");
62static u_int g_eli_tries = 3;
63TUNABLE_INT("kern.geom.eli.tries", &g_eli_tries);
64SYSCTL_UINT(_kern_geom_eli, OID_AUTO, tries, CTLFLAG_RW, &g_eli_tries, 0,
65 "Number of tries for entering the passphrase");
66static u_int g_eli_visible_passphrase = 0;
67TUNABLE_INT("kern.geom.eli.visible_passphrase", &g_eli_visible_passphrase);
68SYSCTL_UINT(_kern_geom_eli, OID_AUTO, visible_passphrase, CTLFLAG_RW,
69 &g_eli_visible_passphrase, 0,
70 "Turn on echo when entering the passphrase (for debug purposes only!!)");
71u_int g_eli_overwrites = 5;
72TUNABLE_INT("kern.geom.eli.overwrites", &g_eli_overwrites);
73SYSCTL_UINT(_kern_geom_eli, OID_AUTO, overwrites, CTLFLAG_RW, &g_eli_overwrites,
74 0, "Number of times on-disk keys should be overwritten when destroying them");
75static u_int g_eli_threads = 0;
76TUNABLE_INT("kern.geom.eli.threads", &g_eli_threads);
77SYSCTL_UINT(_kern_geom_eli, OID_AUTO, threads, CTLFLAG_RW, &g_eli_threads, 0,
78 "Number of threads doing crypto work");
79
80static int g_eli_destroy_geom(struct gctl_req *req, struct g_class *mp,
81 struct g_geom *gp);
82static void g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp);
83
84static g_taste_t g_eli_taste;
85static g_dumpconf_t g_eli_dumpconf;
86
87struct g_class g_eli_class = {
88 .name = G_ELI_CLASS_NAME,
89 .version = G_VERSION,
90 .ctlreq = g_eli_config,
91 .taste = g_eli_taste,
92 .destroy_geom = g_eli_destroy_geom
93};
94
95
96/*
97 * Code paths:
98 * BIO_READ:
99 * g_eli_start -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
100 * BIO_WRITE:
101 * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
102 */
103
104
105/*
106 * EAGAIN from crypto(9) means, that we were probably balanced to another crypto
107 * accelerator or something like this.
108 * The function updates the SID and rerun the operation.
109 */
110static int
111g_eli_crypto_rerun(struct cryptop *crp)
112{
113 struct g_eli_softc *sc;
114 struct g_eli_worker *wr;
115 struct bio *bp;
116 int error;
117
118 bp = (struct bio *)crp->crp_opaque;
119 sc = bp->bio_to->geom->softc;
120 LIST_FOREACH(wr, &sc->sc_workers, w_next) {
121 if (wr->w_number == bp->bio_pflags)
122 break;
123 }
124 KASSERT(wr != NULL, ("Invalid worker (%u).", bp->bio_pflags));
125 G_ELI_DEBUG(1, "Rerunning crypto %s request (sid: %ju -> %ju).",
126 bp->bio_cmd == BIO_READ ? "READ" : "WRITE", (uintmax_t)wr->w_sid,
127 (uintmax_t)crp->crp_sid);
128 wr->w_sid = crp->crp_sid;
129 crp->crp_etype = 0;
130 error = crypto_dispatch(crp);
131 if (error == 0)
132 return (0);
133 G_ELI_DEBUG(1, "%s: crypto_dispatch() returned %d.", __func__, error);
134 crp->crp_etype = error;
135 return (error);
136}
137
138/*
139 * The function is called afer reading encrypted data from the provider.
140 *
141 * g_eli_start -> g_io_request -> G_ELI_READ_DONE -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
142 */
143static void
144g_eli_read_done(struct bio *bp)
145{
146 struct g_eli_softc *sc;
147 struct bio *pbp;
148
149 G_ELI_LOGREQ(2, bp, "Request done.");
150 pbp = bp->bio_parent;
151 if (pbp->bio_error == 0)
152 pbp->bio_error = bp->bio_error;
153 g_destroy_bio(bp);
154 if (pbp->bio_error != 0) {
155 G_ELI_LOGREQ(0, pbp, "%s() failed", __func__);
156 pbp->bio_completed = 0;
157 g_io_deliver(pbp, pbp->bio_error);
158 return;
159 }
160 sc = pbp->bio_to->geom->softc;
161 mtx_lock(&sc->sc_queue_mtx);
162 bioq_insert_tail(&sc->sc_queue, pbp);
163 mtx_unlock(&sc->sc_queue_mtx);
164 wakeup(sc);
165}
166
167/*
168 * The function is called after we read and decrypt data.
169 *
170 * g_eli_start -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver
171 */
172static int
173g_eli_crypto_read_done(struct cryptop *crp)
174{
175 struct bio *bp;
176
177 if (crp->crp_etype == EAGAIN) {
178 if (g_eli_crypto_rerun(crp) == 0)
179 return (0);
180 }
181 bp = (struct bio *)crp->crp_opaque;
182 bp->bio_inbed++;
183 if (crp->crp_etype == 0) {
184 G_ELI_DEBUG(3, "Crypto READ request done (%d/%d).",
185 bp->bio_inbed, bp->bio_children);
186 bp->bio_completed += crp->crp_olen;
187 } else {
188 G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.",
189 bp->bio_inbed, bp->bio_children, crp->crp_etype);
190 if (bp->bio_error == 0)
191 bp->bio_error = crp->crp_etype;
192 }
193 /*
194 * Do we have all sectors already?
195 */
196 if (bp->bio_inbed < bp->bio_children)
197 return (0);
198 free(bp->bio_driver2, M_ELI);
199 bp->bio_driver2 = NULL;
200 if (bp->bio_error != 0) {
201 G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).",
202 bp->bio_error);
203 bp->bio_completed = 0;
204 }
205 /*
206 * Read is finished, send it up.
207 */
208 g_io_deliver(bp, bp->bio_error);
209 return (0);
210}
211
212/*
213 * The function is called after we encrypt and write data.
214 *
215 * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> G_ELI_WRITE_DONE -> g_io_deliver
216 */
217static void
218g_eli_write_done(struct bio *bp)
219{
220 struct bio *pbp;
221
222 G_ELI_LOGREQ(2, bp, "Request done.");
223 pbp = bp->bio_parent;
224 if (pbp->bio_error == 0)
225 pbp->bio_error = bp->bio_error;
226 free(pbp->bio_driver2, M_ELI);
227 pbp->bio_driver2 = NULL;
228 if (pbp->bio_error == 0)
229 pbp->bio_completed = pbp->bio_length;
230 else {
231 G_ELI_LOGREQ(0, pbp, "Crypto WRITE request failed (error=%d).",
232 pbp->bio_error);
233 pbp->bio_completed = 0;
234 }
235 g_destroy_bio(bp);
236 /*
237 * Write is finished, send it up.
238 */
239 g_io_deliver(pbp, pbp->bio_error);
240}
241
242/*
243 * The function is called after data encryption.
244 *
245 * g_eli_start -> g_eli_crypto_run -> G_ELI_CRYPTO_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver
246 */
247static int
248g_eli_crypto_write_done(struct cryptop *crp)
249{
250 struct g_geom *gp;
251 struct g_consumer *cp;
252 struct bio *bp, *cbp;
253
254 if (crp->crp_etype == EAGAIN) {
255 if (g_eli_crypto_rerun(crp) == 0)
256 return (0);
257 }
258 bp = (struct bio *)crp->crp_opaque;
259 bp->bio_inbed++;
260 if (crp->crp_etype == 0) {
261 G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).",
262 bp->bio_inbed, bp->bio_children);
263 } else {
264 G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.",
265 bp->bio_inbed, bp->bio_children, crp->crp_etype);
266 if (bp->bio_error == 0)
267 bp->bio_error = crp->crp_etype;
268 }
269 /*
270 * All sectors are already encrypted?
271 */
272 if (bp->bio_inbed < bp->bio_children)
273 return (0);
274 bp->bio_inbed = 0;
275 bp->bio_children = 1;
276 cbp = bp->bio_driver1;
277 bp->bio_driver1 = NULL;
278 if (bp->bio_error != 0) {
279 G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
280 bp->bio_error);
281 free(bp->bio_driver2, M_ELI);
282 bp->bio_driver2 = NULL;
283 g_destroy_bio(cbp);
284 g_io_deliver(bp, bp->bio_error);
285 return (0);
286 }
287 cbp->bio_data = bp->bio_driver2;
288 cbp->bio_done = g_eli_write_done;
289 gp = bp->bio_to->geom;
290 cp = LIST_FIRST(&gp->consumer);
291 cbp->bio_to = cp->provider;
292 G_ELI_LOGREQ(2, cbp, "Sending request.");
293 /*
294 * Send encrypted data to the provider.
295 */
296 g_io_request(cbp, cp);
297 return (0);
298}
299
300/*
301 * This function should never be called, but GEOM made as it set ->orphan()
302 * method for every geom.
303 */
304static void
305g_eli_orphan_spoil_assert(struct g_consumer *cp)
306{
307
308 panic("Function %s() called for %s.", __func__, cp->geom->name);
309}
310
311static void
312g_eli_orphan(struct g_consumer *cp)
313{
314 struct g_eli_softc *sc;
315
316 g_topology_assert();
317 sc = cp->geom->softc;
318 if (sc == NULL)
319 return;
320 g_eli_destroy(sc, 1);
321}
322
323/*
324 * BIO_READ : G_ELI_START -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
325 * BIO_WRITE: G_ELI_START -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
326 */
327static void
328g_eli_start(struct bio *bp)
329{
330 struct g_eli_softc *sc;
331 struct g_consumer *cp;
332 struct bio *cbp;
333
334 sc = bp->bio_to->geom->softc;
335 KASSERT(sc != NULL,
336 ("Provider's error should be set (error=%d)(device=%s).",
337 bp->bio_to->error, bp->bio_to->name));
338 G_ELI_LOGREQ(2, bp, "Request received.");
339
340 switch (bp->bio_cmd) {
341 case BIO_READ:
342 case BIO_WRITE:
343 case BIO_GETATTR:
344 break;
345 case BIO_DELETE:
346 /*
347 * We could eventually support BIO_DELETE request.
348 * It could be done by overwritting requested sector with
349 * random data g_eli_overwrites number of times.
350 */
351 default:
352 g_io_deliver(bp, EOPNOTSUPP);
353 return;
354 }
355 cbp = g_clone_bio(bp);
356 if (cbp == NULL) {
357 g_io_deliver(bp, ENOMEM);
358 return;
359 }
360 switch (bp->bio_cmd) {
361 case BIO_READ:
362 cbp->bio_done = g_eli_read_done;
363 cp = LIST_FIRST(&sc->sc_geom->consumer);
364 cbp->bio_to = cp->provider;
365 G_ELI_LOGREQ(2, cbp, "Sending request.");
366 /*
367 * Read encrypted data from provider.
368 */
369 g_io_request(cbp, cp);
370 break;
371 case BIO_WRITE:
372 bp->bio_driver1 = cbp;
373 mtx_lock(&sc->sc_queue_mtx);
374 bioq_insert_tail(&sc->sc_queue, bp);
375 mtx_unlock(&sc->sc_queue_mtx);
376 wakeup(sc);
377 break;
378 case BIO_GETATTR:
379 cbp->bio_done = g_std_done;
380 cp = LIST_FIRST(&sc->sc_geom->consumer);
381 cbp->bio_to = cp->provider;
382 G_ELI_LOGREQ(2, cbp, "Sending request.");
383 g_io_request(cbp, cp);
384 break;
385 }
386}
387
388/*
389 * This is the main function for kernel worker thread when we don't have
390 * hardware acceleration and we have to do cryptography in software.
391 * Dedicated thread is needed, so we don't slow down g_up/g_down GEOM
392 * threads with crypto work.
393 */
394static void
395g_eli_worker(void *arg)
396{
397 struct g_eli_softc *sc;
398 struct g_eli_worker *wr;
399 struct bio *bp;
400
401 wr = arg;
402 sc = wr->w_softc;
403 mtx_lock_spin(&sched_lock);
404 sched_prio(curthread, PRIBIO);
405 if (sc->sc_crypto == G_ELI_CRYPTO_SW && g_eli_threads == 0)
406 sched_bind(curthread, wr->w_number);
407 mtx_unlock_spin(&sched_lock);
408
409 G_ELI_DEBUG(1, "Thread %s started.", curthread->td_proc->p_comm);
410
411 for (;;) {
412 mtx_lock(&sc->sc_queue_mtx);
413 bp = bioq_takefirst(&sc->sc_queue);
414 if (bp == NULL) {
415 if ((sc->sc_flags & G_ELI_FLAG_DESTROY) != 0) {
416 LIST_REMOVE(wr, w_next);
417 crypto_freesession(wr->w_sid);
418 free(wr, M_ELI);
419 G_ELI_DEBUG(1, "Thread %s exiting.",
420 curthread->td_proc->p_comm);
421 wakeup(&sc->sc_workers);
422 mtx_unlock(&sc->sc_queue_mtx);
423 kthread_exit(0);
424 }
425 msleep(sc, &sc->sc_queue_mtx, PRIBIO | PDROP,
426 "geli:w", 0);
427 continue;
428 }
429 mtx_unlock(&sc->sc_queue_mtx);
430 g_eli_crypto_run(wr, bp);
431 }
432}
433
434/*
435 * Here we generate IV. It is unique for every sector.
436 */
437static void
438g_eli_crypto_ivgen(struct g_eli_softc *sc, off_t offset, u_char *iv,
439 size_t size)
440{
441 u_char hash[SHA256_DIGEST_LENGTH];
442 SHA256_CTX ctx;
443
444 /* Copy precalculated SHA256 context for IV-Key. */
445 bcopy(&sc->sc_ivctx, &ctx, sizeof(ctx));
446 SHA256_Update(&ctx, (uint8_t *)&offset, sizeof(offset));
447 SHA256_Final(hash, &ctx);
448 bcopy(hash, iv, size);
449}
450
451/*
452 * This is the main function responsible for cryptography (ie. communication
453 * with crypto(9) subsystem).
454 */
455static void
456g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp)
457{
458 struct g_eli_softc *sc;
459 struct cryptop *crp;
460 struct cryptodesc *crd;
461 struct uio *uio;
462 struct iovec *iov;
463 u_int i, nsec, add, secsize;
464 int err, error;
465 size_t size;
466 u_char *p, *data;
467
468 G_ELI_LOGREQ(3, bp, "%s", __func__);
469
470 bp->bio_pflags = wr->w_number;
471 sc = wr->w_softc;
472 secsize = LIST_FIRST(&sc->sc_geom->provider)->sectorsize;
473 nsec = bp->bio_length / secsize;
474
475 /*
476 * Calculate how much memory do we need.
477 * We need separate crypto operation for every single sector.
478 * It is much faster to calculate total amount of needed memory here and
479 * do the allocation once instead of allocating memory in pieces (many,
480 * many pieces).
481 */
482 size = sizeof(*crp) * nsec;
483 size += sizeof(*crd) * nsec;
484 size += sizeof(*uio) * nsec;
485 size += sizeof(*iov) * nsec;
486 /*
487 * If we write the data we cannot destroy current bio_data content,
488 * so we need to allocate more memory for encrypted data.
489 */
490 if (bp->bio_cmd == BIO_WRITE)
491 size += bp->bio_length;
492 p = malloc(size, M_ELI, M_WAITOK);
493
494 bp->bio_inbed = 0;
495 bp->bio_children = nsec;
496 bp->bio_driver2 = p;
497
498 if (bp->bio_cmd == BIO_READ)
499 data = bp->bio_data;
500 else {
501 data = p;
502 p += bp->bio_length;
503 bcopy(bp->bio_data, data, bp->bio_length);
504 }
505
506 error = 0;
507 for (i = 0, add = 0; i < nsec; i++, add += secsize) {
508 crp = (struct cryptop *)p; p += sizeof(*crp);
509 crd = (struct cryptodesc *)p; p += sizeof(*crd);
510 uio = (struct uio *)p; p += sizeof(*uio);
511 iov = (struct iovec *)p; p += sizeof(*iov);
512
513 iov->iov_len = secsize;
514 iov->iov_base = data;
515 data += secsize;
516
517 uio->uio_iov = iov;
518 uio->uio_iovcnt = 1;
519 uio->uio_segflg = UIO_SYSSPACE;
520 uio->uio_resid = secsize;
521
522 crp->crp_sid = wr->w_sid;
523 crp->crp_ilen = secsize;
524 crp->crp_olen = secsize;
525 crp->crp_opaque = (void *)bp;
526 crp->crp_buf = (void *)uio;
527 if (bp->bio_cmd == BIO_WRITE)
528 crp->crp_callback = g_eli_crypto_write_done;
529 else /* if (bp->bio_cmd == BIO_READ) */
530 crp->crp_callback = g_eli_crypto_read_done;
531 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC | CRYPTO_F_REL;
532 crp->crp_desc = crd;
533
534 crd->crd_skip = 0;
535 crd->crd_len = secsize;
536 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
537 if (bp->bio_cmd == BIO_WRITE)
538 crd->crd_flags |= CRD_F_ENCRYPT;
539 crd->crd_alg = sc->sc_algo;
540 crd->crd_key = sc->sc_datakey;
541 crd->crd_klen = sc->sc_keylen;
542 g_eli_crypto_ivgen(sc, bp->bio_offset + add, crd->crd_iv,
543 sizeof(crd->crd_iv));
544 crd->crd_next = NULL;
545
546 crp->crp_etype = 0;
547 err = crypto_dispatch(crp);
548 if (error == 0)
549 error = err;
550 }
551 if (bp->bio_error == 0)
552 bp->bio_error = error;
553}
554
555int
556g_eli_read_metadata(struct g_class *mp, struct g_provider *pp,
557 struct g_eli_metadata *md)
558{
559 struct g_geom *gp;
560 struct g_consumer *cp;
561 u_char *buf = NULL;
562 int error;
563
564 g_topology_assert();
565
566 gp = g_new_geomf(mp, "eli:taste");
567 gp->start = g_eli_start;
568 gp->access = g_std_access;
569 /*
570 * g_eli_read_metadata() is always called from the event thread.
571 * Our geom is created and destroyed in the same event, so there
572 * could be no orphan nor spoil event in the meantime.
573 */
574 gp->orphan = g_eli_orphan_spoil_assert;
575 gp->spoiled = g_eli_orphan_spoil_assert;
576 cp = g_new_consumer(gp);
577 error = g_attach(cp, pp);
578 if (error != 0)
579 goto end;
580 error = g_access(cp, 1, 0, 0);
581 if (error != 0)
582 goto end;
583 g_topology_unlock();
584 buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
585 &error);
586 g_topology_lock();
587 if (buf == NULL)
588 goto end;
589 eli_metadata_decode(buf, md);
590end:
591 if (buf != NULL)
592 g_free(buf);
593 if (cp->provider != NULL) {
594 if (cp->acr == 1)
595 g_access(cp, -1, 0, 0);
596 g_detach(cp);
597 }
598 g_destroy_consumer(cp);
599 g_destroy_geom(gp);
600 return (error);
601}
602
603/*
604 * The function is called when we had last close on provider and user requested
605 * to close it when this situation occur.
606 */
607static void
608g_eli_last_close(struct g_eli_softc *sc)
609{
610 struct g_geom *gp;
611 struct g_provider *pp;
612 char ppname[64];
613 int error;
614
615 g_topology_assert();
616 gp = sc->sc_geom;
617 pp = LIST_FIRST(&gp->provider);
618 strlcpy(ppname, pp->name, sizeof(ppname));
619 error = g_eli_destroy(sc, 1);
620 KASSERT(error == 0, ("Cannot detach %s on last close (error=%d).",
621 ppname, error));
622 G_ELI_DEBUG(0, "Detached %s on last close.", ppname);
623}
624
625int
626g_eli_access(struct g_provider *pp, int dr, int dw, int de)
627{
628 struct g_eli_softc *sc;
629 struct g_geom *gp;
630
631 gp = pp->geom;
632 sc = gp->softc;
633
634 if (dw > 0) {
635 /* Someone is opening us for write, we need to remember that. */
636 sc->sc_flags |= G_ELI_FLAG_WOPEN;
637 return (0);
638 }
639 /* Is this the last close? */
640 if (pp->acr + dr > 0 || pp->acw + dw > 0 || pp->ace + de > 0)
641 return (0);
642
643 /*
644 * Automatically detach on last close if requested.
645 */
646 if ((sc->sc_flags & G_ELI_FLAG_RW_DETACH) ||
647 (sc->sc_flags & G_ELI_FLAG_WOPEN)) {
648 g_eli_last_close(sc);
649 }
650 return (0);
651}
652
653struct g_geom *
654g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
655 const struct g_eli_metadata *md, const u_char *mkey, int nkey)
656{
657 struct g_eli_softc *sc;
658 struct g_eli_worker *wr;
659 struct g_geom *gp;
660 struct g_provider *pp;
661 struct g_consumer *cp;
662 struct cryptoini cri;
663 u_int i, threads;
664 int error;
665
666 G_ELI_DEBUG(1, "Creating device %s%s.", bpp->name, G_ELI_SUFFIX);
667
668 gp = g_new_geomf(mp, "%s%s", bpp->name, G_ELI_SUFFIX);
669 gp->softc = NULL; /* for a moment */
670
671 sc = malloc(sizeof(*sc), M_ELI, M_WAITOK | M_ZERO);
672 gp->start = g_eli_start;
673 /*
674 * Spoiling cannot happen actually, because we keep provider open for
675 * writing all the time.
676 */
677 gp->spoiled = g_eli_orphan_spoil_assert;
678 gp->orphan = g_eli_orphan;
679 /*
680 * If detach-on-last-close feature is not enabled, we can simply use
681 * g_std_access().
682 */
683 if (md->md_flags & G_ELI_FLAG_WO_DETACH)
684 gp->access = g_eli_access;
685 else
686 gp->access = g_std_access;
687 gp->dumpconf = g_eli_dumpconf;
688
689 sc->sc_crypto = G_ELI_CRYPTO_SW;
690 sc->sc_flags = md->md_flags;
691 sc->sc_algo = md->md_algo;
692 sc->sc_nkey = nkey;
693 /*
694 * Remember the keys in our softc structure.
695 */
696 bcopy(mkey, sc->sc_ivkey, sizeof(sc->sc_ivkey));
697 mkey += sizeof(sc->sc_ivkey);
698 bcopy(mkey, sc->sc_datakey, sizeof(sc->sc_datakey));
699 sc->sc_keylen = md->md_keylen;
700
701 /*
702 * Precalculate SHA256 for IV generation.
703 * This is expensive operation and we can do it only once now or for
704 * every access to sector, so now will be much better.
705 */
706 SHA256_Init(&sc->sc_ivctx);
707 SHA256_Update(&sc->sc_ivctx, sc->sc_ivkey, sizeof(sc->sc_ivkey));
708
709 gp->softc = sc;
710 sc->sc_geom = gp;
711
712 bioq_init(&sc->sc_queue);
713 mtx_init(&sc->sc_queue_mtx, "geli:queue", NULL, MTX_DEF);
714
715 pp = NULL;
716 cp = g_new_consumer(gp);
717 error = g_attach(cp, bpp);
718 if (error != 0) {
719 if (req != NULL) {
720 gctl_error(req, "Cannot attach to %s (error=%d).",
721 bpp->name, error);
722 } else {
723 G_ELI_DEBUG(1, "Cannot attach to %s (error=%d).",
724 bpp->name, error);
725 }
726 goto failed;
727 }
728 /*
729 * Keep provider open all the time, so we can run critical tasks,
730 * like Master Keys deletion, without wondering if we can open
731 * provider or not.
732 */
733 error = g_access(cp, 1, 1, 1);
734 if (error != 0) {
735 if (req != NULL) {
736 gctl_error(req, "Cannot access %s (error=%d).",
737 bpp->name, error);
738 } else {
739 G_ELI_DEBUG(1, "Cannot access %s (error=%d).",
740 bpp->name, error);
741 }
742 goto failed;
743 }
744
745 LIST_INIT(&sc->sc_workers);
746
747 bzero(&cri, sizeof(cri));
748 cri.cri_alg = sc->sc_algo;
749 cri.cri_klen = sc->sc_keylen;
750 cri.cri_key = sc->sc_datakey;
751
752 threads = g_eli_threads;
753 if (threads == 0)
754 threads = mp_ncpus;
755 else if (threads > mp_ncpus) {
756 /* There is really no need for too many worker threads. */
757 threads = mp_ncpus;
758 G_ELI_DEBUG(0, "Reducing number of threads to %u.", threads);
759 }
760 for (i = 0; i < threads; i++) {
761 wr = malloc(sizeof(*wr), M_ELI, M_WAITOK | M_ZERO);
762 wr->w_softc = sc;
763 wr->w_number = i;
764
765 /*
766 * If this is the first pass, try to get hardware support.
767 * Use software cryptography, if we cannot get it.
768 */
769 if (i == 0) {
770 error = crypto_newsession(&wr->w_sid, &cri, 1);
771 if (error == 0)
772 sc->sc_crypto = G_ELI_CRYPTO_HW;
773 }
774 if (sc->sc_crypto == G_ELI_CRYPTO_SW)
775 error = crypto_newsession(&wr->w_sid, &cri, 0);
776 if (error != 0) {
777 free(wr, M_ELI);
778 if (req != NULL) {
779 gctl_error(req, "Cannot set up crypto session "
780 "for %s (error=%d).", bpp->name, error);
781 } else {
782 G_ELI_DEBUG(1, "Cannot set up crypto session "
783 "for %s (error=%d).", bpp->name, error);
784 }
785 goto failed;
786 }
787
788 error = kthread_create(g_eli_worker, wr, &wr->w_proc, 0, 0,
789 "g_eli[%u] %s", i, bpp->name);
790 if (error != 0) {
791 crypto_freesession(wr->w_sid);
792 free(wr, M_ELI);
793 if (req != NULL) {
794 gctl_error(req, "Cannot create kernel thread "
795 "for %s (error=%d).", bpp->name, error);
796 } else {
797 G_ELI_DEBUG(1, "Cannot create kernel thread "
798 "for %s (error=%d).", bpp->name, error);
799 }
800 goto failed;
801 }
802 LIST_INSERT_HEAD(&sc->sc_workers, wr, w_next);
803 /* If we have hardware support, one thread is enough. */
804 if (sc->sc_crypto == G_ELI_CRYPTO_HW)
805 break;
806 }
807
808 /*
809 * Create decrypted provider.
810 */
811 pp = g_new_providerf(gp, "%s%s", bpp->name, G_ELI_SUFFIX);
812 pp->sectorsize = md->md_sectorsize;
813 pp->mediasize = bpp->mediasize;
814 if ((sc->sc_flags & G_ELI_FLAG_ONETIME) == 0)
815 pp->mediasize -= bpp->sectorsize;
816 pp->mediasize -= (pp->mediasize % pp->sectorsize);
817 g_error_provider(pp, 0);
818
819 G_ELI_DEBUG(0, "Device %s created.", pp->name);
820 G_ELI_DEBUG(0, " Cipher: %s", g_eli_algo2str(sc->sc_algo));
821 G_ELI_DEBUG(0, "Key length: %u", sc->sc_keylen);
822 G_ELI_DEBUG(0, " Crypto: %s",
823 sc->sc_crypto == G_ELI_CRYPTO_SW ? "software" : "hardware");
824 return (gp);
825failed:
826 mtx_lock(&sc->sc_queue_mtx);
827 sc->sc_flags |= G_ELI_FLAG_DESTROY;
828 wakeup(sc);
829 /*
830 * Wait for kernel threads self destruction.
831 */
832 while (!LIST_EMPTY(&sc->sc_workers)) {
833 msleep(&sc->sc_workers, &sc->sc_queue_mtx, PRIBIO,
834 "geli:destroy", 0);
835 }
836 mtx_destroy(&sc->sc_queue_mtx);
837 if (cp->provider != NULL) {
838 if (cp->acr == 1)
839 g_access(cp, -1, -1, -1);
840 g_detach(cp);
841 }
842 g_destroy_consumer(cp);
843 g_destroy_geom(gp);
844 bzero(sc, sizeof(*sc));
845 free(sc, M_ELI);
846 return (NULL);
847}
848
849int
850g_eli_destroy(struct g_eli_softc *sc, boolean_t force)
851{
852 struct g_geom *gp;
853 struct g_provider *pp;
854
855 g_topology_assert();
856
857 if (sc == NULL)
858 return (ENXIO);
859
860 gp = sc->sc_geom;
861 pp = LIST_FIRST(&gp->provider);
862 if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
863 if (force) {
864 G_ELI_DEBUG(1, "Device %s is still open, so it "
865 "cannot be definitely removed.", pp->name);
866 } else {
867 G_ELI_DEBUG(1,
868 "Device %s is still open (r%dw%de%d).", pp->name,
869 pp->acr, pp->acw, pp->ace);
870 return (EBUSY);
871 }
872 }
873
874 mtx_lock(&sc->sc_queue_mtx);
875 sc->sc_flags |= G_ELI_FLAG_DESTROY;
876 wakeup(sc);
877 while (!LIST_EMPTY(&sc->sc_workers)) {
878 msleep(&sc->sc_workers, &sc->sc_queue_mtx, PRIBIO,
879 "geli:destroy", 0);
880 }
881 mtx_destroy(&sc->sc_queue_mtx);
882 gp->softc = NULL;
883 bzero(sc, sizeof(*sc));
884 free(sc, M_ELI);
885
886 if (pp == NULL || (pp->acr == 0 && pp->acw == 0 && pp->ace == 0))
887 G_ELI_DEBUG(0, "Device %s destroyed.", gp->name);
888 g_wither_geom_close(gp, ENXIO);
889
890 return (0);
891}
892
893static int
894g_eli_destroy_geom(struct gctl_req *req __unused,
895 struct g_class *mp __unused, struct g_geom *gp)
896{
897 struct g_eli_softc *sc;
898
899 sc = gp->softc;
900 return (g_eli_destroy(sc, 0));
901}
902
903static int
904g_eli_keyfiles_load(struct hmac_ctx *ctx, const char *provider)
905{
906 u_char *keyfile, *data, *size;
907 char *file, name[64];
908 int i;
909
910 for (i = 0; ; i++) {
911 snprintf(name, sizeof(name), "%s:geli_keyfile%d", provider, i);
912 keyfile = preload_search_by_type(name);
913 if (keyfile == NULL)
914 return (i); /* Return number of loaded keyfiles. */
915 data = preload_search_info(keyfile, MODINFO_ADDR);
916 if (data == NULL) {
917 G_ELI_DEBUG(0, "Cannot find key file data for %s.",
918 name);
919 return (0);
920 }
921 data = *(void **)data;
922 size = preload_search_info(keyfile, MODINFO_SIZE);
923 if (size == NULL) {
924 G_ELI_DEBUG(0, "Cannot find key file size for %s.",
925 name);
926 return (0);
927 }
928 file = preload_search_info(keyfile, MODINFO_NAME);
929 if (file == NULL) {
930 G_ELI_DEBUG(0, "Cannot find key file name for %s.",
931 name);
932 return (0);
933 }
934 G_ELI_DEBUG(1, "Loaded keyfile %s for %s (type: %s).", file,
935 provider, name);
936 g_eli_crypto_hmac_update(ctx, data, *(size_t *)size);
937 }
938}
939
940static void
941g_eli_keyfiles_clear(const char *provider)
942{
943 u_char *keyfile, *data, *size;
944 char name[64];
945 int i;
946
947 for (i = 0; ; i++) {
948 snprintf(name, sizeof(name), "%s:geli_keyfile%d", provider, i);
949 keyfile = preload_search_by_type(name);
950 if (keyfile == NULL)
951 return;
952 data = preload_search_info(keyfile, MODINFO_ADDR);
953 size = preload_search_info(keyfile, MODINFO_SIZE);
954 if (data == NULL || size == NULL)
955 continue;
956 data = *(void **)data;
957 bzero(data, *(size_t *)size);
958 }
959}
960
961/*
962 * Tasting is only made on boot.
963 * We detect providers which should be attached before root is mounted.
964 */
965static struct g_geom *
966g_eli_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
967{
968 struct g_eli_metadata md;
969 struct g_geom *gp;
970 struct hmac_ctx ctx;
971 char passphrase[256];
972 u_char key[G_ELI_USERKEYLEN], mkey[G_ELI_DATAIVKEYLEN];
973 u_int i, nkey, nkeyfiles, tries;
974 int error;
975
976 g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
977 g_topology_assert();
978
979 if (rootvnode != NULL || g_eli_tries == 0)
980 return (NULL);
981
982 G_ELI_DEBUG(3, "Tasting %s.", pp->name);
983
984 error = g_eli_read_metadata(mp, pp, &md);
985 if (error != 0)
986 return (NULL);
987 gp = NULL;
988
989 if (strcmp(md.md_magic, G_ELI_MAGIC) != 0)
990 return (NULL);
991 if (md.md_version > G_ELI_VERSION) {
992 printf("geom_eli.ko module is too old to handle %s.\n",
993 pp->name);
994 return (NULL);
995 }
996 if (md.md_provsize != pp->mediasize)
997 return (NULL);
998 /* Should we attach it on boot? */
999 if ((md.md_flags & G_ELI_FLAG_BOOT) == 0)
1000 return (NULL);
1001 if (md.md_keys == 0x00) {
1002 G_ELI_DEBUG(0, "No valid keys on %s.", pp->name);
1003 return (NULL);
1004 }
1005 if (md.md_iterations == -1) {
1006 /* If there is no passphrase, we try only once. */
1007 tries = 1;
1008 } else {
1009 /* Ask for the passphrase no more than g_eli_tries times. */
1010 tries = g_eli_tries;
1011 }
1012
1013 for (i = 0; i < tries; i++) {
1014 g_eli_crypto_hmac_init(&ctx, NULL, 0);
1015
1016 /*
1017 * Load all key files.
1018 */
1019 nkeyfiles = g_eli_keyfiles_load(&ctx, pp->name);
1020
1021 if (nkeyfiles == 0 && md.md_iterations == -1) {
1022 /*
1023 * No key files and no passphrase, something is
1024 * definitely wrong here.
1025 * geli(8) doesn't allow for such situation, so assume
1026 * that there was really no passphrase and in that case
1027 * key files are no properly defined in loader.conf.
1028 */
1029 G_ELI_DEBUG(0,
1030 "Found no key files in loader.conf for %s.",
1031 pp->name);
1032 return (NULL);
1033 }
1034
1035 /* Ask for the passphrase if defined. */
1036 if (md.md_iterations >= 0) {
1037 printf("Enter passphrase for %s: ", pp->name);
1038 gets(passphrase, sizeof(passphrase),
1039 g_eli_visible_passphrase);
1040 }
1041
1042 /*
1043 * Prepare Derived-Key from the user passphrase.
1044 */
1045 if (md.md_iterations == 0) {
1046 g_eli_crypto_hmac_update(&ctx, md.md_salt,
1047 sizeof(md.md_salt));
1048 g_eli_crypto_hmac_update(&ctx, passphrase,
1049 strlen(passphrase));
1050 } else if (md.md_iterations > 0) {
1051 u_char dkey[G_ELI_USERKEYLEN];
1052
1053 pkcs5v2_genkey(dkey, sizeof(dkey), md.md_salt,
1054 sizeof(md.md_salt), passphrase, md.md_iterations);
1055 g_eli_crypto_hmac_update(&ctx, dkey, sizeof(dkey));
1056 bzero(dkey, sizeof(dkey));
1057 }
1058
1059 g_eli_crypto_hmac_final(&ctx, key, 0);
1060
1061 /*
1062 * Decrypt Master-Key.
1063 */
1064 error = g_eli_mkey_decrypt(&md, key, mkey, &nkey);
1065 bzero(key, sizeof(key));
1066 if (error == -1) {
1067 if (i == tries - 1) {
1068 G_ELI_DEBUG(0,
1069 "Wrong key for %s. No tries left.",
1070 pp->name);
1071 g_eli_keyfiles_clear(pp->name);
1072 return (NULL);
1073 }
1074 G_ELI_DEBUG(0, "Wrong key for %s. Tries left: %u.",
1075 pp->name, tries - i - 1);
1076 /* Try again. */
1077 continue;
1078 } else if (error > 0) {
1079 G_ELI_DEBUG(0, "Cannot decrypt Master Key for %s (error=%d).",
1080 pp->name, error);
1081 g_eli_keyfiles_clear(pp->name);
1082 return (NULL);
1083 }
1084 G_ELI_DEBUG(1, "Using Master Key %u for %s.", nkey, pp->name);
1085 break;
1086 }
1087
1088 /*
1089 * We have correct key, let's attach provider.
1090 */
1091 gp = g_eli_create(NULL, mp, pp, &md, mkey, nkey);
1092 bzero(mkey, sizeof(mkey));
1093 bzero(&md, sizeof(md));
1094 if (gp == NULL) {
1095 G_ELI_DEBUG(0, "Cannot create device %s%s.", pp->name,
1096 G_ELI_SUFFIX);
1097 return (NULL);
1098 }
1099 return (gp);
1100}
1101
1102static void
1103g_eli_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
1104 struct g_consumer *cp, struct g_provider *pp)
1105{
1106 struct g_eli_softc *sc;
1107
1108 g_topology_assert();
1109 sc = gp->softc;
1110 if (sc == NULL)
1111 return;
1112 if (pp != NULL || cp != NULL)
1113 return; /* Nothing here. */
1114 sbuf_printf(sb, "%s<Flags>", indent);
1115 if (sc->sc_flags == 0)
1116 sbuf_printf(sb, "NONE");
1117 else {
1118 int first = 1;
1119
1120#define ADD_FLAG(flag, name) do { \
1121 if ((sc->sc_flags & (flag)) != 0) { \
1122 if (!first) \
1123 sbuf_printf(sb, ", "); \
1124 else \
1125 first = 0; \
1126 sbuf_printf(sb, name); \
1127 } \
1128} while (0)
1129 ADD_FLAG(G_ELI_FLAG_ONETIME, "ONETIME");
1130 ADD_FLAG(G_ELI_FLAG_BOOT, "BOOT");
1131 ADD_FLAG(G_ELI_FLAG_WO_DETACH, "W-DETACH");
1132 ADD_FLAG(G_ELI_FLAG_RW_DETACH, "RW-DETACH");
1133 ADD_FLAG(G_ELI_FLAG_WOPEN, "W-OPEN");
1134 ADD_FLAG(G_ELI_FLAG_DESTROY, "DESTROY");
1135#undef ADD_FLAG
1136 }
1137 sbuf_printf(sb, "</Flags>\n");
1138
1139 if ((sc->sc_flags & G_ELI_FLAG_ONETIME) == 0) {
1140 sbuf_printf(sb, "%s<UsedKey>%u</UsedKey>\n", indent,
1141 sc->sc_nkey);
1142 }
1143 sbuf_printf(sb, "%s<Crypto>", indent);
1144 switch (sc->sc_crypto) {
1145 case G_ELI_CRYPTO_HW:
1146 sbuf_printf(sb, "hardware");
1147 break;
1148 case G_ELI_CRYPTO_SW:
1149 sbuf_printf(sb, "software");
1150 break;
1151 default:
1152 sbuf_printf(sb, "UNKNOWN");
1153 break;
1154 }
1155 sbuf_printf(sb, "</Crypto>\n");
1156 sbuf_printf(sb, "%s<KeyLength>%u</KeyLength>\n", indent, sc->sc_keylen);
1157 sbuf_printf(sb, "%s<Cipher>%s</Cipher>\n", indent,
1158 g_eli_algo2str(sc->sc_algo));
1159}
1160
1161DECLARE_GEOM_CLASS(g_eli_class, g_eli);
1162MODULE_DEPEND(geom_eli, crypto, 1, 1, 1);
2 * Copyright (c) 2005 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/geom/eli/g_eli.c 159306 2006-06-05 21:25:19Z pjd $");
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/kernel.h>
33#include <sys/linker.h>
34#include <sys/module.h>
35#include <sys/lock.h>
36#include <sys/mutex.h>
37#include <sys/bio.h>
38#include <sys/sysctl.h>
39#include <sys/malloc.h>
40#include <sys/kthread.h>
41#include <sys/proc.h>
42#include <sys/sched.h>
43#include <sys/smp.h>
44#include <sys/uio.h>
45#include <sys/vnode.h>
46
47#include <vm/uma.h>
48
49#include <geom/geom.h>
50#include <geom/eli/g_eli.h>
51#include <geom/eli/pkcs5v2.h>
52
53
54MALLOC_DEFINE(M_ELI, "eli data", "GEOM_ELI Data");
55
56SYSCTL_DECL(_kern_geom);
57SYSCTL_NODE(_kern_geom, OID_AUTO, eli, CTLFLAG_RW, 0, "GEOM_ELI stuff");
58u_int g_eli_debug = 0;
59TUNABLE_INT("kern.geom.eli.debug", &g_eli_debug);
60SYSCTL_UINT(_kern_geom_eli, OID_AUTO, debug, CTLFLAG_RW, &g_eli_debug, 0,
61 "Debug level");
62static u_int g_eli_tries = 3;
63TUNABLE_INT("kern.geom.eli.tries", &g_eli_tries);
64SYSCTL_UINT(_kern_geom_eli, OID_AUTO, tries, CTLFLAG_RW, &g_eli_tries, 0,
65 "Number of tries for entering the passphrase");
66static u_int g_eli_visible_passphrase = 0;
67TUNABLE_INT("kern.geom.eli.visible_passphrase", &g_eli_visible_passphrase);
68SYSCTL_UINT(_kern_geom_eli, OID_AUTO, visible_passphrase, CTLFLAG_RW,
69 &g_eli_visible_passphrase, 0,
70 "Turn on echo when entering the passphrase (for debug purposes only!!)");
71u_int g_eli_overwrites = 5;
72TUNABLE_INT("kern.geom.eli.overwrites", &g_eli_overwrites);
73SYSCTL_UINT(_kern_geom_eli, OID_AUTO, overwrites, CTLFLAG_RW, &g_eli_overwrites,
74 0, "Number of times on-disk keys should be overwritten when destroying them");
75static u_int g_eli_threads = 0;
76TUNABLE_INT("kern.geom.eli.threads", &g_eli_threads);
77SYSCTL_UINT(_kern_geom_eli, OID_AUTO, threads, CTLFLAG_RW, &g_eli_threads, 0,
78 "Number of threads doing crypto work");
79
80static int g_eli_destroy_geom(struct gctl_req *req, struct g_class *mp,
81 struct g_geom *gp);
82static void g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp);
83
84static g_taste_t g_eli_taste;
85static g_dumpconf_t g_eli_dumpconf;
86
87struct g_class g_eli_class = {
88 .name = G_ELI_CLASS_NAME,
89 .version = G_VERSION,
90 .ctlreq = g_eli_config,
91 .taste = g_eli_taste,
92 .destroy_geom = g_eli_destroy_geom
93};
94
95
96/*
97 * Code paths:
98 * BIO_READ:
99 * g_eli_start -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
100 * BIO_WRITE:
101 * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
102 */
103
104
105/*
106 * EAGAIN from crypto(9) means, that we were probably balanced to another crypto
107 * accelerator or something like this.
108 * The function updates the SID and rerun the operation.
109 */
110static int
111g_eli_crypto_rerun(struct cryptop *crp)
112{
113 struct g_eli_softc *sc;
114 struct g_eli_worker *wr;
115 struct bio *bp;
116 int error;
117
118 bp = (struct bio *)crp->crp_opaque;
119 sc = bp->bio_to->geom->softc;
120 LIST_FOREACH(wr, &sc->sc_workers, w_next) {
121 if (wr->w_number == bp->bio_pflags)
122 break;
123 }
124 KASSERT(wr != NULL, ("Invalid worker (%u).", bp->bio_pflags));
125 G_ELI_DEBUG(1, "Rerunning crypto %s request (sid: %ju -> %ju).",
126 bp->bio_cmd == BIO_READ ? "READ" : "WRITE", (uintmax_t)wr->w_sid,
127 (uintmax_t)crp->crp_sid);
128 wr->w_sid = crp->crp_sid;
129 crp->crp_etype = 0;
130 error = crypto_dispatch(crp);
131 if (error == 0)
132 return (0);
133 G_ELI_DEBUG(1, "%s: crypto_dispatch() returned %d.", __func__, error);
134 crp->crp_etype = error;
135 return (error);
136}
137
138/*
139 * The function is called afer reading encrypted data from the provider.
140 *
141 * g_eli_start -> g_io_request -> G_ELI_READ_DONE -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
142 */
143static void
144g_eli_read_done(struct bio *bp)
145{
146 struct g_eli_softc *sc;
147 struct bio *pbp;
148
149 G_ELI_LOGREQ(2, bp, "Request done.");
150 pbp = bp->bio_parent;
151 if (pbp->bio_error == 0)
152 pbp->bio_error = bp->bio_error;
153 g_destroy_bio(bp);
154 if (pbp->bio_error != 0) {
155 G_ELI_LOGREQ(0, pbp, "%s() failed", __func__);
156 pbp->bio_completed = 0;
157 g_io_deliver(pbp, pbp->bio_error);
158 return;
159 }
160 sc = pbp->bio_to->geom->softc;
161 mtx_lock(&sc->sc_queue_mtx);
162 bioq_insert_tail(&sc->sc_queue, pbp);
163 mtx_unlock(&sc->sc_queue_mtx);
164 wakeup(sc);
165}
166
167/*
168 * The function is called after we read and decrypt data.
169 *
170 * g_eli_start -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver
171 */
172static int
173g_eli_crypto_read_done(struct cryptop *crp)
174{
175 struct bio *bp;
176
177 if (crp->crp_etype == EAGAIN) {
178 if (g_eli_crypto_rerun(crp) == 0)
179 return (0);
180 }
181 bp = (struct bio *)crp->crp_opaque;
182 bp->bio_inbed++;
183 if (crp->crp_etype == 0) {
184 G_ELI_DEBUG(3, "Crypto READ request done (%d/%d).",
185 bp->bio_inbed, bp->bio_children);
186 bp->bio_completed += crp->crp_olen;
187 } else {
188 G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.",
189 bp->bio_inbed, bp->bio_children, crp->crp_etype);
190 if (bp->bio_error == 0)
191 bp->bio_error = crp->crp_etype;
192 }
193 /*
194 * Do we have all sectors already?
195 */
196 if (bp->bio_inbed < bp->bio_children)
197 return (0);
198 free(bp->bio_driver2, M_ELI);
199 bp->bio_driver2 = NULL;
200 if (bp->bio_error != 0) {
201 G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).",
202 bp->bio_error);
203 bp->bio_completed = 0;
204 }
205 /*
206 * Read is finished, send it up.
207 */
208 g_io_deliver(bp, bp->bio_error);
209 return (0);
210}
211
212/*
213 * The function is called after we encrypt and write data.
214 *
215 * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> G_ELI_WRITE_DONE -> g_io_deliver
216 */
217static void
218g_eli_write_done(struct bio *bp)
219{
220 struct bio *pbp;
221
222 G_ELI_LOGREQ(2, bp, "Request done.");
223 pbp = bp->bio_parent;
224 if (pbp->bio_error == 0)
225 pbp->bio_error = bp->bio_error;
226 free(pbp->bio_driver2, M_ELI);
227 pbp->bio_driver2 = NULL;
228 if (pbp->bio_error == 0)
229 pbp->bio_completed = pbp->bio_length;
230 else {
231 G_ELI_LOGREQ(0, pbp, "Crypto WRITE request failed (error=%d).",
232 pbp->bio_error);
233 pbp->bio_completed = 0;
234 }
235 g_destroy_bio(bp);
236 /*
237 * Write is finished, send it up.
238 */
239 g_io_deliver(pbp, pbp->bio_error);
240}
241
242/*
243 * The function is called after data encryption.
244 *
245 * g_eli_start -> g_eli_crypto_run -> G_ELI_CRYPTO_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver
246 */
247static int
248g_eli_crypto_write_done(struct cryptop *crp)
249{
250 struct g_geom *gp;
251 struct g_consumer *cp;
252 struct bio *bp, *cbp;
253
254 if (crp->crp_etype == EAGAIN) {
255 if (g_eli_crypto_rerun(crp) == 0)
256 return (0);
257 }
258 bp = (struct bio *)crp->crp_opaque;
259 bp->bio_inbed++;
260 if (crp->crp_etype == 0) {
261 G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).",
262 bp->bio_inbed, bp->bio_children);
263 } else {
264 G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.",
265 bp->bio_inbed, bp->bio_children, crp->crp_etype);
266 if (bp->bio_error == 0)
267 bp->bio_error = crp->crp_etype;
268 }
269 /*
270 * All sectors are already encrypted?
271 */
272 if (bp->bio_inbed < bp->bio_children)
273 return (0);
274 bp->bio_inbed = 0;
275 bp->bio_children = 1;
276 cbp = bp->bio_driver1;
277 bp->bio_driver1 = NULL;
278 if (bp->bio_error != 0) {
279 G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
280 bp->bio_error);
281 free(bp->bio_driver2, M_ELI);
282 bp->bio_driver2 = NULL;
283 g_destroy_bio(cbp);
284 g_io_deliver(bp, bp->bio_error);
285 return (0);
286 }
287 cbp->bio_data = bp->bio_driver2;
288 cbp->bio_done = g_eli_write_done;
289 gp = bp->bio_to->geom;
290 cp = LIST_FIRST(&gp->consumer);
291 cbp->bio_to = cp->provider;
292 G_ELI_LOGREQ(2, cbp, "Sending request.");
293 /*
294 * Send encrypted data to the provider.
295 */
296 g_io_request(cbp, cp);
297 return (0);
298}
299
300/*
301 * This function should never be called, but GEOM made as it set ->orphan()
302 * method for every geom.
303 */
304static void
305g_eli_orphan_spoil_assert(struct g_consumer *cp)
306{
307
308 panic("Function %s() called for %s.", __func__, cp->geom->name);
309}
310
311static void
312g_eli_orphan(struct g_consumer *cp)
313{
314 struct g_eli_softc *sc;
315
316 g_topology_assert();
317 sc = cp->geom->softc;
318 if (sc == NULL)
319 return;
320 g_eli_destroy(sc, 1);
321}
322
323/*
324 * BIO_READ : G_ELI_START -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
325 * BIO_WRITE: G_ELI_START -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
326 */
327static void
328g_eli_start(struct bio *bp)
329{
330 struct g_eli_softc *sc;
331 struct g_consumer *cp;
332 struct bio *cbp;
333
334 sc = bp->bio_to->geom->softc;
335 KASSERT(sc != NULL,
336 ("Provider's error should be set (error=%d)(device=%s).",
337 bp->bio_to->error, bp->bio_to->name));
338 G_ELI_LOGREQ(2, bp, "Request received.");
339
340 switch (bp->bio_cmd) {
341 case BIO_READ:
342 case BIO_WRITE:
343 case BIO_GETATTR:
344 break;
345 case BIO_DELETE:
346 /*
347 * We could eventually support BIO_DELETE request.
348 * It could be done by overwritting requested sector with
349 * random data g_eli_overwrites number of times.
350 */
351 default:
352 g_io_deliver(bp, EOPNOTSUPP);
353 return;
354 }
355 cbp = g_clone_bio(bp);
356 if (cbp == NULL) {
357 g_io_deliver(bp, ENOMEM);
358 return;
359 }
360 switch (bp->bio_cmd) {
361 case BIO_READ:
362 cbp->bio_done = g_eli_read_done;
363 cp = LIST_FIRST(&sc->sc_geom->consumer);
364 cbp->bio_to = cp->provider;
365 G_ELI_LOGREQ(2, cbp, "Sending request.");
366 /*
367 * Read encrypted data from provider.
368 */
369 g_io_request(cbp, cp);
370 break;
371 case BIO_WRITE:
372 bp->bio_driver1 = cbp;
373 mtx_lock(&sc->sc_queue_mtx);
374 bioq_insert_tail(&sc->sc_queue, bp);
375 mtx_unlock(&sc->sc_queue_mtx);
376 wakeup(sc);
377 break;
378 case BIO_GETATTR:
379 cbp->bio_done = g_std_done;
380 cp = LIST_FIRST(&sc->sc_geom->consumer);
381 cbp->bio_to = cp->provider;
382 G_ELI_LOGREQ(2, cbp, "Sending request.");
383 g_io_request(cbp, cp);
384 break;
385 }
386}
387
388/*
389 * This is the main function for kernel worker thread when we don't have
390 * hardware acceleration and we have to do cryptography in software.
391 * Dedicated thread is needed, so we don't slow down g_up/g_down GEOM
392 * threads with crypto work.
393 */
394static void
395g_eli_worker(void *arg)
396{
397 struct g_eli_softc *sc;
398 struct g_eli_worker *wr;
399 struct bio *bp;
400
401 wr = arg;
402 sc = wr->w_softc;
403 mtx_lock_spin(&sched_lock);
404 sched_prio(curthread, PRIBIO);
405 if (sc->sc_crypto == G_ELI_CRYPTO_SW && g_eli_threads == 0)
406 sched_bind(curthread, wr->w_number);
407 mtx_unlock_spin(&sched_lock);
408
409 G_ELI_DEBUG(1, "Thread %s started.", curthread->td_proc->p_comm);
410
411 for (;;) {
412 mtx_lock(&sc->sc_queue_mtx);
413 bp = bioq_takefirst(&sc->sc_queue);
414 if (bp == NULL) {
415 if ((sc->sc_flags & G_ELI_FLAG_DESTROY) != 0) {
416 LIST_REMOVE(wr, w_next);
417 crypto_freesession(wr->w_sid);
418 free(wr, M_ELI);
419 G_ELI_DEBUG(1, "Thread %s exiting.",
420 curthread->td_proc->p_comm);
421 wakeup(&sc->sc_workers);
422 mtx_unlock(&sc->sc_queue_mtx);
423 kthread_exit(0);
424 }
425 msleep(sc, &sc->sc_queue_mtx, PRIBIO | PDROP,
426 "geli:w", 0);
427 continue;
428 }
429 mtx_unlock(&sc->sc_queue_mtx);
430 g_eli_crypto_run(wr, bp);
431 }
432}
433
434/*
435 * Here we generate IV. It is unique for every sector.
436 */
437static void
438g_eli_crypto_ivgen(struct g_eli_softc *sc, off_t offset, u_char *iv,
439 size_t size)
440{
441 u_char hash[SHA256_DIGEST_LENGTH];
442 SHA256_CTX ctx;
443
444 /* Copy precalculated SHA256 context for IV-Key. */
445 bcopy(&sc->sc_ivctx, &ctx, sizeof(ctx));
446 SHA256_Update(&ctx, (uint8_t *)&offset, sizeof(offset));
447 SHA256_Final(hash, &ctx);
448 bcopy(hash, iv, size);
449}
450
451/*
452 * This is the main function responsible for cryptography (ie. communication
453 * with crypto(9) subsystem).
454 */
455static void
456g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp)
457{
458 struct g_eli_softc *sc;
459 struct cryptop *crp;
460 struct cryptodesc *crd;
461 struct uio *uio;
462 struct iovec *iov;
463 u_int i, nsec, add, secsize;
464 int err, error;
465 size_t size;
466 u_char *p, *data;
467
468 G_ELI_LOGREQ(3, bp, "%s", __func__);
469
470 bp->bio_pflags = wr->w_number;
471 sc = wr->w_softc;
472 secsize = LIST_FIRST(&sc->sc_geom->provider)->sectorsize;
473 nsec = bp->bio_length / secsize;
474
475 /*
476 * Calculate how much memory do we need.
477 * We need separate crypto operation for every single sector.
478 * It is much faster to calculate total amount of needed memory here and
479 * do the allocation once instead of allocating memory in pieces (many,
480 * many pieces).
481 */
482 size = sizeof(*crp) * nsec;
483 size += sizeof(*crd) * nsec;
484 size += sizeof(*uio) * nsec;
485 size += sizeof(*iov) * nsec;
486 /*
487 * If we write the data we cannot destroy current bio_data content,
488 * so we need to allocate more memory for encrypted data.
489 */
490 if (bp->bio_cmd == BIO_WRITE)
491 size += bp->bio_length;
492 p = malloc(size, M_ELI, M_WAITOK);
493
494 bp->bio_inbed = 0;
495 bp->bio_children = nsec;
496 bp->bio_driver2 = p;
497
498 if (bp->bio_cmd == BIO_READ)
499 data = bp->bio_data;
500 else {
501 data = p;
502 p += bp->bio_length;
503 bcopy(bp->bio_data, data, bp->bio_length);
504 }
505
506 error = 0;
507 for (i = 0, add = 0; i < nsec; i++, add += secsize) {
508 crp = (struct cryptop *)p; p += sizeof(*crp);
509 crd = (struct cryptodesc *)p; p += sizeof(*crd);
510 uio = (struct uio *)p; p += sizeof(*uio);
511 iov = (struct iovec *)p; p += sizeof(*iov);
512
513 iov->iov_len = secsize;
514 iov->iov_base = data;
515 data += secsize;
516
517 uio->uio_iov = iov;
518 uio->uio_iovcnt = 1;
519 uio->uio_segflg = UIO_SYSSPACE;
520 uio->uio_resid = secsize;
521
522 crp->crp_sid = wr->w_sid;
523 crp->crp_ilen = secsize;
524 crp->crp_olen = secsize;
525 crp->crp_opaque = (void *)bp;
526 crp->crp_buf = (void *)uio;
527 if (bp->bio_cmd == BIO_WRITE)
528 crp->crp_callback = g_eli_crypto_write_done;
529 else /* if (bp->bio_cmd == BIO_READ) */
530 crp->crp_callback = g_eli_crypto_read_done;
531 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC | CRYPTO_F_REL;
532 crp->crp_desc = crd;
533
534 crd->crd_skip = 0;
535 crd->crd_len = secsize;
536 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
537 if (bp->bio_cmd == BIO_WRITE)
538 crd->crd_flags |= CRD_F_ENCRYPT;
539 crd->crd_alg = sc->sc_algo;
540 crd->crd_key = sc->sc_datakey;
541 crd->crd_klen = sc->sc_keylen;
542 g_eli_crypto_ivgen(sc, bp->bio_offset + add, crd->crd_iv,
543 sizeof(crd->crd_iv));
544 crd->crd_next = NULL;
545
546 crp->crp_etype = 0;
547 err = crypto_dispatch(crp);
548 if (error == 0)
549 error = err;
550 }
551 if (bp->bio_error == 0)
552 bp->bio_error = error;
553}
554
555int
556g_eli_read_metadata(struct g_class *mp, struct g_provider *pp,
557 struct g_eli_metadata *md)
558{
559 struct g_geom *gp;
560 struct g_consumer *cp;
561 u_char *buf = NULL;
562 int error;
563
564 g_topology_assert();
565
566 gp = g_new_geomf(mp, "eli:taste");
567 gp->start = g_eli_start;
568 gp->access = g_std_access;
569 /*
570 * g_eli_read_metadata() is always called from the event thread.
571 * Our geom is created and destroyed in the same event, so there
572 * could be no orphan nor spoil event in the meantime.
573 */
574 gp->orphan = g_eli_orphan_spoil_assert;
575 gp->spoiled = g_eli_orphan_spoil_assert;
576 cp = g_new_consumer(gp);
577 error = g_attach(cp, pp);
578 if (error != 0)
579 goto end;
580 error = g_access(cp, 1, 0, 0);
581 if (error != 0)
582 goto end;
583 g_topology_unlock();
584 buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
585 &error);
586 g_topology_lock();
587 if (buf == NULL)
588 goto end;
589 eli_metadata_decode(buf, md);
590end:
591 if (buf != NULL)
592 g_free(buf);
593 if (cp->provider != NULL) {
594 if (cp->acr == 1)
595 g_access(cp, -1, 0, 0);
596 g_detach(cp);
597 }
598 g_destroy_consumer(cp);
599 g_destroy_geom(gp);
600 return (error);
601}
602
603/*
604 * The function is called when we had last close on provider and user requested
605 * to close it when this situation occur.
606 */
607static void
608g_eli_last_close(struct g_eli_softc *sc)
609{
610 struct g_geom *gp;
611 struct g_provider *pp;
612 char ppname[64];
613 int error;
614
615 g_topology_assert();
616 gp = sc->sc_geom;
617 pp = LIST_FIRST(&gp->provider);
618 strlcpy(ppname, pp->name, sizeof(ppname));
619 error = g_eli_destroy(sc, 1);
620 KASSERT(error == 0, ("Cannot detach %s on last close (error=%d).",
621 ppname, error));
622 G_ELI_DEBUG(0, "Detached %s on last close.", ppname);
623}
624
625int
626g_eli_access(struct g_provider *pp, int dr, int dw, int de)
627{
628 struct g_eli_softc *sc;
629 struct g_geom *gp;
630
631 gp = pp->geom;
632 sc = gp->softc;
633
634 if (dw > 0) {
635 /* Someone is opening us for write, we need to remember that. */
636 sc->sc_flags |= G_ELI_FLAG_WOPEN;
637 return (0);
638 }
639 /* Is this the last close? */
640 if (pp->acr + dr > 0 || pp->acw + dw > 0 || pp->ace + de > 0)
641 return (0);
642
643 /*
644 * Automatically detach on last close if requested.
645 */
646 if ((sc->sc_flags & G_ELI_FLAG_RW_DETACH) ||
647 (sc->sc_flags & G_ELI_FLAG_WOPEN)) {
648 g_eli_last_close(sc);
649 }
650 return (0);
651}
652
653struct g_geom *
654g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
655 const struct g_eli_metadata *md, const u_char *mkey, int nkey)
656{
657 struct g_eli_softc *sc;
658 struct g_eli_worker *wr;
659 struct g_geom *gp;
660 struct g_provider *pp;
661 struct g_consumer *cp;
662 struct cryptoini cri;
663 u_int i, threads;
664 int error;
665
666 G_ELI_DEBUG(1, "Creating device %s%s.", bpp->name, G_ELI_SUFFIX);
667
668 gp = g_new_geomf(mp, "%s%s", bpp->name, G_ELI_SUFFIX);
669 gp->softc = NULL; /* for a moment */
670
671 sc = malloc(sizeof(*sc), M_ELI, M_WAITOK | M_ZERO);
672 gp->start = g_eli_start;
673 /*
674 * Spoiling cannot happen actually, because we keep provider open for
675 * writing all the time.
676 */
677 gp->spoiled = g_eli_orphan_spoil_assert;
678 gp->orphan = g_eli_orphan;
679 /*
680 * If detach-on-last-close feature is not enabled, we can simply use
681 * g_std_access().
682 */
683 if (md->md_flags & G_ELI_FLAG_WO_DETACH)
684 gp->access = g_eli_access;
685 else
686 gp->access = g_std_access;
687 gp->dumpconf = g_eli_dumpconf;
688
689 sc->sc_crypto = G_ELI_CRYPTO_SW;
690 sc->sc_flags = md->md_flags;
691 sc->sc_algo = md->md_algo;
692 sc->sc_nkey = nkey;
693 /*
694 * Remember the keys in our softc structure.
695 */
696 bcopy(mkey, sc->sc_ivkey, sizeof(sc->sc_ivkey));
697 mkey += sizeof(sc->sc_ivkey);
698 bcopy(mkey, sc->sc_datakey, sizeof(sc->sc_datakey));
699 sc->sc_keylen = md->md_keylen;
700
701 /*
702 * Precalculate SHA256 for IV generation.
703 * This is expensive operation and we can do it only once now or for
704 * every access to sector, so now will be much better.
705 */
706 SHA256_Init(&sc->sc_ivctx);
707 SHA256_Update(&sc->sc_ivctx, sc->sc_ivkey, sizeof(sc->sc_ivkey));
708
709 gp->softc = sc;
710 sc->sc_geom = gp;
711
712 bioq_init(&sc->sc_queue);
713 mtx_init(&sc->sc_queue_mtx, "geli:queue", NULL, MTX_DEF);
714
715 pp = NULL;
716 cp = g_new_consumer(gp);
717 error = g_attach(cp, bpp);
718 if (error != 0) {
719 if (req != NULL) {
720 gctl_error(req, "Cannot attach to %s (error=%d).",
721 bpp->name, error);
722 } else {
723 G_ELI_DEBUG(1, "Cannot attach to %s (error=%d).",
724 bpp->name, error);
725 }
726 goto failed;
727 }
728 /*
729 * Keep provider open all the time, so we can run critical tasks,
730 * like Master Keys deletion, without wondering if we can open
731 * provider or not.
732 */
733 error = g_access(cp, 1, 1, 1);
734 if (error != 0) {
735 if (req != NULL) {
736 gctl_error(req, "Cannot access %s (error=%d).",
737 bpp->name, error);
738 } else {
739 G_ELI_DEBUG(1, "Cannot access %s (error=%d).",
740 bpp->name, error);
741 }
742 goto failed;
743 }
744
745 LIST_INIT(&sc->sc_workers);
746
747 bzero(&cri, sizeof(cri));
748 cri.cri_alg = sc->sc_algo;
749 cri.cri_klen = sc->sc_keylen;
750 cri.cri_key = sc->sc_datakey;
751
752 threads = g_eli_threads;
753 if (threads == 0)
754 threads = mp_ncpus;
755 else if (threads > mp_ncpus) {
756 /* There is really no need for too many worker threads. */
757 threads = mp_ncpus;
758 G_ELI_DEBUG(0, "Reducing number of threads to %u.", threads);
759 }
760 for (i = 0; i < threads; i++) {
761 wr = malloc(sizeof(*wr), M_ELI, M_WAITOK | M_ZERO);
762 wr->w_softc = sc;
763 wr->w_number = i;
764
765 /*
766 * If this is the first pass, try to get hardware support.
767 * Use software cryptography, if we cannot get it.
768 */
769 if (i == 0) {
770 error = crypto_newsession(&wr->w_sid, &cri, 1);
771 if (error == 0)
772 sc->sc_crypto = G_ELI_CRYPTO_HW;
773 }
774 if (sc->sc_crypto == G_ELI_CRYPTO_SW)
775 error = crypto_newsession(&wr->w_sid, &cri, 0);
776 if (error != 0) {
777 free(wr, M_ELI);
778 if (req != NULL) {
779 gctl_error(req, "Cannot set up crypto session "
780 "for %s (error=%d).", bpp->name, error);
781 } else {
782 G_ELI_DEBUG(1, "Cannot set up crypto session "
783 "for %s (error=%d).", bpp->name, error);
784 }
785 goto failed;
786 }
787
788 error = kthread_create(g_eli_worker, wr, &wr->w_proc, 0, 0,
789 "g_eli[%u] %s", i, bpp->name);
790 if (error != 0) {
791 crypto_freesession(wr->w_sid);
792 free(wr, M_ELI);
793 if (req != NULL) {
794 gctl_error(req, "Cannot create kernel thread "
795 "for %s (error=%d).", bpp->name, error);
796 } else {
797 G_ELI_DEBUG(1, "Cannot create kernel thread "
798 "for %s (error=%d).", bpp->name, error);
799 }
800 goto failed;
801 }
802 LIST_INSERT_HEAD(&sc->sc_workers, wr, w_next);
803 /* If we have hardware support, one thread is enough. */
804 if (sc->sc_crypto == G_ELI_CRYPTO_HW)
805 break;
806 }
807
808 /*
809 * Create decrypted provider.
810 */
811 pp = g_new_providerf(gp, "%s%s", bpp->name, G_ELI_SUFFIX);
812 pp->sectorsize = md->md_sectorsize;
813 pp->mediasize = bpp->mediasize;
814 if ((sc->sc_flags & G_ELI_FLAG_ONETIME) == 0)
815 pp->mediasize -= bpp->sectorsize;
816 pp->mediasize -= (pp->mediasize % pp->sectorsize);
817 g_error_provider(pp, 0);
818
819 G_ELI_DEBUG(0, "Device %s created.", pp->name);
820 G_ELI_DEBUG(0, " Cipher: %s", g_eli_algo2str(sc->sc_algo));
821 G_ELI_DEBUG(0, "Key length: %u", sc->sc_keylen);
822 G_ELI_DEBUG(0, " Crypto: %s",
823 sc->sc_crypto == G_ELI_CRYPTO_SW ? "software" : "hardware");
824 return (gp);
825failed:
826 mtx_lock(&sc->sc_queue_mtx);
827 sc->sc_flags |= G_ELI_FLAG_DESTROY;
828 wakeup(sc);
829 /*
830 * Wait for kernel threads self destruction.
831 */
832 while (!LIST_EMPTY(&sc->sc_workers)) {
833 msleep(&sc->sc_workers, &sc->sc_queue_mtx, PRIBIO,
834 "geli:destroy", 0);
835 }
836 mtx_destroy(&sc->sc_queue_mtx);
837 if (cp->provider != NULL) {
838 if (cp->acr == 1)
839 g_access(cp, -1, -1, -1);
840 g_detach(cp);
841 }
842 g_destroy_consumer(cp);
843 g_destroy_geom(gp);
844 bzero(sc, sizeof(*sc));
845 free(sc, M_ELI);
846 return (NULL);
847}
848
849int
850g_eli_destroy(struct g_eli_softc *sc, boolean_t force)
851{
852 struct g_geom *gp;
853 struct g_provider *pp;
854
855 g_topology_assert();
856
857 if (sc == NULL)
858 return (ENXIO);
859
860 gp = sc->sc_geom;
861 pp = LIST_FIRST(&gp->provider);
862 if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
863 if (force) {
864 G_ELI_DEBUG(1, "Device %s is still open, so it "
865 "cannot be definitely removed.", pp->name);
866 } else {
867 G_ELI_DEBUG(1,
868 "Device %s is still open (r%dw%de%d).", pp->name,
869 pp->acr, pp->acw, pp->ace);
870 return (EBUSY);
871 }
872 }
873
874 mtx_lock(&sc->sc_queue_mtx);
875 sc->sc_flags |= G_ELI_FLAG_DESTROY;
876 wakeup(sc);
877 while (!LIST_EMPTY(&sc->sc_workers)) {
878 msleep(&sc->sc_workers, &sc->sc_queue_mtx, PRIBIO,
879 "geli:destroy", 0);
880 }
881 mtx_destroy(&sc->sc_queue_mtx);
882 gp->softc = NULL;
883 bzero(sc, sizeof(*sc));
884 free(sc, M_ELI);
885
886 if (pp == NULL || (pp->acr == 0 && pp->acw == 0 && pp->ace == 0))
887 G_ELI_DEBUG(0, "Device %s destroyed.", gp->name);
888 g_wither_geom_close(gp, ENXIO);
889
890 return (0);
891}
892
893static int
894g_eli_destroy_geom(struct gctl_req *req __unused,
895 struct g_class *mp __unused, struct g_geom *gp)
896{
897 struct g_eli_softc *sc;
898
899 sc = gp->softc;
900 return (g_eli_destroy(sc, 0));
901}
902
903static int
904g_eli_keyfiles_load(struct hmac_ctx *ctx, const char *provider)
905{
906 u_char *keyfile, *data, *size;
907 char *file, name[64];
908 int i;
909
910 for (i = 0; ; i++) {
911 snprintf(name, sizeof(name), "%s:geli_keyfile%d", provider, i);
912 keyfile = preload_search_by_type(name);
913 if (keyfile == NULL)
914 return (i); /* Return number of loaded keyfiles. */
915 data = preload_search_info(keyfile, MODINFO_ADDR);
916 if (data == NULL) {
917 G_ELI_DEBUG(0, "Cannot find key file data for %s.",
918 name);
919 return (0);
920 }
921 data = *(void **)data;
922 size = preload_search_info(keyfile, MODINFO_SIZE);
923 if (size == NULL) {
924 G_ELI_DEBUG(0, "Cannot find key file size for %s.",
925 name);
926 return (0);
927 }
928 file = preload_search_info(keyfile, MODINFO_NAME);
929 if (file == NULL) {
930 G_ELI_DEBUG(0, "Cannot find key file name for %s.",
931 name);
932 return (0);
933 }
934 G_ELI_DEBUG(1, "Loaded keyfile %s for %s (type: %s).", file,
935 provider, name);
936 g_eli_crypto_hmac_update(ctx, data, *(size_t *)size);
937 }
938}
939
940static void
941g_eli_keyfiles_clear(const char *provider)
942{
943 u_char *keyfile, *data, *size;
944 char name[64];
945 int i;
946
947 for (i = 0; ; i++) {
948 snprintf(name, sizeof(name), "%s:geli_keyfile%d", provider, i);
949 keyfile = preload_search_by_type(name);
950 if (keyfile == NULL)
951 return;
952 data = preload_search_info(keyfile, MODINFO_ADDR);
953 size = preload_search_info(keyfile, MODINFO_SIZE);
954 if (data == NULL || size == NULL)
955 continue;
956 data = *(void **)data;
957 bzero(data, *(size_t *)size);
958 }
959}
960
961/*
962 * Tasting is only made on boot.
963 * We detect providers which should be attached before root is mounted.
964 */
965static struct g_geom *
966g_eli_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
967{
968 struct g_eli_metadata md;
969 struct g_geom *gp;
970 struct hmac_ctx ctx;
971 char passphrase[256];
972 u_char key[G_ELI_USERKEYLEN], mkey[G_ELI_DATAIVKEYLEN];
973 u_int i, nkey, nkeyfiles, tries;
974 int error;
975
976 g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
977 g_topology_assert();
978
979 if (rootvnode != NULL || g_eli_tries == 0)
980 return (NULL);
981
982 G_ELI_DEBUG(3, "Tasting %s.", pp->name);
983
984 error = g_eli_read_metadata(mp, pp, &md);
985 if (error != 0)
986 return (NULL);
987 gp = NULL;
988
989 if (strcmp(md.md_magic, G_ELI_MAGIC) != 0)
990 return (NULL);
991 if (md.md_version > G_ELI_VERSION) {
992 printf("geom_eli.ko module is too old to handle %s.\n",
993 pp->name);
994 return (NULL);
995 }
996 if (md.md_provsize != pp->mediasize)
997 return (NULL);
998 /* Should we attach it on boot? */
999 if ((md.md_flags & G_ELI_FLAG_BOOT) == 0)
1000 return (NULL);
1001 if (md.md_keys == 0x00) {
1002 G_ELI_DEBUG(0, "No valid keys on %s.", pp->name);
1003 return (NULL);
1004 }
1005 if (md.md_iterations == -1) {
1006 /* If there is no passphrase, we try only once. */
1007 tries = 1;
1008 } else {
1009 /* Ask for the passphrase no more than g_eli_tries times. */
1010 tries = g_eli_tries;
1011 }
1012
1013 for (i = 0; i < tries; i++) {
1014 g_eli_crypto_hmac_init(&ctx, NULL, 0);
1015
1016 /*
1017 * Load all key files.
1018 */
1019 nkeyfiles = g_eli_keyfiles_load(&ctx, pp->name);
1020
1021 if (nkeyfiles == 0 && md.md_iterations == -1) {
1022 /*
1023 * No key files and no passphrase, something is
1024 * definitely wrong here.
1025 * geli(8) doesn't allow for such situation, so assume
1026 * that there was really no passphrase and in that case
1027 * key files are no properly defined in loader.conf.
1028 */
1029 G_ELI_DEBUG(0,
1030 "Found no key files in loader.conf for %s.",
1031 pp->name);
1032 return (NULL);
1033 }
1034
1035 /* Ask for the passphrase if defined. */
1036 if (md.md_iterations >= 0) {
1037 printf("Enter passphrase for %s: ", pp->name);
1038 gets(passphrase, sizeof(passphrase),
1039 g_eli_visible_passphrase);
1040 }
1041
1042 /*
1043 * Prepare Derived-Key from the user passphrase.
1044 */
1045 if (md.md_iterations == 0) {
1046 g_eli_crypto_hmac_update(&ctx, md.md_salt,
1047 sizeof(md.md_salt));
1048 g_eli_crypto_hmac_update(&ctx, passphrase,
1049 strlen(passphrase));
1050 } else if (md.md_iterations > 0) {
1051 u_char dkey[G_ELI_USERKEYLEN];
1052
1053 pkcs5v2_genkey(dkey, sizeof(dkey), md.md_salt,
1054 sizeof(md.md_salt), passphrase, md.md_iterations);
1055 g_eli_crypto_hmac_update(&ctx, dkey, sizeof(dkey));
1056 bzero(dkey, sizeof(dkey));
1057 }
1058
1059 g_eli_crypto_hmac_final(&ctx, key, 0);
1060
1061 /*
1062 * Decrypt Master-Key.
1063 */
1064 error = g_eli_mkey_decrypt(&md, key, mkey, &nkey);
1065 bzero(key, sizeof(key));
1066 if (error == -1) {
1067 if (i == tries - 1) {
1068 G_ELI_DEBUG(0,
1069 "Wrong key for %s. No tries left.",
1070 pp->name);
1071 g_eli_keyfiles_clear(pp->name);
1072 return (NULL);
1073 }
1074 G_ELI_DEBUG(0, "Wrong key for %s. Tries left: %u.",
1075 pp->name, tries - i - 1);
1076 /* Try again. */
1077 continue;
1078 } else if (error > 0) {
1079 G_ELI_DEBUG(0, "Cannot decrypt Master Key for %s (error=%d).",
1080 pp->name, error);
1081 g_eli_keyfiles_clear(pp->name);
1082 return (NULL);
1083 }
1084 G_ELI_DEBUG(1, "Using Master Key %u for %s.", nkey, pp->name);
1085 break;
1086 }
1087
1088 /*
1089 * We have correct key, let's attach provider.
1090 */
1091 gp = g_eli_create(NULL, mp, pp, &md, mkey, nkey);
1092 bzero(mkey, sizeof(mkey));
1093 bzero(&md, sizeof(md));
1094 if (gp == NULL) {
1095 G_ELI_DEBUG(0, "Cannot create device %s%s.", pp->name,
1096 G_ELI_SUFFIX);
1097 return (NULL);
1098 }
1099 return (gp);
1100}
1101
1102static void
1103g_eli_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
1104 struct g_consumer *cp, struct g_provider *pp)
1105{
1106 struct g_eli_softc *sc;
1107
1108 g_topology_assert();
1109 sc = gp->softc;
1110 if (sc == NULL)
1111 return;
1112 if (pp != NULL || cp != NULL)
1113 return; /* Nothing here. */
1114 sbuf_printf(sb, "%s<Flags>", indent);
1115 if (sc->sc_flags == 0)
1116 sbuf_printf(sb, "NONE");
1117 else {
1118 int first = 1;
1119
1120#define ADD_FLAG(flag, name) do { \
1121 if ((sc->sc_flags & (flag)) != 0) { \
1122 if (!first) \
1123 sbuf_printf(sb, ", "); \
1124 else \
1125 first = 0; \
1126 sbuf_printf(sb, name); \
1127 } \
1128} while (0)
1129 ADD_FLAG(G_ELI_FLAG_ONETIME, "ONETIME");
1130 ADD_FLAG(G_ELI_FLAG_BOOT, "BOOT");
1131 ADD_FLAG(G_ELI_FLAG_WO_DETACH, "W-DETACH");
1132 ADD_FLAG(G_ELI_FLAG_RW_DETACH, "RW-DETACH");
1133 ADD_FLAG(G_ELI_FLAG_WOPEN, "W-OPEN");
1134 ADD_FLAG(G_ELI_FLAG_DESTROY, "DESTROY");
1135#undef ADD_FLAG
1136 }
1137 sbuf_printf(sb, "</Flags>\n");
1138
1139 if ((sc->sc_flags & G_ELI_FLAG_ONETIME) == 0) {
1140 sbuf_printf(sb, "%s<UsedKey>%u</UsedKey>\n", indent,
1141 sc->sc_nkey);
1142 }
1143 sbuf_printf(sb, "%s<Crypto>", indent);
1144 switch (sc->sc_crypto) {
1145 case G_ELI_CRYPTO_HW:
1146 sbuf_printf(sb, "hardware");
1147 break;
1148 case G_ELI_CRYPTO_SW:
1149 sbuf_printf(sb, "software");
1150 break;
1151 default:
1152 sbuf_printf(sb, "UNKNOWN");
1153 break;
1154 }
1155 sbuf_printf(sb, "</Crypto>\n");
1156 sbuf_printf(sb, "%s<KeyLength>%u</KeyLength>\n", indent, sc->sc_keylen);
1157 sbuf_printf(sb, "%s<Cipher>%s</Cipher>\n", indent,
1158 g_eli_algo2str(sc->sc_algo));
1159}
1160
1161DECLARE_GEOM_CLASS(g_eli_class, g_eli);
1162MODULE_DEPEND(geom_eli, crypto, 1, 1, 1);