1/*-
2 * Copyright (c) 2005 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
--- 11 unchanged lines hidden (view full) ---
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/geom/eli/g_eli.c 159306 2006-06-05 21:25:19Z pjd $");
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/kernel.h>
33#include <sys/linker.h>
34#include <sys/module.h>
35#include <sys/lock.h>
36#include <sys/mutex.h>
--- 34 unchanged lines hidden (view full) ---
71u_int g_eli_overwrites = 5;
72TUNABLE_INT("kern.geom.eli.overwrites", &g_eli_overwrites);
73SYSCTL_UINT(_kern_geom_eli, OID_AUTO, overwrites, CTLFLAG_RW, &g_eli_overwrites,
74 0, "Number of times on-disk keys should be overwritten when destroying them");
75static u_int g_eli_threads = 0;
76TUNABLE_INT("kern.geom.eli.threads", &g_eli_threads);
77SYSCTL_UINT(_kern_geom_eli, OID_AUTO, threads, CTLFLAG_RW, &g_eli_threads, 0,
78 "Number of threads doing crypto work");
79
80static int g_eli_destroy_geom(struct gctl_req *req, struct g_class *mp,
81 struct g_geom *gp);
82static void g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp);
83
84static g_taste_t g_eli_taste;
85static g_dumpconf_t g_eli_dumpconf;
86
87struct g_class g_eli_class = {
88 .name = G_ELI_CLASS_NAME,
89 .version = G_VERSION,
90 .ctlreq = g_eli_config,
--- 11 unchanged lines hidden (view full) ---
102 */
103
104
105/*
106 * EAGAIN from crypto(9) means, that we were probably balanced to another crypto
107 * accelerator or something like this.
108 * The function updates the SID and rerun the operation.
109 */
110static int
111g_eli_crypto_rerun(struct cryptop *crp)
112{
113 struct g_eli_softc *sc;
114 struct g_eli_worker *wr;
115 struct bio *bp;
116 int error;
117
118 bp = (struct bio *)crp->crp_opaque;
--- 16 unchanged lines hidden (view full) ---
135 return (error);
136}
137
138/*
139 * The function is called afer reading encrypted data from the provider.
140 *
141 * g_eli_start -> g_io_request -> G_ELI_READ_DONE -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
142 */
143static void
144g_eli_read_done(struct bio *bp)
145{
146 struct g_eli_softc *sc;
147 struct bio *pbp;
148
149 G_ELI_LOGREQ(2, bp, "Request done.");
150 pbp = bp->bio_parent;
151 if (pbp->bio_error == 0)
152 pbp->bio_error = bp->bio_error;
153 g_destroy_bio(bp);
154 if (pbp->bio_error != 0) {
155 G_ELI_LOGREQ(0, pbp, "%s() failed", __func__);
156 pbp->bio_completed = 0;
157 g_io_deliver(pbp, pbp->bio_error);
158 return;
159 }
160 sc = pbp->bio_to->geom->softc;
161 mtx_lock(&sc->sc_queue_mtx);
162 bioq_insert_tail(&sc->sc_queue, pbp);
163 mtx_unlock(&sc->sc_queue_mtx);
164 wakeup(sc);
165}
166
167/*
168 * The function is called after we read and decrypt data.
169 *
170 * g_eli_start -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver
171 */
172static int
173g_eli_crypto_read_done(struct cryptop *crp)
174{
175 struct bio *bp;
176
177 if (crp->crp_etype == EAGAIN) {
178 if (g_eli_crypto_rerun(crp) == 0)
179 return (0);
180 }
181 bp = (struct bio *)crp->crp_opaque;
182 bp->bio_inbed++;
183 if (crp->crp_etype == 0) {
184 G_ELI_DEBUG(3, "Crypto READ request done (%d/%d).",
185 bp->bio_inbed, bp->bio_children);
186 bp->bio_completed += crp->crp_olen;
187 } else {
188 G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.",
189 bp->bio_inbed, bp->bio_children, crp->crp_etype);
190 if (bp->bio_error == 0)
191 bp->bio_error = crp->crp_etype;
192 }
193 /*
194 * Do we have all sectors already?
195 */
196 if (bp->bio_inbed < bp->bio_children)
197 return (0);
198 free(bp->bio_driver2, M_ELI);
199 bp->bio_driver2 = NULL;
200 if (bp->bio_error != 0) {
201 G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).",
202 bp->bio_error);
203 bp->bio_completed = 0;
204 }
205 /*
206 * Read is finished, send it up.
207 */
208 g_io_deliver(bp, bp->bio_error);
209 return (0);
210}
211
212/*
213 * The function is called after we encrypt and write data.
214 *
215 * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> G_ELI_WRITE_DONE -> g_io_deliver
216 */
217static void
218g_eli_write_done(struct bio *bp)
219{
220 struct bio *pbp;
221
222 G_ELI_LOGREQ(2, bp, "Request done.");
223 pbp = bp->bio_parent;
224 if (pbp->bio_error == 0)
225 pbp->bio_error = bp->bio_error;
226 free(pbp->bio_driver2, M_ELI);
227 pbp->bio_driver2 = NULL;
228 if (pbp->bio_error == 0)
229 pbp->bio_completed = pbp->bio_length;
230 else {
231 G_ELI_LOGREQ(0, pbp, "Crypto WRITE request failed (error=%d).",
232 pbp->bio_error);
233 pbp->bio_completed = 0;
234 }
235 g_destroy_bio(bp);
236 /*
237 * Write is finished, send it up.
238 */
239 g_io_deliver(pbp, pbp->bio_error);
240}
241
242/*
243 * The function is called after data encryption.
244 *
245 * g_eli_start -> g_eli_crypto_run -> G_ELI_CRYPTO_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver
246 */
247static int
248g_eli_crypto_write_done(struct cryptop *crp)
249{
250 struct g_geom *gp;
251 struct g_consumer *cp;
252 struct bio *bp, *cbp;
253
254 if (crp->crp_etype == EAGAIN) {
255 if (g_eli_crypto_rerun(crp) == 0)
256 return (0);
257 }
258 bp = (struct bio *)crp->crp_opaque;
259 bp->bio_inbed++;
260 if (crp->crp_etype == 0) {
261 G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).",
262 bp->bio_inbed, bp->bio_children);
263 } else {
264 G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.",
265 bp->bio_inbed, bp->bio_children, crp->crp_etype);
266 if (bp->bio_error == 0)
267 bp->bio_error = crp->crp_etype;
268 }
269 /*
270 * All sectors are already encrypted?
271 */
272 if (bp->bio_inbed < bp->bio_children)
273 return (0);
274 bp->bio_inbed = 0;
275 bp->bio_children = 1;
276 cbp = bp->bio_driver1;
277 bp->bio_driver1 = NULL;
278 if (bp->bio_error != 0) {
279 G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
280 bp->bio_error);
281 free(bp->bio_driver2, M_ELI);
282 bp->bio_driver2 = NULL;
283 g_destroy_bio(cbp);
284 g_io_deliver(bp, bp->bio_error);
285 return (0);
286 }
287 cbp->bio_data = bp->bio_driver2;
288 cbp->bio_done = g_eli_write_done;
289 gp = bp->bio_to->geom;
290 cp = LIST_FIRST(&gp->consumer);
291 cbp->bio_to = cp->provider;
292 G_ELI_LOGREQ(2, cbp, "Sending request.");
293 /*
294 * Send encrypted data to the provider.
295 */
296 g_io_request(cbp, cp);
297 return (0);
298}
299
300/*
301 * This function should never be called, but GEOM made as it set ->orphan()
302 * method for every geom.
303 */
304static void
305g_eli_orphan_spoil_assert(struct g_consumer *cp)
306{
307
308 panic("Function %s() called for %s.", __func__, cp->geom->name);
--- 45 unchanged lines hidden (view full) ---
354 }
355 cbp = g_clone_bio(bp);
356 if (cbp == NULL) {
357 g_io_deliver(bp, ENOMEM);
358 return;
359 }
360 switch (bp->bio_cmd) {
361 case BIO_READ:
362 cbp->bio_done = g_eli_read_done;
363 cp = LIST_FIRST(&sc->sc_geom->consumer);
364 cbp->bio_to = cp->provider;
365 G_ELI_LOGREQ(2, cbp, "Sending request.");
366 /*
367 * Read encrypted data from provider.
368 */
369 g_io_request(cbp, cp);
370 break;
371 case BIO_WRITE:
372 bp->bio_driver1 = cbp;
373 mtx_lock(&sc->sc_queue_mtx);
374 bioq_insert_tail(&sc->sc_queue, bp);
375 mtx_unlock(&sc->sc_queue_mtx);
376 wakeup(sc);
377 break;
378 case BIO_GETATTR:
--- 28 unchanged lines hidden (view full) ---
407 mtx_unlock_spin(&sched_lock);
408
409 G_ELI_DEBUG(1, "Thread %s started.", curthread->td_proc->p_comm);
410
411 for (;;) {
412 mtx_lock(&sc->sc_queue_mtx);
413 bp = bioq_takefirst(&sc->sc_queue);
414 if (bp == NULL) {
415 if ((sc->sc_flags & G_ELI_FLAG_DESTROY) != 0) {
416 LIST_REMOVE(wr, w_next);
417 crypto_freesession(wr->w_sid);
418 free(wr, M_ELI);
419 G_ELI_DEBUG(1, "Thread %s exiting.",
420 curthread->td_proc->p_comm);
421 wakeup(&sc->sc_workers);
422 mtx_unlock(&sc->sc_queue_mtx);
423 kthread_exit(0);
424 }
425 msleep(sc, &sc->sc_queue_mtx, PRIBIO | PDROP,
426 "geli:w", 0);
427 continue;
428 }
429 mtx_unlock(&sc->sc_queue_mtx);
430 g_eli_crypto_run(wr, bp);
431 }
432}
433
434/*
435 * Here we generate IV. It is unique for every sector.
436 */
437static void
438g_eli_crypto_ivgen(struct g_eli_softc *sc, off_t offset, u_char *iv,
439 size_t size)
440{
441 u_char hash[SHA256_DIGEST_LENGTH];
442 SHA256_CTX ctx;
443
444 /* Copy precalculated SHA256 context for IV-Key. */
445 bcopy(&sc->sc_ivctx, &ctx, sizeof(ctx));
446 SHA256_Update(&ctx, (uint8_t *)&offset, sizeof(offset));
447 SHA256_Final(hash, &ctx);
448 bcopy(hash, iv, size);
449}
450
451/*
452 * This is the main function responsible for cryptography (ie. communication
453 * with crypto(9) subsystem).
454 */
455static void
456g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp)
457{
458 struct g_eli_softc *sc;
459 struct cryptop *crp;
460 struct cryptodesc *crd;
461 struct uio *uio;
462 struct iovec *iov;
463 u_int i, nsec, add, secsize;
464 int err, error;
465 size_t size;
466 u_char *p, *data;
467
468 G_ELI_LOGREQ(3, bp, "%s", __func__);
469
470 bp->bio_pflags = wr->w_number;
471 sc = wr->w_softc;
472 secsize = LIST_FIRST(&sc->sc_geom->provider)->sectorsize;
473 nsec = bp->bio_length / secsize;
474
475 /*
476 * Calculate how much memory do we need.
477 * We need separate crypto operation for every single sector.
478 * It is much faster to calculate total amount of needed memory here and
479 * do the allocation once instead of allocating memory in pieces (many,
480 * many pieces).
481 */
482 size = sizeof(*crp) * nsec;
483 size += sizeof(*crd) * nsec;
484 size += sizeof(*uio) * nsec;
485 size += sizeof(*iov) * nsec;
486 /*
487 * If we write the data we cannot destroy current bio_data content,
488 * so we need to allocate more memory for encrypted data.
489 */
490 if (bp->bio_cmd == BIO_WRITE)
491 size += bp->bio_length;
492 p = malloc(size, M_ELI, M_WAITOK);
493
494 bp->bio_inbed = 0;
495 bp->bio_children = nsec;
496 bp->bio_driver2 = p;
497
498 if (bp->bio_cmd == BIO_READ)
499 data = bp->bio_data;
500 else {
501 data = p;
502 p += bp->bio_length;
503 bcopy(bp->bio_data, data, bp->bio_length);
504 }
505
506 error = 0;
507 for (i = 0, add = 0; i < nsec; i++, add += secsize) {
508 crp = (struct cryptop *)p; p += sizeof(*crp);
509 crd = (struct cryptodesc *)p; p += sizeof(*crd);
510 uio = (struct uio *)p; p += sizeof(*uio);
511 iov = (struct iovec *)p; p += sizeof(*iov);
512
513 iov->iov_len = secsize;
514 iov->iov_base = data;
515 data += secsize;
516
517 uio->uio_iov = iov;
518 uio->uio_iovcnt = 1;
519 uio->uio_segflg = UIO_SYSSPACE;
520 uio->uio_resid = secsize;
521
522 crp->crp_sid = wr->w_sid;
523 crp->crp_ilen = secsize;
524 crp->crp_olen = secsize;
525 crp->crp_opaque = (void *)bp;
526 crp->crp_buf = (void *)uio;
527 if (bp->bio_cmd == BIO_WRITE)
528 crp->crp_callback = g_eli_crypto_write_done;
529 else /* if (bp->bio_cmd == BIO_READ) */
530 crp->crp_callback = g_eli_crypto_read_done;
531 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC | CRYPTO_F_REL;
532 crp->crp_desc = crd;
533
534 crd->crd_skip = 0;
535 crd->crd_len = secsize;
536 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
537 if (bp->bio_cmd == BIO_WRITE)
538 crd->crd_flags |= CRD_F_ENCRYPT;
539 crd->crd_alg = sc->sc_algo;
540 crd->crd_key = sc->sc_datakey;
541 crd->crd_klen = sc->sc_keylen;
542 g_eli_crypto_ivgen(sc, bp->bio_offset + add, crd->crd_iv,
543 sizeof(crd->crd_iv));
544 crd->crd_next = NULL;
545
546 crp->crp_etype = 0;
547 err = crypto_dispatch(crp);
548 if (error == 0)
549 error = err;
550 }
551 if (bp->bio_error == 0)
552 bp->bio_error = error;
553}
554
555int
556g_eli_read_metadata(struct g_class *mp, struct g_provider *pp,
557 struct g_eli_metadata *md)
558{
559 struct g_geom *gp;
560 struct g_consumer *cp;
561 u_char *buf = NULL;
562 int error;
--- 91 unchanged lines hidden (view full) ---
654g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
655 const struct g_eli_metadata *md, const u_char *mkey, int nkey)
656{
657 struct g_eli_softc *sc;
658 struct g_eli_worker *wr;
659 struct g_geom *gp;
660 struct g_provider *pp;
661 struct g_consumer *cp;
662 struct cryptoini cri;
663 u_int i, threads;
664 int error;
665
666 G_ELI_DEBUG(1, "Creating device %s%s.", bpp->name, G_ELI_SUFFIX);
667
668 gp = g_new_geomf(mp, "%s%s", bpp->name, G_ELI_SUFFIX);
669 gp->softc = NULL; /* for a moment */
670
--- 12 unchanged lines hidden (view full) ---
683 if (md->md_flags & G_ELI_FLAG_WO_DETACH)
684 gp->access = g_eli_access;
685 else
686 gp->access = g_std_access;
687 gp->dumpconf = g_eli_dumpconf;
688
689 sc->sc_crypto = G_ELI_CRYPTO_SW;
690 sc->sc_flags = md->md_flags;
691 sc->sc_algo = md->md_algo;
692 sc->sc_nkey = nkey;
693 /*
694 * Remember the keys in our softc structure.
695 */
696 bcopy(mkey, sc->sc_ivkey, sizeof(sc->sc_ivkey));
697 mkey += sizeof(sc->sc_ivkey);
698 bcopy(mkey, sc->sc_datakey, sizeof(sc->sc_datakey));
699 sc->sc_keylen = md->md_keylen;
700
701 /*
702 * Precalculate SHA256 for IV generation.
703 * This is expensive operation and we can do it only once now or for
704 * every access to sector, so now will be much better.
705 */
706 SHA256_Init(&sc->sc_ivctx);
707 SHA256_Update(&sc->sc_ivctx, sc->sc_ivkey, sizeof(sc->sc_ivkey));
708
--- 30 unchanged lines hidden (view full) ---
739 G_ELI_DEBUG(1, "Cannot access %s (error=%d).",
740 bpp->name, error);
741 }
742 goto failed;
743 }
744
745 LIST_INIT(&sc->sc_workers);
746
747 bzero(&cri, sizeof(cri));
748 cri.cri_alg = sc->sc_algo;
749 cri.cri_klen = sc->sc_keylen;
750 cri.cri_key = sc->sc_datakey;
751
752 threads = g_eli_threads;
753 if (threads == 0)
754 threads = mp_ncpus;
755 else if (threads > mp_ncpus) {
756 /* There is really no need for too many worker threads. */
757 threads = mp_ncpus;
758 G_ELI_DEBUG(0, "Reducing number of threads to %u.", threads);
759 }
760 for (i = 0; i < threads; i++) {
761 wr = malloc(sizeof(*wr), M_ELI, M_WAITOK | M_ZERO);
762 wr->w_softc = sc;
763 wr->w_number = i;
764
765 /*
766 * If this is the first pass, try to get hardware support.
767 * Use software cryptography, if we cannot get it.
768 */
769 if (i == 0) {
770 error = crypto_newsession(&wr->w_sid, &cri, 1);
771 if (error == 0)
772 sc->sc_crypto = G_ELI_CRYPTO_HW;
773 }
774 if (sc->sc_crypto == G_ELI_CRYPTO_SW)
775 error = crypto_newsession(&wr->w_sid, &cri, 0);
776 if (error != 0) {
777 free(wr, M_ELI);
778 if (req != NULL) {
779 gctl_error(req, "Cannot set up crypto session "
780 "for %s (error=%d).", bpp->name, error);
781 } else {
782 G_ELI_DEBUG(1, "Cannot set up crypto session "
783 "for %s (error=%d).", bpp->name, error);
--- 22 unchanged lines hidden (view full) ---
806 }
807
808 /*
809 * Create decrypted provider.
810 */
811 pp = g_new_providerf(gp, "%s%s", bpp->name, G_ELI_SUFFIX);
812 pp->sectorsize = md->md_sectorsize;
813 pp->mediasize = bpp->mediasize;
814 if ((sc->sc_flags & G_ELI_FLAG_ONETIME) == 0)
815 pp->mediasize -= bpp->sectorsize;
816 pp->mediasize -= (pp->mediasize % pp->sectorsize);
817 g_error_provider(pp, 0);
818
819 G_ELI_DEBUG(0, "Device %s created.", pp->name);
820 G_ELI_DEBUG(0, " Cipher: %s", g_eli_algo2str(sc->sc_algo));
821 G_ELI_DEBUG(0, "Key length: %u", sc->sc_keylen);
822 G_ELI_DEBUG(0, " Crypto: %s",
823 sc->sc_crypto == G_ELI_CRYPTO_SW ? "software" : "hardware");
824 return (gp);
825failed:
826 mtx_lock(&sc->sc_queue_mtx);
827 sc->sc_flags |= G_ELI_FLAG_DESTROY;
828 wakeup(sc);
829 /*
--- 161 unchanged lines hidden (view full) ---
991 if (md.md_version > G_ELI_VERSION) {
992 printf("geom_eli.ko module is too old to handle %s.\n",
993 pp->name);
994 return (NULL);
995 }
996 if (md.md_provsize != pp->mediasize)
997 return (NULL);
998 /* Should we attach it on boot? */
999 if ((md.md_flags & G_ELI_FLAG_BOOT) == 0)
1000 return (NULL);
1001 if (md.md_keys == 0x00) {
1002 G_ELI_DEBUG(0, "No valid keys on %s.", pp->name);
1003 return (NULL);
1004 }
1005 if (md.md_iterations == -1) {
1006 /* If there is no passphrase, we try only once. */
1007 tries = 1;
--- 105 unchanged lines hidden (view full) ---
1113 return; /* Nothing here. */
1114 sbuf_printf(sb, "%s<Flags>", indent);
1115 if (sc->sc_flags == 0)
1116 sbuf_printf(sb, "NONE");
1117 else {
1118 int first = 1;
1119
1120#define ADD_FLAG(flag, name) do { \
1121 if ((sc->sc_flags & (flag)) != 0) { \
1122 if (!first) \
1123 sbuf_printf(sb, ", "); \
1124 else \
1125 first = 0; \
1126 sbuf_printf(sb, name); \
1127 } \
1128} while (0)
1129 ADD_FLAG(G_ELI_FLAG_ONETIME, "ONETIME");
1130 ADD_FLAG(G_ELI_FLAG_BOOT, "BOOT");
1131 ADD_FLAG(G_ELI_FLAG_WO_DETACH, "W-DETACH");
1132 ADD_FLAG(G_ELI_FLAG_RW_DETACH, "RW-DETACH");
1133 ADD_FLAG(G_ELI_FLAG_WOPEN, "W-OPEN");
1134 ADD_FLAG(G_ELI_FLAG_DESTROY, "DESTROY");
1135#undef ADD_FLAG
1136 }
1137 sbuf_printf(sb, "</Flags>\n");
1138
1139 if ((sc->sc_flags & G_ELI_FLAG_ONETIME) == 0) {
1140 sbuf_printf(sb, "%s<UsedKey>%u</UsedKey>\n", indent,
1141 sc->sc_nkey);
1142 }
1143 sbuf_printf(sb, "%s<Crypto>", indent);
1144 switch (sc->sc_crypto) {
1145 case G_ELI_CRYPTO_HW:
1146 sbuf_printf(sb, "hardware");
1147 break;
1148 case G_ELI_CRYPTO_SW:
1149 sbuf_printf(sb, "software");
1150 break;
1151 default:
1152 sbuf_printf(sb, "UNKNOWN");
1153 break;
1154 }
1155 sbuf_printf(sb, "</Crypto>\n");
1156 sbuf_printf(sb, "%s<KeyLength>%u</KeyLength>\n", indent, sc->sc_keylen);
1157 sbuf_printf(sb, "%s<Cipher>%s</Cipher>\n", indent,
1158 g_eli_algo2str(sc->sc_algo));
1159}
1160
1161DECLARE_GEOM_CLASS(g_eli_class, g_eli);
1162MODULE_DEPEND(geom_eli, crypto, 1, 1, 1);
2 * Copyright (c) 2005 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
--- 11 unchanged lines hidden (view full) ---
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/geom/eli/g_eli.c 159306 2006-06-05 21:25:19Z pjd $");
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/kernel.h>
33#include <sys/linker.h>
34#include <sys/module.h>
35#include <sys/lock.h>
36#include <sys/mutex.h>
--- 34 unchanged lines hidden (view full) ---
71u_int g_eli_overwrites = 5;
72TUNABLE_INT("kern.geom.eli.overwrites", &g_eli_overwrites);
73SYSCTL_UINT(_kern_geom_eli, OID_AUTO, overwrites, CTLFLAG_RW, &g_eli_overwrites,
74 0, "Number of times on-disk keys should be overwritten when destroying them");
75static u_int g_eli_threads = 0;
76TUNABLE_INT("kern.geom.eli.threads", &g_eli_threads);
77SYSCTL_UINT(_kern_geom_eli, OID_AUTO, threads, CTLFLAG_RW, &g_eli_threads, 0,
78 "Number of threads doing crypto work");
79
80static int g_eli_destroy_geom(struct gctl_req *req, struct g_class *mp,
81 struct g_geom *gp);
82static void g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp);
83
84static g_taste_t g_eli_taste;
85static g_dumpconf_t g_eli_dumpconf;
86
87struct g_class g_eli_class = {
88 .name = G_ELI_CLASS_NAME,
89 .version = G_VERSION,
90 .ctlreq = g_eli_config,
--- 11 unchanged lines hidden (view full) ---
102 */
103
104
105/*
106 * EAGAIN from crypto(9) means, that we were probably balanced to another crypto
107 * accelerator or something like this.
108 * The function updates the SID and rerun the operation.
109 */
110static int
111g_eli_crypto_rerun(struct cryptop *crp)
112{
113 struct g_eli_softc *sc;
114 struct g_eli_worker *wr;
115 struct bio *bp;
116 int error;
117
118 bp = (struct bio *)crp->crp_opaque;
--- 16 unchanged lines hidden (view full) ---
135 return (error);
136}
137
138/*
139 * The function is called afer reading encrypted data from the provider.
140 *
141 * g_eli_start -> g_io_request -> G_ELI_READ_DONE -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
142 */
143static void
144g_eli_read_done(struct bio *bp)
145{
146 struct g_eli_softc *sc;
147 struct bio *pbp;
148
149 G_ELI_LOGREQ(2, bp, "Request done.");
150 pbp = bp->bio_parent;
151 if (pbp->bio_error == 0)
152 pbp->bio_error = bp->bio_error;
153 g_destroy_bio(bp);
154 if (pbp->bio_error != 0) {
155 G_ELI_LOGREQ(0, pbp, "%s() failed", __func__);
156 pbp->bio_completed = 0;
157 g_io_deliver(pbp, pbp->bio_error);
158 return;
159 }
160 sc = pbp->bio_to->geom->softc;
161 mtx_lock(&sc->sc_queue_mtx);
162 bioq_insert_tail(&sc->sc_queue, pbp);
163 mtx_unlock(&sc->sc_queue_mtx);
164 wakeup(sc);
165}
166
167/*
168 * The function is called after we read and decrypt data.
169 *
170 * g_eli_start -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver
171 */
172static int
173g_eli_crypto_read_done(struct cryptop *crp)
174{
175 struct bio *bp;
176
177 if (crp->crp_etype == EAGAIN) {
178 if (g_eli_crypto_rerun(crp) == 0)
179 return (0);
180 }
181 bp = (struct bio *)crp->crp_opaque;
182 bp->bio_inbed++;
183 if (crp->crp_etype == 0) {
184 G_ELI_DEBUG(3, "Crypto READ request done (%d/%d).",
185 bp->bio_inbed, bp->bio_children);
186 bp->bio_completed += crp->crp_olen;
187 } else {
188 G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.",
189 bp->bio_inbed, bp->bio_children, crp->crp_etype);
190 if (bp->bio_error == 0)
191 bp->bio_error = crp->crp_etype;
192 }
193 /*
194 * Do we have all sectors already?
195 */
196 if (bp->bio_inbed < bp->bio_children)
197 return (0);
198 free(bp->bio_driver2, M_ELI);
199 bp->bio_driver2 = NULL;
200 if (bp->bio_error != 0) {
201 G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).",
202 bp->bio_error);
203 bp->bio_completed = 0;
204 }
205 /*
206 * Read is finished, send it up.
207 */
208 g_io_deliver(bp, bp->bio_error);
209 return (0);
210}
211
212/*
213 * The function is called after we encrypt and write data.
214 *
215 * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> G_ELI_WRITE_DONE -> g_io_deliver
216 */
217static void
218g_eli_write_done(struct bio *bp)
219{
220 struct bio *pbp;
221
222 G_ELI_LOGREQ(2, bp, "Request done.");
223 pbp = bp->bio_parent;
224 if (pbp->bio_error == 0)
225 pbp->bio_error = bp->bio_error;
226 free(pbp->bio_driver2, M_ELI);
227 pbp->bio_driver2 = NULL;
228 if (pbp->bio_error == 0)
229 pbp->bio_completed = pbp->bio_length;
230 else {
231 G_ELI_LOGREQ(0, pbp, "Crypto WRITE request failed (error=%d).",
232 pbp->bio_error);
233 pbp->bio_completed = 0;
234 }
235 g_destroy_bio(bp);
236 /*
237 * Write is finished, send it up.
238 */
239 g_io_deliver(pbp, pbp->bio_error);
240}
241
242/*
243 * The function is called after data encryption.
244 *
245 * g_eli_start -> g_eli_crypto_run -> G_ELI_CRYPTO_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver
246 */
247static int
248g_eli_crypto_write_done(struct cryptop *crp)
249{
250 struct g_geom *gp;
251 struct g_consumer *cp;
252 struct bio *bp, *cbp;
253
254 if (crp->crp_etype == EAGAIN) {
255 if (g_eli_crypto_rerun(crp) == 0)
256 return (0);
257 }
258 bp = (struct bio *)crp->crp_opaque;
259 bp->bio_inbed++;
260 if (crp->crp_etype == 0) {
261 G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).",
262 bp->bio_inbed, bp->bio_children);
263 } else {
264 G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.",
265 bp->bio_inbed, bp->bio_children, crp->crp_etype);
266 if (bp->bio_error == 0)
267 bp->bio_error = crp->crp_etype;
268 }
269 /*
270 * All sectors are already encrypted?
271 */
272 if (bp->bio_inbed < bp->bio_children)
273 return (0);
274 bp->bio_inbed = 0;
275 bp->bio_children = 1;
276 cbp = bp->bio_driver1;
277 bp->bio_driver1 = NULL;
278 if (bp->bio_error != 0) {
279 G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
280 bp->bio_error);
281 free(bp->bio_driver2, M_ELI);
282 bp->bio_driver2 = NULL;
283 g_destroy_bio(cbp);
284 g_io_deliver(bp, bp->bio_error);
285 return (0);
286 }
287 cbp->bio_data = bp->bio_driver2;
288 cbp->bio_done = g_eli_write_done;
289 gp = bp->bio_to->geom;
290 cp = LIST_FIRST(&gp->consumer);
291 cbp->bio_to = cp->provider;
292 G_ELI_LOGREQ(2, cbp, "Sending request.");
293 /*
294 * Send encrypted data to the provider.
295 */
296 g_io_request(cbp, cp);
297 return (0);
298}
299
300/*
301 * This function should never be called, but GEOM made as it set ->orphan()
302 * method for every geom.
303 */
304static void
305g_eli_orphan_spoil_assert(struct g_consumer *cp)
306{
307
308 panic("Function %s() called for %s.", __func__, cp->geom->name);
--- 45 unchanged lines hidden (view full) ---
354 }
355 cbp = g_clone_bio(bp);
356 if (cbp == NULL) {
357 g_io_deliver(bp, ENOMEM);
358 return;
359 }
360 switch (bp->bio_cmd) {
361 case BIO_READ:
362 cbp->bio_done = g_eli_read_done;
363 cp = LIST_FIRST(&sc->sc_geom->consumer);
364 cbp->bio_to = cp->provider;
365 G_ELI_LOGREQ(2, cbp, "Sending request.");
366 /*
367 * Read encrypted data from provider.
368 */
369 g_io_request(cbp, cp);
370 break;
371 case BIO_WRITE:
372 bp->bio_driver1 = cbp;
373 mtx_lock(&sc->sc_queue_mtx);
374 bioq_insert_tail(&sc->sc_queue, bp);
375 mtx_unlock(&sc->sc_queue_mtx);
376 wakeup(sc);
377 break;
378 case BIO_GETATTR:
--- 28 unchanged lines hidden (view full) ---
407 mtx_unlock_spin(&sched_lock);
408
409 G_ELI_DEBUG(1, "Thread %s started.", curthread->td_proc->p_comm);
410
411 for (;;) {
412 mtx_lock(&sc->sc_queue_mtx);
413 bp = bioq_takefirst(&sc->sc_queue);
414 if (bp == NULL) {
415 if ((sc->sc_flags & G_ELI_FLAG_DESTROY) != 0) {
416 LIST_REMOVE(wr, w_next);
417 crypto_freesession(wr->w_sid);
418 free(wr, M_ELI);
419 G_ELI_DEBUG(1, "Thread %s exiting.",
420 curthread->td_proc->p_comm);
421 wakeup(&sc->sc_workers);
422 mtx_unlock(&sc->sc_queue_mtx);
423 kthread_exit(0);
424 }
425 msleep(sc, &sc->sc_queue_mtx, PRIBIO | PDROP,
426 "geli:w", 0);
427 continue;
428 }
429 mtx_unlock(&sc->sc_queue_mtx);
430 g_eli_crypto_run(wr, bp);
431 }
432}
433
434/*
435 * Here we generate IV. It is unique for every sector.
436 */
437static void
438g_eli_crypto_ivgen(struct g_eli_softc *sc, off_t offset, u_char *iv,
439 size_t size)
440{
441 u_char hash[SHA256_DIGEST_LENGTH];
442 SHA256_CTX ctx;
443
444 /* Copy precalculated SHA256 context for IV-Key. */
445 bcopy(&sc->sc_ivctx, &ctx, sizeof(ctx));
446 SHA256_Update(&ctx, (uint8_t *)&offset, sizeof(offset));
447 SHA256_Final(hash, &ctx);
448 bcopy(hash, iv, size);
449}
450
451/*
452 * This is the main function responsible for cryptography (ie. communication
453 * with crypto(9) subsystem).
454 */
455static void
456g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp)
457{
458 struct g_eli_softc *sc;
459 struct cryptop *crp;
460 struct cryptodesc *crd;
461 struct uio *uio;
462 struct iovec *iov;
463 u_int i, nsec, add, secsize;
464 int err, error;
465 size_t size;
466 u_char *p, *data;
467
468 G_ELI_LOGREQ(3, bp, "%s", __func__);
469
470 bp->bio_pflags = wr->w_number;
471 sc = wr->w_softc;
472 secsize = LIST_FIRST(&sc->sc_geom->provider)->sectorsize;
473 nsec = bp->bio_length / secsize;
474
475 /*
476 * Calculate how much memory do we need.
477 * We need separate crypto operation for every single sector.
478 * It is much faster to calculate total amount of needed memory here and
479 * do the allocation once instead of allocating memory in pieces (many,
480 * many pieces).
481 */
482 size = sizeof(*crp) * nsec;
483 size += sizeof(*crd) * nsec;
484 size += sizeof(*uio) * nsec;
485 size += sizeof(*iov) * nsec;
486 /*
487 * If we write the data we cannot destroy current bio_data content,
488 * so we need to allocate more memory for encrypted data.
489 */
490 if (bp->bio_cmd == BIO_WRITE)
491 size += bp->bio_length;
492 p = malloc(size, M_ELI, M_WAITOK);
493
494 bp->bio_inbed = 0;
495 bp->bio_children = nsec;
496 bp->bio_driver2 = p;
497
498 if (bp->bio_cmd == BIO_READ)
499 data = bp->bio_data;
500 else {
501 data = p;
502 p += bp->bio_length;
503 bcopy(bp->bio_data, data, bp->bio_length);
504 }
505
506 error = 0;
507 for (i = 0, add = 0; i < nsec; i++, add += secsize) {
508 crp = (struct cryptop *)p; p += sizeof(*crp);
509 crd = (struct cryptodesc *)p; p += sizeof(*crd);
510 uio = (struct uio *)p; p += sizeof(*uio);
511 iov = (struct iovec *)p; p += sizeof(*iov);
512
513 iov->iov_len = secsize;
514 iov->iov_base = data;
515 data += secsize;
516
517 uio->uio_iov = iov;
518 uio->uio_iovcnt = 1;
519 uio->uio_segflg = UIO_SYSSPACE;
520 uio->uio_resid = secsize;
521
522 crp->crp_sid = wr->w_sid;
523 crp->crp_ilen = secsize;
524 crp->crp_olen = secsize;
525 crp->crp_opaque = (void *)bp;
526 crp->crp_buf = (void *)uio;
527 if (bp->bio_cmd == BIO_WRITE)
528 crp->crp_callback = g_eli_crypto_write_done;
529 else /* if (bp->bio_cmd == BIO_READ) */
530 crp->crp_callback = g_eli_crypto_read_done;
531 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC | CRYPTO_F_REL;
532 crp->crp_desc = crd;
533
534 crd->crd_skip = 0;
535 crd->crd_len = secsize;
536 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
537 if (bp->bio_cmd == BIO_WRITE)
538 crd->crd_flags |= CRD_F_ENCRYPT;
539 crd->crd_alg = sc->sc_algo;
540 crd->crd_key = sc->sc_datakey;
541 crd->crd_klen = sc->sc_keylen;
542 g_eli_crypto_ivgen(sc, bp->bio_offset + add, crd->crd_iv,
543 sizeof(crd->crd_iv));
544 crd->crd_next = NULL;
545
546 crp->crp_etype = 0;
547 err = crypto_dispatch(crp);
548 if (error == 0)
549 error = err;
550 }
551 if (bp->bio_error == 0)
552 bp->bio_error = error;
553}
554
555int
556g_eli_read_metadata(struct g_class *mp, struct g_provider *pp,
557 struct g_eli_metadata *md)
558{
559 struct g_geom *gp;
560 struct g_consumer *cp;
561 u_char *buf = NULL;
562 int error;
--- 91 unchanged lines hidden (view full) ---
654g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
655 const struct g_eli_metadata *md, const u_char *mkey, int nkey)
656{
657 struct g_eli_softc *sc;
658 struct g_eli_worker *wr;
659 struct g_geom *gp;
660 struct g_provider *pp;
661 struct g_consumer *cp;
662 struct cryptoini cri;
663 u_int i, threads;
664 int error;
665
666 G_ELI_DEBUG(1, "Creating device %s%s.", bpp->name, G_ELI_SUFFIX);
667
668 gp = g_new_geomf(mp, "%s%s", bpp->name, G_ELI_SUFFIX);
669 gp->softc = NULL; /* for a moment */
670
--- 12 unchanged lines hidden (view full) ---
683 if (md->md_flags & G_ELI_FLAG_WO_DETACH)
684 gp->access = g_eli_access;
685 else
686 gp->access = g_std_access;
687 gp->dumpconf = g_eli_dumpconf;
688
689 sc->sc_crypto = G_ELI_CRYPTO_SW;
690 sc->sc_flags = md->md_flags;
691 sc->sc_algo = md->md_algo;
692 sc->sc_nkey = nkey;
693 /*
694 * Remember the keys in our softc structure.
695 */
696 bcopy(mkey, sc->sc_ivkey, sizeof(sc->sc_ivkey));
697 mkey += sizeof(sc->sc_ivkey);
698 bcopy(mkey, sc->sc_datakey, sizeof(sc->sc_datakey));
699 sc->sc_keylen = md->md_keylen;
700
701 /*
702 * Precalculate SHA256 for IV generation.
703 * This is expensive operation and we can do it only once now or for
704 * every access to sector, so now will be much better.
705 */
706 SHA256_Init(&sc->sc_ivctx);
707 SHA256_Update(&sc->sc_ivctx, sc->sc_ivkey, sizeof(sc->sc_ivkey));
708
--- 30 unchanged lines hidden (view full) ---
739 G_ELI_DEBUG(1, "Cannot access %s (error=%d).",
740 bpp->name, error);
741 }
742 goto failed;
743 }
744
745 LIST_INIT(&sc->sc_workers);
746
747 bzero(&cri, sizeof(cri));
748 cri.cri_alg = sc->sc_algo;
749 cri.cri_klen = sc->sc_keylen;
750 cri.cri_key = sc->sc_datakey;
751
752 threads = g_eli_threads;
753 if (threads == 0)
754 threads = mp_ncpus;
755 else if (threads > mp_ncpus) {
756 /* There is really no need for too many worker threads. */
757 threads = mp_ncpus;
758 G_ELI_DEBUG(0, "Reducing number of threads to %u.", threads);
759 }
760 for (i = 0; i < threads; i++) {
761 wr = malloc(sizeof(*wr), M_ELI, M_WAITOK | M_ZERO);
762 wr->w_softc = sc;
763 wr->w_number = i;
764
765 /*
766 * If this is the first pass, try to get hardware support.
767 * Use software cryptography, if we cannot get it.
768 */
769 if (i == 0) {
770 error = crypto_newsession(&wr->w_sid, &cri, 1);
771 if (error == 0)
772 sc->sc_crypto = G_ELI_CRYPTO_HW;
773 }
774 if (sc->sc_crypto == G_ELI_CRYPTO_SW)
775 error = crypto_newsession(&wr->w_sid, &cri, 0);
776 if (error != 0) {
777 free(wr, M_ELI);
778 if (req != NULL) {
779 gctl_error(req, "Cannot set up crypto session "
780 "for %s (error=%d).", bpp->name, error);
781 } else {
782 G_ELI_DEBUG(1, "Cannot set up crypto session "
783 "for %s (error=%d).", bpp->name, error);
--- 22 unchanged lines hidden (view full) ---
806 }
807
808 /*
809 * Create decrypted provider.
810 */
811 pp = g_new_providerf(gp, "%s%s", bpp->name, G_ELI_SUFFIX);
812 pp->sectorsize = md->md_sectorsize;
813 pp->mediasize = bpp->mediasize;
814 if ((sc->sc_flags & G_ELI_FLAG_ONETIME) == 0)
815 pp->mediasize -= bpp->sectorsize;
816 pp->mediasize -= (pp->mediasize % pp->sectorsize);
817 g_error_provider(pp, 0);
818
819 G_ELI_DEBUG(0, "Device %s created.", pp->name);
820 G_ELI_DEBUG(0, " Cipher: %s", g_eli_algo2str(sc->sc_algo));
821 G_ELI_DEBUG(0, "Key length: %u", sc->sc_keylen);
822 G_ELI_DEBUG(0, " Crypto: %s",
823 sc->sc_crypto == G_ELI_CRYPTO_SW ? "software" : "hardware");
824 return (gp);
825failed:
826 mtx_lock(&sc->sc_queue_mtx);
827 sc->sc_flags |= G_ELI_FLAG_DESTROY;
828 wakeup(sc);
829 /*
--- 161 unchanged lines hidden (view full) ---
991 if (md.md_version > G_ELI_VERSION) {
992 printf("geom_eli.ko module is too old to handle %s.\n",
993 pp->name);
994 return (NULL);
995 }
996 if (md.md_provsize != pp->mediasize)
997 return (NULL);
998 /* Should we attach it on boot? */
999 if ((md.md_flags & G_ELI_FLAG_BOOT) == 0)
1000 return (NULL);
1001 if (md.md_keys == 0x00) {
1002 G_ELI_DEBUG(0, "No valid keys on %s.", pp->name);
1003 return (NULL);
1004 }
1005 if (md.md_iterations == -1) {
1006 /* If there is no passphrase, we try only once. */
1007 tries = 1;
--- 105 unchanged lines hidden (view full) ---
1113 return; /* Nothing here. */
1114 sbuf_printf(sb, "%s<Flags>", indent);
1115 if (sc->sc_flags == 0)
1116 sbuf_printf(sb, "NONE");
1117 else {
1118 int first = 1;
1119
1120#define ADD_FLAG(flag, name) do { \
1121 if ((sc->sc_flags & (flag)) != 0) { \
1122 if (!first) \
1123 sbuf_printf(sb, ", "); \
1124 else \
1125 first = 0; \
1126 sbuf_printf(sb, name); \
1127 } \
1128} while (0)
1129 ADD_FLAG(G_ELI_FLAG_ONETIME, "ONETIME");
1130 ADD_FLAG(G_ELI_FLAG_BOOT, "BOOT");
1131 ADD_FLAG(G_ELI_FLAG_WO_DETACH, "W-DETACH");
1132 ADD_FLAG(G_ELI_FLAG_RW_DETACH, "RW-DETACH");
1133 ADD_FLAG(G_ELI_FLAG_WOPEN, "W-OPEN");
1134 ADD_FLAG(G_ELI_FLAG_DESTROY, "DESTROY");
1135#undef ADD_FLAG
1136 }
1137 sbuf_printf(sb, "</Flags>\n");
1138
1139 if ((sc->sc_flags & G_ELI_FLAG_ONETIME) == 0) {
1140 sbuf_printf(sb, "%s<UsedKey>%u</UsedKey>\n", indent,
1141 sc->sc_nkey);
1142 }
1143 sbuf_printf(sb, "%s<Crypto>", indent);
1144 switch (sc->sc_crypto) {
1145 case G_ELI_CRYPTO_HW:
1146 sbuf_printf(sb, "hardware");
1147 break;
1148 case G_ELI_CRYPTO_SW:
1149 sbuf_printf(sb, "software");
1150 break;
1151 default:
1152 sbuf_printf(sb, "UNKNOWN");
1153 break;
1154 }
1155 sbuf_printf(sb, "</Crypto>\n");
1156 sbuf_printf(sb, "%s<KeyLength>%u</KeyLength>\n", indent, sc->sc_keylen);
1157 sbuf_printf(sb, "%s<Cipher>%s</Cipher>\n", indent,
1158 g_eli_algo2str(sc->sc_algo));
1159}
1160
1161DECLARE_GEOM_CLASS(g_eli_class, g_eli);
1162MODULE_DEPEND(geom_eli, crypto, 1, 1, 1);