g_eli_privacy.c revision 214118
1159307Spjd/*- 2213072Spjd * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3159307Spjd * All rights reserved. 4159307Spjd * 5159307Spjd * Redistribution and use in source and binary forms, with or without 6159307Spjd * modification, are permitted provided that the following conditions 7159307Spjd * are met: 8159307Spjd * 1. Redistributions of source code must retain the above copyright 9159307Spjd * notice, this list of conditions and the following disclaimer. 10159307Spjd * 2. Redistributions in binary form must reproduce the above copyright 11159307Spjd * notice, this list of conditions and the following disclaimer in the 12159307Spjd * documentation and/or other materials provided with the distribution. 13159307Spjd * 14159307Spjd * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 15159307Spjd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16159307Spjd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17159307Spjd * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 18159307Spjd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19159307Spjd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20159307Spjd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21159307Spjd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22159307Spjd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23159307Spjd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24159307Spjd * SUCH DAMAGE. 25159307Spjd */ 26159307Spjd 27159307Spjd#include <sys/cdefs.h> 28159307Spjd__FBSDID("$FreeBSD: head/sys/geom/eli/g_eli_privacy.c 214118 2010-10-20 20:50:55Z pjd $"); 29159307Spjd 30159307Spjd#include <sys/param.h> 31159307Spjd#include <sys/systm.h> 32159307Spjd#include <sys/kernel.h> 33159307Spjd#include <sys/linker.h> 34159307Spjd#include <sys/module.h> 35159307Spjd#include <sys/lock.h> 36159307Spjd#include <sys/mutex.h> 37159307Spjd#include <sys/bio.h> 38159307Spjd#include <sys/sysctl.h> 39159307Spjd#include <sys/malloc.h> 40159307Spjd#include <sys/kthread.h> 41159307Spjd#include <sys/proc.h> 42159307Spjd#include <sys/sched.h> 43159307Spjd#include <sys/smp.h> 44159307Spjd#include <sys/uio.h> 45159307Spjd#include <sys/vnode.h> 46159307Spjd 47159307Spjd#include <vm/uma.h> 48159307Spjd 49159307Spjd#include <geom/geom.h> 50159307Spjd#include <geom/eli/g_eli.h> 51159307Spjd#include <geom/eli/pkcs5v2.h> 52159307Spjd 53159307Spjd/* 54159307Spjd * Code paths: 55159307Spjd * BIO_READ: 56214118Spjd * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver 57159307Spjd * BIO_WRITE: 58159307Spjd * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver 59159307Spjd */ 60159307Spjd 61159307SpjdMALLOC_DECLARE(M_ELI); 62159307Spjd 63159307Spjd/* 64159307Spjd * The function is called after we read and decrypt data. 65159307Spjd * 66214118Spjd * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver 67159307Spjd */ 68159307Spjdstatic int 69159307Spjdg_eli_crypto_read_done(struct cryptop *crp) 70159307Spjd{ 71214118Spjd struct g_eli_softc *sc; 72159307Spjd struct bio *bp; 73159307Spjd 74159307Spjd if (crp->crp_etype == EAGAIN) { 75159307Spjd if (g_eli_crypto_rerun(crp) == 0) 76159307Spjd return (0); 77159307Spjd } 78159307Spjd bp = (struct bio *)crp->crp_opaque; 79159307Spjd bp->bio_inbed++; 80159307Spjd if (crp->crp_etype == 0) { 81159307Spjd G_ELI_DEBUG(3, "Crypto READ request done (%d/%d).", 82159307Spjd bp->bio_inbed, bp->bio_children); 83159307Spjd bp->bio_completed += crp->crp_olen; 84159307Spjd } else { 85159307Spjd G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.", 86159307Spjd bp->bio_inbed, bp->bio_children, crp->crp_etype); 87159307Spjd if (bp->bio_error == 0) 88159307Spjd bp->bio_error = crp->crp_etype; 89159307Spjd } 90159307Spjd /* 91159307Spjd * Do we have all sectors already? 92159307Spjd */ 93159307Spjd if (bp->bio_inbed < bp->bio_children) 94159307Spjd return (0); 95159307Spjd free(bp->bio_driver2, M_ELI); 96159307Spjd bp->bio_driver2 = NULL; 97159307Spjd if (bp->bio_error != 0) { 98159307Spjd G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).", 99159307Spjd bp->bio_error); 100159307Spjd bp->bio_completed = 0; 101159307Spjd } 102159307Spjd /* 103159307Spjd * Read is finished, send it up. 104159307Spjd */ 105214118Spjd sc = bp->bio_to->geom->softc; 106159307Spjd g_io_deliver(bp, bp->bio_error); 107214118Spjd atomic_subtract_int(&sc->sc_inflight, 1); 108159307Spjd return (0); 109159307Spjd} 110159307Spjd 111159307Spjd/* 112159307Spjd * The function is called after data encryption. 113159307Spjd * 114159307Spjd * g_eli_start -> g_eli_crypto_run -> G_ELI_CRYPTO_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver 115159307Spjd */ 116159307Spjdstatic int 117159307Spjdg_eli_crypto_write_done(struct cryptop *crp) 118159307Spjd{ 119214118Spjd struct g_eli_softc *sc; 120159307Spjd struct g_geom *gp; 121159307Spjd struct g_consumer *cp; 122159307Spjd struct bio *bp, *cbp; 123159307Spjd 124159307Spjd if (crp->crp_etype == EAGAIN) { 125159307Spjd if (g_eli_crypto_rerun(crp) == 0) 126159307Spjd return (0); 127159307Spjd } 128159307Spjd bp = (struct bio *)crp->crp_opaque; 129159307Spjd bp->bio_inbed++; 130159307Spjd if (crp->crp_etype == 0) { 131159307Spjd G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).", 132159307Spjd bp->bio_inbed, bp->bio_children); 133159307Spjd } else { 134159307Spjd G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.", 135159307Spjd bp->bio_inbed, bp->bio_children, crp->crp_etype); 136159307Spjd if (bp->bio_error == 0) 137159307Spjd bp->bio_error = crp->crp_etype; 138159307Spjd } 139159307Spjd /* 140159307Spjd * All sectors are already encrypted? 141159307Spjd */ 142159307Spjd if (bp->bio_inbed < bp->bio_children) 143159307Spjd return (0); 144159307Spjd bp->bio_inbed = 0; 145159307Spjd bp->bio_children = 1; 146159307Spjd cbp = bp->bio_driver1; 147159307Spjd bp->bio_driver1 = NULL; 148214118Spjd gp = bp->bio_to->geom; 149159307Spjd if (bp->bio_error != 0) { 150159307Spjd G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).", 151159307Spjd bp->bio_error); 152159307Spjd free(bp->bio_driver2, M_ELI); 153159307Spjd bp->bio_driver2 = NULL; 154159307Spjd g_destroy_bio(cbp); 155214118Spjd sc = gp->softc; 156159307Spjd g_io_deliver(bp, bp->bio_error); 157214118Spjd atomic_subtract_int(&sc->sc_inflight, 1); 158159307Spjd return (0); 159159307Spjd } 160159307Spjd cbp->bio_data = bp->bio_driver2; 161159307Spjd cbp->bio_done = g_eli_write_done; 162159307Spjd cp = LIST_FIRST(&gp->consumer); 163159307Spjd cbp->bio_to = cp->provider; 164159307Spjd G_ELI_LOGREQ(2, cbp, "Sending request."); 165159307Spjd /* 166159307Spjd * Send encrypted data to the provider. 167159307Spjd */ 168159307Spjd g_io_request(cbp, cp); 169159307Spjd return (0); 170159307Spjd} 171159307Spjd 172159307Spjd/* 173214118Spjd * The function is called to read encrypted data. 174214118Spjd * 175214118Spjd * g_eli_start -> G_ELI_CRYPTO_READ -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver 176214118Spjd */ 177214118Spjdvoid 178214118Spjdg_eli_crypto_read(struct g_eli_softc *sc, struct bio *bp, boolean_t fromworker) 179214118Spjd{ 180214118Spjd struct g_consumer *cp; 181214118Spjd struct bio *cbp; 182214118Spjd 183214118Spjd if (!fromworker) { 184214118Spjd /* 185214118Spjd * We are not called from the worker thread, so check if 186214118Spjd * device is suspended. 187214118Spjd */ 188214118Spjd mtx_lock(&sc->sc_queue_mtx); 189214118Spjd if (sc->sc_flags & G_ELI_FLAG_SUSPEND) { 190214118Spjd /* 191214118Spjd * If device is suspended, we place the request onto 192214118Spjd * the queue, so it can be handled after resume. 193214118Spjd */ 194214118Spjd G_ELI_DEBUG(0, "device suspended, move onto queue"); 195214118Spjd bioq_insert_tail(&sc->sc_queue, bp); 196214118Spjd mtx_unlock(&sc->sc_queue_mtx); 197214118Spjd wakeup(sc); 198214118Spjd return; 199214118Spjd } 200214118Spjd atomic_add_int(&sc->sc_inflight, 1); 201214118Spjd mtx_unlock(&sc->sc_queue_mtx); 202214118Spjd } 203214118Spjd bp->bio_pflags = 0; 204214118Spjd bp->bio_driver2 = NULL; 205214118Spjd cbp = bp->bio_driver1; 206214118Spjd cbp->bio_done = g_eli_read_done; 207214118Spjd cp = LIST_FIRST(&sc->sc_geom->consumer); 208214118Spjd cbp->bio_to = cp->provider; 209214118Spjd G_ELI_LOGREQ(2, cbp, "Sending request."); 210214118Spjd /* 211214118Spjd * Read encrypted data from provider. 212214118Spjd */ 213214118Spjd g_io_request(cbp, cp); 214214118Spjd} 215214118Spjd 216214118Spjd/* 217159307Spjd * This is the main function responsible for cryptography (ie. communication 218159307Spjd * with crypto(9) subsystem). 219214116Spjd * 220214116Spjd * BIO_READ: 221214118Spjd * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> G_ELI_CRYPTO_RUN -> g_eli_crypto_read_done -> g_io_deliver 222214116Spjd * BIO_WRITE: 223214116Spjd * g_eli_start -> G_ELI_CRYPTO_RUN -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver 224159307Spjd */ 225159307Spjdvoid 226159307Spjdg_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp) 227159307Spjd{ 228159307Spjd struct g_eli_softc *sc; 229159307Spjd struct cryptop *crp; 230159307Spjd struct cryptodesc *crd; 231159307Spjd struct uio *uio; 232159307Spjd struct iovec *iov; 233213063Spjd u_int i, nsec, secsize; 234159307Spjd int err, error; 235213063Spjd off_t dstoff; 236159307Spjd size_t size; 237159307Spjd u_char *p, *data; 238159307Spjd 239159307Spjd G_ELI_LOGREQ(3, bp, "%s", __func__); 240159307Spjd 241159307Spjd bp->bio_pflags = wr->w_number; 242159307Spjd sc = wr->w_softc; 243159307Spjd secsize = LIST_FIRST(&sc->sc_geom->provider)->sectorsize; 244159307Spjd nsec = bp->bio_length / secsize; 245159307Spjd 246159307Spjd /* 247159307Spjd * Calculate how much memory do we need. 248159307Spjd * We need separate crypto operation for every single sector. 249159307Spjd * It is much faster to calculate total amount of needed memory here and 250159307Spjd * do the allocation once instead of allocating memory in pieces (many, 251159307Spjd * many pieces). 252159307Spjd */ 253159307Spjd size = sizeof(*crp) * nsec; 254159307Spjd size += sizeof(*crd) * nsec; 255159307Spjd size += sizeof(*uio) * nsec; 256159307Spjd size += sizeof(*iov) * nsec; 257159307Spjd /* 258159307Spjd * If we write the data we cannot destroy current bio_data content, 259159307Spjd * so we need to allocate more memory for encrypted data. 260159307Spjd */ 261159307Spjd if (bp->bio_cmd == BIO_WRITE) 262159307Spjd size += bp->bio_length; 263159307Spjd p = malloc(size, M_ELI, M_WAITOK); 264159307Spjd 265159307Spjd bp->bio_inbed = 0; 266159307Spjd bp->bio_children = nsec; 267159307Spjd bp->bio_driver2 = p; 268159307Spjd 269159307Spjd if (bp->bio_cmd == BIO_READ) 270159307Spjd data = bp->bio_data; 271159307Spjd else { 272159307Spjd data = p; 273159307Spjd p += bp->bio_length; 274159307Spjd bcopy(bp->bio_data, data, bp->bio_length); 275159307Spjd } 276159307Spjd 277159307Spjd error = 0; 278213063Spjd for (i = 0, dstoff = bp->bio_offset; i < nsec; i++, dstoff += secsize) { 279159307Spjd crp = (struct cryptop *)p; p += sizeof(*crp); 280159307Spjd crd = (struct cryptodesc *)p; p += sizeof(*crd); 281159307Spjd uio = (struct uio *)p; p += sizeof(*uio); 282159307Spjd iov = (struct iovec *)p; p += sizeof(*iov); 283159307Spjd 284159307Spjd iov->iov_len = secsize; 285159307Spjd iov->iov_base = data; 286159307Spjd data += secsize; 287159307Spjd 288159307Spjd uio->uio_iov = iov; 289159307Spjd uio->uio_iovcnt = 1; 290159307Spjd uio->uio_segflg = UIO_SYSSPACE; 291159307Spjd uio->uio_resid = secsize; 292159307Spjd 293159307Spjd crp->crp_sid = wr->w_sid; 294159307Spjd crp->crp_ilen = secsize; 295159307Spjd crp->crp_olen = secsize; 296159307Spjd crp->crp_opaque = (void *)bp; 297159307Spjd crp->crp_buf = (void *)uio; 298159307Spjd if (bp->bio_cmd == BIO_WRITE) 299159307Spjd crp->crp_callback = g_eli_crypto_write_done; 300159307Spjd else /* if (bp->bio_cmd == BIO_READ) */ 301159307Spjd crp->crp_callback = g_eli_crypto_read_done; 302159307Spjd crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC | CRYPTO_F_REL; 303159307Spjd if (g_eli_batch) 304159307Spjd crp->crp_flags |= CRYPTO_F_BATCH; 305159307Spjd crp->crp_desc = crd; 306159307Spjd 307159307Spjd crd->crd_skip = 0; 308159307Spjd crd->crd_len = secsize; 309159307Spjd crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 310213067Spjd if (sc->sc_nekeys > 1) 311213067Spjd crd->crd_flags |= CRD_F_KEY_EXPLICIT; 312159307Spjd if (bp->bio_cmd == BIO_WRITE) 313159307Spjd crd->crd_flags |= CRD_F_ENCRYPT; 314159307Spjd crd->crd_alg = sc->sc_ealgo; 315213067Spjd crd->crd_key = g_eli_crypto_key(sc, dstoff, secsize); 316159307Spjd crd->crd_klen = sc->sc_ekeylen; 317213070Spjd if (sc->sc_ealgo == CRYPTO_AES_XTS) 318213070Spjd crd->crd_klen <<= 1; 319213063Spjd g_eli_crypto_ivgen(sc, dstoff, crd->crd_iv, 320159307Spjd sizeof(crd->crd_iv)); 321159307Spjd crd->crd_next = NULL; 322159307Spjd 323159307Spjd crp->crp_etype = 0; 324159307Spjd err = crypto_dispatch(crp); 325159307Spjd if (error == 0) 326159307Spjd error = err; 327159307Spjd } 328159307Spjd if (bp->bio_error == 0) 329159307Spjd bp->bio_error = error; 330159307Spjd} 331