g_eli_privacy.c revision 275732
1159307Spjd/*- 2220922Spjd * Copyright (c) 2005-2011 Pawel Jakub Dawidek <pawel@dawidek.net> 3159307Spjd * All rights reserved. 4159307Spjd * 5159307Spjd * Redistribution and use in source and binary forms, with or without 6159307Spjd * modification, are permitted provided that the following conditions 7159307Spjd * are met: 8159307Spjd * 1. Redistributions of source code must retain the above copyright 9159307Spjd * notice, this list of conditions and the following disclaimer. 10159307Spjd * 2. Redistributions in binary form must reproduce the above copyright 11159307Spjd * notice, this list of conditions and the following disclaimer in the 12159307Spjd * documentation and/or other materials provided with the distribution. 13159307Spjd * 14159307Spjd * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 15159307Spjd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16159307Spjd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17159307Spjd * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 18159307Spjd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19159307Spjd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20159307Spjd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21159307Spjd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22159307Spjd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23159307Spjd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24159307Spjd * SUCH DAMAGE. 25159307Spjd */ 26159307Spjd 27159307Spjd#include <sys/cdefs.h> 28159307Spjd__FBSDID("$FreeBSD: head/sys/geom/eli/g_eli_privacy.c 275732 2014-12-12 19:56:36Z jmg $"); 29159307Spjd 30159307Spjd#include <sys/param.h> 31159307Spjd#include <sys/systm.h> 32159307Spjd#include <sys/kernel.h> 33159307Spjd#include <sys/linker.h> 34159307Spjd#include <sys/module.h> 35159307Spjd#include <sys/lock.h> 36159307Spjd#include <sys/mutex.h> 37159307Spjd#include <sys/bio.h> 38159307Spjd#include <sys/sysctl.h> 39159307Spjd#include <sys/malloc.h> 40159307Spjd#include <sys/kthread.h> 41159307Spjd#include <sys/proc.h> 42159307Spjd#include <sys/sched.h> 43159307Spjd#include <sys/smp.h> 44159307Spjd#include <sys/vnode.h> 45159307Spjd 46159307Spjd#include <vm/uma.h> 47159307Spjd 48159307Spjd#include <geom/geom.h> 49159307Spjd#include <geom/eli/g_eli.h> 50159307Spjd#include <geom/eli/pkcs5v2.h> 51159307Spjd 52159307Spjd/* 53159307Spjd * Code paths: 54159307Spjd * BIO_READ: 55214118Spjd * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver 56159307Spjd * BIO_WRITE: 57159307Spjd * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver 58159307Spjd */ 59159307Spjd 60159307SpjdMALLOC_DECLARE(M_ELI); 61159307Spjd 62159307Spjd/* 63159307Spjd * The function is called after we read and decrypt data. 64159307Spjd * 65214118Spjd * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver 66159307Spjd */ 67159307Spjdstatic int 68159307Spjdg_eli_crypto_read_done(struct cryptop *crp) 69159307Spjd{ 70214118Spjd struct g_eli_softc *sc; 71159307Spjd struct bio *bp; 72159307Spjd 73159307Spjd if (crp->crp_etype == EAGAIN) { 74159307Spjd if (g_eli_crypto_rerun(crp) == 0) 75159307Spjd return (0); 76159307Spjd } 77159307Spjd bp = (struct bio *)crp->crp_opaque; 78159307Spjd bp->bio_inbed++; 79159307Spjd if (crp->crp_etype == 0) { 80159307Spjd G_ELI_DEBUG(3, "Crypto READ request done (%d/%d).", 81159307Spjd bp->bio_inbed, bp->bio_children); 82159307Spjd bp->bio_completed += crp->crp_olen; 83159307Spjd } else { 84159307Spjd G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.", 85159307Spjd bp->bio_inbed, bp->bio_children, crp->crp_etype); 86159307Spjd if (bp->bio_error == 0) 87159307Spjd bp->bio_error = crp->crp_etype; 88159307Spjd } 89220922Spjd sc = bp->bio_to->geom->softc; 90220922Spjd g_eli_key_drop(sc, crp->crp_desc->crd_key); 91159307Spjd /* 92159307Spjd * Do we have all sectors already? 93159307Spjd */ 94159307Spjd if (bp->bio_inbed < bp->bio_children) 95159307Spjd return (0); 96159307Spjd free(bp->bio_driver2, M_ELI); 97159307Spjd bp->bio_driver2 = NULL; 98159307Spjd if (bp->bio_error != 0) { 99159307Spjd G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).", 100159307Spjd bp->bio_error); 101159307Spjd bp->bio_completed = 0; 102159307Spjd } 103159307Spjd /* 104159307Spjd * Read is finished, send it up. 105159307Spjd */ 106159307Spjd g_io_deliver(bp, bp->bio_error); 107214118Spjd atomic_subtract_int(&sc->sc_inflight, 1); 108159307Spjd return (0); 109159307Spjd} 110159307Spjd 111159307Spjd/* 112159307Spjd * The function is called after data encryption. 113159307Spjd * 114159307Spjd * g_eli_start -> g_eli_crypto_run -> G_ELI_CRYPTO_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver 115159307Spjd */ 116159307Spjdstatic int 117159307Spjdg_eli_crypto_write_done(struct cryptop *crp) 118159307Spjd{ 119214118Spjd struct g_eli_softc *sc; 120159307Spjd struct g_geom *gp; 121159307Spjd struct g_consumer *cp; 122159307Spjd struct bio *bp, *cbp; 123159307Spjd 124159307Spjd if (crp->crp_etype == EAGAIN) { 125159307Spjd if (g_eli_crypto_rerun(crp) == 0) 126159307Spjd return (0); 127159307Spjd } 128159307Spjd bp = (struct bio *)crp->crp_opaque; 129159307Spjd bp->bio_inbed++; 130159307Spjd if (crp->crp_etype == 0) { 131159307Spjd G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).", 132159307Spjd bp->bio_inbed, bp->bio_children); 133159307Spjd } else { 134159307Spjd G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.", 135159307Spjd bp->bio_inbed, bp->bio_children, crp->crp_etype); 136159307Spjd if (bp->bio_error == 0) 137159307Spjd bp->bio_error = crp->crp_etype; 138159307Spjd } 139220922Spjd gp = bp->bio_to->geom; 140220922Spjd sc = gp->softc; 141220922Spjd g_eli_key_drop(sc, crp->crp_desc->crd_key); 142159307Spjd /* 143159307Spjd * All sectors are already encrypted? 144159307Spjd */ 145159307Spjd if (bp->bio_inbed < bp->bio_children) 146159307Spjd return (0); 147159307Spjd bp->bio_inbed = 0; 148159307Spjd bp->bio_children = 1; 149159307Spjd cbp = bp->bio_driver1; 150159307Spjd bp->bio_driver1 = NULL; 151159307Spjd if (bp->bio_error != 0) { 152159307Spjd G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).", 153159307Spjd bp->bio_error); 154159307Spjd free(bp->bio_driver2, M_ELI); 155159307Spjd bp->bio_driver2 = NULL; 156159307Spjd g_destroy_bio(cbp); 157159307Spjd g_io_deliver(bp, bp->bio_error); 158214118Spjd atomic_subtract_int(&sc->sc_inflight, 1); 159159307Spjd return (0); 160159307Spjd } 161159307Spjd cbp->bio_data = bp->bio_driver2; 162159307Spjd cbp->bio_done = g_eli_write_done; 163159307Spjd cp = LIST_FIRST(&gp->consumer); 164159307Spjd cbp->bio_to = cp->provider; 165159307Spjd G_ELI_LOGREQ(2, cbp, "Sending request."); 166159307Spjd /* 167159307Spjd * Send encrypted data to the provider. 168159307Spjd */ 169159307Spjd g_io_request(cbp, cp); 170159307Spjd return (0); 171159307Spjd} 172159307Spjd 173159307Spjd/* 174214118Spjd * The function is called to read encrypted data. 175214118Spjd * 176214118Spjd * g_eli_start -> G_ELI_CRYPTO_READ -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver 177214118Spjd */ 178214118Spjdvoid 179214118Spjdg_eli_crypto_read(struct g_eli_softc *sc, struct bio *bp, boolean_t fromworker) 180214118Spjd{ 181214118Spjd struct g_consumer *cp; 182214118Spjd struct bio *cbp; 183214118Spjd 184214118Spjd if (!fromworker) { 185214118Spjd /* 186214118Spjd * We are not called from the worker thread, so check if 187214118Spjd * device is suspended. 188214118Spjd */ 189214118Spjd mtx_lock(&sc->sc_queue_mtx); 190214118Spjd if (sc->sc_flags & G_ELI_FLAG_SUSPEND) { 191214118Spjd /* 192214118Spjd * If device is suspended, we place the request onto 193214118Spjd * the queue, so it can be handled after resume. 194214118Spjd */ 195214118Spjd G_ELI_DEBUG(0, "device suspended, move onto queue"); 196214118Spjd bioq_insert_tail(&sc->sc_queue, bp); 197214118Spjd mtx_unlock(&sc->sc_queue_mtx); 198214118Spjd wakeup(sc); 199214118Spjd return; 200214118Spjd } 201214118Spjd atomic_add_int(&sc->sc_inflight, 1); 202214118Spjd mtx_unlock(&sc->sc_queue_mtx); 203214118Spjd } 204214118Spjd bp->bio_pflags = 0; 205214118Spjd bp->bio_driver2 = NULL; 206214118Spjd cbp = bp->bio_driver1; 207214118Spjd cbp->bio_done = g_eli_read_done; 208214118Spjd cp = LIST_FIRST(&sc->sc_geom->consumer); 209214118Spjd cbp->bio_to = cp->provider; 210214118Spjd G_ELI_LOGREQ(2, cbp, "Sending request."); 211214118Spjd /* 212214118Spjd * Read encrypted data from provider. 213214118Spjd */ 214214118Spjd g_io_request(cbp, cp); 215214118Spjd} 216214118Spjd 217214118Spjd/* 218159307Spjd * This is the main function responsible for cryptography (ie. communication 219159307Spjd * with crypto(9) subsystem). 220214116Spjd * 221214116Spjd * BIO_READ: 222214118Spjd * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> G_ELI_CRYPTO_RUN -> g_eli_crypto_read_done -> g_io_deliver 223214116Spjd * BIO_WRITE: 224214116Spjd * g_eli_start -> G_ELI_CRYPTO_RUN -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver 225159307Spjd */ 226159307Spjdvoid 227159307Spjdg_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp) 228159307Spjd{ 229159307Spjd struct g_eli_softc *sc; 230159307Spjd struct cryptop *crp; 231159307Spjd struct cryptodesc *crd; 232213063Spjd u_int i, nsec, secsize; 233159307Spjd int err, error; 234213063Spjd off_t dstoff; 235159307Spjd size_t size; 236159307Spjd u_char *p, *data; 237159307Spjd 238159307Spjd G_ELI_LOGREQ(3, bp, "%s", __func__); 239159307Spjd 240159307Spjd bp->bio_pflags = wr->w_number; 241159307Spjd sc = wr->w_softc; 242159307Spjd secsize = LIST_FIRST(&sc->sc_geom->provider)->sectorsize; 243159307Spjd nsec = bp->bio_length / secsize; 244159307Spjd 245159307Spjd /* 246159307Spjd * Calculate how much memory do we need. 247159307Spjd * We need separate crypto operation for every single sector. 248159307Spjd * It is much faster to calculate total amount of needed memory here and 249159307Spjd * do the allocation once instead of allocating memory in pieces (many, 250159307Spjd * many pieces). 251159307Spjd */ 252159307Spjd size = sizeof(*crp) * nsec; 253159307Spjd size += sizeof(*crd) * nsec; 254159307Spjd /* 255159307Spjd * If we write the data we cannot destroy current bio_data content, 256159307Spjd * so we need to allocate more memory for encrypted data. 257159307Spjd */ 258159307Spjd if (bp->bio_cmd == BIO_WRITE) 259159307Spjd size += bp->bio_length; 260159307Spjd p = malloc(size, M_ELI, M_WAITOK); 261159307Spjd 262159307Spjd bp->bio_inbed = 0; 263159307Spjd bp->bio_children = nsec; 264159307Spjd bp->bio_driver2 = p; 265159307Spjd 266159307Spjd if (bp->bio_cmd == BIO_READ) 267159307Spjd data = bp->bio_data; 268159307Spjd else { 269159307Spjd data = p; 270159307Spjd p += bp->bio_length; 271159307Spjd bcopy(bp->bio_data, data, bp->bio_length); 272159307Spjd } 273159307Spjd 274159307Spjd error = 0; 275213063Spjd for (i = 0, dstoff = bp->bio_offset; i < nsec; i++, dstoff += secsize) { 276159307Spjd crp = (struct cryptop *)p; p += sizeof(*crp); 277159307Spjd crd = (struct cryptodesc *)p; p += sizeof(*crd); 278159307Spjd 279159307Spjd crp->crp_sid = wr->w_sid; 280159307Spjd crp->crp_ilen = secsize; 281159307Spjd crp->crp_olen = secsize; 282159307Spjd crp->crp_opaque = (void *)bp; 283271148Sjmg crp->crp_buf = (void *)data; 284271148Sjmg data += secsize; 285159307Spjd if (bp->bio_cmd == BIO_WRITE) 286159307Spjd crp->crp_callback = g_eli_crypto_write_done; 287159307Spjd else /* if (bp->bio_cmd == BIO_READ) */ 288159307Spjd crp->crp_callback = g_eli_crypto_read_done; 289275732Sjmg crp->crp_flags = CRYPTO_F_CBIFSYNC; 290159307Spjd if (g_eli_batch) 291159307Spjd crp->crp_flags |= CRYPTO_F_BATCH; 292159307Spjd crp->crp_desc = crd; 293159307Spjd 294159307Spjd crd->crd_skip = 0; 295159307Spjd crd->crd_len = secsize; 296159307Spjd crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 297220922Spjd if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) == 0) 298213067Spjd crd->crd_flags |= CRD_F_KEY_EXPLICIT; 299159307Spjd if (bp->bio_cmd == BIO_WRITE) 300159307Spjd crd->crd_flags |= CRD_F_ENCRYPT; 301159307Spjd crd->crd_alg = sc->sc_ealgo; 302220922Spjd crd->crd_key = g_eli_key_hold(sc, dstoff, secsize); 303159307Spjd crd->crd_klen = sc->sc_ekeylen; 304213070Spjd if (sc->sc_ealgo == CRYPTO_AES_XTS) 305213070Spjd crd->crd_klen <<= 1; 306213063Spjd g_eli_crypto_ivgen(sc, dstoff, crd->crd_iv, 307159307Spjd sizeof(crd->crd_iv)); 308159307Spjd crd->crd_next = NULL; 309159307Spjd 310159307Spjd crp->crp_etype = 0; 311159307Spjd err = crypto_dispatch(crp); 312159307Spjd if (error == 0) 313159307Spjd error = err; 314159307Spjd } 315159307Spjd if (bp->bio_error == 0) 316159307Spjd bp->bio_error = error; 317159307Spjd} 318