1159307Spjd/*- 2220922Spjd * Copyright (c) 2005-2011 Pawel Jakub Dawidek <pawel@dawidek.net> 3159307Spjd * All rights reserved. 4159307Spjd * 5159307Spjd * Redistribution and use in source and binary forms, with or without 6159307Spjd * modification, are permitted provided that the following conditions 7159307Spjd * are met: 8159307Spjd * 1. Redistributions of source code must retain the above copyright 9159307Spjd * notice, this list of conditions and the following disclaimer. 10159307Spjd * 2. Redistributions in binary form must reproduce the above copyright 11159307Spjd * notice, this list of conditions and the following disclaimer in the 12159307Spjd * documentation and/or other materials provided with the distribution. 13159307Spjd * 14159307Spjd * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 15159307Spjd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16159307Spjd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17159307Spjd * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 18159307Spjd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19159307Spjd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20159307Spjd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21159307Spjd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22159307Spjd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23159307Spjd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24159307Spjd * SUCH DAMAGE. 25159307Spjd */ 26159307Spjd 27159307Spjd#include <sys/cdefs.h> 28159307Spjd__FBSDID("$FreeBSD: stable/11/sys/geom/eli/g_eli_privacy.c 339023 2018-09-30 12:25:38Z oshogbo $"); 29159307Spjd 30159307Spjd#include <sys/param.h> 31159307Spjd#include <sys/systm.h> 32159307Spjd#include <sys/kernel.h> 33159307Spjd#include <sys/linker.h> 34159307Spjd#include <sys/module.h> 35159307Spjd#include <sys/lock.h> 36159307Spjd#include <sys/mutex.h> 37159307Spjd#include <sys/bio.h> 38159307Spjd#include <sys/sysctl.h> 39159307Spjd#include <sys/malloc.h> 40159307Spjd#include <sys/kthread.h> 41159307Spjd#include <sys/proc.h> 42159307Spjd#include <sys/sched.h> 43159307Spjd#include <sys/smp.h> 44159307Spjd#include <sys/vnode.h> 45159307Spjd 46159307Spjd#include <vm/uma.h> 47159307Spjd 48159307Spjd#include <geom/geom.h> 49159307Spjd#include <geom/eli/g_eli.h> 50159307Spjd#include <geom/eli/pkcs5v2.h> 51159307Spjd 52159307Spjd/* 53159307Spjd * Code paths: 54159307Spjd * BIO_READ: 55214118Spjd * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver 56159307Spjd * BIO_WRITE: 57159307Spjd * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver 58159307Spjd */ 59159307Spjd 60159307SpjdMALLOC_DECLARE(M_ELI); 61159307Spjd 62159307Spjd/* 63159307Spjd * The function is called after we read and decrypt data. 64159307Spjd * 65214118Spjd * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver 66159307Spjd */ 67159307Spjdstatic int 68159307Spjdg_eli_crypto_read_done(struct cryptop *crp) 69159307Spjd{ 70214118Spjd struct g_eli_softc *sc; 71159307Spjd struct bio *bp; 72159307Spjd 73159307Spjd if (crp->crp_etype == EAGAIN) { 74159307Spjd if (g_eli_crypto_rerun(crp) == 0) 75159307Spjd return (0); 76159307Spjd } 77159307Spjd bp = (struct bio *)crp->crp_opaque; 78159307Spjd bp->bio_inbed++; 79159307Spjd if (crp->crp_etype == 0) { 80159307Spjd G_ELI_DEBUG(3, "Crypto READ request done (%d/%d).", 81159307Spjd bp->bio_inbed, bp->bio_children); 82159307Spjd bp->bio_completed += crp->crp_olen; 83159307Spjd } else { 84159307Spjd G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.", 85159307Spjd bp->bio_inbed, bp->bio_children, crp->crp_etype); 86159307Spjd if (bp->bio_error == 0) 87159307Spjd bp->bio_error = crp->crp_etype; 88159307Spjd } 89220922Spjd sc = bp->bio_to->geom->softc; 90339023Soshogbo if (sc != NULL) 91339023Soshogbo g_eli_key_drop(sc, crp->crp_desc->crd_key); 92159307Spjd /* 93159307Spjd * Do we have all sectors already? 94159307Spjd */ 95159307Spjd if (bp->bio_inbed < bp->bio_children) 96159307Spjd return (0); 97159307Spjd free(bp->bio_driver2, M_ELI); 98159307Spjd bp->bio_driver2 = NULL; 99159307Spjd if (bp->bio_error != 0) { 100159307Spjd G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).", 101159307Spjd bp->bio_error); 102159307Spjd bp->bio_completed = 0; 103159307Spjd } 104159307Spjd /* 105159307Spjd * Read is finished, send it up. 106159307Spjd */ 107159307Spjd g_io_deliver(bp, bp->bio_error); 108339023Soshogbo if (sc != NULL) 109339023Soshogbo atomic_subtract_int(&sc->sc_inflight, 1); 110159307Spjd return (0); 111159307Spjd} 112159307Spjd 113159307Spjd/* 114159307Spjd * The function is called after data encryption. 115159307Spjd * 116159307Spjd * g_eli_start -> g_eli_crypto_run -> G_ELI_CRYPTO_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver 117159307Spjd */ 118159307Spjdstatic int 119159307Spjdg_eli_crypto_write_done(struct cryptop *crp) 120159307Spjd{ 121214118Spjd struct g_eli_softc *sc; 122159307Spjd struct g_geom *gp; 123159307Spjd struct g_consumer *cp; 124159307Spjd struct bio *bp, *cbp; 125159307Spjd 126159307Spjd if (crp->crp_etype == EAGAIN) { 127159307Spjd if (g_eli_crypto_rerun(crp) == 0) 128159307Spjd return (0); 129159307Spjd } 130159307Spjd bp = (struct bio *)crp->crp_opaque; 131159307Spjd bp->bio_inbed++; 132159307Spjd if (crp->crp_etype == 0) { 133159307Spjd G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).", 134159307Spjd bp->bio_inbed, bp->bio_children); 135159307Spjd } else { 136159307Spjd G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.", 137159307Spjd bp->bio_inbed, bp->bio_children, crp->crp_etype); 138159307Spjd if (bp->bio_error == 0) 139159307Spjd bp->bio_error = crp->crp_etype; 140159307Spjd } 141220922Spjd gp = bp->bio_to->geom; 142220922Spjd sc = gp->softc; 143220922Spjd g_eli_key_drop(sc, crp->crp_desc->crd_key); 144159307Spjd /* 145159307Spjd * All sectors are already encrypted? 146159307Spjd */ 147159307Spjd if (bp->bio_inbed < bp->bio_children) 148159307Spjd return (0); 149159307Spjd bp->bio_inbed = 0; 150159307Spjd bp->bio_children = 1; 151159307Spjd cbp = bp->bio_driver1; 152159307Spjd bp->bio_driver1 = NULL; 153159307Spjd if (bp->bio_error != 0) { 154159307Spjd G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).", 155159307Spjd bp->bio_error); 156159307Spjd free(bp->bio_driver2, M_ELI); 157159307Spjd bp->bio_driver2 = NULL; 158159307Spjd g_destroy_bio(cbp); 159159307Spjd g_io_deliver(bp, bp->bio_error); 160214118Spjd atomic_subtract_int(&sc->sc_inflight, 1); 161159307Spjd return (0); 162159307Spjd } 163159307Spjd cbp->bio_data = bp->bio_driver2; 164159307Spjd cbp->bio_done = g_eli_write_done; 165159307Spjd cp = LIST_FIRST(&gp->consumer); 166159307Spjd cbp->bio_to = cp->provider; 167159307Spjd G_ELI_LOGREQ(2, cbp, "Sending request."); 168159307Spjd /* 169159307Spjd * Send encrypted data to the provider. 170159307Spjd */ 171159307Spjd g_io_request(cbp, cp); 172159307Spjd return (0); 173159307Spjd} 174159307Spjd 175159307Spjd/* 176214118Spjd * The function is called to read encrypted data. 177214118Spjd * 178214118Spjd * g_eli_start -> G_ELI_CRYPTO_READ -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver 179214118Spjd */ 180214118Spjdvoid 181214118Spjdg_eli_crypto_read(struct g_eli_softc *sc, struct bio *bp, boolean_t fromworker) 182214118Spjd{ 183214118Spjd struct g_consumer *cp; 184214118Spjd struct bio *cbp; 185214118Spjd 186214118Spjd if (!fromworker) { 187214118Spjd /* 188214118Spjd * We are not called from the worker thread, so check if 189214118Spjd * device is suspended. 190214118Spjd */ 191214118Spjd mtx_lock(&sc->sc_queue_mtx); 192214118Spjd if (sc->sc_flags & G_ELI_FLAG_SUSPEND) { 193214118Spjd /* 194214118Spjd * If device is suspended, we place the request onto 195214118Spjd * the queue, so it can be handled after resume. 196214118Spjd */ 197214118Spjd G_ELI_DEBUG(0, "device suspended, move onto queue"); 198214118Spjd bioq_insert_tail(&sc->sc_queue, bp); 199214118Spjd mtx_unlock(&sc->sc_queue_mtx); 200214118Spjd wakeup(sc); 201214118Spjd return; 202214118Spjd } 203214118Spjd atomic_add_int(&sc->sc_inflight, 1); 204214118Spjd mtx_unlock(&sc->sc_queue_mtx); 205214118Spjd } 206214118Spjd bp->bio_pflags = 0; 207214118Spjd bp->bio_driver2 = NULL; 208214118Spjd cbp = bp->bio_driver1; 209214118Spjd cbp->bio_done = g_eli_read_done; 210214118Spjd cp = LIST_FIRST(&sc->sc_geom->consumer); 211214118Spjd cbp->bio_to = cp->provider; 212214118Spjd G_ELI_LOGREQ(2, cbp, "Sending request."); 213214118Spjd /* 214214118Spjd * Read encrypted data from provider. 215214118Spjd */ 216214118Spjd g_io_request(cbp, cp); 217214118Spjd} 218214118Spjd 219214118Spjd/* 220159307Spjd * This is the main function responsible for cryptography (ie. communication 221159307Spjd * with crypto(9) subsystem). 222214116Spjd * 223214116Spjd * BIO_READ: 224214118Spjd * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> G_ELI_CRYPTO_RUN -> g_eli_crypto_read_done -> g_io_deliver 225214116Spjd * BIO_WRITE: 226214116Spjd * g_eli_start -> G_ELI_CRYPTO_RUN -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver 227159307Spjd */ 228159307Spjdvoid 229159307Spjdg_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp) 230159307Spjd{ 231159307Spjd struct g_eli_softc *sc; 232159307Spjd struct cryptop *crp; 233159307Spjd struct cryptodesc *crd; 234213063Spjd u_int i, nsec, secsize; 235213063Spjd off_t dstoff; 236159307Spjd size_t size; 237159307Spjd u_char *p, *data; 238286373Spjd int error; 239159307Spjd 240159307Spjd G_ELI_LOGREQ(3, bp, "%s", __func__); 241159307Spjd 242159307Spjd bp->bio_pflags = wr->w_number; 243159307Spjd sc = wr->w_softc; 244159307Spjd secsize = LIST_FIRST(&sc->sc_geom->provider)->sectorsize; 245159307Spjd nsec = bp->bio_length / secsize; 246159307Spjd 247159307Spjd /* 248159307Spjd * Calculate how much memory do we need. 249159307Spjd * We need separate crypto operation for every single sector. 250159307Spjd * It is much faster to calculate total amount of needed memory here and 251159307Spjd * do the allocation once instead of allocating memory in pieces (many, 252159307Spjd * many pieces). 253159307Spjd */ 254159307Spjd size = sizeof(*crp) * nsec; 255159307Spjd size += sizeof(*crd) * nsec; 256159307Spjd /* 257159307Spjd * If we write the data we cannot destroy current bio_data content, 258159307Spjd * so we need to allocate more memory for encrypted data. 259159307Spjd */ 260159307Spjd if (bp->bio_cmd == BIO_WRITE) 261159307Spjd size += bp->bio_length; 262159307Spjd p = malloc(size, M_ELI, M_WAITOK); 263159307Spjd 264159307Spjd bp->bio_inbed = 0; 265159307Spjd bp->bio_children = nsec; 266159307Spjd bp->bio_driver2 = p; 267159307Spjd 268159307Spjd if (bp->bio_cmd == BIO_READ) 269159307Spjd data = bp->bio_data; 270159307Spjd else { 271159307Spjd data = p; 272159307Spjd p += bp->bio_length; 273159307Spjd bcopy(bp->bio_data, data, bp->bio_length); 274159307Spjd } 275159307Spjd 276213063Spjd for (i = 0, dstoff = bp->bio_offset; i < nsec; i++, dstoff += secsize) { 277159307Spjd crp = (struct cryptop *)p; p += sizeof(*crp); 278159307Spjd crd = (struct cryptodesc *)p; p += sizeof(*crd); 279159307Spjd 280159307Spjd crp->crp_sid = wr->w_sid; 281159307Spjd crp->crp_ilen = secsize; 282159307Spjd crp->crp_olen = secsize; 283159307Spjd crp->crp_opaque = (void *)bp; 284271148Sjmg crp->crp_buf = (void *)data; 285271148Sjmg data += secsize; 286159307Spjd if (bp->bio_cmd == BIO_WRITE) 287159307Spjd crp->crp_callback = g_eli_crypto_write_done; 288159307Spjd else /* if (bp->bio_cmd == BIO_READ) */ 289159307Spjd crp->crp_callback = g_eli_crypto_read_done; 290275732Sjmg crp->crp_flags = CRYPTO_F_CBIFSYNC; 291159307Spjd if (g_eli_batch) 292159307Spjd crp->crp_flags |= CRYPTO_F_BATCH; 293159307Spjd crp->crp_desc = crd; 294159307Spjd 295159307Spjd crd->crd_skip = 0; 296159307Spjd crd->crd_len = secsize; 297159307Spjd crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 298220922Spjd if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) == 0) 299213067Spjd crd->crd_flags |= CRD_F_KEY_EXPLICIT; 300159307Spjd if (bp->bio_cmd == BIO_WRITE) 301159307Spjd crd->crd_flags |= CRD_F_ENCRYPT; 302159307Spjd crd->crd_alg = sc->sc_ealgo; 303220922Spjd crd->crd_key = g_eli_key_hold(sc, dstoff, secsize); 304159307Spjd crd->crd_klen = sc->sc_ekeylen; 305213070Spjd if (sc->sc_ealgo == CRYPTO_AES_XTS) 306213070Spjd crd->crd_klen <<= 1; 307213063Spjd g_eli_crypto_ivgen(sc, dstoff, crd->crd_iv, 308159307Spjd sizeof(crd->crd_iv)); 309159307Spjd crd->crd_next = NULL; 310159307Spjd 311159307Spjd crp->crp_etype = 0; 312286373Spjd error = crypto_dispatch(crp); 313286373Spjd KASSERT(error == 0, ("crypto_dispatch() failed (error=%d)", 314286373Spjd error)); 315159307Spjd } 316159307Spjd} 317