geom_io.c revision 156170
1191783Srmacklem/*- 2191783Srmacklem * Copyright (c) 2002 Poul-Henning Kamp 3191783Srmacklem * Copyright (c) 2002 Networks Associates Technology, Inc. 4191783Srmacklem * All rights reserved. 5191783Srmacklem * 6191783Srmacklem * This software was developed for the FreeBSD Project by Poul-Henning Kamp 7191783Srmacklem * and NAI Labs, the Security Research Division of Network Associates, Inc. 8191783Srmacklem * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the 9191783Srmacklem * DARPA CHATS research program. 10191783Srmacklem * 11191783Srmacklem * Redistribution and use in source and binary forms, with or without 12191783Srmacklem * modification, are permitted provided that the following conditions 13191783Srmacklem * are met: 14191783Srmacklem * 1. Redistributions of source code must retain the above copyright 15191783Srmacklem * notice, this list of conditions and the following disclaimer. 16191783Srmacklem * 2. Redistributions in binary form must reproduce the above copyright 17191783Srmacklem * notice, this list of conditions and the following disclaimer in the 18191783Srmacklem * documentation and/or other materials provided with the distribution. 19191783Srmacklem * 3. The names of the authors may not be used to endorse or promote 20191783Srmacklem * products derived from this software without specific prior written 21191783Srmacklem * permission. 22191783Srmacklem * 23191783Srmacklem * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 24191783Srmacklem * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25191783Srmacklem * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26191783Srmacklem * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 27191783Srmacklem * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28191783Srmacklem * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29191783Srmacklem * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30191783Srmacklem * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31191783Srmacklem * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32191783Srmacklem * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33191783Srmacklem * SUCH DAMAGE. 34191783Srmacklem */ 35191783Srmacklem 36191783Srmacklem#include <sys/cdefs.h> 37191783Srmacklem__FBSDID("$FreeBSD: head/sys/geom/geom_io.c 156170 2006-03-01 19:01:58Z pjd $"); 38191783Srmacklem 39191783Srmacklem#include <sys/param.h> 40191783Srmacklem#include <sys/systm.h> 41191783Srmacklem#include <sys/kernel.h> 42191783Srmacklem#include <sys/malloc.h> 43191783Srmacklem#include <sys/bio.h> 44191783Srmacklem#include <sys/ktr.h> 45191783Srmacklem#include <sys/proc.h> 46191783Srmacklem#include <sys/stack.h> 47191783Srmacklem 48193066Sjamie#include <sys/errno.h> 49220739Srmacklem#include <geom/geom.h> 50191783Srmacklem#include <geom/geom_int.h> 51191783Srmacklem#include <sys/devicestat.h> 52191783Srmacklem 53191783Srmacklem#include <vm/uma.h> 54191783Srmacklem 55191783Srmacklemstatic struct g_bioq g_bio_run_down; 56191783Srmacklemstatic struct g_bioq g_bio_run_up; 57191783Srmacklemstatic struct g_bioq g_bio_run_task; 58191783Srmacklem 59191783Srmacklemstatic u_int pace; 60191783Srmacklemstatic uma_zone_t biozone; 61191783Srmacklem 62191783Srmacklem#include <machine/atomic.h> 63191783Srmacklem 64191783Srmacklemstatic void 65191783Srmacklemg_bioq_lock(struct g_bioq *bq) 66191783Srmacklem{ 67191783Srmacklem 68191783Srmacklem mtx_lock(&bq->bio_queue_lock); 69191783Srmacklem} 70191783Srmacklem 71191783Srmacklemstatic void 72191783Srmacklemg_bioq_unlock(struct g_bioq *bq) 73191783Srmacklem{ 74191783Srmacklem 75221040Srmacklem mtx_unlock(&bq->bio_queue_lock); 76191783Srmacklem} 77219028Snetchild 78219028Snetchild#if 0 79191783Srmacklemstatic void 80191783Srmacklemg_bioq_destroy(struct g_bioq *bq) 81191783Srmacklem{ 82191783Srmacklem 83191783Srmacklem mtx_destroy(&bq->bio_queue_lock); 84191783Srmacklem} 85191783Srmacklem#endif 86191783Srmacklem 87191783Srmacklemstatic void 88191783Srmacklemg_bioq_init(struct g_bioq *bq) 89191783Srmacklem{ 90191783Srmacklem 91191783Srmacklem TAILQ_INIT(&bq->bio_queue); 92191783Srmacklem mtx_init(&bq->bio_queue_lock, "bio queue", NULL, MTX_DEF); 93191783Srmacklem} 94191783Srmacklem 95191783Srmacklemstatic struct bio * 96191783Srmacklemg_bioq_first(struct g_bioq *bq) 97191783Srmacklem{ 98191783Srmacklem struct bio *bp; 99191783Srmacklem 100221040Srmacklem bp = TAILQ_FIRST(&bq->bio_queue); 101192585Srmacklem if (bp != NULL) { 102191783Srmacklem KASSERT((bp->bio_flags & BIO_ONQUEUE), 103214048Srmacklem ("Bio not on queue bp=%p target %p", bp, bq)); 104214048Srmacklem bp->bio_flags &= ~BIO_ONQUEUE; 105191783Srmacklem TAILQ_REMOVE(&bq->bio_queue, bp, bio_queue); 106221014Srmacklem bq->bio_queue_length--; 107221014Srmacklem } 108221014Srmacklem return (bp); 109214053Srmacklem} 110216931Srmacklem 111216931Srmacklemstruct bio * 112191783Srmacklemg_new_bio(void) 113191783Srmacklem{ 114191783Srmacklem struct bio *bp; 115191783Srmacklem 116191783Srmacklem bp = uma_zalloc(biozone, M_NOWAIT | M_ZERO); 117191783Srmacklem#ifdef KTR 118191783Srmacklem if (KTR_COMPILE & KTR_GEOM) { 119191783Srmacklem struct stack st; 120191783Srmacklem 121191783Srmacklem CTR1(KTR_GEOM, "g_new_bio(): %p", bp); 122191783Srmacklem stack_save(&st); 123191783Srmacklem CTRSTACK(KTR_GEOM, &st, 3, 0); 124191783Srmacklem } 125191783Srmacklem#endif 126191783Srmacklem return (bp); 127191783Srmacklem} 128191783Srmacklem 129191783Srmacklemstruct bio * 130191783Srmacklemg_alloc_bio(void) 131191783Srmacklem{ 132191783Srmacklem struct bio *bp; 133191783Srmacklem 134221124Srmacklem bp = uma_zalloc(biozone, M_WAITOK | M_ZERO); 135191783Srmacklem#ifdef KTR 136191783Srmacklem if (KTR_COMPILE & KTR_GEOM) { 137221139Srmacklem struct stack st; 138221139Srmacklem 139221139Srmacklem CTR1(KTR_GEOM, "g_alloc_bio(): %p", bp); 140221139Srmacklem stack_save(&st); 141221139Srmacklem CTRSTACK(KTR_GEOM, &st, 3, 0); 142191783Srmacklem } 143191783Srmacklem#endif 144221066Srmacklem return (bp); 145221066Srmacklem} 146221066Srmacklem 147221066Srmacklemvoid 148191783Srmacklemg_destroy_bio(struct bio *bp) 149221066Srmacklem{ 150221066Srmacklem#ifdef KTR 151221066Srmacklem if (KTR_COMPILE & KTR_GEOM) { 152221066Srmacklem struct stack st; 153221066Srmacklem 154221066Srmacklem CTR1(KTR_GEOM, "g_destroy_bio(): %p", bp); 155191783Srmacklem stack_save(&st); 156221040Srmacklem CTRSTACK(KTR_GEOM, &st, 3, 0); 157192145Srmacklem } 158191783Srmacklem#endif 159191783Srmacklem uma_zfree(biozone, bp); 160221040Srmacklem} 161191783Srmacklem 162191783Srmacklemstruct bio * 163221040Srmacklemg_clone_bio(struct bio *bp) 164192145Srmacklem{ 165191783Srmacklem struct bio *bp2; 166191783Srmacklem 167191783Srmacklem bp2 = uma_zalloc(biozone, M_NOWAIT | M_ZERO); 168191783Srmacklem if (bp2 != NULL) { 169191783Srmacklem bp2->bio_parent = bp; 170191783Srmacklem bp2->bio_cmd = bp->bio_cmd; 171191783Srmacklem bp2->bio_length = bp->bio_length; 172191783Srmacklem bp2->bio_offset = bp->bio_offset; 173191783Srmacklem bp2->bio_data = bp->bio_data; 174191783Srmacklem bp2->bio_attribute = bp->bio_attribute; 175191783Srmacklem bp->bio_children++; 176191783Srmacklem } 177191783Srmacklem#ifdef KTR 178191783Srmacklem if (KTR_COMPILE & KTR_GEOM) { 179191783Srmacklem struct stack st; 180191783Srmacklem 181191783Srmacklem CTR2(KTR_GEOM, "g_close_bio(%p): %p", bp, bp2); 182191783Srmacklem stack_save(&st); 183191783Srmacklem CTRSTACK(KTR_GEOM, &st, 3, 0); 184191783Srmacklem } 185191783Srmacklem#endif 186191783Srmacklem return(bp2); 187191783Srmacklem} 188191783Srmacklem 189191783Srmacklemvoid 190191783Srmacklemg_io_init() 191191783Srmacklem{ 192191783Srmacklem 193191783Srmacklem g_bioq_init(&g_bio_run_down); 194191783Srmacklem g_bioq_init(&g_bio_run_up); 195191783Srmacklem g_bioq_init(&g_bio_run_task); 196191783Srmacklem biozone = uma_zcreate("g_bio", sizeof (struct bio), 197191783Srmacklem NULL, NULL, 198191783Srmacklem NULL, NULL, 199191783Srmacklem 0, 0); 200191783Srmacklem} 201191783Srmacklem 202191783Srmacklemint 203191783Srmacklemg_io_getattr(const char *attr, struct g_consumer *cp, int *len, void *ptr) 204191783Srmacklem{ 205191783Srmacklem struct bio *bp; 206191783Srmacklem int error; 207191783Srmacklem 208191783Srmacklem g_trace(G_T_BIO, "bio_getattr(%s)", attr); 209191783Srmacklem bp = g_alloc_bio(); 210191783Srmacklem bp->bio_cmd = BIO_GETATTR; 211191783Srmacklem bp->bio_done = NULL; 212191783Srmacklem bp->bio_attribute = attr; 213191783Srmacklem bp->bio_length = *len; 214191783Srmacklem bp->bio_data = ptr; 215191783Srmacklem g_io_request(bp, cp); 216191783Srmacklem error = biowait(bp, "ggetattr"); 217191783Srmacklem *len = bp->bio_completed; 218191783Srmacklem g_destroy_bio(bp); 219191783Srmacklem return (error); 220191783Srmacklem} 221191783Srmacklem 222191783Srmacklemstatic int 223191783Srmacklemg_io_check(struct bio *bp) 224191783Srmacklem{ 225191783Srmacklem struct g_consumer *cp; 226191783Srmacklem struct g_provider *pp; 227191783Srmacklem 228191783Srmacklem cp = bp->bio_from; 229191783Srmacklem pp = bp->bio_to; 230191783Srmacklem 231191783Srmacklem /* Fail if access counters dont allow the operation */ 232191783Srmacklem switch(bp->bio_cmd) { 233191783Srmacklem case BIO_READ: 234191783Srmacklem case BIO_GETATTR: 235191783Srmacklem if (cp->acr == 0) 236191783Srmacklem return (EPERM); 237191783Srmacklem break; 238191783Srmacklem case BIO_WRITE: 239191783Srmacklem case BIO_DELETE: 240191783Srmacklem if (cp->acw == 0) 241221040Srmacklem return (EPERM); 242221040Srmacklem break; 243221040Srmacklem default: 244221040Srmacklem return (EPERM); 245221040Srmacklem } 246221040Srmacklem /* if provider is marked for error, don't disturb. */ 247221040Srmacklem if (pp->error) 248221040Srmacklem return (pp->error); 249191783Srmacklem 250221040Srmacklem switch(bp->bio_cmd) { 251221040Srmacklem case BIO_READ: 252191783Srmacklem case BIO_WRITE: 253221040Srmacklem case BIO_DELETE: 254221040Srmacklem /* Zero sectorsize is a probably lack of media */ 255221040Srmacklem if (pp->sectorsize == 0) 256221040Srmacklem return (ENXIO); 257221040Srmacklem /* Reject I/O not on sector boundary */ 258221040Srmacklem if (bp->bio_offset % pp->sectorsize) 259221040Srmacklem return (EINVAL); 260191783Srmacklem /* Reject I/O not integral sector long */ 261191783Srmacklem if (bp->bio_length % pp->sectorsize) 262191783Srmacklem return (EINVAL); 263191783Srmacklem /* Reject requests before or past the end of media. */ 264191783Srmacklem if (bp->bio_offset < 0) 265191783Srmacklem return (EIO); 266191990Sattilio if (bp->bio_offset > pp->mediasize) 267191783Srmacklem return (EIO); 268191783Srmacklem break; 269191990Sattilio default: 270191783Srmacklem break; 271191783Srmacklem } 272191783Srmacklem return (0); 273191783Srmacklem} 274191783Srmacklem 275191783Srmacklemvoid 276191783Srmacklemg_io_request(struct bio *bp, struct g_consumer *cp) 277191990Sattilio{ 278191990Sattilio struct g_provider *pp; 279191783Srmacklem 280191783Srmacklem KASSERT(cp != NULL, ("NULL cp in g_io_request")); 281191783Srmacklem KASSERT(bp != NULL, ("NULL bp in g_io_request")); 282220732Srmacklem KASSERT(bp->bio_data != NULL, ("NULL bp->data in g_io_request")); 283191783Srmacklem pp = cp->provider; 284191783Srmacklem KASSERT(pp != NULL, ("consumer not attached in g_io_request")); 285191783Srmacklem#ifdef DIAGNOSTIC 286191783Srmacklem KASSERT(bp->bio_driver1 == NULL, 287191783Srmacklem ("bio_driver1 used by the consumer (geom %s)", cp->geom->name)); 288191783Srmacklem KASSERT(bp->bio_driver2 == NULL, 289191783Srmacklem ("bio_driver2 used by the consumer (geom %s)", cp->geom->name)); 290191783Srmacklem KASSERT(bp->bio_pflags == 0, 291191783Srmacklem ("bio_pflags used by the consumer (geom %s)", cp->geom->name)); 292191783Srmacklem /* 293191783Srmacklem * Remember consumer's private fields, so we can detect if they were 294191783Srmacklem * modified by the provider. 295191783Srmacklem */ 296191783Srmacklem bp->_bio_caller1 = bp->bio_caller1; 297191783Srmacklem bp->_bio_caller2 = bp->bio_caller2; 298191783Srmacklem bp->_bio_cflags = bp->bio_cflags; 299191783Srmacklem#endif 300191783Srmacklem 301191783Srmacklem if (bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_DELETE)) { 302191783Srmacklem KASSERT(bp->bio_offset % cp->provider->sectorsize == 0, 303191783Srmacklem ("wrong offset %jd for sectorsize %u", 304191783Srmacklem bp->bio_offset, cp->provider->sectorsize)); 305191783Srmacklem KASSERT(bp->bio_length % cp->provider->sectorsize == 0, 306191783Srmacklem ("wrong length %jd for sectorsize %u", 307191783Srmacklem bp->bio_length, cp->provider->sectorsize)); 308191783Srmacklem } 309191783Srmacklem 310191783Srmacklem g_trace(G_T_BIO, "bio_request(%p) from %p(%s) to %p(%s) cmd %d", 311191783Srmacklem bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd); 312191783Srmacklem 313191783Srmacklem bp->bio_from = cp; 314191783Srmacklem bp->bio_to = pp; 315191783Srmacklem bp->bio_error = 0; 316191783Srmacklem bp->bio_completed = 0; 317191783Srmacklem 318191783Srmacklem KASSERT(!(bp->bio_flags & BIO_ONQUEUE), 319191783Srmacklem ("Bio already on queue bp=%p", bp)); 320191783Srmacklem bp->bio_flags |= BIO_ONQUEUE; 321191783Srmacklem 322191783Srmacklem binuptime(&bp->bio_t0); 323191783Srmacklem 324191783Srmacklem /* 325191783Srmacklem * The statistics collection is lockless, as such, but we 326191783Srmacklem * can not update one instance of the statistics from more 327191783Srmacklem * than one thread at a time, so grab the lock first. 328191783Srmacklem */ 329191783Srmacklem g_bioq_lock(&g_bio_run_down); 330191783Srmacklem if (g_collectstats & 1) 331191783Srmacklem devstat_start_transaction(pp->stat, &bp->bio_t0); 332191783Srmacklem if (g_collectstats & 2) 333191783Srmacklem devstat_start_transaction(cp->stat, &bp->bio_t0); 334191783Srmacklem 335191783Srmacklem pp->nstart++; 336191783Srmacklem cp->nstart++; 337191783Srmacklem TAILQ_INSERT_TAIL(&g_bio_run_down.bio_queue, bp, bio_queue); 338191783Srmacklem g_bio_run_down.bio_queue_length++; 339191783Srmacklem g_bioq_unlock(&g_bio_run_down); 340191783Srmacklem 341191783Srmacklem /* Pass it on down. */ 342191783Srmacklem wakeup(&g_wait_down); 343191783Srmacklem} 344191783Srmacklem 345191783Srmacklemvoid 346191783Srmacklemg_io_deliver(struct bio *bp, int error) 347191783Srmacklem{ 348191783Srmacklem struct g_consumer *cp; 349191783Srmacklem struct g_provider *pp; 350191783Srmacklem 351191783Srmacklem KASSERT(bp != NULL, ("NULL bp in g_io_deliver")); 352191783Srmacklem pp = bp->bio_to; 353191783Srmacklem KASSERT(pp != NULL, ("NULL bio_to in g_io_deliver")); 354191783Srmacklem#ifdef DIAGNOSTIC 355191783Srmacklem KASSERT(bp->bio_caller1 == bp->_bio_caller1, 356191783Srmacklem ("bio_caller1 used by the provider %s", pp->name)); 357191783Srmacklem KASSERT(bp->bio_caller2 == bp->_bio_caller2, 358191783Srmacklem ("bio_caller2 used by the provider %s", pp->name)); 359191783Srmacklem KASSERT(bp->bio_cflags == bp->_bio_cflags, 360191783Srmacklem ("bio_cflags used by the provider %s", pp->name)); 361191783Srmacklem#endif 362191783Srmacklem cp = bp->bio_from; 363191783Srmacklem if (cp == NULL) { 364191783Srmacklem bp->bio_error = error; 365221040Srmacklem bp->bio_done(bp); 366191783Srmacklem return; 367191783Srmacklem } 368191783Srmacklem KASSERT(cp != NULL, ("NULL bio_from in g_io_deliver")); 369191783Srmacklem KASSERT(cp->geom != NULL, ("NULL bio_from->geom in g_io_deliver")); 370221040Srmacklem KASSERT(bp->bio_completed >= 0, ("bio_completed can't be less than 0")); 371191783Srmacklem KASSERT(bp->bio_completed <= bp->bio_length, 372191783Srmacklem ("bio_completed can't be greater than bio_length")); 373191783Srmacklem 374191783Srmacklem g_trace(G_T_BIO, 375191783Srmacklem"g_io_deliver(%p) from %p(%s) to %p(%s) cmd %d error %d off %jd len %jd", 376192145Srmacklem bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd, error, 377191783Srmacklem (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length); 378191783Srmacklem 379221040Srmacklem KASSERT(!(bp->bio_flags & BIO_ONQUEUE), 380221040Srmacklem ("Bio already on queue bp=%p", bp)); 381191783Srmacklem 382192145Srmacklem /* 383221040Srmacklem * XXX: next two doesn't belong here 384191783Srmacklem */ 385191783Srmacklem bp->bio_bcount = bp->bio_length; 386191783Srmacklem bp->bio_resid = bp->bio_bcount - bp->bio_completed; 387193066Sjamie 388191783Srmacklem /* 389191783Srmacklem * The statistics collection is lockless, as such, but we 390191783Srmacklem * can not update one instance of the statistics from more 391191783Srmacklem * than one thread at a time, so grab the lock first. 392191783Srmacklem */ 393192145Srmacklem g_bioq_lock(&g_bio_run_up); 394191783Srmacklem if (g_collectstats & 1) 395191783Srmacklem devstat_end_transaction_bio(pp->stat, bp); 396191783Srmacklem if (g_collectstats & 2) 397191783Srmacklem devstat_end_transaction_bio(cp->stat, bp); 398221040Srmacklem 399191783Srmacklem cp->nend++; 400221040Srmacklem pp->nend++; 401191783Srmacklem if (error != ENOMEM) { 402191783Srmacklem bp->bio_error = error; 403191783Srmacklem TAILQ_INSERT_TAIL(&g_bio_run_up.bio_queue, bp, bio_queue); 404191783Srmacklem bp->bio_flags |= BIO_ONQUEUE; 405191783Srmacklem g_bio_run_up.bio_queue_length++; 406191783Srmacklem g_bioq_unlock(&g_bio_run_up); 407191783Srmacklem wakeup(&g_wait_up); 408191783Srmacklem return; 409191783Srmacklem } 410191783Srmacklem g_bioq_unlock(&g_bio_run_up); 411191783Srmacklem 412191783Srmacklem if (bootverbose) 413191783Srmacklem printf("ENOMEM %p on %p(%s)\n", bp, pp, pp->name); 414191783Srmacklem bp->bio_children = 0; 415192145Srmacklem bp->bio_inbed = 0; 416191783Srmacklem g_io_request(bp, cp); 417191783Srmacklem pace++; 418191783Srmacklem return; 419191783Srmacklem} 420191783Srmacklem 421191783Srmacklemvoid 422191783Srmacklemg_io_schedule_down(struct thread *tp __unused) 423191783Srmacklem{ 424191783Srmacklem struct bio *bp; 425191783Srmacklem off_t excess; 426191783Srmacklem int error; 427191783Srmacklem 428191783Srmacklem for(;;) { 429191783Srmacklem g_bioq_lock(&g_bio_run_down); 430191783Srmacklem bp = g_bioq_first(&g_bio_run_down); 431191783Srmacklem if (bp == NULL) { 432191783Srmacklem CTR0(KTR_GEOM, "g_down going to sleep"); 433191783Srmacklem msleep(&g_wait_down, &g_bio_run_down.bio_queue_lock, 434191783Srmacklem PRIBIO | PDROP, "-", hz/10); 435191783Srmacklem continue; 436192145Srmacklem } 437191783Srmacklem CTR0(KTR_GEOM, "g_down has work to do"); 438191783Srmacklem g_bioq_unlock(&g_bio_run_down); 439191783Srmacklem if (pace > 0) { 440191783Srmacklem CTR1(KTR_GEOM, "g_down pacing self (pace %d)", pace); 441191783Srmacklem msleep(&error, NULL, PRIBIO, "g_down", hz/10); 442191783Srmacklem pace--; 443192145Srmacklem } 444191783Srmacklem error = g_io_check(bp); 445191783Srmacklem if (error) { 446191783Srmacklem CTR3(KTR_GEOM, "g_down g_io_check on bp %p provider " 447191783Srmacklem "%s returned %d", bp, bp->bio_to->name, error); 448191783Srmacklem g_io_deliver(bp, error); 449191783Srmacklem continue; 450191783Srmacklem } 451191783Srmacklem CTR2(KTR_GEOM, "g_down processing bp %p provider %s", bp, 452191783Srmacklem bp->bio_to->name); 453191783Srmacklem switch (bp->bio_cmd) { 454191783Srmacklem case BIO_READ: 455191783Srmacklem case BIO_WRITE: 456191783Srmacklem case BIO_DELETE: 457191783Srmacklem /* Truncate requests to the end of providers media. */ 458191783Srmacklem /* 459191783Srmacklem * XXX: What if we truncate because of offset being 460191783Srmacklem * bad, not length? 461192145Srmacklem */ 462218757Sbz excess = bp->bio_offset + bp->bio_length; 463191783Srmacklem if (excess > bp->bio_to->mediasize) { 464191783Srmacklem excess -= bp->bio_to->mediasize; 465191783Srmacklem bp->bio_length -= excess; 466191783Srmacklem if (excess > 0) 467218757Sbz CTR3(KTR_GEOM, "g_down truncated bio " 468191783Srmacklem "%p provider %s by %d", bp, 469192145Srmacklem bp->bio_to->name, excess); 470191783Srmacklem } 471191783Srmacklem /* Deliver zero length transfers right here. */ 472191783Srmacklem if (bp->bio_length == 0) { 473191783Srmacklem g_io_deliver(bp, 0); 474191783Srmacklem CTR2(KTR_GEOM, "g_down terminated 0-length " 475191783Srmacklem "bp %p provider %s", bp, bp->bio_to->name); 476191783Srmacklem continue; 477191783Srmacklem } 478191783Srmacklem break; 479191783Srmacklem default: 480191783Srmacklem break; 481191783Srmacklem } 482192145Srmacklem THREAD_NO_SLEEPING(); 483191783Srmacklem CTR4(KTR_GEOM, "g_down starting bp %p provider %s off %ld " 484191783Srmacklem "len %ld", bp, bp->bio_to->name, bp->bio_offset, 485191783Srmacklem bp->bio_length); 486191783Srmacklem bp->bio_to->geom->start(bp); 487191783Srmacklem THREAD_SLEEPING_OK(); 488191783Srmacklem } 489191783Srmacklem} 490191783Srmacklem 491191783Srmacklemvoid 492191783Srmacklembio_taskqueue(struct bio *bp, bio_task_t *func, void *arg) 493193066Sjamie{ 494194118Sjamie bp->bio_task = func; 495194118Sjamie bp->bio_task_arg = arg; 496193066Sjamie /* 497191783Srmacklem * The taskqueue is actually just a second queue off the "up" 498191783Srmacklem * queue, so we use the same lock. 499191783Srmacklem */ 500191783Srmacklem g_bioq_lock(&g_bio_run_up); 501191783Srmacklem KASSERT(!(bp->bio_flags & BIO_ONQUEUE), 502191783Srmacklem ("Bio already on queue bp=%p target taskq", bp)); 503191783Srmacklem bp->bio_flags |= BIO_ONQUEUE; 504191783Srmacklem TAILQ_INSERT_TAIL(&g_bio_run_task.bio_queue, bp, bio_queue); 505191783Srmacklem g_bio_run_task.bio_queue_length++; 506191783Srmacklem wakeup(&g_wait_up); 507191783Srmacklem g_bioq_unlock(&g_bio_run_up); 508191783Srmacklem} 509191783Srmacklem 510221014Srmacklem 511221014Srmacklemvoid 512191783Srmacklemg_io_schedule_up(struct thread *tp __unused) 513221014Srmacklem{ 514221014Srmacklem struct bio *bp; 515221014Srmacklem for(;;) { 516221014Srmacklem g_bioq_lock(&g_bio_run_up); 517221014Srmacklem bp = g_bioq_first(&g_bio_run_task); 518221014Srmacklem if (bp != NULL) { 519221014Srmacklem g_bioq_unlock(&g_bio_run_up); 520221014Srmacklem THREAD_NO_SLEEPING(); 521221014Srmacklem CTR1(KTR_GEOM, "g_up processing task bp %p", bp); 522191783Srmacklem bp->bio_task(bp->bio_task_arg); 523221014Srmacklem THREAD_SLEEPING_OK(); 524221014Srmacklem continue; 525192145Srmacklem } 526191783Srmacklem bp = g_bioq_first(&g_bio_run_up); 527191783Srmacklem if (bp != NULL) { 528191783Srmacklem g_bioq_unlock(&g_bio_run_up); 529191783Srmacklem THREAD_NO_SLEEPING(); 530191783Srmacklem CTR4(KTR_GEOM, "g_up biodone bp %p provider %s off " 531191783Srmacklem "%ld len %ld", bp, bp->bio_to->name, 532192585Srmacklem bp->bio_offset, bp->bio_length); 533192585Srmacklem biodone(bp); 534192585Srmacklem THREAD_SLEEPING_OK(); 535192585Srmacklem continue; 536192585Srmacklem } 537192585Srmacklem CTR0(KTR_GEOM, "g_up going to sleep"); 538192585Srmacklem msleep(&g_wait_up, &g_bio_run_up.bio_queue_lock, 539192585Srmacklem PRIBIO | PDROP, "-", hz/10); 540192585Srmacklem } 541192585Srmacklem} 542192585Srmacklem 543191783Srmacklemvoid * 544214048Srmacklemg_read_data(struct g_consumer *cp, off_t offset, off_t length, int *error) 545191783Srmacklem{ 546191783Srmacklem struct bio *bp; 547191783Srmacklem void *ptr; 548214048Srmacklem int errorc; 549191783Srmacklem 550191783Srmacklem KASSERT(length > 0 && length >= cp->provider->sectorsize && 551191783Srmacklem length <= MAXPHYS, ("g_read_data(): invalid length %jd", 552191783Srmacklem (intmax_t)length)); 553191783Srmacklem 554191783Srmacklem bp = g_alloc_bio(); 555191783Srmacklem bp->bio_cmd = BIO_READ; 556191783Srmacklem bp->bio_done = NULL; 557191783Srmacklem bp->bio_offset = offset; 558191783Srmacklem bp->bio_length = length; 559191783Srmacklem ptr = g_malloc(length, M_WAITOK); 560191783Srmacklem bp->bio_data = ptr; 561191783Srmacklem g_io_request(bp, cp); 562191783Srmacklem errorc = biowait(bp, "gread"); 563191783Srmacklem if (error != NULL) 564191783Srmacklem *error = errorc; 565191783Srmacklem g_destroy_bio(bp); 566191783Srmacklem if (errorc) { 567191783Srmacklem g_free(ptr); 568191783Srmacklem ptr = NULL; 569191783Srmacklem } 570191783Srmacklem return (ptr); 571191783Srmacklem} 572191783Srmacklem 573191783Srmacklemint 574191783Srmacklemg_write_data(struct g_consumer *cp, off_t offset, void *ptr, off_t length) 575191783Srmacklem{ 576220739Srmacklem struct bio *bp; 577220739Srmacklem int error; 578220739Srmacklem 579220739Srmacklem KASSERT(length > 0 && length >= cp->provider->sectorsize && 580191783Srmacklem length <= MAXPHYS, ("g_write_data(): invalid length %jd", 581191783Srmacklem (intmax_t)length)); 582220739Srmacklem 583220739Srmacklem bp = g_alloc_bio(); 584220739Srmacklem bp->bio_cmd = BIO_WRITE; 585191783Srmacklem bp->bio_done = NULL; 586220739Srmacklem bp->bio_offset = offset; 587191783Srmacklem bp->bio_length = length; 588220739Srmacklem bp->bio_data = ptr; 589220739Srmacklem g_io_request(bp, cp); 590220739Srmacklem error = biowait(bp, "gwrite"); 591220739Srmacklem g_destroy_bio(bp); 592220739Srmacklem return (error); 593220739Srmacklem} 594220739Srmacklem 595220739Srmacklemvoid 596220739Srmacklemg_print_bio(struct bio *bp) 597191783Srmacklem{ 598220739Srmacklem const char *pname, *cmd = NULL; 599191783Srmacklem 600191783Srmacklem if (bp->bio_to != NULL) 601191783Srmacklem pname = bp->bio_to->name; 602191783Srmacklem else 603191783Srmacklem pname = "[unknown]"; 604191783Srmacklem 605191783Srmacklem switch (bp->bio_cmd) { 606191783Srmacklem case BIO_GETATTR: 607191783Srmacklem cmd = "GETATTR"; 608191783Srmacklem printf("%s[%s(attr=%s)]", pname, cmd, bp->bio_attribute); 609191783Srmacklem return; 610191783Srmacklem case BIO_READ: 611191783Srmacklem cmd = "READ"; 612191783Srmacklem case BIO_WRITE: 613191783Srmacklem if (cmd == NULL) 614191783Srmacklem cmd = "WRITE"; 615191783Srmacklem case BIO_DELETE: 616191783Srmacklem if (cmd == NULL) 617191783Srmacklem cmd = "DELETE"; 618191783Srmacklem printf("%s[%s(offset=%jd, length=%jd)]", pname, cmd, 619191783Srmacklem (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length); 620191783Srmacklem return; 621191783Srmacklem default: 622191783Srmacklem cmd = "UNKNOWN"; 623191783Srmacklem printf("%s[%s()]", pname, cmd); 624191783Srmacklem return; 625191783Srmacklem } 626191783Srmacklem /* NOTREACHED */ 627191783Srmacklem} 628191783Srmacklem