geom_io.c revision 116196
1139969Simp/*- 21556Srgrimes * Copyright (c) 2002 Poul-Henning Kamp 31556Srgrimes * Copyright (c) 2002 Networks Associates Technology, Inc. 41556Srgrimes * All rights reserved. 51556Srgrimes * 61556Srgrimes * This software was developed for the FreeBSD Project by Poul-Henning Kamp 71556Srgrimes * and NAI Labs, the Security Research Division of Network Associates, Inc. 81556Srgrimes * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the 91556Srgrimes * DARPA CHATS research program. 101556Srgrimes * 111556Srgrimes * Redistribution and use in source and binary forms, with or without 121556Srgrimes * modification, are permitted provided that the following conditions 131556Srgrimes * are met: 141556Srgrimes * 1. Redistributions of source code must retain the above copyright 151556Srgrimes * notice, this list of conditions and the following disclaimer. 161556Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 171556Srgrimes * notice, this list of conditions and the following disclaimer in the 181556Srgrimes * documentation and/or other materials provided with the distribution. 191556Srgrimes * 3. The names of the authors may not be used to endorse or promote 201556Srgrimes * products derived from this software without specific prior written 211556Srgrimes * permission. 221556Srgrimes * 231556Srgrimes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 241556Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 251556Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 261556Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 271556Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 281556Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 291556Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 301556Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 311556Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 321556Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 331556Srgrimes * SUCH DAMAGE. 341556Srgrimes */ 35114433Sobrien 361556Srgrimes#include <sys/cdefs.h> 3727962Ssteve__FBSDID("$FreeBSD: head/sys/geom/geom_io.c 116196 2003-06-11 06:49:16Z obrien $"); 381556Srgrimes 391556Srgrimes#include <sys/param.h> 401556Srgrimes#include <sys/systm.h> 411556Srgrimes#include <sys/kernel.h> 421556Srgrimes#include <sys/malloc.h> 4327962Ssteve#include <sys/bio.h> 44114433Sobrien 4527962Ssteve#include <sys/errno.h> 4699109Sobrien#include <geom/geom.h> 4799109Sobrien#include <geom/geom_int.h> 481556Srgrimes#include <sys/devicestat.h> 491556Srgrimes 501556Srgrimes#include <vm/uma.h> 511556Srgrimes 5277734Spirzykstatic struct g_bioq g_bio_run_down; 5323852Sbdestatic struct g_bioq g_bio_run_up; 541556Srgrimes 55129678Spjdstatic u_int pace; 56243049Sgrogstatic uma_zone_t biozone; 57125611Siedowse 581556Srgrimes#include <machine/atomic.h> 591556Srgrimes 601556Srgrimesstatic void 6154621Smharog_bioq_lock(struct g_bioq *bq) 621556Srgrimes{ 631556Srgrimes 6496470Sphk mtx_lock(&bq->bio_queue_lock); 6596470Sphk} 66129678Spjd 67129678Spjdstatic void 6854621Smharog_bioq_unlock(struct g_bioq *bq) 6993246Siedowse{ 7093246Siedowse 71125611Siedowse mtx_unlock(&bq->bio_queue_lock); 72185200Spjd} 73125611Siedowse 74125611Siedowse#if 0 75125611Siedowsestatic void 76125611Siedowseg_bioq_destroy(struct g_bioq *bq) 77125611Siedowse{ 7893246Siedowse 7993246Siedowse mtx_destroy(&bq->bio_queue_lock); 80128555Sobrien} 81114579Smarkm#endif 82125611Siedowse 8396470Sphkstatic void 84122537Smckusickg_bioq_init(struct g_bioq *bq) 85129678Spjd{ 86130060Sdas 8796470Sphk TAILQ_INIT(&bq->bio_queue); 88120037Sobrien mtx_init(&bq->bio_queue_lock, "bio queue", NULL, MTX_DEF); 89114579Smarkm} 9096470Sphk 911556Srgrimesstatic struct bio * 92125611Siedowseg_bioq_first(struct g_bioq *bq) 93125611Siedowse{ 9493246Siedowse struct bio *bp; 95114579Smarkm 9693246Siedowse bp = TAILQ_FIRST(&bq->bio_queue); 9793246Siedowse if (bp != NULL) { 98185200Spjd TAILQ_REMOVE(&bq->bio_queue, bp, bio_queue); 99243049Sgrog bq->bio_queue_length--; 10096470Sphk } 10196470Sphk return (bp); 1021556Srgrimes} 10390108Simp 1041556Srgrimesstatic void 1051556Srgrimesg_bioq_enqueue_tail(struct bio *bp, struct g_bioq *rq) 106128555Sobrien{ 10793246Siedowse 108128555Sobrien g_bioq_lock(rq); 10976873Skris TAILQ_INSERT_TAIL(&rq->bio_queue, bp, bio_queue); 11096470Sphk rq->bio_queue_length++; 11196470Sphk g_bioq_unlock(rq); 112226502Sdes} 113120037Sobrien 1141556Srgrimesstruct bio * 11576404Skrisg_new_bio(void) 116243049Sgrog{ 117249698Suqs struct bio *bp; 118128555Sobrien 119128410Sobrien bp = uma_zalloc(biozone, M_NOWAIT | M_ZERO); 120161470Simp return (bp); 12123852Sbde} 122243049Sgrog 1231556Srgrimesvoid 12452735Sjuliang_destroy_bio(struct bio *bp) 12552735Sjulian{ 12652735Sjulian 12754621Smharo uma_zfree(biozone, bp); 12854621Smharo} 12954621Smharo 130162483Scsjpstruct bio * 131218909Sbrucecg_clone_bio(struct bio *bp) 132162483Scsjp{ 133162483Scsjp struct bio *bp2; 134162483Scsjp 135162483Scsjp bp2 = uma_zalloc(biozone, M_NOWAIT | M_ZERO); 136162483Scsjp if (bp2 != NULL) { 137162483Scsjp bp2->bio_parent = bp; 138171195Sscf bp2->bio_cmd = bp->bio_cmd; 13954621Smharo bp2->bio_length = bp->bio_length; 14054621Smharo bp2->bio_offset = bp->bio_offset; 141128555Sobrien bp2->bio_data = bp->bio_data; 142128555Sobrien bp2->bio_attribute = bp->bio_attribute; 143128555Sobrien bp->bio_children++; 14461227Sjwd } 145171195Sscf return(bp2); 14661227Sjwd} 14761227Sjwd 14854621Smharovoid 14954621Smharog_io_init() 15054621Smharo{ 15154621Smharo 15254621Smharo g_bioq_init(&g_bio_run_down); 15354621Smharo g_bioq_init(&g_bio_run_up); 1541556Srgrimes biozone = uma_zcreate("g_bio", sizeof (struct bio), 1551556Srgrimes NULL, NULL, 1561556Srgrimes NULL, NULL, 1572008Swollman 0, 0); 158162483Scsjp} 159171195Sscf 16054621Smharoint 1612008Swollmang_io_getattr(const char *attr, struct g_consumer *cp, int *len, void *ptr) 16277734Spirzyk{ 16377734Spirzyk struct bio *bp; 16477734Spirzyk int error; 16577734Spirzyk 166167326Swill g_trace(G_T_BIO, "bio_getattr(%s)", attr); 16777734Spirzyk bp = g_new_bio(); 16854621Smharo bp->bio_cmd = BIO_GETATTR; 169171195Sscf bp->bio_done = NULL; 17054621Smharo bp->bio_attribute = attr; 17154621Smharo bp->bio_length = *len; 1721556Srgrimes bp->bio_data = ptr; 1731556Srgrimes g_io_request(bp, cp); 1741556Srgrimes error = biowait(bp, "ggetattr"); 1751556Srgrimes *len = bp->bio_completed; 176167326Swill g_destroy_bio(bp); 177167326Swill return (error); 17823852Sbde} 17987666Scharnier 18080795Sobrienstatic int 18123852Sbdeg_io_check(struct bio *bp) 1821556Srgrimes{ 183185200Spjd struct g_consumer *cp; 184185200Spjd struct g_provider *pp; 185185200Spjd 186243049Sgrog cp = bp->bio_from; 187243049Sgrog pp = bp->bio_to; 188243049Sgrog 1891556Srgrimes /* Fail if access counters dont allow the operation */ 1901556Srgrimes switch(bp->bio_cmd) { 1911556Srgrimes case BIO_READ: 1921556Srgrimes case BIO_GETATTR: 1931556Srgrimes if (cp->acr == 0) 1941556Srgrimes return (EPERM); 1951556Srgrimes break; 19630340Sjoerg case BIO_WRITE: 1971556Srgrimes case BIO_DELETE: 198226502Sdes if (cp->acw == 0) 199226502Sdes return (EPERM); 20023852Sbde break; 201226502Sdes default: 202226502Sdes return (EPERM); 203226502Sdes } 204249698Suqs /* if provider is marked for error, don't disturb. */ 205226502Sdes if (pp->error) 206226502Sdes return (pp->error); 207226502Sdes 2081556Srgrimes switch(bp->bio_cmd) { 2091556Srgrimes case BIO_READ: 210226502Sdes case BIO_WRITE: 2111556Srgrimes case BIO_DELETE: 2121556Srgrimes /* Reject I/O not on sector boundary */ 213249698Suqs if (bp->bio_offset % pp->sectorsize) 2141556Srgrimes return (EINVAL); 21530340Sjoerg /* Reject I/O not integral sector long */ 2161556Srgrimes if (bp->bio_length % pp->sectorsize) 2171556Srgrimes return (EINVAL); 21876404Skris /* Reject requests past the end of media. */ 219249698Suqs if (bp->bio_offset > pp->mediasize) 22076404Skris return (EIO); 22176404Skris break; 22276404Skris default: 22376404Skris break; 22476404Skris } 22576404Skris return (0); 22676404Skris} 22776404Skris 22876404Skrisvoid 22976404Skrisg_io_request(struct bio *bp, struct g_consumer *cp) 23076404Skris{ 23176404Skris struct g_provider *pp; 23276404Skris 23376404Skris pp = cp->provider; 23476404Skris KASSERT(cp != NULL, ("NULL cp in g_io_request")); 23576404Skris KASSERT(bp != NULL, ("NULL bp in g_io_request")); 23696470Sphk KASSERT(bp->bio_data != NULL, ("NULL bp->data in g_io_request")); 23796470Sphk KASSERT(pp != NULL, ("consumer not attached in g_io_request")); 23876404Skris 23976404Skris bp->bio_from = cp; 24076404Skris bp->bio_to = pp; 24176404Skris bp->bio_error = 0; 24276404Skris bp->bio_completed = 0; 24393246Siedowse 244128410Sobrien if (g_collectstats) { 245128410Sobrien devstat_start_transaction_bio(cp->stat, bp); 24676404Skris devstat_start_transaction_bio(pp->stat, bp); 24776404Skris } 24876404Skris cp->nstart++; 24976404Skris pp->nstart++; 25076404Skris 25176404Skris /* Pass it on down. */ 25276404Skris g_trace(G_T_BIO, "bio_request(%p) from %p(%s) to %p(%s) cmd %d", 25376404Skris bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd); 25476404Skris g_bioq_enqueue_tail(bp, &g_bio_run_down); 2551556Srgrimes wakeup(&g_wait_down); 2561556Srgrimes} 257115769Sbde 2581556Srgrimesvoid 2591556Srgrimesg_io_deliver(struct bio *bp, int error) 2601556Srgrimes{ 2611556Srgrimes struct g_consumer *cp; 2621556Srgrimes struct g_provider *pp; 2631556Srgrimes 26430340Sjoerg cp = bp->bio_from; 2651556Srgrimes pp = bp->bio_to; 2661556Srgrimes KASSERT(bp != NULL, ("NULL bp in g_io_deliver")); 267115769Sbde KASSERT(cp != NULL, ("NULL bio_from in g_io_deliver")); 268115769Sbde KASSERT(cp->geom != NULL, ("NULL bio_from->geom in g_io_deliver")); 269115769Sbde KASSERT(pp != NULL, ("NULL bio_to in g_io_deliver")); 270115769Sbde 271115769Sbde g_trace(G_T_BIO, 272115769Sbde"g_io_deliver(%p) from %p(%s) to %p(%s) cmd %d error %d off %jd len %jd", 273115744Sjkh bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd, error, 274115744Sjkh (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length); 275115769Sbde 276115744Sjkh bp->bio_bcount = bp->bio_length; 277115744Sjkh if (g_collectstats) { 278115769Sbde bp->bio_resid = bp->bio_bcount - bp->bio_completed; 279226502Sdes devstat_end_transaction_bio(cp->stat, bp); 280226502Sdes devstat_end_transaction_bio(pp->stat, bp); 281226502Sdes } 282226502Sdes cp->nend++; 283226502Sdes pp->nend++; 284226502Sdes 285226502Sdes if (error == ENOMEM) { 286249698Suqs if (bootverbose) 287226502Sdes printf("ENOMEM %p on %p(%s)\n", bp, pp, pp->name); 288226502Sdes g_io_request(bp, cp); 289226502Sdes pace++; 290226502Sdes return; 291226502Sdes } 29293246Siedowse bp->bio_error = error; 2931556Srgrimes g_bioq_enqueue_tail(bp, &g_bio_run_up); 294226502Sdes wakeup(&g_wait_up); 295226502Sdes} 296226502Sdes 297128410Sobrienvoid 298128410Sobrieng_io_schedule_down(struct thread *tp __unused) 299287790Sdelphij{ 3001556Srgrimes struct bio *bp; 3011556Srgrimes off_t excess; 30296470Sphk int error; 303114579Smarkm struct mtx mymutex; 3041556Srgrimes 305120037Sobrien bzero(&mymutex, sizeof mymutex); 3061556Srgrimes mtx_init(&mymutex, "g_xdown", MTX_DEF, 0); 3071556Srgrimes 3081556Srgrimes for(;;) { 3091556Srgrimes g_bioq_lock(&g_bio_run_down); 3101556Srgrimes bp = g_bioq_first(&g_bio_run_down); 3111556Srgrimes if (bp == NULL) { 3121556Srgrimes msleep(&g_wait_down, &g_bio_run_down.bio_queue_lock, 313249698Suqs PRIBIO | PDROP, "g_down", hz/10); 3141556Srgrimes continue; 3151556Srgrimes } 3161556Srgrimes g_bioq_unlock(&g_bio_run_down); 317102230Strhodes if (pace > 0) { 318102230Strhodes msleep(&error, NULL, PRIBIO, "g_down", hz/10); 3191556Srgrimes pace--; 3201556Srgrimes } 321120037Sobrien error = g_io_check(bp); 32296470Sphk if (error) { 3231556Srgrimes g_io_deliver(bp, error); 324132465Scsjp continue; 3251556Srgrimes } 3261556Srgrimes switch (bp->bio_cmd) { 32723852Sbde case BIO_READ: 3281556Srgrimes case BIO_WRITE: 3291556Srgrimes case BIO_DELETE: 3301556Srgrimes /* Truncate requests to the end of providers media. */ 33123852Sbde excess = bp->bio_offset + bp->bio_length; 33223852Sbde if (excess > bp->bio_to->mediasize) { 33323852Sbde excess -= bp->bio_to->mediasize; 334132465Scsjp bp->bio_length -= excess; 335132465Scsjp } 336132465Scsjp /* Deliver zero length transfers right here. */ 337132465Scsjp if (bp->bio_length == 0) { 338132465Scsjp g_io_deliver(bp, 0); 339132465Scsjp continue; 340132465Scsjp } 341132465Scsjp break; 342132465Scsjp default: 343132465Scsjp break; 344132465Scsjp } 345132465Scsjp mtx_lock(&mymutex); 346132465Scsjp bp->bio_to->geom->start(bp); 347132465Scsjp mtx_unlock(&mymutex); 34823852Sbde } 3491556Srgrimes} 3501556Srgrimes 3511556Srgrimesvoid 3521556Srgrimesg_io_schedule_up(struct thread *tp __unused) 35396470Sphk{ 354122537Smckusick struct bio *bp; 35554621Smharo struct mtx mymutex; 35654621Smharo 357129678Spjd bzero(&mymutex, sizeof mymutex); 358129678Spjd mtx_init(&mymutex, "g_xup", MTX_DEF, 0); 359129678Spjd for(;;) { 36054621Smharo g_bioq_lock(&g_bio_run_up); 36154621Smharo bp = g_bioq_first(&g_bio_run_up); 36296470Sphk if (bp != NULL) { 363129678Spjd g_bioq_unlock(&g_bio_run_up); 36454621Smharo mtx_lock(&mymutex); 365129678Spjd biodone(bp); 366129678Spjd mtx_unlock(&mymutex); 36754621Smharo continue; 368129678Spjd } 369129678Spjd msleep(&g_wait_up, &g_bio_run_up.bio_queue_lock, 370129678Spjd PRIBIO | PDROP, "g_up", hz/10); 37154621Smharo } 372129678Spjd} 373129678Spjd 374129678Spjdvoid * 375129678Spjdg_read_data(struct g_consumer *cp, off_t offset, off_t length, int *error) 37654621Smharo{ 37754621Smharo struct bio *bp; 37854621Smharo void *ptr; 379193629Ssimon int errorc; 380193629Ssimon 381193629Ssimon bp = g_new_bio(); 382193629Ssimon bp->bio_cmd = BIO_READ; 383193629Ssimon bp->bio_done = NULL; 384193629Ssimon bp->bio_offset = offset; 385193629Ssimon bp->bio_length = length; 386193629Ssimon ptr = g_malloc(length, M_WAITOK); 387193629Ssimon bp->bio_data = ptr; 388193629Ssimon g_io_request(bp, cp); 389193629Ssimon errorc = biowait(bp, "gread"); 390193629Ssimon if (error != NULL) 391193629Ssimon *error = errorc; 392193629Ssimon g_destroy_bio(bp); 393193629Ssimon if (errorc) { 394193629Ssimon g_free(ptr); 395193629Ssimon ptr = NULL; 396102230Strhodes } 3971556Srgrimes return (ptr); 398130060Sdas} 399130060Sdas 400130060Sdasint 401244134Sgrogg_write_data(struct g_consumer *cp, off_t offset, void *ptr, off_t length) 402130060Sdas{ 403130060Sdas struct bio *bp; 4041556Srgrimes int error; 405102230Strhodes 4061556Srgrimes bp = g_new_bio(); 40796470Sphk bp->bio_cmd = BIO_WRITE; 40893246Siedowse bp->bio_done = NULL; 4091556Srgrimes bp->bio_offset = offset; 410168428Skan bp->bio_length = length; 411114579Smarkm bp->bio_data = ptr; 41276873Skris g_io_request(bp, cp); 413122537Smckusick error = biowait(bp, "gwrite"); 414243049Sgrog g_destroy_bio(bp); 4151556Srgrimes return (error); 4161556Srgrimes} 417125611Siedowse