geom_fox.c revision 133314
1116518Sphk/*- 2116518Sphk * Copyright (c) 2003 Poul-Henning Kamp 3116518Sphk * All rights reserved. 4116518Sphk * 5116518Sphk * Redistribution and use in source and binary forms, with or without 6116518Sphk * modification, are permitted provided that the following conditions 7116518Sphk * are met: 8116518Sphk * 1. Redistributions of source code must retain the above copyright 9116518Sphk * notice, this list of conditions and the following disclaimer. 10116518Sphk * 2. Redistributions in binary form must reproduce the above copyright 11116518Sphk * notice, this list of conditions and the following disclaimer in the 12116518Sphk * documentation and/or other materials provided with the distribution. 13116518Sphk * 3. The names of the authors may not be used to endorse or promote 14116518Sphk * products derived from this software without specific prior written 15116518Sphk * permission. 16116518Sphk * 17116518Sphk * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18116518Sphk * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19116518Sphk * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20116518Sphk * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21116518Sphk * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22116518Sphk * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23116518Sphk * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24116518Sphk * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25116518Sphk * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26116518Sphk * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27116518Sphk * SUCH DAMAGE. 28116518Sphk * 29116518Sphk * $FreeBSD: head/sys/geom/geom_fox.c 133314 2004-08-08 06:49:07Z phk $ 30116518Sphk * 31116518Sphk * This is a GEOM module for handling path selection for multi-path 32116518Sphk * storage devices. It is named "fox" because it, like they, prefer 33116518Sphk * to have multiple exits to choose from. 34116518Sphk * 35116518Sphk */ 36116518Sphk 37116518Sphk#include <sys/param.h> 38116518Sphk#include <sys/systm.h> 39116518Sphk#include <sys/kernel.h> 40116518Sphk#include <sys/conf.h> 41116518Sphk#include <sys/bio.h> 42116518Sphk#include <sys/malloc.h> 43116518Sphk#include <sys/lock.h> 44116518Sphk#include <sys/mutex.h> 45116518Sphk#include <sys/libkern.h> 46116518Sphk#include <sys/endian.h> 47116518Sphk#include <sys/md5.h> 48116518Sphk#include <sys/errno.h> 49116518Sphk#include <geom/geom.h> 50116518Sphk 51116518Sphk#define FOX_CLASS_NAME "FOX" 52116518Sphk#define FOX_MAGIC "GEOM::FOX" 53116518Sphk 54116518Sphkstruct g_fox_softc { 55116518Sphk off_t mediasize; 56116518Sphk u_int sectorsize; 57116518Sphk TAILQ_HEAD(, bio) queue; 58116518Sphk struct mtx lock; 59116518Sphk u_char magic[16]; 60116518Sphk struct g_consumer *path; 61116518Sphk struct g_consumer *opath; 62116518Sphk int waiting; 63116518Sphk int cr, cw, ce; 64116518Sphk}; 65116518Sphk 66116518Sphk/* 67116518Sphk * This function is called whenever we need to select a new path. 68116518Sphk */ 69116518Sphkstatic void 70116518Sphkg_fox_select_path(void *arg, int flag) 71116518Sphk{ 72116518Sphk struct g_geom *gp; 73116518Sphk struct g_fox_softc *sc; 74116518Sphk struct g_consumer *cp1; 75116518Sphk struct bio *bp; 76116518Sphk int error; 77116518Sphk 78116518Sphk g_topology_assert(); 79116518Sphk if (flag == EV_CANCEL) 80116518Sphk return; 81116518Sphk gp = arg; 82116518Sphk sc = gp->softc; 83116518Sphk 84116518Sphk if (sc->opath != NULL) { 85116518Sphk /* 86116518Sphk * First, close the old path entirely. 87116518Sphk */ 88116518Sphk printf("Closing old path (%s) on fox (%s)\n", 89116518Sphk sc->opath->provider->name, gp->name); 90116518Sphk 91116518Sphk cp1 = LIST_NEXT(sc->opath, consumer); 92116518Sphk 93125803Sphk g_access(sc->opath, -sc->cr, -sc->cw, -(sc->ce + 1)); 94116518Sphk 95116518Sphk /* 96116518Sphk * The attempt to reopen it with a exclusive count 97116518Sphk */ 98125755Sphk error = g_access(sc->opath, 0, 0, 1); 99116518Sphk if (error) { 100116518Sphk /* 101116518Sphk * Ok, ditch this consumer, we can't use it. 102116518Sphk */ 103116518Sphk printf("Drop old path (%s) on fox (%s)\n", 104116518Sphk sc->opath->provider->name, gp->name); 105116518Sphk g_detach(sc->opath); 106116518Sphk g_destroy_consumer(sc->opath); 107116518Sphk if (LIST_EMPTY(&gp->consumer)) { 108116518Sphk /* No consumers left */ 109116518Sphk g_wither_geom(gp, ENXIO); 110116518Sphk for (;;) { 111116518Sphk bp = TAILQ_FIRST(&sc->queue); 112116518Sphk if (bp == NULL) 113116518Sphk break; 114116518Sphk TAILQ_REMOVE(&sc->queue, bp, bio_queue); 115116518Sphk bp->bio_error = ENXIO; 116116518Sphk g_std_done(bp); 117116518Sphk } 118116518Sphk return; 119116518Sphk } 120116518Sphk } else { 121116518Sphk printf("Got e-bit on old path (%s) on fox (%s)\n", 122116518Sphk sc->opath->provider->name, gp->name); 123116518Sphk } 124116518Sphk sc->opath = NULL; 125116518Sphk } else { 126116518Sphk cp1 = LIST_FIRST(&gp->consumer); 127116518Sphk } 128116518Sphk if (cp1 == NULL) 129116518Sphk cp1 = LIST_FIRST(&gp->consumer); 130116518Sphk printf("Open new path (%s) on fox (%s)\n", 131116518Sphk cp1->provider->name, gp->name); 132125755Sphk error = g_access(cp1, sc->cr, sc->cw, sc->ce); 133116518Sphk if (error) { 134116518Sphk /* 135116518Sphk * If we failed, we take another trip through here 136116518Sphk */ 137116518Sphk printf("Open new path (%s) on fox (%s) failed, reselect.\n", 138116518Sphk cp1->provider->name, gp->name); 139116518Sphk sc->opath = cp1; 140116518Sphk g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL); 141116518Sphk } else { 142116518Sphk printf("Open new path (%s) on fox (%s) succeeded\n", 143116518Sphk cp1->provider->name, gp->name); 144116518Sphk mtx_lock(&sc->lock); 145116518Sphk sc->path = cp1; 146116518Sphk sc->waiting = 0; 147116518Sphk for (;;) { 148116518Sphk bp = TAILQ_FIRST(&sc->queue); 149116518Sphk if (bp == NULL) 150116518Sphk break; 151116518Sphk TAILQ_REMOVE(&sc->queue, bp, bio_queue); 152116518Sphk g_io_request(bp, sc->path); 153116518Sphk } 154116518Sphk mtx_unlock(&sc->lock); 155116518Sphk } 156116518Sphk} 157116518Sphk 158116518Sphkstatic void 159116518Sphkg_fox_orphan(struct g_consumer *cp) 160116518Sphk{ 161116518Sphk struct g_geom *gp; 162116518Sphk struct g_fox_softc *sc; 163116518Sphk int error, mark; 164116518Sphk 165116518Sphk g_topology_assert(); 166116518Sphk gp = cp->geom; 167116518Sphk sc = gp->softc; 168116518Sphk printf("Removing path (%s) from fox (%s)\n", 169116518Sphk cp->provider->name, gp->name); 170116518Sphk mtx_lock(&sc->lock); 171116518Sphk if (cp == sc->path) { 172116518Sphk sc->opath = NULL; 173116518Sphk sc->path = NULL; 174116518Sphk sc->waiting = 1; 175116518Sphk mark = 1; 176116518Sphk } else { 177116518Sphk mark = 0; 178116518Sphk } 179116518Sphk mtx_unlock(&sc->lock); 180116518Sphk 181125755Sphk g_access(cp, -cp->acr, -cp->acw, -cp->ace); 182116518Sphk error = cp->provider->error; 183116518Sphk g_detach(cp); 184116518Sphk g_destroy_consumer(cp); 185116518Sphk if (!LIST_EMPTY(&gp->consumer)) { 186116518Sphk if (mark) 187116518Sphk g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL); 188116518Sphk return; 189116518Sphk } 190116518Sphk 191116518Sphk mtx_destroy(&sc->lock); 192121366Sphk g_free(gp->softc); 193116518Sphk gp->softc = NULL; 194116518Sphk g_wither_geom(gp, ENXIO); 195116518Sphk} 196116518Sphk 197116518Sphkstatic void 198116518Sphkg_fox_done(struct bio *bp) 199116518Sphk{ 200116518Sphk struct g_geom *gp; 201116518Sphk struct g_fox_softc *sc; 202116518Sphk int error; 203116518Sphk 204116518Sphk if (bp->bio_error == 0) { 205116518Sphk g_std_done(bp); 206116518Sphk return; 207116518Sphk } 208116518Sphk gp = bp->bio_from->geom; 209116518Sphk sc = gp->softc; 210116518Sphk if (bp->bio_from != sc->path) { 211116518Sphk g_io_request(bp, sc->path); 212116518Sphk return; 213116518Sphk } 214116518Sphk mtx_lock(&sc->lock); 215116518Sphk sc->opath = sc->path; 216116518Sphk sc->path = NULL; 217116518Sphk error = g_post_event(g_fox_select_path, gp, M_NOWAIT, gp, NULL); 218116518Sphk if (error) { 219116518Sphk bp->bio_error = ENOMEM; 220116518Sphk g_std_done(bp); 221116518Sphk } else { 222116518Sphk sc->waiting = 1; 223116518Sphk TAILQ_INSERT_TAIL(&sc->queue, bp, bio_queue); 224116518Sphk } 225116518Sphk mtx_unlock(&sc->lock); 226116518Sphk} 227116518Sphk 228116518Sphkstatic void 229116518Sphkg_fox_start(struct bio *bp) 230116518Sphk{ 231116518Sphk struct g_geom *gp; 232116518Sphk struct bio *bp2; 233116518Sphk struct g_fox_softc *sc; 234116518Sphk int error; 235116518Sphk 236116518Sphk gp = bp->bio_to->geom; 237116518Sphk sc = gp->softc; 238116518Sphk if (sc == NULL) { 239116518Sphk g_io_deliver(bp, ENXIO); 240116518Sphk return; 241116518Sphk } 242116518Sphk switch(bp->bio_cmd) { 243116518Sphk case BIO_READ: 244116518Sphk case BIO_WRITE: 245116518Sphk case BIO_DELETE: 246116518Sphk bp2 = g_clone_bio(bp); 247116518Sphk if (bp2 == NULL) { 248116518Sphk g_io_deliver(bp, ENOMEM); 249116518Sphk break; 250116518Sphk } 251116518Sphk bp2->bio_offset += sc->sectorsize; 252116518Sphk bp2->bio_done = g_fox_done; 253116518Sphk mtx_lock(&sc->lock); 254116518Sphk if (sc->path == NULL || !TAILQ_EMPTY(&sc->queue)) { 255116518Sphk if (sc->waiting == 0) { 256116518Sphk error = g_post_event(g_fox_select_path, gp, 257116518Sphk M_NOWAIT, gp, NULL); 258116518Sphk if (error) { 259116518Sphk g_destroy_bio(bp2); 260116518Sphk bp2 = NULL; 261116518Sphk g_io_deliver(bp, error); 262116518Sphk } else { 263116518Sphk sc->waiting = 1; 264116518Sphk } 265116518Sphk } 266116518Sphk if (bp2 != NULL) 267116518Sphk TAILQ_INSERT_TAIL(&sc->queue, bp2, 268116518Sphk bio_queue); 269116518Sphk } else { 270116518Sphk g_io_request(bp2, sc->path); 271116518Sphk } 272116518Sphk mtx_unlock(&sc->lock); 273116518Sphk break; 274116518Sphk default: 275116518Sphk g_io_deliver(bp, EOPNOTSUPP); 276116518Sphk break; 277116518Sphk } 278116518Sphk return; 279116518Sphk} 280116518Sphk 281116518Sphkstatic int 282116518Sphkg_fox_access(struct g_provider *pp, int dr, int dw, int de) 283116518Sphk{ 284116518Sphk struct g_geom *gp; 285116518Sphk struct g_fox_softc *sc; 286116518Sphk struct g_consumer *cp1; 287116518Sphk int error; 288116518Sphk 289116518Sphk g_topology_assert(); 290116518Sphk gp = pp->geom; 291116518Sphk sc = gp->softc; 292125803Sphk if (sc == NULL) { 293125803Sphk if (dr <= 0 && dw <= 0 && de <= 0) 294125803Sphk return (0); 295125803Sphk else 296125803Sphk return (ENXIO); 297125803Sphk } 298116518Sphk 299116518Sphk if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) { 300116518Sphk /* 301116518Sphk * First open, open all consumers with an exclusive bit 302116518Sphk */ 303116518Sphk error = 0; 304116518Sphk LIST_FOREACH(cp1, &gp->consumer, consumer) { 305125755Sphk error = g_access(cp1, 0, 0, 1); 306116518Sphk if (error) { 307116518Sphk printf("FOX: access(%s,0,0,1) = %d\n", 308116518Sphk cp1->provider->name, error); 309116518Sphk break; 310116518Sphk } 311116518Sphk } 312116518Sphk if (error) { 313116518Sphk LIST_FOREACH(cp1, &gp->consumer, consumer) { 314116518Sphk if (cp1->ace) 315125755Sphk g_access(cp1, 0, 0, -1); 316116518Sphk } 317116518Sphk return (error); 318116518Sphk } 319116518Sphk } 320116518Sphk if (sc->path == NULL) 321116518Sphk g_fox_select_path(gp, 0); 322116518Sphk if (sc->path == NULL) 323116518Sphk error = ENXIO; 324116518Sphk else 325125755Sphk error = g_access(sc->path, dr, dw, de); 326116518Sphk if (error == 0) { 327116518Sphk sc->cr += dr; 328116518Sphk sc->cw += dw; 329116518Sphk sc->ce += de; 330116518Sphk if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) { 331116518Sphk /* 332116518Sphk * Last close, remove e-bit on all consumers 333116518Sphk */ 334116518Sphk LIST_FOREACH(cp1, &gp->consumer, consumer) 335125755Sphk g_access(cp1, 0, 0, -1); 336116518Sphk } 337116518Sphk } 338116518Sphk return (error); 339116518Sphk} 340116518Sphk 341116518Sphkstatic struct g_geom * 342116518Sphkg_fox_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) 343116518Sphk{ 344116518Sphk struct g_geom *gp, *gp2; 345116518Sphk struct g_provider *pp2; 346116518Sphk struct g_consumer *cp, *cp2; 347116518Sphk struct g_fox_softc *sc, *sc2; 348116518Sphk int error; 349116518Sphk u_int sectorsize; 350116518Sphk u_char *buf; 351116518Sphk 352116518Sphk g_trace(G_T_TOPOLOGY, "fox_taste(%s, %s)", mp->name, pp->name); 353116518Sphk g_topology_assert(); 354116518Sphk if (!strcmp(pp->geom->class->name, mp->name)) 355116518Sphk return (NULL); 356116518Sphk gp = g_new_geomf(mp, "%s.fox", pp->name); 357116518Sphk gp->softc = g_malloc(sizeof(struct g_fox_softc), M_WAITOK | M_ZERO); 358116518Sphk sc = gp->softc; 359116518Sphk 360116518Sphk cp = g_new_consumer(gp); 361116518Sphk g_attach(cp, pp); 362125755Sphk error = g_access(cp, 1, 0, 0); 363116518Sphk if (error) { 364116518Sphk g_free(sc); 365116518Sphk g_detach(cp); 366116518Sphk g_destroy_consumer(cp); 367116518Sphk g_destroy_geom(gp); 368116518Sphk return(NULL); 369116518Sphk } 370116518Sphk do { 371116518Sphk sectorsize = cp->provider->sectorsize; 372116518Sphk g_topology_unlock(); 373116518Sphk buf = g_read_data(cp, 0, sectorsize, &error); 374116518Sphk g_topology_lock(); 375116518Sphk if (buf == NULL || error != 0) 376116518Sphk break; 377116518Sphk if (memcmp(buf, FOX_MAGIC, strlen(FOX_MAGIC))) 378116518Sphk break; 379116518Sphk 380116518Sphk /* 381116518Sphk * First we need to see if this a new path for an existing fox. 382116518Sphk */ 383116518Sphk LIST_FOREACH(gp2, &mp->geom, geom) { 384116518Sphk sc2 = gp2->softc; 385121475Sphk if (sc2 == NULL) 386116518Sphk continue; 387116518Sphk if (memcmp(buf + 16, sc2->magic, sizeof sc2->magic)) 388116518Sphk continue; 389116518Sphk break; 390116518Sphk } 391116518Sphk if (gp2 != NULL) { 392116518Sphk /* 393116518Sphk * It was. Create a new consumer for that fox, 394116518Sphk * attach it, and if the fox is open, open this 395116518Sphk * path with an exclusive count of one. 396116518Sphk */ 397116518Sphk printf("Adding path (%s) to fox (%s)\n", 398116518Sphk pp->name, gp2->name); 399116518Sphk cp2 = g_new_consumer(gp2); 400116518Sphk g_attach(cp2, pp); 401116518Sphk pp2 = LIST_FIRST(&gp2->provider); 402116518Sphk if (pp2->acr > 0 || pp2->acw > 0 || pp2->ace > 0) { 403125755Sphk error = g_access(cp2, 0, 0, 1); 404116518Sphk if (error) { 405116518Sphk /* 406116518Sphk * This is bad, or more likely, 407116518Sphk * the user is doing something stupid 408116518Sphk */ 409116518Sphk printf( 410116518Sphk "WARNING: New path (%s) to fox(%s) not added: %s\n%s", 411121475Sphk cp2->provider->name, gp2->name, 412116518Sphk "Could not get exclusive bit.", 413116518Sphk "WARNING: This indicates a risk of data inconsistency." 414116518Sphk ); 415116518Sphk g_detach(cp2); 416116518Sphk g_destroy_consumer(cp2); 417116518Sphk } 418116518Sphk } 419116518Sphk break; 420116518Sphk } 421116518Sphk printf("Creating new fox (%s)\n", pp->name); 422116518Sphk sc->path = cp; 423116518Sphk memcpy(sc->magic, buf + 16, sizeof sc->magic); 424116518Sphk pp2 = g_new_providerf(gp, "%s", gp->name); 425116518Sphk pp2->mediasize = sc->mediasize = pp->mediasize - pp->sectorsize; 426116518Sphk pp2->sectorsize = sc->sectorsize = pp->sectorsize; 427116518Sphkprintf("fox %s lock %p\n", gp->name, &sc->lock); 428116518Sphk 429116518Sphk mtx_init(&sc->lock, "fox queue", NULL, MTX_DEF); 430116518Sphk TAILQ_INIT(&sc->queue); 431116518Sphk g_error_provider(pp2, 0); 432116518Sphk } while (0); 433116518Sphk if (buf != NULL) 434116518Sphk g_free(buf); 435125755Sphk g_access(cp, -1, 0, 0); 436116518Sphk 437116518Sphk if (!LIST_EMPTY(&gp->provider)) 438116518Sphk return (gp); 439116518Sphk 440116518Sphk g_free(gp->softc); 441116518Sphk g_detach(cp); 442116518Sphk g_destroy_consumer(cp); 443116518Sphk g_destroy_geom(gp); 444116518Sphk return (NULL); 445116518Sphk} 446116518Sphk 447116518Sphkstatic int 448116518Sphkg_fox_destroy_geom(struct gctl_req *req, struct g_class *mp, struct g_geom *gp) 449116518Sphk{ 450116518Sphk struct g_fox_softc *sc; 451116518Sphk 452116518Sphk g_topology_assert(); 453116518Sphk sc = gp->softc; 454116518Sphk mtx_destroy(&sc->lock); 455116518Sphk g_free(gp->softc); 456125538Sle gp->softc = NULL; 457116518Sphk g_wither_geom(gp, ENXIO); 458116518Sphk return (0); 459116518Sphk} 460116518Sphk 461116518Sphkstatic struct g_class g_fox_class = { 462116518Sphk .name = FOX_CLASS_NAME, 463116518Sphk .taste = g_fox_taste, 464116518Sphk .destroy_geom = g_fox_destroy_geom, 465133314Sphk .start = g_fox_start, 466133314Sphk .spoiled = g_fox_orphan, 467133314Sphk .orphan = g_fox_orphan, 468133314Sphk .access= g_fox_access, 469116518Sphk}; 470116518Sphk 471116518SphkDECLARE_GEOM_CLASS(g_fox_class, g_fox); 472