geom_fox.c revision 152971
1116518Sphk/*- 2116518Sphk * Copyright (c) 2003 Poul-Henning Kamp 3116518Sphk * All rights reserved. 4116518Sphk * 5116518Sphk * Redistribution and use in source and binary forms, with or without 6116518Sphk * modification, are permitted provided that the following conditions 7116518Sphk * are met: 8116518Sphk * 1. Redistributions of source code must retain the above copyright 9116518Sphk * notice, this list of conditions and the following disclaimer. 10116518Sphk * 2. Redistributions in binary form must reproduce the above copyright 11116518Sphk * notice, this list of conditions and the following disclaimer in the 12116518Sphk * documentation and/or other materials provided with the distribution. 13116518Sphk * 3. The names of the authors may not be used to endorse or promote 14116518Sphk * products derived from this software without specific prior written 15116518Sphk * permission. 16116518Sphk * 17116518Sphk * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18116518Sphk * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19116518Sphk * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20116518Sphk * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21116518Sphk * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22116518Sphk * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23116518Sphk * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24116518Sphk * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25116518Sphk * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26116518Sphk * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27116518Sphk * SUCH DAMAGE. 28116518Sphk * 29116518Sphk * $FreeBSD: head/sys/geom/geom_fox.c 152971 2005-11-30 22:15:00Z sobomax $ 30139778Simp */ 31139778Simp 32139778Simp/* This is a GEOM module for handling path selection for multi-path 33116518Sphk * storage devices. It is named "fox" because it, like they, prefer 34116518Sphk * to have multiple exits to choose from. 35116518Sphk * 36116518Sphk */ 37116518Sphk 38116518Sphk#include <sys/param.h> 39116518Sphk#include <sys/systm.h> 40116518Sphk#include <sys/kernel.h> 41116518Sphk#include <sys/conf.h> 42116518Sphk#include <sys/bio.h> 43116518Sphk#include <sys/malloc.h> 44116518Sphk#include <sys/lock.h> 45116518Sphk#include <sys/mutex.h> 46116518Sphk#include <sys/libkern.h> 47116518Sphk#include <sys/endian.h> 48116518Sphk#include <sys/md5.h> 49116518Sphk#include <sys/errno.h> 50116518Sphk#include <geom/geom.h> 51116518Sphk 52116518Sphk#define FOX_CLASS_NAME "FOX" 53116518Sphk#define FOX_MAGIC "GEOM::FOX" 54116518Sphk 55116518Sphkstruct g_fox_softc { 56116518Sphk off_t mediasize; 57116518Sphk u_int sectorsize; 58116518Sphk TAILQ_HEAD(, bio) queue; 59116518Sphk struct mtx lock; 60116518Sphk u_char magic[16]; 61116518Sphk struct g_consumer *path; 62116518Sphk struct g_consumer *opath; 63116518Sphk int waiting; 64116518Sphk int cr, cw, ce; 65116518Sphk}; 66116518Sphk 67116518Sphk/* 68116518Sphk * This function is called whenever we need to select a new path. 69116518Sphk */ 70116518Sphkstatic void 71116518Sphkg_fox_select_path(void *arg, int flag) 72116518Sphk{ 73116518Sphk struct g_geom *gp; 74116518Sphk struct g_fox_softc *sc; 75116518Sphk struct g_consumer *cp1; 76116518Sphk struct bio *bp; 77116518Sphk int error; 78116518Sphk 79116518Sphk g_topology_assert(); 80116518Sphk if (flag == EV_CANCEL) 81116518Sphk return; 82116518Sphk gp = arg; 83116518Sphk sc = gp->softc; 84116518Sphk 85116518Sphk if (sc->opath != NULL) { 86116518Sphk /* 87116518Sphk * First, close the old path entirely. 88116518Sphk */ 89116518Sphk printf("Closing old path (%s) on fox (%s)\n", 90116518Sphk sc->opath->provider->name, gp->name); 91116518Sphk 92116518Sphk cp1 = LIST_NEXT(sc->opath, consumer); 93116518Sphk 94125803Sphk g_access(sc->opath, -sc->cr, -sc->cw, -(sc->ce + 1)); 95116518Sphk 96116518Sphk /* 97116518Sphk * The attempt to reopen it with a exclusive count 98116518Sphk */ 99125755Sphk error = g_access(sc->opath, 0, 0, 1); 100116518Sphk if (error) { 101116518Sphk /* 102116518Sphk * Ok, ditch this consumer, we can't use it. 103116518Sphk */ 104116518Sphk printf("Drop old path (%s) on fox (%s)\n", 105116518Sphk sc->opath->provider->name, gp->name); 106116518Sphk g_detach(sc->opath); 107116518Sphk g_destroy_consumer(sc->opath); 108116518Sphk if (LIST_EMPTY(&gp->consumer)) { 109116518Sphk /* No consumers left */ 110116518Sphk g_wither_geom(gp, ENXIO); 111116518Sphk for (;;) { 112116518Sphk bp = TAILQ_FIRST(&sc->queue); 113116518Sphk if (bp == NULL) 114116518Sphk break; 115116518Sphk TAILQ_REMOVE(&sc->queue, bp, bio_queue); 116116518Sphk bp->bio_error = ENXIO; 117116518Sphk g_std_done(bp); 118116518Sphk } 119116518Sphk return; 120116518Sphk } 121116518Sphk } else { 122116518Sphk printf("Got e-bit on old path (%s) on fox (%s)\n", 123116518Sphk sc->opath->provider->name, gp->name); 124116518Sphk } 125116518Sphk sc->opath = NULL; 126116518Sphk } else { 127116518Sphk cp1 = LIST_FIRST(&gp->consumer); 128116518Sphk } 129116518Sphk if (cp1 == NULL) 130116518Sphk cp1 = LIST_FIRST(&gp->consumer); 131116518Sphk printf("Open new path (%s) on fox (%s)\n", 132116518Sphk cp1->provider->name, gp->name); 133125755Sphk error = g_access(cp1, sc->cr, sc->cw, sc->ce); 134116518Sphk if (error) { 135116518Sphk /* 136116518Sphk * If we failed, we take another trip through here 137116518Sphk */ 138116518Sphk printf("Open new path (%s) on fox (%s) failed, reselect.\n", 139116518Sphk cp1->provider->name, gp->name); 140116518Sphk sc->opath = cp1; 141116518Sphk g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL); 142116518Sphk } else { 143116518Sphk printf("Open new path (%s) on fox (%s) succeeded\n", 144116518Sphk cp1->provider->name, gp->name); 145116518Sphk mtx_lock(&sc->lock); 146116518Sphk sc->path = cp1; 147116518Sphk sc->waiting = 0; 148116518Sphk for (;;) { 149116518Sphk bp = TAILQ_FIRST(&sc->queue); 150116518Sphk if (bp == NULL) 151116518Sphk break; 152116518Sphk TAILQ_REMOVE(&sc->queue, bp, bio_queue); 153116518Sphk g_io_request(bp, sc->path); 154116518Sphk } 155116518Sphk mtx_unlock(&sc->lock); 156116518Sphk } 157116518Sphk} 158116518Sphk 159116518Sphkstatic void 160116518Sphkg_fox_orphan(struct g_consumer *cp) 161116518Sphk{ 162116518Sphk struct g_geom *gp; 163116518Sphk struct g_fox_softc *sc; 164116518Sphk int error, mark; 165116518Sphk 166116518Sphk g_topology_assert(); 167116518Sphk gp = cp->geom; 168116518Sphk sc = gp->softc; 169116518Sphk printf("Removing path (%s) from fox (%s)\n", 170116518Sphk cp->provider->name, gp->name); 171116518Sphk mtx_lock(&sc->lock); 172116518Sphk if (cp == sc->path) { 173116518Sphk sc->opath = NULL; 174116518Sphk sc->path = NULL; 175116518Sphk sc->waiting = 1; 176116518Sphk mark = 1; 177116518Sphk } else { 178116518Sphk mark = 0; 179116518Sphk } 180116518Sphk mtx_unlock(&sc->lock); 181116518Sphk 182125755Sphk g_access(cp, -cp->acr, -cp->acw, -cp->ace); 183116518Sphk error = cp->provider->error; 184116518Sphk g_detach(cp); 185116518Sphk g_destroy_consumer(cp); 186116518Sphk if (!LIST_EMPTY(&gp->consumer)) { 187116518Sphk if (mark) 188116518Sphk g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL); 189116518Sphk return; 190116518Sphk } 191116518Sphk 192116518Sphk mtx_destroy(&sc->lock); 193121366Sphk g_free(gp->softc); 194116518Sphk gp->softc = NULL; 195116518Sphk g_wither_geom(gp, ENXIO); 196116518Sphk} 197116518Sphk 198116518Sphkstatic void 199116518Sphkg_fox_done(struct bio *bp) 200116518Sphk{ 201116518Sphk struct g_geom *gp; 202116518Sphk struct g_fox_softc *sc; 203116518Sphk int error; 204116518Sphk 205116518Sphk if (bp->bio_error == 0) { 206116518Sphk g_std_done(bp); 207116518Sphk return; 208116518Sphk } 209116518Sphk gp = bp->bio_from->geom; 210116518Sphk sc = gp->softc; 211116518Sphk if (bp->bio_from != sc->path) { 212116518Sphk g_io_request(bp, sc->path); 213116518Sphk return; 214116518Sphk } 215116518Sphk mtx_lock(&sc->lock); 216116518Sphk sc->opath = sc->path; 217116518Sphk sc->path = NULL; 218116518Sphk error = g_post_event(g_fox_select_path, gp, M_NOWAIT, gp, NULL); 219116518Sphk if (error) { 220116518Sphk bp->bio_error = ENOMEM; 221116518Sphk g_std_done(bp); 222116518Sphk } else { 223116518Sphk sc->waiting = 1; 224116518Sphk TAILQ_INSERT_TAIL(&sc->queue, bp, bio_queue); 225116518Sphk } 226116518Sphk mtx_unlock(&sc->lock); 227116518Sphk} 228116518Sphk 229116518Sphkstatic void 230116518Sphkg_fox_start(struct bio *bp) 231116518Sphk{ 232116518Sphk struct g_geom *gp; 233116518Sphk struct bio *bp2; 234116518Sphk struct g_fox_softc *sc; 235116518Sphk int error; 236116518Sphk 237116518Sphk gp = bp->bio_to->geom; 238116518Sphk sc = gp->softc; 239116518Sphk if (sc == NULL) { 240116518Sphk g_io_deliver(bp, ENXIO); 241116518Sphk return; 242116518Sphk } 243116518Sphk switch(bp->bio_cmd) { 244116518Sphk case BIO_READ: 245116518Sphk case BIO_WRITE: 246116518Sphk case BIO_DELETE: 247116518Sphk bp2 = g_clone_bio(bp); 248116518Sphk if (bp2 == NULL) { 249116518Sphk g_io_deliver(bp, ENOMEM); 250116518Sphk break; 251116518Sphk } 252116518Sphk bp2->bio_offset += sc->sectorsize; 253116518Sphk bp2->bio_done = g_fox_done; 254116518Sphk mtx_lock(&sc->lock); 255116518Sphk if (sc->path == NULL || !TAILQ_EMPTY(&sc->queue)) { 256116518Sphk if (sc->waiting == 0) { 257116518Sphk error = g_post_event(g_fox_select_path, gp, 258116518Sphk M_NOWAIT, gp, NULL); 259116518Sphk if (error) { 260116518Sphk g_destroy_bio(bp2); 261116518Sphk bp2 = NULL; 262116518Sphk g_io_deliver(bp, error); 263116518Sphk } else { 264116518Sphk sc->waiting = 1; 265116518Sphk } 266116518Sphk } 267116518Sphk if (bp2 != NULL) 268116518Sphk TAILQ_INSERT_TAIL(&sc->queue, bp2, 269116518Sphk bio_queue); 270116518Sphk } else { 271116518Sphk g_io_request(bp2, sc->path); 272116518Sphk } 273116518Sphk mtx_unlock(&sc->lock); 274116518Sphk break; 275116518Sphk default: 276116518Sphk g_io_deliver(bp, EOPNOTSUPP); 277116518Sphk break; 278116518Sphk } 279116518Sphk return; 280116518Sphk} 281116518Sphk 282116518Sphkstatic int 283116518Sphkg_fox_access(struct g_provider *pp, int dr, int dw, int de) 284116518Sphk{ 285116518Sphk struct g_geom *gp; 286116518Sphk struct g_fox_softc *sc; 287116518Sphk struct g_consumer *cp1; 288116518Sphk int error; 289116518Sphk 290116518Sphk g_topology_assert(); 291116518Sphk gp = pp->geom; 292116518Sphk sc = gp->softc; 293125803Sphk if (sc == NULL) { 294125803Sphk if (dr <= 0 && dw <= 0 && de <= 0) 295125803Sphk return (0); 296125803Sphk else 297125803Sphk return (ENXIO); 298125803Sphk } 299116518Sphk 300116518Sphk if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) { 301116518Sphk /* 302116518Sphk * First open, open all consumers with an exclusive bit 303116518Sphk */ 304116518Sphk error = 0; 305116518Sphk LIST_FOREACH(cp1, &gp->consumer, consumer) { 306125755Sphk error = g_access(cp1, 0, 0, 1); 307116518Sphk if (error) { 308116518Sphk printf("FOX: access(%s,0,0,1) = %d\n", 309116518Sphk cp1->provider->name, error); 310116518Sphk break; 311116518Sphk } 312116518Sphk } 313116518Sphk if (error) { 314116518Sphk LIST_FOREACH(cp1, &gp->consumer, consumer) { 315116518Sphk if (cp1->ace) 316125755Sphk g_access(cp1, 0, 0, -1); 317116518Sphk } 318116518Sphk return (error); 319116518Sphk } 320116518Sphk } 321116518Sphk if (sc->path == NULL) 322116518Sphk g_fox_select_path(gp, 0); 323116518Sphk if (sc->path == NULL) 324116518Sphk error = ENXIO; 325116518Sphk else 326125755Sphk error = g_access(sc->path, dr, dw, de); 327116518Sphk if (error == 0) { 328116518Sphk sc->cr += dr; 329116518Sphk sc->cw += dw; 330116518Sphk sc->ce += de; 331116518Sphk if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) { 332116518Sphk /* 333116518Sphk * Last close, remove e-bit on all consumers 334116518Sphk */ 335116518Sphk LIST_FOREACH(cp1, &gp->consumer, consumer) 336125755Sphk g_access(cp1, 0, 0, -1); 337116518Sphk } 338116518Sphk } 339116518Sphk return (error); 340116518Sphk} 341116518Sphk 342116518Sphkstatic struct g_geom * 343116518Sphkg_fox_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) 344116518Sphk{ 345116518Sphk struct g_geom *gp, *gp2; 346116518Sphk struct g_provider *pp2; 347116518Sphk struct g_consumer *cp, *cp2; 348116518Sphk struct g_fox_softc *sc, *sc2; 349116518Sphk int error; 350116518Sphk u_int sectorsize; 351116518Sphk u_char *buf; 352116518Sphk 353116518Sphk g_trace(G_T_TOPOLOGY, "fox_taste(%s, %s)", mp->name, pp->name); 354116518Sphk g_topology_assert(); 355116518Sphk if (!strcmp(pp->geom->class->name, mp->name)) 356116518Sphk return (NULL); 357116518Sphk gp = g_new_geomf(mp, "%s.fox", pp->name); 358116518Sphk gp->softc = g_malloc(sizeof(struct g_fox_softc), M_WAITOK | M_ZERO); 359116518Sphk sc = gp->softc; 360116518Sphk 361116518Sphk cp = g_new_consumer(gp); 362116518Sphk g_attach(cp, pp); 363125755Sphk error = g_access(cp, 1, 0, 0); 364116518Sphk if (error) { 365116518Sphk g_free(sc); 366116518Sphk g_detach(cp); 367116518Sphk g_destroy_consumer(cp); 368116518Sphk g_destroy_geom(gp); 369116518Sphk return(NULL); 370116518Sphk } 371116518Sphk do { 372116518Sphk sectorsize = cp->provider->sectorsize; 373116518Sphk g_topology_unlock(); 374152971Ssobomax buf = g_read_data(cp, 0, sectorsize, NULL); 375116518Sphk g_topology_lock(); 376152967Ssobomax if (buf == NULL) 377116518Sphk break; 378116518Sphk if (memcmp(buf, FOX_MAGIC, strlen(FOX_MAGIC))) 379116518Sphk break; 380116518Sphk 381116518Sphk /* 382116518Sphk * First we need to see if this a new path for an existing fox. 383116518Sphk */ 384116518Sphk LIST_FOREACH(gp2, &mp->geom, geom) { 385116518Sphk sc2 = gp2->softc; 386121475Sphk if (sc2 == NULL) 387116518Sphk continue; 388116518Sphk if (memcmp(buf + 16, sc2->magic, sizeof sc2->magic)) 389116518Sphk continue; 390116518Sphk break; 391116518Sphk } 392116518Sphk if (gp2 != NULL) { 393116518Sphk /* 394116518Sphk * It was. Create a new consumer for that fox, 395116518Sphk * attach it, and if the fox is open, open this 396116518Sphk * path with an exclusive count of one. 397116518Sphk */ 398116518Sphk printf("Adding path (%s) to fox (%s)\n", 399116518Sphk pp->name, gp2->name); 400116518Sphk cp2 = g_new_consumer(gp2); 401116518Sphk g_attach(cp2, pp); 402116518Sphk pp2 = LIST_FIRST(&gp2->provider); 403116518Sphk if (pp2->acr > 0 || pp2->acw > 0 || pp2->ace > 0) { 404125755Sphk error = g_access(cp2, 0, 0, 1); 405116518Sphk if (error) { 406116518Sphk /* 407116518Sphk * This is bad, or more likely, 408116518Sphk * the user is doing something stupid 409116518Sphk */ 410116518Sphk printf( 411116518Sphk "WARNING: New path (%s) to fox(%s) not added: %s\n%s", 412121475Sphk cp2->provider->name, gp2->name, 413116518Sphk "Could not get exclusive bit.", 414116518Sphk "WARNING: This indicates a risk of data inconsistency." 415116518Sphk ); 416116518Sphk g_detach(cp2); 417116518Sphk g_destroy_consumer(cp2); 418116518Sphk } 419116518Sphk } 420116518Sphk break; 421116518Sphk } 422116518Sphk printf("Creating new fox (%s)\n", pp->name); 423116518Sphk sc->path = cp; 424116518Sphk memcpy(sc->magic, buf + 16, sizeof sc->magic); 425116518Sphk pp2 = g_new_providerf(gp, "%s", gp->name); 426116518Sphk pp2->mediasize = sc->mediasize = pp->mediasize - pp->sectorsize; 427116518Sphk pp2->sectorsize = sc->sectorsize = pp->sectorsize; 428116518Sphkprintf("fox %s lock %p\n", gp->name, &sc->lock); 429116518Sphk 430116518Sphk mtx_init(&sc->lock, "fox queue", NULL, MTX_DEF); 431116518Sphk TAILQ_INIT(&sc->queue); 432116518Sphk g_error_provider(pp2, 0); 433116518Sphk } while (0); 434116518Sphk if (buf != NULL) 435116518Sphk g_free(buf); 436125755Sphk g_access(cp, -1, 0, 0); 437116518Sphk 438116518Sphk if (!LIST_EMPTY(&gp->provider)) 439116518Sphk return (gp); 440116518Sphk 441116518Sphk g_free(gp->softc); 442116518Sphk g_detach(cp); 443116518Sphk g_destroy_consumer(cp); 444116518Sphk g_destroy_geom(gp); 445116518Sphk return (NULL); 446116518Sphk} 447116518Sphk 448116518Sphkstatic int 449116518Sphkg_fox_destroy_geom(struct gctl_req *req, struct g_class *mp, struct g_geom *gp) 450116518Sphk{ 451116518Sphk struct g_fox_softc *sc; 452116518Sphk 453116518Sphk g_topology_assert(); 454116518Sphk sc = gp->softc; 455116518Sphk mtx_destroy(&sc->lock); 456116518Sphk g_free(gp->softc); 457125538Sle gp->softc = NULL; 458116518Sphk g_wither_geom(gp, ENXIO); 459116518Sphk return (0); 460116518Sphk} 461116518Sphk 462116518Sphkstatic struct g_class g_fox_class = { 463116518Sphk .name = FOX_CLASS_NAME, 464133318Sphk .version = G_VERSION, 465116518Sphk .taste = g_fox_taste, 466116518Sphk .destroy_geom = g_fox_destroy_geom, 467133314Sphk .start = g_fox_start, 468133314Sphk .spoiled = g_fox_orphan, 469133314Sphk .orphan = g_fox_orphan, 470133314Sphk .access= g_fox_access, 471116518Sphk}; 472116518Sphk 473116518SphkDECLARE_GEOM_CLASS(g_fox_class, g_fox); 474