geom_fox.c revision 152967 
1/*-
2 * Copyright (c) 2003 Poul-Henning Kamp
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 * 3. The names of the authors may not be used to endorse or promote
14 *    products derived from this software without specific prior written
15 *    permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD: head/sys/geom/geom_fox.c 152967 2005-11-30 19:24:51Z sobomax $
30 */
31
32/* This is a GEOM module for handling path selection for multi-path
33 * storage devices.  It is named "fox" because it, like they, prefer
34 * to have multiple exits to choose from.
35 *
36 */
37
38#include <sys/param.h>
39#include <sys/systm.h>
40#include <sys/kernel.h>
41#include <sys/conf.h>
42#include <sys/bio.h>
43#include <sys/malloc.h>
44#include <sys/lock.h>
45#include <sys/mutex.h>
46#include <sys/libkern.h>
47#include <sys/endian.h>
48#include <sys/md5.h>
49#include <sys/errno.h>
50#include <geom/geom.h>
51
52#define FOX_CLASS_NAME "FOX"
53#define FOX_MAGIC	"GEOM::FOX"
54
55struct g_fox_softc {
56	off_t			mediasize;
57	u_int			sectorsize;
58	TAILQ_HEAD(, bio)	queue;
59	struct mtx		lock;
60	u_char 			magic[16];
61	struct g_consumer 	*path;
62	struct g_consumer 	*opath;
63	int			waiting;
64	int			cr, cw, ce;
65};
66
67/*
68 * This function is called whenever we need to select a new path.
69 */
70static void
71g_fox_select_path(void *arg, int flag)
72{
73	struct g_geom *gp;
74	struct g_fox_softc *sc;
75	struct g_consumer *cp1;
76	struct bio *bp;
77	int error;
78
79	g_topology_assert();
80	if (flag == EV_CANCEL)
81		return;
82	gp = arg;
83	sc = gp->softc;
84
85	if (sc->opath != NULL) {
86		/*
87		 * First, close the old path entirely.
88		 */
89		printf("Closing old path (%s) on fox (%s)\n",
90			sc->opath->provider->name, gp->name);
91
92		cp1 = LIST_NEXT(sc->opath, consumer);
93
94		g_access(sc->opath, -sc->cr, -sc->cw, -(sc->ce + 1));
95
96		/*
97		 * The attempt to reopen it with a exclusive count
98		 */
99		error = g_access(sc->opath, 0, 0, 1);
100		if (error) {
101			/*
102			 * Ok, ditch this consumer, we can't use it.
103			 */
104			printf("Drop old path (%s) on fox (%s)\n",
105				sc->opath->provider->name, gp->name);
106			g_detach(sc->opath);
107			g_destroy_consumer(sc->opath);
108			if (LIST_EMPTY(&gp->consumer)) {
109				/* No consumers left */
110				g_wither_geom(gp, ENXIO);
111				for (;;) {
112					bp = TAILQ_FIRST(&sc->queue);
113					if (bp == NULL)
114						break;
115					TAILQ_REMOVE(&sc->queue, bp, bio_queue);
116					bp->bio_error = ENXIO;
117					g_std_done(bp);
118				}
119				return;
120			}
121		} else {
122			printf("Got e-bit on old path (%s) on fox (%s)\n",
123				sc->opath->provider->name, gp->name);
124		}
125		sc->opath = NULL;
126	} else {
127		cp1 = LIST_FIRST(&gp->consumer);
128	}
129	if (cp1 == NULL)
130		cp1 = LIST_FIRST(&gp->consumer);
131	printf("Open new path (%s) on fox (%s)\n",
132		cp1->provider->name, gp->name);
133	error = g_access(cp1, sc->cr, sc->cw, sc->ce);
134	if (error) {
135		/*
136		 * If we failed, we take another trip through here
137		 */
138		printf("Open new path (%s) on fox (%s) failed, reselect.\n",
139			cp1->provider->name, gp->name);
140		sc->opath = cp1;
141		g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL);
142	} else {
143		printf("Open new path (%s) on fox (%s) succeeded\n",
144			cp1->provider->name, gp->name);
145		mtx_lock(&sc->lock);
146		sc->path = cp1;
147		sc->waiting = 0;
148		for (;;) {
149			bp = TAILQ_FIRST(&sc->queue);
150			if (bp == NULL)
151				break;
152			TAILQ_REMOVE(&sc->queue, bp, bio_queue);
153			g_io_request(bp, sc->path);
154		}
155		mtx_unlock(&sc->lock);
156	}
157}
158
159static void
160g_fox_orphan(struct g_consumer *cp)
161{
162	struct g_geom *gp;
163	struct g_fox_softc *sc;
164	int error, mark;
165
166	g_topology_assert();
167	gp = cp->geom;
168	sc = gp->softc;
169	printf("Removing path (%s) from fox (%s)\n",
170	    cp->provider->name, gp->name);
171	mtx_lock(&sc->lock);
172	if (cp == sc->path) {
173		sc->opath = NULL;
174		sc->path = NULL;
175		sc->waiting = 1;
176		mark = 1;
177	} else {
178		mark = 0;
179	}
180	mtx_unlock(&sc->lock);
181
182	g_access(cp, -cp->acr, -cp->acw, -cp->ace);
183	error = cp->provider->error;
184	g_detach(cp);
185	g_destroy_consumer(cp);
186	if (!LIST_EMPTY(&gp->consumer)) {
187		if (mark)
188			g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL);
189		return;
190	}
191
192	mtx_destroy(&sc->lock);
193	g_free(gp->softc);
194	gp->softc = NULL;
195	g_wither_geom(gp, ENXIO);
196}
197
198static void
199g_fox_done(struct bio *bp)
200{
201	struct g_geom *gp;
202	struct g_fox_softc *sc;
203	int error;
204
205	if (bp->bio_error == 0) {
206		g_std_done(bp);
207		return;
208	}
209	gp = bp->bio_from->geom;
210	sc = gp->softc;
211	if (bp->bio_from != sc->path) {
212		g_io_request(bp, sc->path);
213		return;
214	}
215	mtx_lock(&sc->lock);
216	sc->opath = sc->path;
217	sc->path = NULL;
218	error = g_post_event(g_fox_select_path, gp, M_NOWAIT, gp, NULL);
219	if (error) {
220		bp->bio_error = ENOMEM;
221		g_std_done(bp);
222	} else {
223		sc->waiting = 1;
224		TAILQ_INSERT_TAIL(&sc->queue, bp, bio_queue);
225	}
226	mtx_unlock(&sc->lock);
227}
228
229static void
230g_fox_start(struct bio *bp)
231{
232	struct g_geom *gp;
233	struct bio *bp2;
234	struct g_fox_softc *sc;
235	int error;
236
237	gp = bp->bio_to->geom;
238	sc = gp->softc;
239	if (sc == NULL) {
240		g_io_deliver(bp, ENXIO);
241		return;
242	}
243	switch(bp->bio_cmd) {
244	case BIO_READ:
245	case BIO_WRITE:
246	case BIO_DELETE:
247		bp2 = g_clone_bio(bp);
248		if (bp2 == NULL) {
249			g_io_deliver(bp, ENOMEM);
250			break;
251		}
252		bp2->bio_offset += sc->sectorsize;
253		bp2->bio_done = g_fox_done;
254		mtx_lock(&sc->lock);
255		if (sc->path == NULL || !TAILQ_EMPTY(&sc->queue)) {
256			if (sc->waiting == 0) {
257				error = g_post_event(g_fox_select_path, gp,
258				    M_NOWAIT, gp, NULL);
259				if (error) {
260					g_destroy_bio(bp2);
261					bp2 = NULL;
262					g_io_deliver(bp, error);
263				} else {
264					sc->waiting = 1;
265				}
266			}
267			if (bp2 != NULL)
268				TAILQ_INSERT_TAIL(&sc->queue, bp2,
269				    bio_queue);
270		} else {
271			g_io_request(bp2, sc->path);
272		}
273		mtx_unlock(&sc->lock);
274		break;
275	default:
276		g_io_deliver(bp, EOPNOTSUPP);
277		break;
278	}
279	return;
280}
281
282static int
283g_fox_access(struct g_provider *pp, int dr, int dw, int de)
284{
285	struct g_geom *gp;
286	struct g_fox_softc *sc;
287	struct g_consumer *cp1;
288	int error;
289
290	g_topology_assert();
291	gp = pp->geom;
292	sc = gp->softc;
293	if (sc == NULL) {
294		if (dr <= 0 && dw <= 0 && de <= 0)
295			return (0);
296		else
297			return (ENXIO);
298	}
299
300	if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) {
301		/*
302		 * First open, open all consumers with an exclusive bit
303		 */
304		error = 0;
305		LIST_FOREACH(cp1, &gp->consumer, consumer) {
306			error = g_access(cp1, 0, 0, 1);
307			if (error) {
308				printf("FOX: access(%s,0,0,1) = %d\n",
309				    cp1->provider->name, error);
310				break;
311			}
312		}
313		if (error) {
314			LIST_FOREACH(cp1, &gp->consumer, consumer) {
315				if (cp1->ace)
316					g_access(cp1, 0, 0, -1);
317			}
318			return (error);
319		}
320	}
321	if (sc->path == NULL)
322		g_fox_select_path(gp, 0);
323	if (sc->path == NULL)
324		error = ENXIO;
325	else
326		error = g_access(sc->path, dr, dw, de);
327	if (error == 0) {
328		sc->cr += dr;
329		sc->cw += dw;
330		sc->ce += de;
331		if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) {
332			/*
333			 * Last close, remove e-bit on all consumers
334			 */
335			LIST_FOREACH(cp1, &gp->consumer, consumer)
336				g_access(cp1, 0, 0, -1);
337		}
338	}
339	return (error);
340}
341
342static struct g_geom *
343g_fox_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
344{
345	struct g_geom *gp, *gp2;
346	struct g_provider *pp2;
347	struct g_consumer *cp, *cp2;
348	struct g_fox_softc *sc, *sc2;
349	int error;
350	u_int sectorsize;
351	u_char *buf;
352
353	g_trace(G_T_TOPOLOGY, "fox_taste(%s, %s)", mp->name, pp->name);
354	g_topology_assert();
355	if (!strcmp(pp->geom->class->name, mp->name))
356		return (NULL);
357	gp = g_new_geomf(mp, "%s.fox", pp->name);
358	gp->softc = g_malloc(sizeof(struct g_fox_softc), M_WAITOK | M_ZERO);
359	sc = gp->softc;
360
361	cp = g_new_consumer(gp);
362	g_attach(cp, pp);
363	error = g_access(cp, 1, 0, 0);
364	if (error) {
365		g_free(sc);
366		g_detach(cp);
367		g_destroy_consumer(cp);
368		g_destroy_geom(gp);
369		return(NULL);
370	}
371	do {
372		sectorsize = cp->provider->sectorsize;
373		g_topology_unlock();
374		buf = g_read_data(cp, 0, sectorsize, &error);
375		g_topology_lock();
376		if (buf == NULL)
377			break;
378		if (memcmp(buf, FOX_MAGIC, strlen(FOX_MAGIC)))
379			break;
380
381		/*
382		 * First we need to see if this a new path for an existing fox.
383		 */
384		LIST_FOREACH(gp2, &mp->geom, geom) {
385			sc2 = gp2->softc;
386			if (sc2 == NULL)
387				continue;
388			if (memcmp(buf + 16, sc2->magic, sizeof sc2->magic))
389				continue;
390			break;
391		}
392		if (gp2 != NULL) {
393			/*
394			 * It was.  Create a new consumer for that fox,
395			 * attach it, and if the fox is open, open this
396			 * path with an exclusive count of one.
397			 */
398			printf("Adding path (%s) to fox (%s)\n",
399			    pp->name, gp2->name);
400			cp2 = g_new_consumer(gp2);
401			g_attach(cp2, pp);
402			pp2 = LIST_FIRST(&gp2->provider);
403			if (pp2->acr > 0 || pp2->acw > 0 || pp2->ace > 0) {
404				error = g_access(cp2, 0, 0, 1);
405				if (error) {
406					/*
407					 * This is bad, or more likely,
408					 * the user is doing something stupid
409					 */
410					printf(
411	"WARNING: New path (%s) to fox(%s) not added: %s\n%s",
412					    cp2->provider->name, gp2->name,
413	"Could not get exclusive bit.",
414	"WARNING: This indicates a risk of data inconsistency."
415					);
416					g_detach(cp2);
417					g_destroy_consumer(cp2);
418				}
419			}
420			break;
421		}
422		printf("Creating new fox (%s)\n", pp->name);
423		sc->path = cp;
424		memcpy(sc->magic, buf + 16, sizeof sc->magic);
425		pp2 = g_new_providerf(gp, "%s", gp->name);
426		pp2->mediasize = sc->mediasize = pp->mediasize - pp->sectorsize;
427		pp2->sectorsize = sc->sectorsize = pp->sectorsize;
428printf("fox %s lock %p\n", gp->name, &sc->lock);
429
430		mtx_init(&sc->lock, "fox queue", NULL, MTX_DEF);
431		TAILQ_INIT(&sc->queue);
432		g_error_provider(pp2, 0);
433	} while (0);
434	if (buf != NULL)
435		g_free(buf);
436	g_access(cp, -1, 0, 0);
437
438	if (!LIST_EMPTY(&gp->provider))
439		return (gp);
440
441	g_free(gp->softc);
442	g_detach(cp);
443	g_destroy_consumer(cp);
444	g_destroy_geom(gp);
445	return (NULL);
446}
447
448static int
449g_fox_destroy_geom(struct gctl_req *req, struct g_class *mp, struct g_geom *gp)
450{
451	struct g_fox_softc *sc;
452
453	g_topology_assert();
454	sc = gp->softc;
455	mtx_destroy(&sc->lock);
456	g_free(gp->softc);
457	gp->softc = NULL;
458	g_wither_geom(gp, ENXIO);
459	return (0);
460}
461
462static struct g_class g_fox_class	= {
463	.name = FOX_CLASS_NAME,
464	.version = G_VERSION,
465	.taste = g_fox_taste,
466	.destroy_geom = g_fox_destroy_geom,
467	.start = g_fox_start,
468	.spoiled = g_fox_orphan,
469	.orphan = g_fox_orphan,
470	.access= g_fox_access,
471};
472
473DECLARE_GEOM_CLASS(g_fox_class, g_fox);
474