geom_fox.c revision 133314 
1/*-
2 * Copyright (c) 2003 Poul-Henning Kamp
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 * 3. The names of the authors may not be used to endorse or promote
14 *    products derived from this software without specific prior written
15 *    permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD: head/sys/geom/geom_fox.c 133314 2004-08-08 06:49:07Z phk $
30 *
31 * This is a GEOM module for handling path selection for multi-path
32 * storage devices.  It is named "fox" because it, like they, prefer
33 * to have multiple exits to choose from.
34 *
35 */
36
37#include <sys/param.h>
38#include <sys/systm.h>
39#include <sys/kernel.h>
40#include <sys/conf.h>
41#include <sys/bio.h>
42#include <sys/malloc.h>
43#include <sys/lock.h>
44#include <sys/mutex.h>
45#include <sys/libkern.h>
46#include <sys/endian.h>
47#include <sys/md5.h>
48#include <sys/errno.h>
49#include <geom/geom.h>
50
51#define FOX_CLASS_NAME "FOX"
52#define FOX_MAGIC	"GEOM::FOX"
53
54struct g_fox_softc {
55	off_t			mediasize;
56	u_int			sectorsize;
57	TAILQ_HEAD(, bio)	queue;
58	struct mtx		lock;
59	u_char 			magic[16];
60	struct g_consumer 	*path;
61	struct g_consumer 	*opath;
62	int			waiting;
63	int			cr, cw, ce;
64};
65
66/*
67 * This function is called whenever we need to select a new path.
68 */
69static void
70g_fox_select_path(void *arg, int flag)
71{
72	struct g_geom *gp;
73	struct g_fox_softc *sc;
74	struct g_consumer *cp1;
75	struct bio *bp;
76	int error;
77
78	g_topology_assert();
79	if (flag == EV_CANCEL)
80		return;
81	gp = arg;
82	sc = gp->softc;
83
84	if (sc->opath != NULL) {
85		/*
86		 * First, close the old path entirely.
87		 */
88		printf("Closing old path (%s) on fox (%s)\n",
89			sc->opath->provider->name, gp->name);
90
91		cp1 = LIST_NEXT(sc->opath, consumer);
92
93		g_access(sc->opath, -sc->cr, -sc->cw, -(sc->ce + 1));
94
95		/*
96		 * The attempt to reopen it with a exclusive count
97		 */
98		error = g_access(sc->opath, 0, 0, 1);
99		if (error) {
100			/*
101			 * Ok, ditch this consumer, we can't use it.
102			 */
103			printf("Drop old path (%s) on fox (%s)\n",
104				sc->opath->provider->name, gp->name);
105			g_detach(sc->opath);
106			g_destroy_consumer(sc->opath);
107			if (LIST_EMPTY(&gp->consumer)) {
108				/* No consumers left */
109				g_wither_geom(gp, ENXIO);
110				for (;;) {
111					bp = TAILQ_FIRST(&sc->queue);
112					if (bp == NULL)
113						break;
114					TAILQ_REMOVE(&sc->queue, bp, bio_queue);
115					bp->bio_error = ENXIO;
116					g_std_done(bp);
117				}
118				return;
119			}
120		} else {
121			printf("Got e-bit on old path (%s) on fox (%s)\n",
122				sc->opath->provider->name, gp->name);
123		}
124		sc->opath = NULL;
125	} else {
126		cp1 = LIST_FIRST(&gp->consumer);
127	}
128	if (cp1 == NULL)
129		cp1 = LIST_FIRST(&gp->consumer);
130	printf("Open new path (%s) on fox (%s)\n",
131		cp1->provider->name, gp->name);
132	error = g_access(cp1, sc->cr, sc->cw, sc->ce);
133	if (error) {
134		/*
135		 * If we failed, we take another trip through here
136		 */
137		printf("Open new path (%s) on fox (%s) failed, reselect.\n",
138			cp1->provider->name, gp->name);
139		sc->opath = cp1;
140		g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL);
141	} else {
142		printf("Open new path (%s) on fox (%s) succeeded\n",
143			cp1->provider->name, gp->name);
144		mtx_lock(&sc->lock);
145		sc->path = cp1;
146		sc->waiting = 0;
147		for (;;) {
148			bp = TAILQ_FIRST(&sc->queue);
149			if (bp == NULL)
150				break;
151			TAILQ_REMOVE(&sc->queue, bp, bio_queue);
152			g_io_request(bp, sc->path);
153		}
154		mtx_unlock(&sc->lock);
155	}
156}
157
158static void
159g_fox_orphan(struct g_consumer *cp)
160{
161	struct g_geom *gp;
162	struct g_fox_softc *sc;
163	int error, mark;
164
165	g_topology_assert();
166	gp = cp->geom;
167	sc = gp->softc;
168	printf("Removing path (%s) from fox (%s)\n",
169	    cp->provider->name, gp->name);
170	mtx_lock(&sc->lock);
171	if (cp == sc->path) {
172		sc->opath = NULL;
173		sc->path = NULL;
174		sc->waiting = 1;
175		mark = 1;
176	} else {
177		mark = 0;
178	}
179	mtx_unlock(&sc->lock);
180
181	g_access(cp, -cp->acr, -cp->acw, -cp->ace);
182	error = cp->provider->error;
183	g_detach(cp);
184	g_destroy_consumer(cp);
185	if (!LIST_EMPTY(&gp->consumer)) {
186		if (mark)
187			g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL);
188		return;
189	}
190
191	mtx_destroy(&sc->lock);
192	g_free(gp->softc);
193	gp->softc = NULL;
194	g_wither_geom(gp, ENXIO);
195}
196
197static void
198g_fox_done(struct bio *bp)
199{
200	struct g_geom *gp;
201	struct g_fox_softc *sc;
202	int error;
203
204	if (bp->bio_error == 0) {
205		g_std_done(bp);
206		return;
207	}
208	gp = bp->bio_from->geom;
209	sc = gp->softc;
210	if (bp->bio_from != sc->path) {
211		g_io_request(bp, sc->path);
212		return;
213	}
214	mtx_lock(&sc->lock);
215	sc->opath = sc->path;
216	sc->path = NULL;
217	error = g_post_event(g_fox_select_path, gp, M_NOWAIT, gp, NULL);
218	if (error) {
219		bp->bio_error = ENOMEM;
220		g_std_done(bp);
221	} else {
222		sc->waiting = 1;
223		TAILQ_INSERT_TAIL(&sc->queue, bp, bio_queue);
224	}
225	mtx_unlock(&sc->lock);
226}
227
228static void
229g_fox_start(struct bio *bp)
230{
231	struct g_geom *gp;
232	struct bio *bp2;
233	struct g_fox_softc *sc;
234	int error;
235
236	gp = bp->bio_to->geom;
237	sc = gp->softc;
238	if (sc == NULL) {
239		g_io_deliver(bp, ENXIO);
240		return;
241	}
242	switch(bp->bio_cmd) {
243	case BIO_READ:
244	case BIO_WRITE:
245	case BIO_DELETE:
246		bp2 = g_clone_bio(bp);
247		if (bp2 == NULL) {
248			g_io_deliver(bp, ENOMEM);
249			break;
250		}
251		bp2->bio_offset += sc->sectorsize;
252		bp2->bio_done = g_fox_done;
253		mtx_lock(&sc->lock);
254		if (sc->path == NULL || !TAILQ_EMPTY(&sc->queue)) {
255			if (sc->waiting == 0) {
256				error = g_post_event(g_fox_select_path, gp,
257				    M_NOWAIT, gp, NULL);
258				if (error) {
259					g_destroy_bio(bp2);
260					bp2 = NULL;
261					g_io_deliver(bp, error);
262				} else {
263					sc->waiting = 1;
264				}
265			}
266			if (bp2 != NULL)
267				TAILQ_INSERT_TAIL(&sc->queue, bp2,
268				    bio_queue);
269		} else {
270			g_io_request(bp2, sc->path);
271		}
272		mtx_unlock(&sc->lock);
273		break;
274	default:
275		g_io_deliver(bp, EOPNOTSUPP);
276		break;
277	}
278	return;
279}
280
281static int
282g_fox_access(struct g_provider *pp, int dr, int dw, int de)
283{
284	struct g_geom *gp;
285	struct g_fox_softc *sc;
286	struct g_consumer *cp1;
287	int error;
288
289	g_topology_assert();
290	gp = pp->geom;
291	sc = gp->softc;
292	if (sc == NULL) {
293		if (dr <= 0 && dw <= 0 && de <= 0)
294			return (0);
295		else
296			return (ENXIO);
297	}
298
299	if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) {
300		/*
301		 * First open, open all consumers with an exclusive bit
302		 */
303		error = 0;
304		LIST_FOREACH(cp1, &gp->consumer, consumer) {
305			error = g_access(cp1, 0, 0, 1);
306			if (error) {
307				printf("FOX: access(%s,0,0,1) = %d\n",
308				    cp1->provider->name, error);
309				break;
310			}
311		}
312		if (error) {
313			LIST_FOREACH(cp1, &gp->consumer, consumer) {
314				if (cp1->ace)
315					g_access(cp1, 0, 0, -1);
316			}
317			return (error);
318		}
319	}
320	if (sc->path == NULL)
321		g_fox_select_path(gp, 0);
322	if (sc->path == NULL)
323		error = ENXIO;
324	else
325		error = g_access(sc->path, dr, dw, de);
326	if (error == 0) {
327		sc->cr += dr;
328		sc->cw += dw;
329		sc->ce += de;
330		if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) {
331			/*
332			 * Last close, remove e-bit on all consumers
333			 */
334			LIST_FOREACH(cp1, &gp->consumer, consumer)
335				g_access(cp1, 0, 0, -1);
336		}
337	}
338	return (error);
339}
340
341static struct g_geom *
342g_fox_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
343{
344	struct g_geom *gp, *gp2;
345	struct g_provider *pp2;
346	struct g_consumer *cp, *cp2;
347	struct g_fox_softc *sc, *sc2;
348	int error;
349	u_int sectorsize;
350	u_char *buf;
351
352	g_trace(G_T_TOPOLOGY, "fox_taste(%s, %s)", mp->name, pp->name);
353	g_topology_assert();
354	if (!strcmp(pp->geom->class->name, mp->name))
355		return (NULL);
356	gp = g_new_geomf(mp, "%s.fox", pp->name);
357	gp->softc = g_malloc(sizeof(struct g_fox_softc), M_WAITOK | M_ZERO);
358	sc = gp->softc;
359
360	cp = g_new_consumer(gp);
361	g_attach(cp, pp);
362	error = g_access(cp, 1, 0, 0);
363	if (error) {
364		g_free(sc);
365		g_detach(cp);
366		g_destroy_consumer(cp);
367		g_destroy_geom(gp);
368		return(NULL);
369	}
370	do {
371		sectorsize = cp->provider->sectorsize;
372		g_topology_unlock();
373		buf = g_read_data(cp, 0, sectorsize, &error);
374		g_topology_lock();
375		if (buf == NULL || error != 0)
376			break;
377		if (memcmp(buf, FOX_MAGIC, strlen(FOX_MAGIC)))
378			break;
379
380		/*
381		 * First we need to see if this a new path for an existing fox.
382		 */
383		LIST_FOREACH(gp2, &mp->geom, geom) {
384			sc2 = gp2->softc;
385			if (sc2 == NULL)
386				continue;
387			if (memcmp(buf + 16, sc2->magic, sizeof sc2->magic))
388				continue;
389			break;
390		}
391		if (gp2 != NULL) {
392			/*
393			 * It was.  Create a new consumer for that fox,
394			 * attach it, and if the fox is open, open this
395			 * path with an exclusive count of one.
396			 */
397			printf("Adding path (%s) to fox (%s)\n",
398			    pp->name, gp2->name);
399			cp2 = g_new_consumer(gp2);
400			g_attach(cp2, pp);
401			pp2 = LIST_FIRST(&gp2->provider);
402			if (pp2->acr > 0 || pp2->acw > 0 || pp2->ace > 0) {
403				error = g_access(cp2, 0, 0, 1);
404				if (error) {
405					/*
406					 * This is bad, or more likely,
407					 * the user is doing something stupid
408					 */
409					printf(
410	"WARNING: New path (%s) to fox(%s) not added: %s\n%s",
411					    cp2->provider->name, gp2->name,
412	"Could not get exclusive bit.",
413	"WARNING: This indicates a risk of data inconsistency."
414					);
415					g_detach(cp2);
416					g_destroy_consumer(cp2);
417				}
418			}
419			break;
420		}
421		printf("Creating new fox (%s)\n", pp->name);
422		sc->path = cp;
423		memcpy(sc->magic, buf + 16, sizeof sc->magic);
424		pp2 = g_new_providerf(gp, "%s", gp->name);
425		pp2->mediasize = sc->mediasize = pp->mediasize - pp->sectorsize;
426		pp2->sectorsize = sc->sectorsize = pp->sectorsize;
427printf("fox %s lock %p\n", gp->name, &sc->lock);
428
429		mtx_init(&sc->lock, "fox queue", NULL, MTX_DEF);
430		TAILQ_INIT(&sc->queue);
431		g_error_provider(pp2, 0);
432	} while (0);
433	if (buf != NULL)
434		g_free(buf);
435	g_access(cp, -1, 0, 0);
436
437	if (!LIST_EMPTY(&gp->provider))
438		return (gp);
439
440	g_free(gp->softc);
441	g_detach(cp);
442	g_destroy_consumer(cp);
443	g_destroy_geom(gp);
444	return (NULL);
445}
446
447static int
448g_fox_destroy_geom(struct gctl_req *req, struct g_class *mp, struct g_geom *gp)
449{
450	struct g_fox_softc *sc;
451
452	g_topology_assert();
453	sc = gp->softc;
454	mtx_destroy(&sc->lock);
455	g_free(gp->softc);
456	gp->softc = NULL;
457	g_wither_geom(gp, ENXIO);
458	return (0);
459}
460
461static struct g_class g_fox_class	= {
462	.name = FOX_CLASS_NAME,
463	.taste = g_fox_taste,
464	.destroy_geom = g_fox_destroy_geom,
465	.start = g_fox_start,
466	.spoiled = g_fox_orphan,
467	.orphan = g_fox_orphan,
468	.access= g_fox_access,
469};
470
471DECLARE_GEOM_CLASS(g_fox_class, g_fox);
472