g_mirror.c revision 280756
1/*-
2 * Copyright (c) 2004-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/geom/mirror/g_mirror.c 280756 2015-03-27 11:14:12Z mav $");
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/kernel.h>
33#include <sys/module.h>
34#include <sys/limits.h>
35#include <sys/lock.h>
36#include <sys/mutex.h>
37#include <sys/bio.h>
38#include <sys/sbuf.h>
39#include <sys/sysctl.h>
40#include <sys/malloc.h>
41#include <sys/eventhandler.h>
42#include <vm/uma.h>
43#include <geom/geom.h>
44#include <sys/proc.h>
45#include <sys/kthread.h>
46#include <sys/sched.h>
47#include <geom/mirror/g_mirror.h>
48
49FEATURE(geom_mirror, "GEOM mirroring support");
50
51static MALLOC_DEFINE(M_MIRROR, "mirror_data", "GEOM_MIRROR Data");
52
53SYSCTL_DECL(_kern_geom);
54static SYSCTL_NODE(_kern_geom, OID_AUTO, mirror, CTLFLAG_RW, 0,
55    "GEOM_MIRROR stuff");
56u_int g_mirror_debug = 0;
57SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, debug, CTLFLAG_RWTUN, &g_mirror_debug, 0,
58    "Debug level");
59static u_int g_mirror_timeout = 4;
60SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, timeout, CTLFLAG_RWTUN, &g_mirror_timeout,
61    0, "Time to wait on all mirror components");
62static u_int g_mirror_idletime = 5;
63SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, idletime, CTLFLAG_RWTUN,
64    &g_mirror_idletime, 0, "Mark components as clean when idling");
65static u_int g_mirror_disconnect_on_failure = 1;
66SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, disconnect_on_failure, CTLFLAG_RWTUN,
67    &g_mirror_disconnect_on_failure, 0, "Disconnect component on I/O failure.");
68static u_int g_mirror_syncreqs = 2;
69SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, sync_requests, CTLFLAG_RDTUN,
70    &g_mirror_syncreqs, 0, "Parallel synchronization I/O requests.");
71
72#define	MSLEEP(ident, mtx, priority, wmesg, timeout)	do {		\
73	G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, (ident));	\
74	msleep((ident), (mtx), (priority), (wmesg), (timeout));		\
75	G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, (ident));	\
76} while (0)
77
78static eventhandler_tag g_mirror_post_sync = NULL;
79static int g_mirror_shutdown = 0;
80
81static int g_mirror_destroy_geom(struct gctl_req *req, struct g_class *mp,
82    struct g_geom *gp);
83static g_taste_t g_mirror_taste;
84static g_resize_t g_mirror_resize;
85static void g_mirror_init(struct g_class *mp);
86static void g_mirror_fini(struct g_class *mp);
87
88struct g_class g_mirror_class = {
89	.name = G_MIRROR_CLASS_NAME,
90	.version = G_VERSION,
91	.ctlreq = g_mirror_config,
92	.taste = g_mirror_taste,
93	.destroy_geom = g_mirror_destroy_geom,
94	.init = g_mirror_init,
95	.fini = g_mirror_fini,
96	.resize = g_mirror_resize
97};
98
99
100static void g_mirror_destroy_provider(struct g_mirror_softc *sc);
101static int g_mirror_update_disk(struct g_mirror_disk *disk, u_int state);
102static void g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force);
103static void g_mirror_dumpconf(struct sbuf *sb, const char *indent,
104    struct g_geom *gp, struct g_consumer *cp, struct g_provider *pp);
105static void g_mirror_sync_stop(struct g_mirror_disk *disk, int type);
106static void g_mirror_register_request(struct bio *bp);
107static void g_mirror_sync_release(struct g_mirror_softc *sc);
108
109
110static const char *
111g_mirror_disk_state2str(int state)
112{
113
114	switch (state) {
115	case G_MIRROR_DISK_STATE_NONE:
116		return ("NONE");
117	case G_MIRROR_DISK_STATE_NEW:
118		return ("NEW");
119	case G_MIRROR_DISK_STATE_ACTIVE:
120		return ("ACTIVE");
121	case G_MIRROR_DISK_STATE_STALE:
122		return ("STALE");
123	case G_MIRROR_DISK_STATE_SYNCHRONIZING:
124		return ("SYNCHRONIZING");
125	case G_MIRROR_DISK_STATE_DISCONNECTED:
126		return ("DISCONNECTED");
127	case G_MIRROR_DISK_STATE_DESTROY:
128		return ("DESTROY");
129	default:
130		return ("INVALID");
131	}
132}
133
134static const char *
135g_mirror_device_state2str(int state)
136{
137
138	switch (state) {
139	case G_MIRROR_DEVICE_STATE_STARTING:
140		return ("STARTING");
141	case G_MIRROR_DEVICE_STATE_RUNNING:
142		return ("RUNNING");
143	default:
144		return ("INVALID");
145	}
146}
147
148static const char *
149g_mirror_get_diskname(struct g_mirror_disk *disk)
150{
151
152	if (disk->d_consumer == NULL || disk->d_consumer->provider == NULL)
153		return ("[unknown]");
154	return (disk->d_name);
155}
156
157/*
158 * --- Events handling functions ---
159 * Events in geom_mirror are used to maintain disks and device status
160 * from one thread to simplify locking.
161 */
162static void
163g_mirror_event_free(struct g_mirror_event *ep)
164{
165
166	free(ep, M_MIRROR);
167}
168
169int
170g_mirror_event_send(void *arg, int state, int flags)
171{
172	struct g_mirror_softc *sc;
173	struct g_mirror_disk *disk;
174	struct g_mirror_event *ep;
175	int error;
176
177	ep = malloc(sizeof(*ep), M_MIRROR, M_WAITOK);
178	G_MIRROR_DEBUG(4, "%s: Sending event %p.", __func__, ep);
179	if ((flags & G_MIRROR_EVENT_DEVICE) != 0) {
180		disk = NULL;
181		sc = arg;
182	} else {
183		disk = arg;
184		sc = disk->d_softc;
185	}
186	ep->e_disk = disk;
187	ep->e_state = state;
188	ep->e_flags = flags;
189	ep->e_error = 0;
190	mtx_lock(&sc->sc_events_mtx);
191	TAILQ_INSERT_TAIL(&sc->sc_events, ep, e_next);
192	mtx_unlock(&sc->sc_events_mtx);
193	G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
194	mtx_lock(&sc->sc_queue_mtx);
195	wakeup(sc);
196	mtx_unlock(&sc->sc_queue_mtx);
197	if ((flags & G_MIRROR_EVENT_DONTWAIT) != 0)
198		return (0);
199	sx_assert(&sc->sc_lock, SX_XLOCKED);
200	G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, ep);
201	sx_xunlock(&sc->sc_lock);
202	while ((ep->e_flags & G_MIRROR_EVENT_DONE) == 0) {
203		mtx_lock(&sc->sc_events_mtx);
204		MSLEEP(ep, &sc->sc_events_mtx, PRIBIO | PDROP, "m:event",
205		    hz * 5);
206	}
207	error = ep->e_error;
208	g_mirror_event_free(ep);
209	sx_xlock(&sc->sc_lock);
210	return (error);
211}
212
213static struct g_mirror_event *
214g_mirror_event_get(struct g_mirror_softc *sc)
215{
216	struct g_mirror_event *ep;
217
218	mtx_lock(&sc->sc_events_mtx);
219	ep = TAILQ_FIRST(&sc->sc_events);
220	mtx_unlock(&sc->sc_events_mtx);
221	return (ep);
222}
223
224static void
225g_mirror_event_remove(struct g_mirror_softc *sc, struct g_mirror_event *ep)
226{
227
228	mtx_lock(&sc->sc_events_mtx);
229	TAILQ_REMOVE(&sc->sc_events, ep, e_next);
230	mtx_unlock(&sc->sc_events_mtx);
231}
232
233static void
234g_mirror_event_cancel(struct g_mirror_disk *disk)
235{
236	struct g_mirror_softc *sc;
237	struct g_mirror_event *ep, *tmpep;
238
239	sc = disk->d_softc;
240	sx_assert(&sc->sc_lock, SX_XLOCKED);
241
242	mtx_lock(&sc->sc_events_mtx);
243	TAILQ_FOREACH_SAFE(ep, &sc->sc_events, e_next, tmpep) {
244		if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0)
245			continue;
246		if (ep->e_disk != disk)
247			continue;
248		TAILQ_REMOVE(&sc->sc_events, ep, e_next);
249		if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
250			g_mirror_event_free(ep);
251		else {
252			ep->e_error = ECANCELED;
253			wakeup(ep);
254		}
255	}
256	mtx_unlock(&sc->sc_events_mtx);
257}
258
259/*
260 * Return the number of disks in given state.
261 * If state is equal to -1, count all connected disks.
262 */
263u_int
264g_mirror_ndisks(struct g_mirror_softc *sc, int state)
265{
266	struct g_mirror_disk *disk;
267	u_int n = 0;
268
269	sx_assert(&sc->sc_lock, SX_LOCKED);
270
271	LIST_FOREACH(disk, &sc->sc_disks, d_next) {
272		if (state == -1 || disk->d_state == state)
273			n++;
274	}
275	return (n);
276}
277
278/*
279 * Find a disk in mirror by its disk ID.
280 */
281static struct g_mirror_disk *
282g_mirror_id2disk(struct g_mirror_softc *sc, uint32_t id)
283{
284	struct g_mirror_disk *disk;
285
286	sx_assert(&sc->sc_lock, SX_XLOCKED);
287
288	LIST_FOREACH(disk, &sc->sc_disks, d_next) {
289		if (disk->d_id == id)
290			return (disk);
291	}
292	return (NULL);
293}
294
295static u_int
296g_mirror_nrequests(struct g_mirror_softc *sc, struct g_consumer *cp)
297{
298	struct bio *bp;
299	u_int nreqs = 0;
300
301	mtx_lock(&sc->sc_queue_mtx);
302	TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
303		if (bp->bio_from == cp)
304			nreqs++;
305	}
306	mtx_unlock(&sc->sc_queue_mtx);
307	return (nreqs);
308}
309
310static int
311g_mirror_is_busy(struct g_mirror_softc *sc, struct g_consumer *cp)
312{
313
314	if (cp->index > 0) {
315		G_MIRROR_DEBUG(2,
316		    "I/O requests for %s exist, can't destroy it now.",
317		    cp->provider->name);
318		return (1);
319	}
320	if (g_mirror_nrequests(sc, cp) > 0) {
321		G_MIRROR_DEBUG(2,
322		    "I/O requests for %s in queue, can't destroy it now.",
323		    cp->provider->name);
324		return (1);
325	}
326	return (0);
327}
328
329static void
330g_mirror_destroy_consumer(void *arg, int flags __unused)
331{
332	struct g_consumer *cp;
333
334	g_topology_assert();
335
336	cp = arg;
337	G_MIRROR_DEBUG(1, "Consumer %s destroyed.", cp->provider->name);
338	g_detach(cp);
339	g_destroy_consumer(cp);
340}
341
342static void
343g_mirror_kill_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
344{
345	struct g_provider *pp;
346	int retaste_wait;
347
348	g_topology_assert();
349
350	cp->private = NULL;
351	if (g_mirror_is_busy(sc, cp))
352		return;
353	pp = cp->provider;
354	retaste_wait = 0;
355	if (cp->acw == 1) {
356		if ((pp->geom->flags & G_GEOM_WITHER) == 0)
357			retaste_wait = 1;
358	}
359	G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", pp->name, -cp->acr,
360	    -cp->acw, -cp->ace, 0);
361	if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
362		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
363	if (retaste_wait) {
364		/*
365		 * After retaste event was send (inside g_access()), we can send
366		 * event to detach and destroy consumer.
367		 * A class, which has consumer to the given provider connected
368		 * will not receive retaste event for the provider.
369		 * This is the way how I ignore retaste events when I close
370		 * consumers opened for write: I detach and destroy consumer
371		 * after retaste event is sent.
372		 */
373		g_post_event(g_mirror_destroy_consumer, cp, M_WAITOK, NULL);
374		return;
375	}
376	G_MIRROR_DEBUG(1, "Consumer %s destroyed.", pp->name);
377	g_detach(cp);
378	g_destroy_consumer(cp);
379}
380
381static int
382g_mirror_connect_disk(struct g_mirror_disk *disk, struct g_provider *pp)
383{
384	struct g_consumer *cp;
385	int error;
386
387	g_topology_assert_not();
388	KASSERT(disk->d_consumer == NULL,
389	    ("Disk already connected (device %s).", disk->d_softc->sc_name));
390
391	g_topology_lock();
392	cp = g_new_consumer(disk->d_softc->sc_geom);
393	cp->flags |= G_CF_DIRECT_RECEIVE;
394	error = g_attach(cp, pp);
395	if (error != 0) {
396		g_destroy_consumer(cp);
397		g_topology_unlock();
398		return (error);
399	}
400	error = g_access(cp, 1, 1, 1);
401	if (error != 0) {
402		g_detach(cp);
403		g_destroy_consumer(cp);
404		g_topology_unlock();
405		G_MIRROR_DEBUG(0, "Cannot open consumer %s (error=%d).",
406		    pp->name, error);
407		return (error);
408	}
409	g_topology_unlock();
410	disk->d_consumer = cp;
411	disk->d_consumer->private = disk;
412	disk->d_consumer->index = 0;
413
414	G_MIRROR_DEBUG(2, "Disk %s connected.", g_mirror_get_diskname(disk));
415	return (0);
416}
417
418static void
419g_mirror_disconnect_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
420{
421
422	g_topology_assert();
423
424	if (cp == NULL)
425		return;
426	if (cp->provider != NULL)
427		g_mirror_kill_consumer(sc, cp);
428	else
429		g_destroy_consumer(cp);
430}
431
432/*
433 * Initialize disk. This means allocate memory, create consumer, attach it
434 * to the provider and open access (r1w1e1) to it.
435 */
436static struct g_mirror_disk *
437g_mirror_init_disk(struct g_mirror_softc *sc, struct g_provider *pp,
438    struct g_mirror_metadata *md, int *errorp)
439{
440	struct g_mirror_disk *disk;
441	int i, error;
442
443	disk = malloc(sizeof(*disk), M_MIRROR, M_NOWAIT | M_ZERO);
444	if (disk == NULL) {
445		error = ENOMEM;
446		goto fail;
447	}
448	disk->d_softc = sc;
449	error = g_mirror_connect_disk(disk, pp);
450	if (error != 0)
451		goto fail;
452	disk->d_id = md->md_did;
453	disk->d_state = G_MIRROR_DISK_STATE_NONE;
454	disk->d_priority = md->md_priority;
455	disk->d_flags = md->md_dflags;
456	error = g_getattr("GEOM::candelete", disk->d_consumer, &i);
457	if (error == 0 && i != 0)
458		disk->d_flags |= G_MIRROR_DISK_FLAG_CANDELETE;
459	if (md->md_provider[0] != '\0')
460		disk->d_flags |= G_MIRROR_DISK_FLAG_HARDCODED;
461	disk->d_sync.ds_consumer = NULL;
462	disk->d_sync.ds_offset = md->md_sync_offset;
463	disk->d_sync.ds_offset_done = md->md_sync_offset;
464	disk->d_genid = md->md_genid;
465	disk->d_sync.ds_syncid = md->md_syncid;
466	if (errorp != NULL)
467		*errorp = 0;
468	return (disk);
469fail:
470	if (errorp != NULL)
471		*errorp = error;
472	if (disk != NULL)
473		free(disk, M_MIRROR);
474	return (NULL);
475}
476
477static void
478g_mirror_destroy_disk(struct g_mirror_disk *disk)
479{
480	struct g_mirror_softc *sc;
481
482	g_topology_assert_not();
483	sc = disk->d_softc;
484	sx_assert(&sc->sc_lock, SX_XLOCKED);
485
486	LIST_REMOVE(disk, d_next);
487	g_mirror_event_cancel(disk);
488	if (sc->sc_hint == disk)
489		sc->sc_hint = NULL;
490	switch (disk->d_state) {
491	case G_MIRROR_DISK_STATE_SYNCHRONIZING:
492		g_mirror_sync_stop(disk, 1);
493		/* FALLTHROUGH */
494	case G_MIRROR_DISK_STATE_NEW:
495	case G_MIRROR_DISK_STATE_STALE:
496	case G_MIRROR_DISK_STATE_ACTIVE:
497		g_topology_lock();
498		g_mirror_disconnect_consumer(sc, disk->d_consumer);
499		g_topology_unlock();
500		free(disk, M_MIRROR);
501		break;
502	default:
503		KASSERT(0 == 1, ("Wrong disk state (%s, %s).",
504		    g_mirror_get_diskname(disk),
505		    g_mirror_disk_state2str(disk->d_state)));
506	}
507}
508
509static void
510g_mirror_destroy_device(struct g_mirror_softc *sc)
511{
512	struct g_mirror_disk *disk;
513	struct g_mirror_event *ep;
514	struct g_geom *gp;
515	struct g_consumer *cp, *tmpcp;
516
517	g_topology_assert_not();
518	sx_assert(&sc->sc_lock, SX_XLOCKED);
519
520	gp = sc->sc_geom;
521	if (sc->sc_provider != NULL)
522		g_mirror_destroy_provider(sc);
523	for (disk = LIST_FIRST(&sc->sc_disks); disk != NULL;
524	    disk = LIST_FIRST(&sc->sc_disks)) {
525		disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
526		g_mirror_update_metadata(disk);
527		g_mirror_destroy_disk(disk);
528	}
529	while ((ep = g_mirror_event_get(sc)) != NULL) {
530		g_mirror_event_remove(sc, ep);
531		if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
532			g_mirror_event_free(ep);
533		else {
534			ep->e_error = ECANCELED;
535			ep->e_flags |= G_MIRROR_EVENT_DONE;
536			G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, ep);
537			mtx_lock(&sc->sc_events_mtx);
538			wakeup(ep);
539			mtx_unlock(&sc->sc_events_mtx);
540		}
541	}
542	callout_drain(&sc->sc_callout);
543
544	g_topology_lock();
545	LIST_FOREACH_SAFE(cp, &sc->sc_sync.ds_geom->consumer, consumer, tmpcp) {
546		g_mirror_disconnect_consumer(sc, cp);
547	}
548	g_wither_geom(sc->sc_sync.ds_geom, ENXIO);
549	G_MIRROR_DEBUG(0, "Device %s destroyed.", gp->name);
550	g_wither_geom(gp, ENXIO);
551	g_topology_unlock();
552	mtx_destroy(&sc->sc_queue_mtx);
553	mtx_destroy(&sc->sc_events_mtx);
554	mtx_destroy(&sc->sc_done_mtx);
555	sx_xunlock(&sc->sc_lock);
556	sx_destroy(&sc->sc_lock);
557}
558
559static void
560g_mirror_orphan(struct g_consumer *cp)
561{
562	struct g_mirror_disk *disk;
563
564	g_topology_assert();
565
566	disk = cp->private;
567	if (disk == NULL)
568		return;
569	disk->d_softc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
570	g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED,
571	    G_MIRROR_EVENT_DONTWAIT);
572}
573
574/*
575 * Function should return the next active disk on the list.
576 * It is possible that it will be the same disk as given.
577 * If there are no active disks on list, NULL is returned.
578 */
579static __inline struct g_mirror_disk *
580g_mirror_find_next(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
581{
582	struct g_mirror_disk *dp;
583
584	for (dp = LIST_NEXT(disk, d_next); dp != disk;
585	    dp = LIST_NEXT(dp, d_next)) {
586		if (dp == NULL)
587			dp = LIST_FIRST(&sc->sc_disks);
588		if (dp->d_state == G_MIRROR_DISK_STATE_ACTIVE)
589			break;
590	}
591	if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
592		return (NULL);
593	return (dp);
594}
595
596static struct g_mirror_disk *
597g_mirror_get_disk(struct g_mirror_softc *sc)
598{
599	struct g_mirror_disk *disk;
600
601	if (sc->sc_hint == NULL) {
602		sc->sc_hint = LIST_FIRST(&sc->sc_disks);
603		if (sc->sc_hint == NULL)
604			return (NULL);
605	}
606	disk = sc->sc_hint;
607	if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE) {
608		disk = g_mirror_find_next(sc, disk);
609		if (disk == NULL)
610			return (NULL);
611	}
612	sc->sc_hint = g_mirror_find_next(sc, disk);
613	return (disk);
614}
615
616static int
617g_mirror_write_metadata(struct g_mirror_disk *disk,
618    struct g_mirror_metadata *md)
619{
620	struct g_mirror_softc *sc;
621	struct g_consumer *cp;
622	off_t offset, length;
623	u_char *sector;
624	int error = 0;
625
626	g_topology_assert_not();
627	sc = disk->d_softc;
628	sx_assert(&sc->sc_lock, SX_LOCKED);
629
630	cp = disk->d_consumer;
631	KASSERT(cp != NULL, ("NULL consumer (%s).", sc->sc_name));
632	KASSERT(cp->provider != NULL, ("NULL provider (%s).", sc->sc_name));
633	KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
634	    ("Consumer %s closed? (r%dw%de%d).", cp->provider->name, cp->acr,
635	    cp->acw, cp->ace));
636	length = cp->provider->sectorsize;
637	offset = cp->provider->mediasize - length;
638	sector = malloc((size_t)length, M_MIRROR, M_WAITOK | M_ZERO);
639	if (md != NULL &&
640	    (sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0) {
641		/*
642		 * Handle the case, when the size of parent provider reduced.
643		 */
644		if (offset < md->md_mediasize)
645			error = ENOSPC;
646		else
647			mirror_metadata_encode(md, sector);
648	}
649	if (error == 0)
650		error = g_write_data(cp, offset, sector, length);
651	free(sector, M_MIRROR);
652	if (error != 0) {
653		if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
654			disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
655			G_MIRROR_DEBUG(0, "Cannot write metadata on %s "
656			    "(device=%s, error=%d).",
657			    g_mirror_get_diskname(disk), sc->sc_name, error);
658		} else {
659			G_MIRROR_DEBUG(1, "Cannot write metadata on %s "
660			    "(device=%s, error=%d).",
661			    g_mirror_get_diskname(disk), sc->sc_name, error);
662		}
663		if (g_mirror_disconnect_on_failure &&
664		    g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1) {
665			sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
666			g_mirror_event_send(disk,
667			    G_MIRROR_DISK_STATE_DISCONNECTED,
668			    G_MIRROR_EVENT_DONTWAIT);
669		}
670	}
671	return (error);
672}
673
674static int
675g_mirror_clear_metadata(struct g_mirror_disk *disk)
676{
677	int error;
678
679	g_topology_assert_not();
680	sx_assert(&disk->d_softc->sc_lock, SX_LOCKED);
681
682	error = g_mirror_write_metadata(disk, NULL);
683	if (error == 0) {
684		G_MIRROR_DEBUG(2, "Metadata on %s cleared.",
685		    g_mirror_get_diskname(disk));
686	} else {
687		G_MIRROR_DEBUG(0,
688		    "Cannot clear metadata on disk %s (error=%d).",
689		    g_mirror_get_diskname(disk), error);
690	}
691	return (error);
692}
693
694void
695g_mirror_fill_metadata(struct g_mirror_softc *sc, struct g_mirror_disk *disk,
696    struct g_mirror_metadata *md)
697{
698
699	strlcpy(md->md_magic, G_MIRROR_MAGIC, sizeof(md->md_magic));
700	md->md_version = G_MIRROR_VERSION;
701	strlcpy(md->md_name, sc->sc_name, sizeof(md->md_name));
702	md->md_mid = sc->sc_id;
703	md->md_all = sc->sc_ndisks;
704	md->md_slice = sc->sc_slice;
705	md->md_balance = sc->sc_balance;
706	md->md_genid = sc->sc_genid;
707	md->md_mediasize = sc->sc_mediasize;
708	md->md_sectorsize = sc->sc_sectorsize;
709	md->md_mflags = (sc->sc_flags & G_MIRROR_DEVICE_FLAG_MASK);
710	bzero(md->md_provider, sizeof(md->md_provider));
711	if (disk == NULL) {
712		md->md_did = arc4random();
713		md->md_priority = 0;
714		md->md_syncid = 0;
715		md->md_dflags = 0;
716		md->md_sync_offset = 0;
717		md->md_provsize = 0;
718	} else {
719		md->md_did = disk->d_id;
720		md->md_priority = disk->d_priority;
721		md->md_syncid = disk->d_sync.ds_syncid;
722		md->md_dflags = (disk->d_flags & G_MIRROR_DISK_FLAG_MASK);
723		if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
724			md->md_sync_offset = disk->d_sync.ds_offset_done;
725		else
726			md->md_sync_offset = 0;
727		if ((disk->d_flags & G_MIRROR_DISK_FLAG_HARDCODED) != 0) {
728			strlcpy(md->md_provider,
729			    disk->d_consumer->provider->name,
730			    sizeof(md->md_provider));
731		}
732		md->md_provsize = disk->d_consumer->provider->mediasize;
733	}
734}
735
736void
737g_mirror_update_metadata(struct g_mirror_disk *disk)
738{
739	struct g_mirror_softc *sc;
740	struct g_mirror_metadata md;
741	int error;
742
743	g_topology_assert_not();
744	sc = disk->d_softc;
745	sx_assert(&sc->sc_lock, SX_LOCKED);
746
747	if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0)
748		g_mirror_fill_metadata(sc, disk, &md);
749	error = g_mirror_write_metadata(disk, &md);
750	if (error == 0) {
751		G_MIRROR_DEBUG(2, "Metadata on %s updated.",
752		    g_mirror_get_diskname(disk));
753	} else {
754		G_MIRROR_DEBUG(0,
755		    "Cannot update metadata on disk %s (error=%d).",
756		    g_mirror_get_diskname(disk), error);
757	}
758}
759
760static void
761g_mirror_bump_syncid(struct g_mirror_softc *sc)
762{
763	struct g_mirror_disk *disk;
764
765	g_topology_assert_not();
766	sx_assert(&sc->sc_lock, SX_XLOCKED);
767	KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
768	    ("%s called with no active disks (device=%s).", __func__,
769	    sc->sc_name));
770
771	sc->sc_syncid++;
772	G_MIRROR_DEBUG(1, "Device %s: syncid bumped to %u.", sc->sc_name,
773	    sc->sc_syncid);
774	LIST_FOREACH(disk, &sc->sc_disks, d_next) {
775		if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
776		    disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
777			disk->d_sync.ds_syncid = sc->sc_syncid;
778			g_mirror_update_metadata(disk);
779		}
780	}
781}
782
783static void
784g_mirror_bump_genid(struct g_mirror_softc *sc)
785{
786	struct g_mirror_disk *disk;
787
788	g_topology_assert_not();
789	sx_assert(&sc->sc_lock, SX_XLOCKED);
790	KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
791	    ("%s called with no active disks (device=%s).", __func__,
792	    sc->sc_name));
793
794	sc->sc_genid++;
795	G_MIRROR_DEBUG(1, "Device %s: genid bumped to %u.", sc->sc_name,
796	    sc->sc_genid);
797	LIST_FOREACH(disk, &sc->sc_disks, d_next) {
798		if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
799		    disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
800			disk->d_genid = sc->sc_genid;
801			g_mirror_update_metadata(disk);
802		}
803	}
804}
805
806static int
807g_mirror_idle(struct g_mirror_softc *sc, int acw)
808{
809	struct g_mirror_disk *disk;
810	int timeout;
811
812	g_topology_assert_not();
813	sx_assert(&sc->sc_lock, SX_XLOCKED);
814
815	if (sc->sc_provider == NULL)
816		return (0);
817	if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
818		return (0);
819	if (sc->sc_idle)
820		return (0);
821	if (sc->sc_writes > 0)
822		return (0);
823	if (acw > 0 || (acw == -1 && sc->sc_provider->acw > 0)) {
824		timeout = g_mirror_idletime - (time_uptime - sc->sc_last_write);
825		if (!g_mirror_shutdown && timeout > 0)
826			return (timeout);
827	}
828	sc->sc_idle = 1;
829	LIST_FOREACH(disk, &sc->sc_disks, d_next) {
830		if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
831			continue;
832		G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as clean.",
833		    g_mirror_get_diskname(disk), sc->sc_name);
834		disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
835		g_mirror_update_metadata(disk);
836	}
837	return (0);
838}
839
840static void
841g_mirror_unidle(struct g_mirror_softc *sc)
842{
843	struct g_mirror_disk *disk;
844
845	g_topology_assert_not();
846	sx_assert(&sc->sc_lock, SX_XLOCKED);
847
848	if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
849		return;
850	sc->sc_idle = 0;
851	sc->sc_last_write = time_uptime;
852	LIST_FOREACH(disk, &sc->sc_disks, d_next) {
853		if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
854			continue;
855		G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as dirty.",
856		    g_mirror_get_diskname(disk), sc->sc_name);
857		disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
858		g_mirror_update_metadata(disk);
859	}
860}
861
862static void
863g_mirror_flush_done(struct bio *bp)
864{
865	struct g_mirror_softc *sc;
866	struct bio *pbp;
867
868	pbp = bp->bio_parent;
869	sc = pbp->bio_to->geom->softc;
870	mtx_lock(&sc->sc_done_mtx);
871	if (pbp->bio_error == 0)
872		pbp->bio_error = bp->bio_error;
873	pbp->bio_completed += bp->bio_completed;
874	pbp->bio_inbed++;
875	if (pbp->bio_children == pbp->bio_inbed) {
876		mtx_unlock(&sc->sc_done_mtx);
877		g_io_deliver(pbp, pbp->bio_error);
878	} else
879		mtx_unlock(&sc->sc_done_mtx);
880	g_destroy_bio(bp);
881}
882
883static void
884g_mirror_done(struct bio *bp)
885{
886	struct g_mirror_softc *sc;
887
888	sc = bp->bio_from->geom->softc;
889	bp->bio_cflags = G_MIRROR_BIO_FLAG_REGULAR;
890	mtx_lock(&sc->sc_queue_mtx);
891	bioq_disksort(&sc->sc_queue, bp);
892	mtx_unlock(&sc->sc_queue_mtx);
893	wakeup(sc);
894}
895
896static void
897g_mirror_regular_request(struct bio *bp)
898{
899	struct g_mirror_softc *sc;
900	struct g_mirror_disk *disk;
901	struct bio *pbp;
902
903	g_topology_assert_not();
904
905	pbp = bp->bio_parent;
906	sc = pbp->bio_to->geom->softc;
907	bp->bio_from->index--;
908	if (bp->bio_cmd == BIO_WRITE)
909		sc->sc_writes--;
910	disk = bp->bio_from->private;
911	if (disk == NULL) {
912		g_topology_lock();
913		g_mirror_kill_consumer(sc, bp->bio_from);
914		g_topology_unlock();
915	}
916
917	pbp->bio_inbed++;
918	KASSERT(pbp->bio_inbed <= pbp->bio_children,
919	    ("bio_inbed (%u) is bigger than bio_children (%u).", pbp->bio_inbed,
920	    pbp->bio_children));
921	if (bp->bio_error == 0 && pbp->bio_error == 0) {
922		G_MIRROR_LOGREQ(3, bp, "Request delivered.");
923		g_destroy_bio(bp);
924		if (pbp->bio_children == pbp->bio_inbed) {
925			G_MIRROR_LOGREQ(3, pbp, "Request delivered.");
926			pbp->bio_completed = pbp->bio_length;
927			if (pbp->bio_cmd == BIO_WRITE ||
928			    pbp->bio_cmd == BIO_DELETE) {
929				bioq_remove(&sc->sc_inflight, pbp);
930				/* Release delayed sync requests if possible. */
931				g_mirror_sync_release(sc);
932			}
933			g_io_deliver(pbp, pbp->bio_error);
934		}
935		return;
936	} else if (bp->bio_error != 0) {
937		if (pbp->bio_error == 0)
938			pbp->bio_error = bp->bio_error;
939		if (disk != NULL) {
940			if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
941				disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
942				G_MIRROR_LOGREQ(0, bp,
943				    "Request failed (error=%d).",
944				    bp->bio_error);
945			} else {
946				G_MIRROR_LOGREQ(1, bp,
947				    "Request failed (error=%d).",
948				    bp->bio_error);
949			}
950			if (g_mirror_disconnect_on_failure &&
951			    g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1)
952			{
953				sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
954				g_mirror_event_send(disk,
955				    G_MIRROR_DISK_STATE_DISCONNECTED,
956				    G_MIRROR_EVENT_DONTWAIT);
957			}
958		}
959		switch (pbp->bio_cmd) {
960		case BIO_DELETE:
961		case BIO_WRITE:
962			pbp->bio_inbed--;
963			pbp->bio_children--;
964			break;
965		}
966	}
967	g_destroy_bio(bp);
968
969	switch (pbp->bio_cmd) {
970	case BIO_READ:
971		if (pbp->bio_inbed < pbp->bio_children)
972			break;
973		if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 1)
974			g_io_deliver(pbp, pbp->bio_error);
975		else {
976			pbp->bio_error = 0;
977			mtx_lock(&sc->sc_queue_mtx);
978			bioq_disksort(&sc->sc_queue, pbp);
979			mtx_unlock(&sc->sc_queue_mtx);
980			G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
981			wakeup(sc);
982		}
983		break;
984	case BIO_DELETE:
985	case BIO_WRITE:
986		if (pbp->bio_children == 0) {
987			/*
988			 * All requests failed.
989			 */
990		} else if (pbp->bio_inbed < pbp->bio_children) {
991			/* Do nothing. */
992			break;
993		} else if (pbp->bio_children == pbp->bio_inbed) {
994			/* Some requests succeeded. */
995			pbp->bio_error = 0;
996			pbp->bio_completed = pbp->bio_length;
997		}
998		bioq_remove(&sc->sc_inflight, pbp);
999		/* Release delayed sync requests if possible. */
1000		g_mirror_sync_release(sc);
1001		g_io_deliver(pbp, pbp->bio_error);
1002		break;
1003	default:
1004		KASSERT(1 == 0, ("Invalid request: %u.", pbp->bio_cmd));
1005		break;
1006	}
1007}
1008
1009static void
1010g_mirror_sync_done(struct bio *bp)
1011{
1012	struct g_mirror_softc *sc;
1013
1014	G_MIRROR_LOGREQ(3, bp, "Synchronization request delivered.");
1015	sc = bp->bio_from->geom->softc;
1016	bp->bio_cflags = G_MIRROR_BIO_FLAG_SYNC;
1017	mtx_lock(&sc->sc_queue_mtx);
1018	bioq_disksort(&sc->sc_queue, bp);
1019	mtx_unlock(&sc->sc_queue_mtx);
1020	wakeup(sc);
1021}
1022
1023static void
1024g_mirror_candelete(struct bio *bp)
1025{
1026	struct g_mirror_softc *sc;
1027	struct g_mirror_disk *disk;
1028	int *val;
1029
1030	sc = bp->bio_to->geom->softc;
1031	LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1032		if (disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE)
1033			break;
1034	}
1035	val = (int *)bp->bio_data;
1036	*val = (disk != NULL);
1037	g_io_deliver(bp, 0);
1038}
1039
1040static void
1041g_mirror_kernel_dump(struct bio *bp)
1042{
1043	struct g_mirror_softc *sc;
1044	struct g_mirror_disk *disk;
1045	struct bio *cbp;
1046	struct g_kerneldump *gkd;
1047
1048	/*
1049	 * We configure dumping to the first component, because this component
1050	 * will be used for reading with 'prefer' balance algorithm.
1051	 * If the component with the higest priority is currently disconnected
1052	 * we will not be able to read the dump after the reboot if it will be
1053	 * connected and synchronized later. Can we do something better?
1054	 */
1055	sc = bp->bio_to->geom->softc;
1056	disk = LIST_FIRST(&sc->sc_disks);
1057
1058	gkd = (struct g_kerneldump *)bp->bio_data;
1059	if (gkd->length > bp->bio_to->mediasize)
1060		gkd->length = bp->bio_to->mediasize;
1061	cbp = g_clone_bio(bp);
1062	if (cbp == NULL) {
1063		g_io_deliver(bp, ENOMEM);
1064		return;
1065	}
1066	cbp->bio_done = g_std_done;
1067	g_io_request(cbp, disk->d_consumer);
1068	G_MIRROR_DEBUG(1, "Kernel dump will go to %s.",
1069	    g_mirror_get_diskname(disk));
1070}
1071
1072static void
1073g_mirror_flush(struct g_mirror_softc *sc, struct bio *bp)
1074{
1075	struct bio_queue_head queue;
1076	struct g_mirror_disk *disk;
1077	struct g_consumer *cp;
1078	struct bio *cbp;
1079
1080	bioq_init(&queue);
1081	LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1082		if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1083			continue;
1084		cbp = g_clone_bio(bp);
1085		if (cbp == NULL) {
1086			while ((cbp = bioq_takefirst(&queue)) != NULL)
1087				g_destroy_bio(cbp);
1088			if (bp->bio_error == 0)
1089				bp->bio_error = ENOMEM;
1090			g_io_deliver(bp, bp->bio_error);
1091			return;
1092		}
1093		bioq_insert_tail(&queue, cbp);
1094		cbp->bio_done = g_mirror_flush_done;
1095		cbp->bio_caller1 = disk;
1096		cbp->bio_to = disk->d_consumer->provider;
1097	}
1098	while ((cbp = bioq_takefirst(&queue)) != NULL) {
1099		G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1100		disk = cbp->bio_caller1;
1101		cbp->bio_caller1 = NULL;
1102		cp = disk->d_consumer;
1103		KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1104		    ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1105		    cp->acr, cp->acw, cp->ace));
1106		g_io_request(cbp, disk->d_consumer);
1107	}
1108}
1109
1110static void
1111g_mirror_start(struct bio *bp)
1112{
1113	struct g_mirror_softc *sc;
1114
1115	sc = bp->bio_to->geom->softc;
1116	/*
1117	 * If sc == NULL or there are no valid disks, provider's error
1118	 * should be set and g_mirror_start() should not be called at all.
1119	 */
1120	KASSERT(sc != NULL && sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
1121	    ("Provider's error should be set (error=%d)(mirror=%s).",
1122	    bp->bio_to->error, bp->bio_to->name));
1123	G_MIRROR_LOGREQ(3, bp, "Request received.");
1124
1125	switch (bp->bio_cmd) {
1126	case BIO_READ:
1127	case BIO_WRITE:
1128	case BIO_DELETE:
1129		break;
1130	case BIO_FLUSH:
1131		g_mirror_flush(sc, bp);
1132		return;
1133	case BIO_GETATTR:
1134		if (!strcmp(bp->bio_attribute, "GEOM::candelete")) {
1135			g_mirror_candelete(bp);
1136			return;
1137		} else if (strcmp("GEOM::kerneldump", bp->bio_attribute) == 0) {
1138			g_mirror_kernel_dump(bp);
1139			return;
1140		}
1141		/* FALLTHROUGH */
1142	default:
1143		g_io_deliver(bp, EOPNOTSUPP);
1144		return;
1145	}
1146	mtx_lock(&sc->sc_queue_mtx);
1147	bioq_disksort(&sc->sc_queue, bp);
1148	mtx_unlock(&sc->sc_queue_mtx);
1149	G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
1150	wakeup(sc);
1151}
1152
1153/*
1154 * Return TRUE if the given request is colliding with a in-progress
1155 * synchronization request.
1156 */
1157static int
1158g_mirror_sync_collision(struct g_mirror_softc *sc, struct bio *bp)
1159{
1160	struct g_mirror_disk *disk;
1161	struct bio *sbp;
1162	off_t rstart, rend, sstart, send;
1163	int i;
1164
1165	if (sc->sc_sync.ds_ndisks == 0)
1166		return (0);
1167	rstart = bp->bio_offset;
1168	rend = bp->bio_offset + bp->bio_length;
1169	LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1170		if (disk->d_state != G_MIRROR_DISK_STATE_SYNCHRONIZING)
1171			continue;
1172		for (i = 0; i < g_mirror_syncreqs; i++) {
1173			sbp = disk->d_sync.ds_bios[i];
1174			if (sbp == NULL)
1175				continue;
1176			sstart = sbp->bio_offset;
1177			send = sbp->bio_offset + sbp->bio_length;
1178			if (rend > sstart && rstart < send)
1179				return (1);
1180		}
1181	}
1182	return (0);
1183}
1184
1185/*
1186 * Return TRUE if the given sync request is colliding with a in-progress regular
1187 * request.
1188 */
1189static int
1190g_mirror_regular_collision(struct g_mirror_softc *sc, struct bio *sbp)
1191{
1192	off_t rstart, rend, sstart, send;
1193	struct bio *bp;
1194
1195	if (sc->sc_sync.ds_ndisks == 0)
1196		return (0);
1197	sstart = sbp->bio_offset;
1198	send = sbp->bio_offset + sbp->bio_length;
1199	TAILQ_FOREACH(bp, &sc->sc_inflight.queue, bio_queue) {
1200		rstart = bp->bio_offset;
1201		rend = bp->bio_offset + bp->bio_length;
1202		if (rend > sstart && rstart < send)
1203			return (1);
1204	}
1205	return (0);
1206}
1207
1208/*
1209 * Puts request onto delayed queue.
1210 */
1211static void
1212g_mirror_regular_delay(struct g_mirror_softc *sc, struct bio *bp)
1213{
1214
1215	G_MIRROR_LOGREQ(2, bp, "Delaying request.");
1216	bioq_insert_head(&sc->sc_regular_delayed, bp);
1217}
1218
1219/*
1220 * Puts synchronization request onto delayed queue.
1221 */
1222static void
1223g_mirror_sync_delay(struct g_mirror_softc *sc, struct bio *bp)
1224{
1225
1226	G_MIRROR_LOGREQ(2, bp, "Delaying synchronization request.");
1227	bioq_insert_tail(&sc->sc_sync_delayed, bp);
1228}
1229
1230/*
1231 * Releases delayed regular requests which don't collide anymore with sync
1232 * requests.
1233 */
1234static void
1235g_mirror_regular_release(struct g_mirror_softc *sc)
1236{
1237	struct bio *bp, *bp2;
1238
1239	TAILQ_FOREACH_SAFE(bp, &sc->sc_regular_delayed.queue, bio_queue, bp2) {
1240		if (g_mirror_sync_collision(sc, bp))
1241			continue;
1242		bioq_remove(&sc->sc_regular_delayed, bp);
1243		G_MIRROR_LOGREQ(2, bp, "Releasing delayed request (%p).", bp);
1244		mtx_lock(&sc->sc_queue_mtx);
1245		bioq_insert_head(&sc->sc_queue, bp);
1246#if 0
1247		/*
1248		 * wakeup() is not needed, because this function is called from
1249		 * the worker thread.
1250		 */
1251		wakeup(&sc->sc_queue);
1252#endif
1253		mtx_unlock(&sc->sc_queue_mtx);
1254	}
1255}
1256
1257/*
1258 * Releases delayed sync requests which don't collide anymore with regular
1259 * requests.
1260 */
1261static void
1262g_mirror_sync_release(struct g_mirror_softc *sc)
1263{
1264	struct bio *bp, *bp2;
1265
1266	TAILQ_FOREACH_SAFE(bp, &sc->sc_sync_delayed.queue, bio_queue, bp2) {
1267		if (g_mirror_regular_collision(sc, bp))
1268			continue;
1269		bioq_remove(&sc->sc_sync_delayed, bp);
1270		G_MIRROR_LOGREQ(2, bp,
1271		    "Releasing delayed synchronization request.");
1272		g_io_request(bp, bp->bio_from);
1273	}
1274}
1275
1276/*
1277 * Handle synchronization requests.
1278 * Every synchronization request is two-steps process: first, READ request is
1279 * send to active provider and then WRITE request (with read data) to the provider
1280 * beeing synchronized. When WRITE is finished, new synchronization request is
1281 * send.
1282 */
1283static void
1284g_mirror_sync_request(struct bio *bp)
1285{
1286	struct g_mirror_softc *sc;
1287	struct g_mirror_disk *disk;
1288
1289	bp->bio_from->index--;
1290	sc = bp->bio_from->geom->softc;
1291	disk = bp->bio_from->private;
1292	if (disk == NULL) {
1293		sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
1294		g_topology_lock();
1295		g_mirror_kill_consumer(sc, bp->bio_from);
1296		g_topology_unlock();
1297		free(bp->bio_data, M_MIRROR);
1298		g_destroy_bio(bp);
1299		sx_xlock(&sc->sc_lock);
1300		return;
1301	}
1302
1303	/*
1304	 * Synchronization request.
1305	 */
1306	switch (bp->bio_cmd) {
1307	case BIO_READ:
1308	    {
1309		struct g_consumer *cp;
1310
1311		if (bp->bio_error != 0) {
1312			G_MIRROR_LOGREQ(0, bp,
1313			    "Synchronization request failed (error=%d).",
1314			    bp->bio_error);
1315			g_destroy_bio(bp);
1316			return;
1317		}
1318		G_MIRROR_LOGREQ(3, bp,
1319		    "Synchronization request half-finished.");
1320		bp->bio_cmd = BIO_WRITE;
1321		bp->bio_cflags = 0;
1322		cp = disk->d_consumer;
1323		KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1324		    ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1325		    cp->acr, cp->acw, cp->ace));
1326		cp->index++;
1327		g_io_request(bp, cp);
1328		return;
1329	    }
1330	case BIO_WRITE:
1331	    {
1332		struct g_mirror_disk_sync *sync;
1333		off_t offset;
1334		void *data;
1335		int i;
1336
1337		if (bp->bio_error != 0) {
1338			G_MIRROR_LOGREQ(0, bp,
1339			    "Synchronization request failed (error=%d).",
1340			    bp->bio_error);
1341			g_destroy_bio(bp);
1342			sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
1343			g_mirror_event_send(disk,
1344			    G_MIRROR_DISK_STATE_DISCONNECTED,
1345			    G_MIRROR_EVENT_DONTWAIT);
1346			return;
1347		}
1348		G_MIRROR_LOGREQ(3, bp, "Synchronization request finished.");
1349		sync = &disk->d_sync;
1350		if (sync->ds_offset >= sc->sc_mediasize ||
1351		    sync->ds_consumer == NULL ||
1352		    (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1353			/* Don't send more synchronization requests. */
1354			sync->ds_inflight--;
1355			if (sync->ds_bios != NULL) {
1356				i = (int)(uintptr_t)bp->bio_caller1;
1357				sync->ds_bios[i] = NULL;
1358			}
1359			free(bp->bio_data, M_MIRROR);
1360			g_destroy_bio(bp);
1361			if (sync->ds_inflight > 0)
1362				return;
1363			if (sync->ds_consumer == NULL ||
1364			    (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1365				return;
1366			}
1367			/* Disk up-to-date, activate it. */
1368			g_mirror_event_send(disk, G_MIRROR_DISK_STATE_ACTIVE,
1369			    G_MIRROR_EVENT_DONTWAIT);
1370			return;
1371		}
1372
1373		/* Send next synchronization request. */
1374		data = bp->bio_data;
1375		bzero(bp, sizeof(*bp));
1376		bp->bio_cmd = BIO_READ;
1377		bp->bio_offset = sync->ds_offset;
1378		bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
1379		sync->ds_offset += bp->bio_length;
1380		bp->bio_done = g_mirror_sync_done;
1381		bp->bio_data = data;
1382		bp->bio_from = sync->ds_consumer;
1383		bp->bio_to = sc->sc_provider;
1384		G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
1385		sync->ds_consumer->index++;
1386		/*
1387		 * Delay the request if it is colliding with a regular request.
1388		 */
1389		if (g_mirror_regular_collision(sc, bp))
1390			g_mirror_sync_delay(sc, bp);
1391		else
1392			g_io_request(bp, sync->ds_consumer);
1393
1394		/* Release delayed requests if possible. */
1395		g_mirror_regular_release(sc);
1396
1397		/* Find the smallest offset */
1398		offset = sc->sc_mediasize;
1399		for (i = 0; i < g_mirror_syncreqs; i++) {
1400			bp = sync->ds_bios[i];
1401			if (bp->bio_offset < offset)
1402				offset = bp->bio_offset;
1403		}
1404		if (sync->ds_offset_done + (MAXPHYS * 100) < offset) {
1405			/* Update offset_done on every 100 blocks. */
1406			sync->ds_offset_done = offset;
1407			g_mirror_update_metadata(disk);
1408		}
1409		return;
1410	    }
1411	default:
1412		KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
1413		    bp->bio_cmd, sc->sc_name));
1414		break;
1415	}
1416}
1417
1418static void
1419g_mirror_request_prefer(struct g_mirror_softc *sc, struct bio *bp)
1420{
1421	struct g_mirror_disk *disk;
1422	struct g_consumer *cp;
1423	struct bio *cbp;
1424
1425	LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1426		if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE)
1427			break;
1428	}
1429	if (disk == NULL) {
1430		if (bp->bio_error == 0)
1431			bp->bio_error = ENXIO;
1432		g_io_deliver(bp, bp->bio_error);
1433		return;
1434	}
1435	cbp = g_clone_bio(bp);
1436	if (cbp == NULL) {
1437		if (bp->bio_error == 0)
1438			bp->bio_error = ENOMEM;
1439		g_io_deliver(bp, bp->bio_error);
1440		return;
1441	}
1442	/*
1443	 * Fill in the component buf structure.
1444	 */
1445	cp = disk->d_consumer;
1446	cbp->bio_done = g_mirror_done;
1447	cbp->bio_to = cp->provider;
1448	G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1449	KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1450	    ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1451	    cp->acw, cp->ace));
1452	cp->index++;
1453	g_io_request(cbp, cp);
1454}
1455
1456static void
1457g_mirror_request_round_robin(struct g_mirror_softc *sc, struct bio *bp)
1458{
1459	struct g_mirror_disk *disk;
1460	struct g_consumer *cp;
1461	struct bio *cbp;
1462
1463	disk = g_mirror_get_disk(sc);
1464	if (disk == NULL) {
1465		if (bp->bio_error == 0)
1466			bp->bio_error = ENXIO;
1467		g_io_deliver(bp, bp->bio_error);
1468		return;
1469	}
1470	cbp = g_clone_bio(bp);
1471	if (cbp == NULL) {
1472		if (bp->bio_error == 0)
1473			bp->bio_error = ENOMEM;
1474		g_io_deliver(bp, bp->bio_error);
1475		return;
1476	}
1477	/*
1478	 * Fill in the component buf structure.
1479	 */
1480	cp = disk->d_consumer;
1481	cbp->bio_done = g_mirror_done;
1482	cbp->bio_to = cp->provider;
1483	G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1484	KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1485	    ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1486	    cp->acw, cp->ace));
1487	cp->index++;
1488	g_io_request(cbp, cp);
1489}
1490
1491#define TRACK_SIZE  (1 * 1024 * 1024)
1492#define LOAD_SCALE	256
1493#define ABS(x)		(((x) >= 0) ? (x) : (-(x)))
1494
1495static void
1496g_mirror_request_load(struct g_mirror_softc *sc, struct bio *bp)
1497{
1498	struct g_mirror_disk *disk, *dp;
1499	struct g_consumer *cp;
1500	struct bio *cbp;
1501	int prio, best;
1502
1503	/* Find a disk with the smallest load. */
1504	disk = NULL;
1505	best = INT_MAX;
1506	LIST_FOREACH(dp, &sc->sc_disks, d_next) {
1507		if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1508			continue;
1509		prio = dp->load;
1510		/* If disk head is precisely in position - highly prefer it. */
1511		if (dp->d_last_offset == bp->bio_offset)
1512			prio -= 2 * LOAD_SCALE;
1513		else
1514		/* If disk head is close to position - prefer it. */
1515		if (ABS(dp->d_last_offset - bp->bio_offset) < TRACK_SIZE)
1516			prio -= 1 * LOAD_SCALE;
1517		if (prio <= best) {
1518			disk = dp;
1519			best = prio;
1520		}
1521	}
1522	KASSERT(disk != NULL, ("NULL disk for %s.", sc->sc_name));
1523	cbp = g_clone_bio(bp);
1524	if (cbp == NULL) {
1525		if (bp->bio_error == 0)
1526			bp->bio_error = ENOMEM;
1527		g_io_deliver(bp, bp->bio_error);
1528		return;
1529	}
1530	/*
1531	 * Fill in the component buf structure.
1532	 */
1533	cp = disk->d_consumer;
1534	cbp->bio_done = g_mirror_done;
1535	cbp->bio_to = cp->provider;
1536	G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1537	KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1538	    ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1539	    cp->acw, cp->ace));
1540	cp->index++;
1541	/* Remember last head position */
1542	disk->d_last_offset = bp->bio_offset + bp->bio_length;
1543	/* Update loads. */
1544	LIST_FOREACH(dp, &sc->sc_disks, d_next) {
1545		dp->load = (dp->d_consumer->index * LOAD_SCALE +
1546		    dp->load * 7) / 8;
1547	}
1548	g_io_request(cbp, cp);
1549}
1550
1551static void
1552g_mirror_request_split(struct g_mirror_softc *sc, struct bio *bp)
1553{
1554	struct bio_queue_head queue;
1555	struct g_mirror_disk *disk;
1556	struct g_consumer *cp;
1557	struct bio *cbp;
1558	off_t left, mod, offset, slice;
1559	u_char *data;
1560	u_int ndisks;
1561
1562	if (bp->bio_length <= sc->sc_slice) {
1563		g_mirror_request_round_robin(sc, bp);
1564		return;
1565	}
1566	ndisks = g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE);
1567	slice = bp->bio_length / ndisks;
1568	mod = slice % sc->sc_provider->sectorsize;
1569	if (mod != 0)
1570		slice += sc->sc_provider->sectorsize - mod;
1571	/*
1572	 * Allocate all bios before sending any request, so we can
1573	 * return ENOMEM in nice and clean way.
1574	 */
1575	left = bp->bio_length;
1576	offset = bp->bio_offset;
1577	data = bp->bio_data;
1578	bioq_init(&queue);
1579	LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1580		if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1581			continue;
1582		cbp = g_clone_bio(bp);
1583		if (cbp == NULL) {
1584			while ((cbp = bioq_takefirst(&queue)) != NULL)
1585				g_destroy_bio(cbp);;
1586			if (bp->bio_error == 0)
1587				bp->bio_error = ENOMEM;
1588			g_io_deliver(bp, bp->bio_error);
1589			return;
1590		}
1591		bioq_insert_tail(&queue, cbp);
1592		cbp->bio_done = g_mirror_done;
1593		cbp->bio_caller1 = disk;
1594		cbp->bio_to = disk->d_consumer->provider;
1595		cbp->bio_offset = offset;
1596		cbp->bio_data = data;
1597		cbp->bio_length = MIN(left, slice);
1598		left -= cbp->bio_length;
1599		if (left == 0)
1600			break;
1601		offset += cbp->bio_length;
1602		data += cbp->bio_length;
1603	}
1604	while ((cbp = bioq_takefirst(&queue)) != NULL) {
1605		G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1606		disk = cbp->bio_caller1;
1607		cbp->bio_caller1 = NULL;
1608		cp = disk->d_consumer;
1609		KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1610		    ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1611		    cp->acr, cp->acw, cp->ace));
1612		disk->d_consumer->index++;
1613		g_io_request(cbp, disk->d_consumer);
1614	}
1615}
1616
1617static void
1618g_mirror_register_request(struct bio *bp)
1619{
1620	struct g_mirror_softc *sc;
1621
1622	sc = bp->bio_to->geom->softc;
1623	switch (bp->bio_cmd) {
1624	case BIO_READ:
1625		switch (sc->sc_balance) {
1626		case G_MIRROR_BALANCE_LOAD:
1627			g_mirror_request_load(sc, bp);
1628			break;
1629		case G_MIRROR_BALANCE_PREFER:
1630			g_mirror_request_prefer(sc, bp);
1631			break;
1632		case G_MIRROR_BALANCE_ROUND_ROBIN:
1633			g_mirror_request_round_robin(sc, bp);
1634			break;
1635		case G_MIRROR_BALANCE_SPLIT:
1636			g_mirror_request_split(sc, bp);
1637			break;
1638		}
1639		return;
1640	case BIO_WRITE:
1641	case BIO_DELETE:
1642	    {
1643		struct g_mirror_disk *disk;
1644		struct g_mirror_disk_sync *sync;
1645		struct bio_queue_head queue;
1646		struct g_consumer *cp;
1647		struct bio *cbp;
1648
1649		/*
1650		 * Delay the request if it is colliding with a synchronization
1651		 * request.
1652		 */
1653		if (g_mirror_sync_collision(sc, bp)) {
1654			g_mirror_regular_delay(sc, bp);
1655			return;
1656		}
1657
1658		if (sc->sc_idle)
1659			g_mirror_unidle(sc);
1660		else
1661			sc->sc_last_write = time_uptime;
1662
1663		/*
1664		 * Allocate all bios before sending any request, so we can
1665		 * return ENOMEM in nice and clean way.
1666		 */
1667		bioq_init(&queue);
1668		LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1669			sync = &disk->d_sync;
1670			switch (disk->d_state) {
1671			case G_MIRROR_DISK_STATE_ACTIVE:
1672				break;
1673			case G_MIRROR_DISK_STATE_SYNCHRONIZING:
1674				if (bp->bio_offset >= sync->ds_offset)
1675					continue;
1676				break;
1677			default:
1678				continue;
1679			}
1680			if (bp->bio_cmd == BIO_DELETE &&
1681			    (disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE) == 0)
1682				continue;
1683			cbp = g_clone_bio(bp);
1684			if (cbp == NULL) {
1685				while ((cbp = bioq_takefirst(&queue)) != NULL)
1686					g_destroy_bio(cbp);
1687				if (bp->bio_error == 0)
1688					bp->bio_error = ENOMEM;
1689				g_io_deliver(bp, bp->bio_error);
1690				return;
1691			}
1692			bioq_insert_tail(&queue, cbp);
1693			cbp->bio_done = g_mirror_done;
1694			cp = disk->d_consumer;
1695			cbp->bio_caller1 = cp;
1696			cbp->bio_to = cp->provider;
1697			KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1698			    ("Consumer %s not opened (r%dw%de%d).",
1699			    cp->provider->name, cp->acr, cp->acw, cp->ace));
1700		}
1701		if (bioq_first(&queue) == NULL) {
1702			g_io_deliver(bp, EOPNOTSUPP);
1703			return;
1704		}
1705		while ((cbp = bioq_takefirst(&queue)) != NULL) {
1706			G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1707			cp = cbp->bio_caller1;
1708			cbp->bio_caller1 = NULL;
1709			cp->index++;
1710			sc->sc_writes++;
1711			g_io_request(cbp, cp);
1712		}
1713		/*
1714		 * Put request onto inflight queue, so we can check if new
1715		 * synchronization requests don't collide with it.
1716		 */
1717		bioq_insert_tail(&sc->sc_inflight, bp);
1718		/*
1719		 * Bump syncid on first write.
1720		 */
1721		if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0) {
1722			sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
1723			g_mirror_bump_syncid(sc);
1724		}
1725		return;
1726	    }
1727	default:
1728		KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
1729		    bp->bio_cmd, sc->sc_name));
1730		break;
1731	}
1732}
1733
1734static int
1735g_mirror_can_destroy(struct g_mirror_softc *sc)
1736{
1737	struct g_geom *gp;
1738	struct g_consumer *cp;
1739
1740	g_topology_assert();
1741	gp = sc->sc_geom;
1742	if (gp->softc == NULL)
1743		return (1);
1744	if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_TASTING) != 0)
1745		return (0);
1746	LIST_FOREACH(cp, &gp->consumer, consumer) {
1747		if (g_mirror_is_busy(sc, cp))
1748			return (0);
1749	}
1750	gp = sc->sc_sync.ds_geom;
1751	LIST_FOREACH(cp, &gp->consumer, consumer) {
1752		if (g_mirror_is_busy(sc, cp))
1753			return (0);
1754	}
1755	G_MIRROR_DEBUG(2, "No I/O requests for %s, it can be destroyed.",
1756	    sc->sc_name);
1757	return (1);
1758}
1759
1760static int
1761g_mirror_try_destroy(struct g_mirror_softc *sc)
1762{
1763
1764	if (sc->sc_rootmount != NULL) {
1765		G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
1766		    sc->sc_rootmount);
1767		root_mount_rel(sc->sc_rootmount);
1768		sc->sc_rootmount = NULL;
1769	}
1770	g_topology_lock();
1771	if (!g_mirror_can_destroy(sc)) {
1772		g_topology_unlock();
1773		return (0);
1774	}
1775	sc->sc_geom->softc = NULL;
1776	sc->sc_sync.ds_geom->softc = NULL;
1777	if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WAIT) != 0) {
1778		g_topology_unlock();
1779		G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
1780		    &sc->sc_worker);
1781		/* Unlock sc_lock here, as it can be destroyed after wakeup. */
1782		sx_xunlock(&sc->sc_lock);
1783		wakeup(&sc->sc_worker);
1784		sc->sc_worker = NULL;
1785	} else {
1786		g_topology_unlock();
1787		g_mirror_destroy_device(sc);
1788		free(sc, M_MIRROR);
1789	}
1790	return (1);
1791}
1792
1793/*
1794 * Worker thread.
1795 */
1796static void
1797g_mirror_worker(void *arg)
1798{
1799	struct g_mirror_softc *sc;
1800	struct g_mirror_event *ep;
1801	struct bio *bp;
1802	int timeout;
1803
1804	sc = arg;
1805	thread_lock(curthread);
1806	sched_prio(curthread, PRIBIO);
1807	thread_unlock(curthread);
1808
1809	sx_xlock(&sc->sc_lock);
1810	for (;;) {
1811		G_MIRROR_DEBUG(5, "%s: Let's see...", __func__);
1812		/*
1813		 * First take a look at events.
1814		 * This is important to handle events before any I/O requests.
1815		 */
1816		ep = g_mirror_event_get(sc);
1817		if (ep != NULL) {
1818			g_mirror_event_remove(sc, ep);
1819			if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0) {
1820				/* Update only device status. */
1821				G_MIRROR_DEBUG(3,
1822				    "Running event for device %s.",
1823				    sc->sc_name);
1824				ep->e_error = 0;
1825				g_mirror_update_device(sc, 1);
1826			} else {
1827				/* Update disk status. */
1828				G_MIRROR_DEBUG(3, "Running event for disk %s.",
1829				     g_mirror_get_diskname(ep->e_disk));
1830				ep->e_error = g_mirror_update_disk(ep->e_disk,
1831				    ep->e_state);
1832				if (ep->e_error == 0)
1833					g_mirror_update_device(sc, 0);
1834			}
1835			if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0) {
1836				KASSERT(ep->e_error == 0,
1837				    ("Error cannot be handled."));
1838				g_mirror_event_free(ep);
1839			} else {
1840				ep->e_flags |= G_MIRROR_EVENT_DONE;
1841				G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
1842				    ep);
1843				mtx_lock(&sc->sc_events_mtx);
1844				wakeup(ep);
1845				mtx_unlock(&sc->sc_events_mtx);
1846			}
1847			if ((sc->sc_flags &
1848			    G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1849				if (g_mirror_try_destroy(sc)) {
1850					curthread->td_pflags &= ~TDP_GEOM;
1851					G_MIRROR_DEBUG(1, "Thread exiting.");
1852					kproc_exit(0);
1853				}
1854			}
1855			G_MIRROR_DEBUG(5, "%s: I'm here 1.", __func__);
1856			continue;
1857		}
1858		/*
1859		 * Check if we can mark array as CLEAN and if we can't take
1860		 * how much seconds should we wait.
1861		 */
1862		timeout = g_mirror_idle(sc, -1);
1863		/*
1864		 * Now I/O requests.
1865		 */
1866		/* Get first request from the queue. */
1867		mtx_lock(&sc->sc_queue_mtx);
1868		bp = bioq_takefirst(&sc->sc_queue);
1869		if (bp == NULL) {
1870			if ((sc->sc_flags &
1871			    G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1872				mtx_unlock(&sc->sc_queue_mtx);
1873				if (g_mirror_try_destroy(sc)) {
1874					curthread->td_pflags &= ~TDP_GEOM;
1875					G_MIRROR_DEBUG(1, "Thread exiting.");
1876					kproc_exit(0);
1877				}
1878				mtx_lock(&sc->sc_queue_mtx);
1879			}
1880			sx_xunlock(&sc->sc_lock);
1881			/*
1882			 * XXX: We can miss an event here, because an event
1883			 *      can be added without sx-device-lock and without
1884			 *      mtx-queue-lock. Maybe I should just stop using
1885			 *      dedicated mutex for events synchronization and
1886			 *      stick with the queue lock?
1887			 *      The event will hang here until next I/O request
1888			 *      or next event is received.
1889			 */
1890			MSLEEP(sc, &sc->sc_queue_mtx, PRIBIO | PDROP, "m:w1",
1891			    timeout * hz);
1892			sx_xlock(&sc->sc_lock);
1893			G_MIRROR_DEBUG(5, "%s: I'm here 4.", __func__);
1894			continue;
1895		}
1896		mtx_unlock(&sc->sc_queue_mtx);
1897
1898		if (bp->bio_from->geom == sc->sc_sync.ds_geom &&
1899		    (bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0) {
1900			g_mirror_sync_request(bp);	/* READ */
1901		} else if (bp->bio_to != sc->sc_provider) {
1902			if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_REGULAR) != 0)
1903				g_mirror_regular_request(bp);
1904			else if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0)
1905				g_mirror_sync_request(bp);	/* WRITE */
1906			else {
1907				KASSERT(0,
1908				    ("Invalid request cflags=0x%hhx to=%s.",
1909				    bp->bio_cflags, bp->bio_to->name));
1910			}
1911		} else {
1912			g_mirror_register_request(bp);
1913		}
1914		G_MIRROR_DEBUG(5, "%s: I'm here 9.", __func__);
1915	}
1916}
1917
1918static void
1919g_mirror_update_idle(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
1920{
1921
1922	sx_assert(&sc->sc_lock, SX_LOCKED);
1923
1924	if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
1925		return;
1926	if (!sc->sc_idle && (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) == 0) {
1927		G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as dirty.",
1928		    g_mirror_get_diskname(disk), sc->sc_name);
1929		disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
1930	} else if (sc->sc_idle &&
1931	    (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
1932		G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as clean.",
1933		    g_mirror_get_diskname(disk), sc->sc_name);
1934		disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
1935	}
1936}
1937
1938static void
1939g_mirror_sync_start(struct g_mirror_disk *disk)
1940{
1941	struct g_mirror_softc *sc;
1942	struct g_consumer *cp;
1943	struct bio *bp;
1944	int error, i;
1945
1946	g_topology_assert_not();
1947	sc = disk->d_softc;
1948	sx_assert(&sc->sc_lock, SX_LOCKED);
1949
1950	KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
1951	    ("Disk %s is not marked for synchronization.",
1952	    g_mirror_get_diskname(disk)));
1953	KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
1954	    ("Device not in RUNNING state (%s, %u).", sc->sc_name,
1955	    sc->sc_state));
1956
1957	sx_xunlock(&sc->sc_lock);
1958	g_topology_lock();
1959	cp = g_new_consumer(sc->sc_sync.ds_geom);
1960	cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
1961	error = g_attach(cp, sc->sc_provider);
1962	KASSERT(error == 0,
1963	    ("Cannot attach to %s (error=%d).", sc->sc_name, error));
1964	error = g_access(cp, 1, 0, 0);
1965	KASSERT(error == 0, ("Cannot open %s (error=%d).", sc->sc_name, error));
1966	g_topology_unlock();
1967	sx_xlock(&sc->sc_lock);
1968
1969	G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s.", sc->sc_name,
1970	    g_mirror_get_diskname(disk));
1971	if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) == 0)
1972		disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
1973	KASSERT(disk->d_sync.ds_consumer == NULL,
1974	    ("Sync consumer already exists (device=%s, disk=%s).",
1975	    sc->sc_name, g_mirror_get_diskname(disk)));
1976
1977	disk->d_sync.ds_consumer = cp;
1978	disk->d_sync.ds_consumer->private = disk;
1979	disk->d_sync.ds_consumer->index = 0;
1980
1981	/*
1982	 * Allocate memory for synchronization bios and initialize them.
1983	 */
1984	disk->d_sync.ds_bios = malloc(sizeof(struct bio *) * g_mirror_syncreqs,
1985	    M_MIRROR, M_WAITOK);
1986	for (i = 0; i < g_mirror_syncreqs; i++) {
1987		bp = g_alloc_bio();
1988		disk->d_sync.ds_bios[i] = bp;
1989		bp->bio_parent = NULL;
1990		bp->bio_cmd = BIO_READ;
1991		bp->bio_data = malloc(MAXPHYS, M_MIRROR, M_WAITOK);
1992		bp->bio_cflags = 0;
1993		bp->bio_offset = disk->d_sync.ds_offset;
1994		bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
1995		disk->d_sync.ds_offset += bp->bio_length;
1996		bp->bio_done = g_mirror_sync_done;
1997		bp->bio_from = disk->d_sync.ds_consumer;
1998		bp->bio_to = sc->sc_provider;
1999		bp->bio_caller1 = (void *)(uintptr_t)i;
2000	}
2001
2002	/* Increase the number of disks in SYNCHRONIZING state. */
2003	sc->sc_sync.ds_ndisks++;
2004	/* Set the number of in-flight synchronization requests. */
2005	disk->d_sync.ds_inflight = g_mirror_syncreqs;
2006
2007	/*
2008	 * Fire off first synchronization requests.
2009	 */
2010	for (i = 0; i < g_mirror_syncreqs; i++) {
2011		bp = disk->d_sync.ds_bios[i];
2012		G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
2013		disk->d_sync.ds_consumer->index++;
2014		/*
2015		 * Delay the request if it is colliding with a regular request.
2016		 */
2017		if (g_mirror_regular_collision(sc, bp))
2018			g_mirror_sync_delay(sc, bp);
2019		else
2020			g_io_request(bp, disk->d_sync.ds_consumer);
2021	}
2022}
2023
2024/*
2025 * Stop synchronization process.
2026 * type: 0 - synchronization finished
2027 *       1 - synchronization stopped
2028 */
2029static void
2030g_mirror_sync_stop(struct g_mirror_disk *disk, int type)
2031{
2032	struct g_mirror_softc *sc;
2033	struct g_consumer *cp;
2034
2035	g_topology_assert_not();
2036	sc = disk->d_softc;
2037	sx_assert(&sc->sc_lock, SX_LOCKED);
2038
2039	KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2040	    ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2041	    g_mirror_disk_state2str(disk->d_state)));
2042	if (disk->d_sync.ds_consumer == NULL)
2043		return;
2044
2045	if (type == 0) {
2046		G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s finished.",
2047		    sc->sc_name, g_mirror_get_diskname(disk));
2048	} else /* if (type == 1) */ {
2049		G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s stopped.",
2050		    sc->sc_name, g_mirror_get_diskname(disk));
2051	}
2052	free(disk->d_sync.ds_bios, M_MIRROR);
2053	disk->d_sync.ds_bios = NULL;
2054	cp = disk->d_sync.ds_consumer;
2055	disk->d_sync.ds_consumer = NULL;
2056	disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2057	sc->sc_sync.ds_ndisks--;
2058	sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
2059	g_topology_lock();
2060	g_mirror_kill_consumer(sc, cp);
2061	g_topology_unlock();
2062	sx_xlock(&sc->sc_lock);
2063}
2064
2065static void
2066g_mirror_launch_provider(struct g_mirror_softc *sc)
2067{
2068	struct g_mirror_disk *disk;
2069	struct g_provider *pp, *dp;
2070
2071	sx_assert(&sc->sc_lock, SX_LOCKED);
2072
2073	g_topology_lock();
2074	pp = g_new_providerf(sc->sc_geom, "mirror/%s", sc->sc_name);
2075	pp->flags |= G_PF_DIRECT_RECEIVE;
2076	pp->mediasize = sc->sc_mediasize;
2077	pp->sectorsize = sc->sc_sectorsize;
2078	pp->stripesize = 0;
2079	pp->stripeoffset = 0;
2080
2081	/* Splitting of unmapped BIO's could work but isn't implemented now */
2082	if (sc->sc_balance != G_MIRROR_BALANCE_SPLIT)
2083		pp->flags |= G_PF_ACCEPT_UNMAPPED;
2084
2085	LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2086		if (disk->d_consumer && disk->d_consumer->provider) {
2087			dp = disk->d_consumer->provider;
2088			if (dp->stripesize > pp->stripesize) {
2089				pp->stripesize = dp->stripesize;
2090				pp->stripeoffset = dp->stripeoffset;
2091			}
2092			/* A provider underneath us doesn't support unmapped */
2093			if ((dp->flags & G_PF_ACCEPT_UNMAPPED) == 0) {
2094				G_MIRROR_DEBUG(0, "Cancelling unmapped "
2095				    "because of %s.", dp->name);
2096				pp->flags &= ~G_PF_ACCEPT_UNMAPPED;
2097			}
2098		}
2099	}
2100	sc->sc_provider = pp;
2101	g_error_provider(pp, 0);
2102	g_topology_unlock();
2103	G_MIRROR_DEBUG(0, "Device %s launched (%u/%u).", pp->name,
2104	    g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE), sc->sc_ndisks);
2105	LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2106		if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
2107			g_mirror_sync_start(disk);
2108	}
2109}
2110
2111static void
2112g_mirror_destroy_provider(struct g_mirror_softc *sc)
2113{
2114	struct g_mirror_disk *disk;
2115	struct bio *bp;
2116
2117	g_topology_assert_not();
2118	KASSERT(sc->sc_provider != NULL, ("NULL provider (device=%s).",
2119	    sc->sc_name));
2120
2121	g_topology_lock();
2122	g_error_provider(sc->sc_provider, ENXIO);
2123	mtx_lock(&sc->sc_queue_mtx);
2124	while ((bp = bioq_takefirst(&sc->sc_queue)) != NULL)
2125		g_io_deliver(bp, ENXIO);
2126	mtx_unlock(&sc->sc_queue_mtx);
2127	G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.", sc->sc_name,
2128	    sc->sc_provider->name);
2129	sc->sc_provider->flags |= G_PF_WITHER;
2130	g_orphan_provider(sc->sc_provider, ENXIO);
2131	g_topology_unlock();
2132	sc->sc_provider = NULL;
2133	LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2134		if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
2135			g_mirror_sync_stop(disk, 1);
2136	}
2137}
2138
2139static void
2140g_mirror_go(void *arg)
2141{
2142	struct g_mirror_softc *sc;
2143
2144	sc = arg;
2145	G_MIRROR_DEBUG(0, "Force device %s start due to timeout.", sc->sc_name);
2146	g_mirror_event_send(sc, 0,
2147	    G_MIRROR_EVENT_DONTWAIT | G_MIRROR_EVENT_DEVICE);
2148}
2149
2150static u_int
2151g_mirror_determine_state(struct g_mirror_disk *disk)
2152{
2153	struct g_mirror_softc *sc;
2154	u_int state;
2155
2156	sc = disk->d_softc;
2157	if (sc->sc_syncid == disk->d_sync.ds_syncid) {
2158		if ((disk->d_flags &
2159		    G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
2160			/* Disk does not need synchronization. */
2161			state = G_MIRROR_DISK_STATE_ACTIVE;
2162		} else {
2163			if ((sc->sc_flags &
2164			     G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2165			    (disk->d_flags &
2166			     G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
2167				/*
2168				 * We can start synchronization from
2169				 * the stored offset.
2170				 */
2171				state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
2172			} else {
2173				state = G_MIRROR_DISK_STATE_STALE;
2174			}
2175		}
2176	} else if (disk->d_sync.ds_syncid < sc->sc_syncid) {
2177		/*
2178		 * Reset all synchronization data for this disk,
2179		 * because if it even was synchronized, it was
2180		 * synchronized to disks with different syncid.
2181		 */
2182		disk->d_flags |= G_MIRROR_DISK_FLAG_SYNCHRONIZING;
2183		disk->d_sync.ds_offset = 0;
2184		disk->d_sync.ds_offset_done = 0;
2185		disk->d_sync.ds_syncid = sc->sc_syncid;
2186		if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2187		    (disk->d_flags & G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
2188			state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
2189		} else {
2190			state = G_MIRROR_DISK_STATE_STALE;
2191		}
2192	} else /* if (sc->sc_syncid < disk->d_sync.ds_syncid) */ {
2193		/*
2194		 * Not good, NOT GOOD!
2195		 * It means that mirror was started on stale disks
2196		 * and more fresh disk just arrive.
2197		 * If there were writes, mirror is broken, sorry.
2198		 * I think the best choice here is don't touch
2199		 * this disk and inform the user loudly.
2200		 */
2201		G_MIRROR_DEBUG(0, "Device %s was started before the freshest "
2202		    "disk (%s) arrives!! It will not be connected to the "
2203		    "running device.", sc->sc_name,
2204		    g_mirror_get_diskname(disk));
2205		g_mirror_destroy_disk(disk);
2206		state = G_MIRROR_DISK_STATE_NONE;
2207		/* Return immediately, because disk was destroyed. */
2208		return (state);
2209	}
2210	G_MIRROR_DEBUG(3, "State for %s disk: %s.",
2211	    g_mirror_get_diskname(disk), g_mirror_disk_state2str(state));
2212	return (state);
2213}
2214
2215/*
2216 * Update device state.
2217 */
2218static void
2219g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force)
2220{
2221	struct g_mirror_disk *disk;
2222	u_int state;
2223
2224	sx_assert(&sc->sc_lock, SX_XLOCKED);
2225
2226	switch (sc->sc_state) {
2227	case G_MIRROR_DEVICE_STATE_STARTING:
2228	    {
2229		struct g_mirror_disk *pdisk, *tdisk;
2230		u_int dirty, ndisks, genid, syncid;
2231
2232		KASSERT(sc->sc_provider == NULL,
2233		    ("Non-NULL provider in STARTING state (%s).", sc->sc_name));
2234		/*
2235		 * Are we ready? We are, if all disks are connected or
2236		 * if we have any disks and 'force' is true.
2237		 */
2238		ndisks = g_mirror_ndisks(sc, -1);
2239		if (sc->sc_ndisks == ndisks || (force && ndisks > 0)) {
2240			;
2241		} else if (ndisks == 0) {
2242			/*
2243			 * Disks went down in starting phase, so destroy
2244			 * device.
2245			 */
2246			callout_drain(&sc->sc_callout);
2247			sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2248			G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
2249			    sc->sc_rootmount);
2250			root_mount_rel(sc->sc_rootmount);
2251			sc->sc_rootmount = NULL;
2252			return;
2253		} else {
2254			return;
2255		}
2256
2257		/*
2258		 * Activate all disks with the biggest syncid.
2259		 */
2260		if (force) {
2261			/*
2262			 * If 'force' is true, we have been called due to
2263			 * timeout, so don't bother canceling timeout.
2264			 */
2265			ndisks = 0;
2266			LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2267				if ((disk->d_flags &
2268				    G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
2269					ndisks++;
2270				}
2271			}
2272			if (ndisks == 0) {
2273				/* No valid disks found, destroy device. */
2274				sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2275				G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
2276				    __LINE__, sc->sc_rootmount);
2277				root_mount_rel(sc->sc_rootmount);
2278				sc->sc_rootmount = NULL;
2279				return;
2280			}
2281		} else {
2282			/* Cancel timeout. */
2283			callout_drain(&sc->sc_callout);
2284		}
2285
2286		/*
2287		 * Find the biggest genid.
2288		 */
2289		genid = 0;
2290		LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2291			if (disk->d_genid > genid)
2292				genid = disk->d_genid;
2293		}
2294		sc->sc_genid = genid;
2295		/*
2296		 * Remove all disks without the biggest genid.
2297		 */
2298		LIST_FOREACH_SAFE(disk, &sc->sc_disks, d_next, tdisk) {
2299			if (disk->d_genid < genid) {
2300				G_MIRROR_DEBUG(0,
2301				    "Component %s (device %s) broken, skipping.",
2302				    g_mirror_get_diskname(disk), sc->sc_name);
2303				g_mirror_destroy_disk(disk);
2304			}
2305		}
2306
2307		/*
2308		 * Find the biggest syncid.
2309		 */
2310		syncid = 0;
2311		LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2312			if (disk->d_sync.ds_syncid > syncid)
2313				syncid = disk->d_sync.ds_syncid;
2314		}
2315
2316		/*
2317		 * Here we need to look for dirty disks and if all disks
2318		 * with the biggest syncid are dirty, we have to choose
2319		 * one with the biggest priority and rebuild the rest.
2320		 */
2321		/*
2322		 * Find the number of dirty disks with the biggest syncid.
2323		 * Find the number of disks with the biggest syncid.
2324		 * While here, find a disk with the biggest priority.
2325		 */
2326		dirty = ndisks = 0;
2327		pdisk = NULL;
2328		LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2329			if (disk->d_sync.ds_syncid != syncid)
2330				continue;
2331			if ((disk->d_flags &
2332			    G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2333				continue;
2334			}
2335			ndisks++;
2336			if ((disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
2337				dirty++;
2338				if (pdisk == NULL ||
2339				    pdisk->d_priority < disk->d_priority) {
2340					pdisk = disk;
2341				}
2342			}
2343		}
2344		if (dirty == 0) {
2345			/* No dirty disks at all, great. */
2346		} else if (dirty == ndisks) {
2347			/*
2348			 * Force synchronization for all dirty disks except one
2349			 * with the biggest priority.
2350			 */
2351			KASSERT(pdisk != NULL, ("pdisk == NULL"));
2352			G_MIRROR_DEBUG(1, "Using disk %s (device %s) as a "
2353			    "master disk for synchronization.",
2354			    g_mirror_get_diskname(pdisk), sc->sc_name);
2355			LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2356				if (disk->d_sync.ds_syncid != syncid)
2357					continue;
2358				if ((disk->d_flags &
2359				    G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2360					continue;
2361				}
2362				KASSERT((disk->d_flags &
2363				    G_MIRROR_DISK_FLAG_DIRTY) != 0,
2364				    ("Disk %s isn't marked as dirty.",
2365				    g_mirror_get_diskname(disk)));
2366				/* Skip the disk with the biggest priority. */
2367				if (disk == pdisk)
2368					continue;
2369				disk->d_sync.ds_syncid = 0;
2370			}
2371		} else if (dirty < ndisks) {
2372			/*
2373			 * Force synchronization for all dirty disks.
2374			 * We have some non-dirty disks.
2375			 */
2376			LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2377				if (disk->d_sync.ds_syncid != syncid)
2378					continue;
2379				if ((disk->d_flags &
2380				    G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2381					continue;
2382				}
2383				if ((disk->d_flags &
2384				    G_MIRROR_DISK_FLAG_DIRTY) == 0) {
2385					continue;
2386				}
2387				disk->d_sync.ds_syncid = 0;
2388			}
2389		}
2390
2391		/* Reset hint. */
2392		sc->sc_hint = NULL;
2393		sc->sc_syncid = syncid;
2394		if (force) {
2395			/* Remember to bump syncid on first write. */
2396			sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2397		}
2398		state = G_MIRROR_DEVICE_STATE_RUNNING;
2399		G_MIRROR_DEBUG(1, "Device %s state changed from %s to %s.",
2400		    sc->sc_name, g_mirror_device_state2str(sc->sc_state),
2401		    g_mirror_device_state2str(state));
2402		sc->sc_state = state;
2403		LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2404			state = g_mirror_determine_state(disk);
2405			g_mirror_event_send(disk, state,
2406			    G_MIRROR_EVENT_DONTWAIT);
2407			if (state == G_MIRROR_DISK_STATE_STALE)
2408				sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2409		}
2410		break;
2411	    }
2412	case G_MIRROR_DEVICE_STATE_RUNNING:
2413		if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 0 &&
2414		    g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
2415			/*
2416			 * No active disks or no disks at all,
2417			 * so destroy device.
2418			 */
2419			if (sc->sc_provider != NULL)
2420				g_mirror_destroy_provider(sc);
2421			sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2422			break;
2423		} else if (g_mirror_ndisks(sc,
2424		    G_MIRROR_DISK_STATE_ACTIVE) > 0 &&
2425		    g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
2426			/*
2427			 * We have active disks, launch provider if it doesn't
2428			 * exist.
2429			 */
2430			if (sc->sc_provider == NULL)
2431				g_mirror_launch_provider(sc);
2432			if (sc->sc_rootmount != NULL) {
2433				G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
2434				    __LINE__, sc->sc_rootmount);
2435				root_mount_rel(sc->sc_rootmount);
2436				sc->sc_rootmount = NULL;
2437			}
2438		}
2439		/*
2440		 * Genid should be bumped immediately, so do it here.
2441		 */
2442		if ((sc->sc_bump_id & G_MIRROR_BUMP_GENID) != 0) {
2443			sc->sc_bump_id &= ~G_MIRROR_BUMP_GENID;
2444			g_mirror_bump_genid(sc);
2445		}
2446		break;
2447	default:
2448		KASSERT(1 == 0, ("Wrong device state (%s, %s).",
2449		    sc->sc_name, g_mirror_device_state2str(sc->sc_state)));
2450		break;
2451	}
2452}
2453
2454/*
2455 * Update disk state and device state if needed.
2456 */
2457#define	DISK_STATE_CHANGED()	G_MIRROR_DEBUG(1,			\
2458	"Disk %s state changed from %s to %s (device %s).",		\
2459	g_mirror_get_diskname(disk),					\
2460	g_mirror_disk_state2str(disk->d_state),				\
2461	g_mirror_disk_state2str(state), sc->sc_name)
2462static int
2463g_mirror_update_disk(struct g_mirror_disk *disk, u_int state)
2464{
2465	struct g_mirror_softc *sc;
2466
2467	sc = disk->d_softc;
2468	sx_assert(&sc->sc_lock, SX_XLOCKED);
2469
2470again:
2471	G_MIRROR_DEBUG(3, "Changing disk %s state from %s to %s.",
2472	    g_mirror_get_diskname(disk), g_mirror_disk_state2str(disk->d_state),
2473	    g_mirror_disk_state2str(state));
2474	switch (state) {
2475	case G_MIRROR_DISK_STATE_NEW:
2476		/*
2477		 * Possible scenarios:
2478		 * 1. New disk arrive.
2479		 */
2480		/* Previous state should be NONE. */
2481		KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NONE,
2482		    ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2483		    g_mirror_disk_state2str(disk->d_state)));
2484		DISK_STATE_CHANGED();
2485
2486		disk->d_state = state;
2487		if (LIST_EMPTY(&sc->sc_disks))
2488			LIST_INSERT_HEAD(&sc->sc_disks, disk, d_next);
2489		else {
2490			struct g_mirror_disk *dp;
2491
2492			LIST_FOREACH(dp, &sc->sc_disks, d_next) {
2493				if (disk->d_priority >= dp->d_priority) {
2494					LIST_INSERT_BEFORE(dp, disk, d_next);
2495					dp = NULL;
2496					break;
2497				}
2498				if (LIST_NEXT(dp, d_next) == NULL)
2499					break;
2500			}
2501			if (dp != NULL)
2502				LIST_INSERT_AFTER(dp, disk, d_next);
2503		}
2504		G_MIRROR_DEBUG(1, "Device %s: provider %s detected.",
2505		    sc->sc_name, g_mirror_get_diskname(disk));
2506		if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
2507			break;
2508		KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2509		    ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2510		    g_mirror_device_state2str(sc->sc_state),
2511		    g_mirror_get_diskname(disk),
2512		    g_mirror_disk_state2str(disk->d_state)));
2513		state = g_mirror_determine_state(disk);
2514		if (state != G_MIRROR_DISK_STATE_NONE)
2515			goto again;
2516		break;
2517	case G_MIRROR_DISK_STATE_ACTIVE:
2518		/*
2519		 * Possible scenarios:
2520		 * 1. New disk does not need synchronization.
2521		 * 2. Synchronization process finished successfully.
2522		 */
2523		KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2524		    ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2525		    g_mirror_device_state2str(sc->sc_state),
2526		    g_mirror_get_diskname(disk),
2527		    g_mirror_disk_state2str(disk->d_state)));
2528		/* Previous state should be NEW or SYNCHRONIZING. */
2529		KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW ||
2530		    disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2531		    ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2532		    g_mirror_disk_state2str(disk->d_state)));
2533		DISK_STATE_CHANGED();
2534
2535		if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
2536			disk->d_flags &= ~G_MIRROR_DISK_FLAG_SYNCHRONIZING;
2537			disk->d_flags &= ~G_MIRROR_DISK_FLAG_FORCE_SYNC;
2538			g_mirror_sync_stop(disk, 0);
2539		}
2540		disk->d_state = state;
2541		disk->d_sync.ds_offset = 0;
2542		disk->d_sync.ds_offset_done = 0;
2543		g_mirror_update_idle(sc, disk);
2544		g_mirror_update_metadata(disk);
2545		G_MIRROR_DEBUG(1, "Device %s: provider %s activated.",
2546		    sc->sc_name, g_mirror_get_diskname(disk));
2547		break;
2548	case G_MIRROR_DISK_STATE_STALE:
2549		/*
2550		 * Possible scenarios:
2551		 * 1. Stale disk was connected.
2552		 */
2553		/* Previous state should be NEW. */
2554		KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2555		    ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2556		    g_mirror_disk_state2str(disk->d_state)));
2557		KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2558		    ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2559		    g_mirror_device_state2str(sc->sc_state),
2560		    g_mirror_get_diskname(disk),
2561		    g_mirror_disk_state2str(disk->d_state)));
2562		/*
2563		 * STALE state is only possible if device is marked
2564		 * NOAUTOSYNC.
2565		 */
2566		KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) != 0,
2567		    ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2568		    g_mirror_device_state2str(sc->sc_state),
2569		    g_mirror_get_diskname(disk),
2570		    g_mirror_disk_state2str(disk->d_state)));
2571		DISK_STATE_CHANGED();
2572
2573		disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2574		disk->d_state = state;
2575		g_mirror_update_metadata(disk);
2576		G_MIRROR_DEBUG(0, "Device %s: provider %s is stale.",
2577		    sc->sc_name, g_mirror_get_diskname(disk));
2578		break;
2579	case G_MIRROR_DISK_STATE_SYNCHRONIZING:
2580		/*
2581		 * Possible scenarios:
2582		 * 1. Disk which needs synchronization was connected.
2583		 */
2584		/* Previous state should be NEW. */
2585		KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2586		    ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2587		    g_mirror_disk_state2str(disk->d_state)));
2588		KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2589		    ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2590		    g_mirror_device_state2str(sc->sc_state),
2591		    g_mirror_get_diskname(disk),
2592		    g_mirror_disk_state2str(disk->d_state)));
2593		DISK_STATE_CHANGED();
2594
2595		if (disk->d_state == G_MIRROR_DISK_STATE_NEW)
2596			disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2597		disk->d_state = state;
2598		if (sc->sc_provider != NULL) {
2599			g_mirror_sync_start(disk);
2600			g_mirror_update_metadata(disk);
2601		}
2602		break;
2603	case G_MIRROR_DISK_STATE_DISCONNECTED:
2604		/*
2605		 * Possible scenarios:
2606		 * 1. Device wasn't running yet, but disk disappear.
2607		 * 2. Disk was active and disapppear.
2608		 * 3. Disk disappear during synchronization process.
2609		 */
2610		if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING) {
2611			/*
2612			 * Previous state should be ACTIVE, STALE or
2613			 * SYNCHRONIZING.
2614			 */
2615			KASSERT(disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
2616			    disk->d_state == G_MIRROR_DISK_STATE_STALE ||
2617			    disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2618			    ("Wrong disk state (%s, %s).",
2619			    g_mirror_get_diskname(disk),
2620			    g_mirror_disk_state2str(disk->d_state)));
2621		} else if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING) {
2622			/* Previous state should be NEW. */
2623			KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2624			    ("Wrong disk state (%s, %s).",
2625			    g_mirror_get_diskname(disk),
2626			    g_mirror_disk_state2str(disk->d_state)));
2627			/*
2628			 * Reset bumping syncid if disk disappeared in STARTING
2629			 * state.
2630			 */
2631			if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0)
2632				sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
2633#ifdef	INVARIANTS
2634		} else {
2635			KASSERT(1 == 0, ("Wrong device state (%s, %s, %s, %s).",
2636			    sc->sc_name,
2637			    g_mirror_device_state2str(sc->sc_state),
2638			    g_mirror_get_diskname(disk),
2639			    g_mirror_disk_state2str(disk->d_state)));
2640#endif
2641		}
2642		DISK_STATE_CHANGED();
2643		G_MIRROR_DEBUG(0, "Device %s: provider %s disconnected.",
2644		    sc->sc_name, g_mirror_get_diskname(disk));
2645
2646		g_mirror_destroy_disk(disk);
2647		break;
2648	case G_MIRROR_DISK_STATE_DESTROY:
2649	    {
2650		int error;
2651
2652		error = g_mirror_clear_metadata(disk);
2653		if (error != 0)
2654			return (error);
2655		DISK_STATE_CHANGED();
2656		G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.",
2657		    sc->sc_name, g_mirror_get_diskname(disk));
2658
2659		g_mirror_destroy_disk(disk);
2660		sc->sc_ndisks--;
2661		LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2662			g_mirror_update_metadata(disk);
2663		}
2664		break;
2665	    }
2666	default:
2667		KASSERT(1 == 0, ("Unknown state (%u).", state));
2668		break;
2669	}
2670	return (0);
2671}
2672#undef	DISK_STATE_CHANGED
2673
2674int
2675g_mirror_read_metadata(struct g_consumer *cp, struct g_mirror_metadata *md)
2676{
2677	struct g_provider *pp;
2678	u_char *buf;
2679	int error;
2680
2681	g_topology_assert();
2682
2683	error = g_access(cp, 1, 0, 0);
2684	if (error != 0)
2685		return (error);
2686	pp = cp->provider;
2687	g_topology_unlock();
2688	/* Metadata are stored on last sector. */
2689	buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
2690	    &error);
2691	g_topology_lock();
2692	g_access(cp, -1, 0, 0);
2693	if (buf == NULL) {
2694		G_MIRROR_DEBUG(1, "Cannot read metadata from %s (error=%d).",
2695		    cp->provider->name, error);
2696		return (error);
2697	}
2698
2699	/* Decode metadata. */
2700	error = mirror_metadata_decode(buf, md);
2701	g_free(buf);
2702	if (strcmp(md->md_magic, G_MIRROR_MAGIC) != 0)
2703		return (EINVAL);
2704	if (md->md_version > G_MIRROR_VERSION) {
2705		G_MIRROR_DEBUG(0,
2706		    "Kernel module is too old to handle metadata from %s.",
2707		    cp->provider->name);
2708		return (EINVAL);
2709	}
2710	if (error != 0) {
2711		G_MIRROR_DEBUG(1, "MD5 metadata hash mismatch for provider %s.",
2712		    cp->provider->name);
2713		return (error);
2714	}
2715
2716	return (0);
2717}
2718
2719static int
2720g_mirror_check_metadata(struct g_mirror_softc *sc, struct g_provider *pp,
2721    struct g_mirror_metadata *md)
2722{
2723
2724	if (g_mirror_id2disk(sc, md->md_did) != NULL) {
2725		G_MIRROR_DEBUG(1, "Disk %s (id=%u) already exists, skipping.",
2726		    pp->name, md->md_did);
2727		return (EEXIST);
2728	}
2729	if (md->md_all != sc->sc_ndisks) {
2730		G_MIRROR_DEBUG(1,
2731		    "Invalid '%s' field on disk %s (device %s), skipping.",
2732		    "md_all", pp->name, sc->sc_name);
2733		return (EINVAL);
2734	}
2735	if (md->md_slice != sc->sc_slice) {
2736		G_MIRROR_DEBUG(1,
2737		    "Invalid '%s' field on disk %s (device %s), skipping.",
2738		    "md_slice", pp->name, sc->sc_name);
2739		return (EINVAL);
2740	}
2741	if (md->md_balance != sc->sc_balance) {
2742		G_MIRROR_DEBUG(1,
2743		    "Invalid '%s' field on disk %s (device %s), skipping.",
2744		    "md_balance", pp->name, sc->sc_name);
2745		return (EINVAL);
2746	}
2747#if 0
2748	if (md->md_mediasize != sc->sc_mediasize) {
2749		G_MIRROR_DEBUG(1,
2750		    "Invalid '%s' field on disk %s (device %s), skipping.",
2751		    "md_mediasize", pp->name, sc->sc_name);
2752		return (EINVAL);
2753	}
2754#endif
2755	if (sc->sc_mediasize > pp->mediasize) {
2756		G_MIRROR_DEBUG(1,
2757		    "Invalid size of disk %s (device %s), skipping.", pp->name,
2758		    sc->sc_name);
2759		return (EINVAL);
2760	}
2761	if (md->md_sectorsize != sc->sc_sectorsize) {
2762		G_MIRROR_DEBUG(1,
2763		    "Invalid '%s' field on disk %s (device %s), skipping.",
2764		    "md_sectorsize", pp->name, sc->sc_name);
2765		return (EINVAL);
2766	}
2767	if ((sc->sc_sectorsize % pp->sectorsize) != 0) {
2768		G_MIRROR_DEBUG(1,
2769		    "Invalid sector size of disk %s (device %s), skipping.",
2770		    pp->name, sc->sc_name);
2771		return (EINVAL);
2772	}
2773	if ((md->md_mflags & ~G_MIRROR_DEVICE_FLAG_MASK) != 0) {
2774		G_MIRROR_DEBUG(1,
2775		    "Invalid device flags on disk %s (device %s), skipping.",
2776		    pp->name, sc->sc_name);
2777		return (EINVAL);
2778	}
2779	if ((md->md_dflags & ~G_MIRROR_DISK_FLAG_MASK) != 0) {
2780		G_MIRROR_DEBUG(1,
2781		    "Invalid disk flags on disk %s (device %s), skipping.",
2782		    pp->name, sc->sc_name);
2783		return (EINVAL);
2784	}
2785	return (0);
2786}
2787
2788int
2789g_mirror_add_disk(struct g_mirror_softc *sc, struct g_provider *pp,
2790    struct g_mirror_metadata *md)
2791{
2792	struct g_mirror_disk *disk;
2793	int error;
2794
2795	g_topology_assert_not();
2796	G_MIRROR_DEBUG(2, "Adding disk %s.", pp->name);
2797
2798	error = g_mirror_check_metadata(sc, pp, md);
2799	if (error != 0)
2800		return (error);
2801	if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING &&
2802	    md->md_genid < sc->sc_genid) {
2803		G_MIRROR_DEBUG(0, "Component %s (device %s) broken, skipping.",
2804		    pp->name, sc->sc_name);
2805		return (EINVAL);
2806	}
2807	disk = g_mirror_init_disk(sc, pp, md, &error);
2808	if (disk == NULL)
2809		return (error);
2810	error = g_mirror_event_send(disk, G_MIRROR_DISK_STATE_NEW,
2811	    G_MIRROR_EVENT_WAIT);
2812	if (error != 0)
2813		return (error);
2814	if (md->md_version < G_MIRROR_VERSION) {
2815		G_MIRROR_DEBUG(0, "Upgrading metadata on %s (v%d->v%d).",
2816		    pp->name, md->md_version, G_MIRROR_VERSION);
2817		g_mirror_update_metadata(disk);
2818	}
2819	return (0);
2820}
2821
2822static void
2823g_mirror_destroy_delayed(void *arg, int flag)
2824{
2825	struct g_mirror_softc *sc;
2826	int error;
2827
2828	if (flag == EV_CANCEL) {
2829		G_MIRROR_DEBUG(1, "Destroying canceled.");
2830		return;
2831	}
2832	sc = arg;
2833	g_topology_unlock();
2834	sx_xlock(&sc->sc_lock);
2835	KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) == 0,
2836	    ("DESTROY flag set on %s.", sc->sc_name));
2837	KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0,
2838	    ("DESTROYING flag not set on %s.", sc->sc_name));
2839	G_MIRROR_DEBUG(1, "Destroying %s (delayed).", sc->sc_name);
2840	error = g_mirror_destroy(sc, G_MIRROR_DESTROY_SOFT);
2841	if (error != 0) {
2842		G_MIRROR_DEBUG(0, "Cannot destroy %s (error=%d).",
2843		    sc->sc_name, error);
2844		sx_xunlock(&sc->sc_lock);
2845	}
2846	g_topology_lock();
2847}
2848
2849static int
2850g_mirror_access(struct g_provider *pp, int acr, int acw, int ace)
2851{
2852	struct g_mirror_softc *sc;
2853	int dcr, dcw, dce, error = 0;
2854
2855	g_topology_assert();
2856	G_MIRROR_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr,
2857	    acw, ace);
2858
2859	sc = pp->geom->softc;
2860	if (sc == NULL && acr <= 0 && acw <= 0 && ace <= 0)
2861		return (0);
2862	KASSERT(sc != NULL, ("NULL softc (provider=%s).", pp->name));
2863
2864	dcr = pp->acr + acr;
2865	dcw = pp->acw + acw;
2866	dce = pp->ace + ace;
2867
2868	g_topology_unlock();
2869	sx_xlock(&sc->sc_lock);
2870	if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0 ||
2871	    LIST_EMPTY(&sc->sc_disks)) {
2872		if (acr > 0 || acw > 0 || ace > 0)
2873			error = ENXIO;
2874		goto end;
2875	}
2876	if (dcw == 0)
2877		g_mirror_idle(sc, dcw);
2878	if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0) {
2879		if (acr > 0 || acw > 0 || ace > 0) {
2880			error = ENXIO;
2881			goto end;
2882		}
2883		if (dcr == 0 && dcw == 0 && dce == 0) {
2884			g_post_event(g_mirror_destroy_delayed, sc, M_WAITOK,
2885			    sc, NULL);
2886		}
2887	}
2888end:
2889	sx_xunlock(&sc->sc_lock);
2890	g_topology_lock();
2891	return (error);
2892}
2893
2894static struct g_geom *
2895g_mirror_create(struct g_class *mp, const struct g_mirror_metadata *md)
2896{
2897	struct g_mirror_softc *sc;
2898	struct g_geom *gp;
2899	int error, timeout;
2900
2901	g_topology_assert();
2902	G_MIRROR_DEBUG(1, "Creating device %s (id=%u).", md->md_name,
2903	    md->md_mid);
2904
2905	/* One disk is minimum. */
2906	if (md->md_all < 1)
2907		return (NULL);
2908	/*
2909	 * Action geom.
2910	 */
2911	gp = g_new_geomf(mp, "%s", md->md_name);
2912	sc = malloc(sizeof(*sc), M_MIRROR, M_WAITOK | M_ZERO);
2913	gp->start = g_mirror_start;
2914	gp->orphan = g_mirror_orphan;
2915	gp->access = g_mirror_access;
2916	gp->dumpconf = g_mirror_dumpconf;
2917
2918	sc->sc_id = md->md_mid;
2919	sc->sc_slice = md->md_slice;
2920	sc->sc_balance = md->md_balance;
2921	sc->sc_mediasize = md->md_mediasize;
2922	sc->sc_sectorsize = md->md_sectorsize;
2923	sc->sc_ndisks = md->md_all;
2924	sc->sc_flags = md->md_mflags;
2925	sc->sc_bump_id = 0;
2926	sc->sc_idle = 1;
2927	sc->sc_last_write = time_uptime;
2928	sc->sc_writes = 0;
2929	sx_init(&sc->sc_lock, "gmirror:lock");
2930	bioq_init(&sc->sc_queue);
2931	mtx_init(&sc->sc_queue_mtx, "gmirror:queue", NULL, MTX_DEF);
2932	bioq_init(&sc->sc_regular_delayed);
2933	bioq_init(&sc->sc_inflight);
2934	bioq_init(&sc->sc_sync_delayed);
2935	LIST_INIT(&sc->sc_disks);
2936	TAILQ_INIT(&sc->sc_events);
2937	mtx_init(&sc->sc_events_mtx, "gmirror:events", NULL, MTX_DEF);
2938	callout_init(&sc->sc_callout, CALLOUT_MPSAFE);
2939	mtx_init(&sc->sc_done_mtx, "gmirror:done", NULL, MTX_DEF);
2940	sc->sc_state = G_MIRROR_DEVICE_STATE_STARTING;
2941	gp->softc = sc;
2942	sc->sc_geom = gp;
2943	sc->sc_provider = NULL;
2944	/*
2945	 * Synchronization geom.
2946	 */
2947	gp = g_new_geomf(mp, "%s.sync", md->md_name);
2948	gp->softc = sc;
2949	gp->orphan = g_mirror_orphan;
2950	sc->sc_sync.ds_geom = gp;
2951	sc->sc_sync.ds_ndisks = 0;
2952	error = kproc_create(g_mirror_worker, sc, &sc->sc_worker, 0, 0,
2953	    "g_mirror %s", md->md_name);
2954	if (error != 0) {
2955		G_MIRROR_DEBUG(1, "Cannot create kernel thread for %s.",
2956		    sc->sc_name);
2957		g_destroy_geom(sc->sc_sync.ds_geom);
2958		mtx_destroy(&sc->sc_done_mtx);
2959		mtx_destroy(&sc->sc_events_mtx);
2960		mtx_destroy(&sc->sc_queue_mtx);
2961		sx_destroy(&sc->sc_lock);
2962		g_destroy_geom(sc->sc_geom);
2963		free(sc, M_MIRROR);
2964		return (NULL);
2965	}
2966
2967	G_MIRROR_DEBUG(1, "Device %s created (%u components, id=%u).",
2968	    sc->sc_name, sc->sc_ndisks, sc->sc_id);
2969
2970	sc->sc_rootmount = root_mount_hold("GMIRROR");
2971	G_MIRROR_DEBUG(1, "root_mount_hold %p", sc->sc_rootmount);
2972	/*
2973	 * Run timeout.
2974	 */
2975	timeout = g_mirror_timeout * hz;
2976	callout_reset(&sc->sc_callout, timeout, g_mirror_go, sc);
2977	return (sc->sc_geom);
2978}
2979
2980int
2981g_mirror_destroy(struct g_mirror_softc *sc, int how)
2982{
2983	struct g_mirror_disk *disk;
2984	struct g_provider *pp;
2985
2986	g_topology_assert_not();
2987	if (sc == NULL)
2988		return (ENXIO);
2989	sx_assert(&sc->sc_lock, SX_XLOCKED);
2990
2991	pp = sc->sc_provider;
2992	if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
2993		switch (how) {
2994		case G_MIRROR_DESTROY_SOFT:
2995			G_MIRROR_DEBUG(1,
2996			    "Device %s is still open (r%dw%de%d).", pp->name,
2997			    pp->acr, pp->acw, pp->ace);
2998			return (EBUSY);
2999		case G_MIRROR_DESTROY_DELAYED:
3000			G_MIRROR_DEBUG(1,
3001			    "Device %s will be destroyed on last close.",
3002			    pp->name);
3003			LIST_FOREACH(disk, &sc->sc_disks, d_next) {
3004				if (disk->d_state ==
3005				    G_MIRROR_DISK_STATE_SYNCHRONIZING) {
3006					g_mirror_sync_stop(disk, 1);
3007				}
3008			}
3009			sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROYING;
3010			return (EBUSY);
3011		case G_MIRROR_DESTROY_HARD:
3012			G_MIRROR_DEBUG(1, "Device %s is still open, so it "
3013			    "can't be definitely removed.", pp->name);
3014		}
3015	}
3016
3017	g_topology_lock();
3018	if (sc->sc_geom->softc == NULL) {
3019		g_topology_unlock();
3020		return (0);
3021	}
3022	sc->sc_geom->softc = NULL;
3023	sc->sc_sync.ds_geom->softc = NULL;
3024	g_topology_unlock();
3025
3026	sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
3027	sc->sc_flags |= G_MIRROR_DEVICE_FLAG_WAIT;
3028	G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
3029	sx_xunlock(&sc->sc_lock);
3030	mtx_lock(&sc->sc_queue_mtx);
3031	wakeup(sc);
3032	mtx_unlock(&sc->sc_queue_mtx);
3033	G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, &sc->sc_worker);
3034	while (sc->sc_worker != NULL)
3035		tsleep(&sc->sc_worker, PRIBIO, "m:destroy", hz / 5);
3036	G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, &sc->sc_worker);
3037	sx_xlock(&sc->sc_lock);
3038	g_mirror_destroy_device(sc);
3039	free(sc, M_MIRROR);
3040	return (0);
3041}
3042
3043static void
3044g_mirror_taste_orphan(struct g_consumer *cp)
3045{
3046
3047	KASSERT(1 == 0, ("%s called while tasting %s.", __func__,
3048	    cp->provider->name));
3049}
3050
3051static struct g_geom *
3052g_mirror_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
3053{
3054	struct g_mirror_metadata md;
3055	struct g_mirror_softc *sc;
3056	struct g_consumer *cp;
3057	struct g_geom *gp;
3058	int error;
3059
3060	g_topology_assert();
3061	g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
3062	G_MIRROR_DEBUG(2, "Tasting %s.", pp->name);
3063
3064	gp = g_new_geomf(mp, "mirror:taste");
3065	/*
3066	 * This orphan function should be never called.
3067	 */
3068	gp->orphan = g_mirror_taste_orphan;
3069	cp = g_new_consumer(gp);
3070	g_attach(cp, pp);
3071	error = g_mirror_read_metadata(cp, &md);
3072	g_detach(cp);
3073	g_destroy_consumer(cp);
3074	g_destroy_geom(gp);
3075	if (error != 0)
3076		return (NULL);
3077	gp = NULL;
3078
3079	if (md.md_provider[0] != '\0' &&
3080	    !g_compare_names(md.md_provider, pp->name))
3081		return (NULL);
3082	if (md.md_provsize != 0 && md.md_provsize != pp->mediasize)
3083		return (NULL);
3084	if ((md.md_dflags & G_MIRROR_DISK_FLAG_INACTIVE) != 0) {
3085		G_MIRROR_DEBUG(0,
3086		    "Device %s: provider %s marked as inactive, skipping.",
3087		    md.md_name, pp->name);
3088		return (NULL);
3089	}
3090	if (g_mirror_debug >= 2)
3091		mirror_metadata_dump(&md);
3092
3093	/*
3094	 * Let's check if device already exists.
3095	 */
3096	sc = NULL;
3097	LIST_FOREACH(gp, &mp->geom, geom) {
3098		sc = gp->softc;
3099		if (sc == NULL)
3100			continue;
3101		if (sc->sc_sync.ds_geom == gp)
3102			continue;
3103		if (strcmp(md.md_name, sc->sc_name) != 0)
3104			continue;
3105		if (md.md_mid != sc->sc_id) {
3106			G_MIRROR_DEBUG(0, "Device %s already configured.",
3107			    sc->sc_name);
3108			return (NULL);
3109		}
3110		break;
3111	}
3112	if (gp == NULL) {
3113		gp = g_mirror_create(mp, &md);
3114		if (gp == NULL) {
3115			G_MIRROR_DEBUG(0, "Cannot create device %s.",
3116			    md.md_name);
3117			return (NULL);
3118		}
3119		sc = gp->softc;
3120	}
3121	G_MIRROR_DEBUG(1, "Adding disk %s to %s.", pp->name, gp->name);
3122	g_topology_unlock();
3123	sx_xlock(&sc->sc_lock);
3124	sc->sc_flags |= G_MIRROR_DEVICE_FLAG_TASTING;
3125	error = g_mirror_add_disk(sc, pp, &md);
3126	if (error != 0) {
3127		G_MIRROR_DEBUG(0, "Cannot add disk %s to %s (error=%d).",
3128		    pp->name, gp->name, error);
3129		if (LIST_EMPTY(&sc->sc_disks)) {
3130			g_cancel_event(sc);
3131			g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
3132			g_topology_lock();
3133			return (NULL);
3134		}
3135		gp = NULL;
3136	}
3137	sc->sc_flags &= ~G_MIRROR_DEVICE_FLAG_TASTING;
3138	if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
3139		g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
3140		g_topology_lock();
3141		return (NULL);
3142	}
3143	sx_xunlock(&sc->sc_lock);
3144	g_topology_lock();
3145	return (gp);
3146}
3147
3148static void
3149g_mirror_resize(struct g_consumer *cp)
3150{
3151	struct g_mirror_disk *disk;
3152
3153	g_topology_assert();
3154	g_trace(G_T_TOPOLOGY, "%s(%s)", __func__, cp->provider->name);
3155
3156	disk = cp->private;
3157	if (disk == NULL)
3158		return;
3159	g_topology_unlock();
3160	g_mirror_update_metadata(disk);
3161	g_topology_lock();
3162}
3163
3164static int
3165g_mirror_destroy_geom(struct gctl_req *req __unused,
3166    struct g_class *mp __unused, struct g_geom *gp)
3167{
3168	struct g_mirror_softc *sc;
3169	int error;
3170
3171	g_topology_unlock();
3172	sc = gp->softc;
3173	sx_xlock(&sc->sc_lock);
3174	g_cancel_event(sc);
3175	error = g_mirror_destroy(gp->softc, G_MIRROR_DESTROY_SOFT);
3176	if (error != 0)
3177		sx_xunlock(&sc->sc_lock);
3178	g_topology_lock();
3179	return (error);
3180}
3181
3182static void
3183g_mirror_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
3184    struct g_consumer *cp, struct g_provider *pp)
3185{
3186	struct g_mirror_softc *sc;
3187
3188	g_topology_assert();
3189
3190	sc = gp->softc;
3191	if (sc == NULL)
3192		return;
3193	/* Skip synchronization geom. */
3194	if (gp == sc->sc_sync.ds_geom)
3195		return;
3196	if (pp != NULL) {
3197		/* Nothing here. */
3198	} else if (cp != NULL) {
3199		struct g_mirror_disk *disk;
3200
3201		disk = cp->private;
3202		if (disk == NULL)
3203			return;
3204		g_topology_unlock();
3205		sx_xlock(&sc->sc_lock);
3206		sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)disk->d_id);
3207		if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
3208			sbuf_printf(sb, "%s<Synchronized>", indent);
3209			if (disk->d_sync.ds_offset == 0)
3210				sbuf_printf(sb, "0%%");
3211			else {
3212				sbuf_printf(sb, "%u%%",
3213				    (u_int)((disk->d_sync.ds_offset * 100) /
3214				    sc->sc_provider->mediasize));
3215			}
3216			sbuf_printf(sb, "</Synchronized>\n");
3217			if (disk->d_sync.ds_offset > 0) {
3218				sbuf_printf(sb, "%s<BytesSynced>%jd"
3219				    "</BytesSynced>\n", indent,
3220				    (intmax_t)disk->d_sync.ds_offset);
3221			}
3222		}
3223		sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent,
3224		    disk->d_sync.ds_syncid);
3225		sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent,
3226		    disk->d_genid);
3227		sbuf_printf(sb, "%s<Flags>", indent);
3228		if (disk->d_flags == 0)
3229			sbuf_printf(sb, "NONE");
3230		else {
3231			int first = 1;
3232
3233#define	ADD_FLAG(flag, name)	do {					\
3234	if ((disk->d_flags & (flag)) != 0) {				\
3235		if (!first)						\
3236			sbuf_printf(sb, ", ");				\
3237		else							\
3238			first = 0;					\
3239		sbuf_printf(sb, name);					\
3240	}								\
3241} while (0)
3242			ADD_FLAG(G_MIRROR_DISK_FLAG_DIRTY, "DIRTY");
3243			ADD_FLAG(G_MIRROR_DISK_FLAG_HARDCODED, "HARDCODED");
3244			ADD_FLAG(G_MIRROR_DISK_FLAG_INACTIVE, "INACTIVE");
3245			ADD_FLAG(G_MIRROR_DISK_FLAG_SYNCHRONIZING,
3246			    "SYNCHRONIZING");
3247			ADD_FLAG(G_MIRROR_DISK_FLAG_FORCE_SYNC, "FORCE_SYNC");
3248			ADD_FLAG(G_MIRROR_DISK_FLAG_BROKEN, "BROKEN");
3249#undef	ADD_FLAG
3250		}
3251		sbuf_printf(sb, "</Flags>\n");
3252		sbuf_printf(sb, "%s<Priority>%u</Priority>\n", indent,
3253		    disk->d_priority);
3254		sbuf_printf(sb, "%s<State>%s</State>\n", indent,
3255		    g_mirror_disk_state2str(disk->d_state));
3256		sx_xunlock(&sc->sc_lock);
3257		g_topology_lock();
3258	} else {
3259		g_topology_unlock();
3260		sx_xlock(&sc->sc_lock);
3261		sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)sc->sc_id);
3262		sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent, sc->sc_syncid);
3263		sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent, sc->sc_genid);
3264		sbuf_printf(sb, "%s<Flags>", indent);
3265		if (sc->sc_flags == 0)
3266			sbuf_printf(sb, "NONE");
3267		else {
3268			int first = 1;
3269
3270#define	ADD_FLAG(flag, name)	do {					\
3271	if ((sc->sc_flags & (flag)) != 0) {				\
3272		if (!first)						\
3273			sbuf_printf(sb, ", ");				\
3274		else							\
3275			first = 0;					\
3276		sbuf_printf(sb, name);					\
3277	}								\
3278} while (0)
3279			ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOFAILSYNC, "NOFAILSYNC");
3280			ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOAUTOSYNC, "NOAUTOSYNC");
3281#undef	ADD_FLAG
3282		}
3283		sbuf_printf(sb, "</Flags>\n");
3284		sbuf_printf(sb, "%s<Slice>%u</Slice>\n", indent,
3285		    (u_int)sc->sc_slice);
3286		sbuf_printf(sb, "%s<Balance>%s</Balance>\n", indent,
3287		    balance_name(sc->sc_balance));
3288		sbuf_printf(sb, "%s<Components>%u</Components>\n", indent,
3289		    sc->sc_ndisks);
3290		sbuf_printf(sb, "%s<State>", indent);
3291		if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
3292			sbuf_printf(sb, "%s", "STARTING");
3293		else if (sc->sc_ndisks ==
3294		    g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE))
3295			sbuf_printf(sb, "%s", "COMPLETE");
3296		else
3297			sbuf_printf(sb, "%s", "DEGRADED");
3298		sbuf_printf(sb, "</State>\n");
3299		sx_xunlock(&sc->sc_lock);
3300		g_topology_lock();
3301	}
3302}
3303
3304static void
3305g_mirror_shutdown_post_sync(void *arg, int howto)
3306{
3307	struct g_class *mp;
3308	struct g_geom *gp, *gp2;
3309	struct g_mirror_softc *sc;
3310	int error;
3311
3312	mp = arg;
3313	DROP_GIANT();
3314	g_topology_lock();
3315	g_mirror_shutdown = 1;
3316	LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
3317		if ((sc = gp->softc) == NULL)
3318			continue;
3319		/* Skip synchronization geom. */
3320		if (gp == sc->sc_sync.ds_geom)
3321			continue;
3322		g_topology_unlock();
3323		sx_xlock(&sc->sc_lock);
3324		g_mirror_idle(sc, -1);
3325		g_cancel_event(sc);
3326		error = g_mirror_destroy(sc, G_MIRROR_DESTROY_DELAYED);
3327		if (error != 0)
3328			sx_xunlock(&sc->sc_lock);
3329		g_topology_lock();
3330	}
3331	g_topology_unlock();
3332	PICKUP_GIANT();
3333}
3334
3335static void
3336g_mirror_init(struct g_class *mp)
3337{
3338
3339	g_mirror_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync,
3340	    g_mirror_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST);
3341	if (g_mirror_post_sync == NULL)
3342		G_MIRROR_DEBUG(0, "Warning! Cannot register shutdown event.");
3343}
3344
3345static void
3346g_mirror_fini(struct g_class *mp)
3347{
3348
3349	if (g_mirror_post_sync != NULL)
3350		EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_mirror_post_sync);
3351}
3352
3353DECLARE_GEOM_CLASS(g_mirror_class, g_mirror);
3354