md.c revision 112827 
1/*
2 * ----------------------------------------------------------------------------
3 * "THE BEER-WARE LICENSE" (Revision 42):
4 * <phk@FreeBSD.ORG> wrote this file.  As long as you retain this notice you
5 * can do whatever you want with this stuff. If we meet some day, and you think
6 * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
7 * ----------------------------------------------------------------------------
8 *
9 * $FreeBSD: head/sys/dev/md/md.c 112827 2003-03-29 22:13:32Z phk $
10 *
11 */
12
13/*
14 * The following functions are based in the vn(4) driver: mdstart_swap(),
15 * mdstart_vnode(), mdcreate_swap(), mdcreate_vnode() and mddestroy(),
16 * and as such under the following copyright:
17 *
18 * Copyright (c) 1988 University of Utah.
19 * Copyright (c) 1990, 1993
20 *	The Regents of the University of California.  All rights reserved.
21 *
22 * This code is derived from software contributed to Berkeley by
23 * the Systems Programming Group of the University of Utah Computer
24 * Science Department.
25 *
26 * Redistribution and use in source and binary forms, with or without
27 * modification, are permitted provided that the following conditions
28 * are met:
29 * 1. Redistributions of source code must retain the above copyright
30 *    notice, this list of conditions and the following disclaimer.
31 * 2. Redistributions in binary form must reproduce the above copyright
32 *    notice, this list of conditions and the following disclaimer in the
33 *    documentation and/or other materials provided with the distribution.
34 * 3. All advertising materials mentioning features or use of this software
35 *    must display the following acknowledgement:
36 *	This product includes software developed by the University of
37 *	California, Berkeley and its contributors.
38 * 4. Neither the name of the University nor the names of its contributors
39 *    may be used to endorse or promote products derived from this software
40 *    without specific prior written permission.
41 *
42 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
43 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
44 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
45 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
46 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
47 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
48 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
49 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
50 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
51 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
52 * SUCH DAMAGE.
53 *
54 * from: Utah Hdr: vn.c 1.13 94/04/02
55 *
56 *	from: @(#)vn.c	8.6 (Berkeley) 4/1/94
57 * From: src/sys/dev/vn/vn.c,v 1.122 2000/12/16 16:06:03
58 */
59
60#include "opt_geom.h"
61#include "opt_md.h"
62
63#include <sys/param.h>
64#include <sys/systm.h>
65#include <sys/bio.h>
66#include <sys/conf.h>
67#include <sys/disk.h>
68#include <sys/fcntl.h>
69#include <sys/kernel.h>
70#include <sys/kthread.h>
71#include <sys/linker.h>
72#include <sys/lock.h>
73#include <sys/malloc.h>
74#include <sys/mdioctl.h>
75#include <sys/mutex.h>
76#include <sys/namei.h>
77#include <sys/proc.h>
78#include <sys/queue.h>
79#include <sys/sysctl.h>
80#include <sys/vnode.h>
81
82#include <geom/geom.h>
83
84#include <vm/vm.h>
85#include <vm/vm_object.h>
86#include <vm/vm_page.h>
87#include <vm/vm_pager.h>
88#include <vm/swap_pager.h>
89#include <vm/uma.h>
90
91#define MD_MODVER 1
92
93#define MD_SHUTDOWN 0x10000	/* Tell worker thread to terminate. */
94
95#ifndef MD_NSECT
96#define MD_NSECT (10000 * 2)
97#endif
98
99static MALLOC_DEFINE(M_MD, "MD disk", "Memory Disk");
100static MALLOC_DEFINE(M_MDSECT, "MD sectors", "Memory Disk Sectors");
101
102static int md_debug;
103SYSCTL_INT(_debug, OID_AUTO, mddebug, CTLFLAG_RW, &md_debug, 0, "");
104
105#if defined(MD_ROOT) && defined(MD_ROOT_SIZE)
106/* Image gets put here: */
107static u_char mfs_root[MD_ROOT_SIZE*1024] = "MFS Filesystem goes here";
108static u_char end_mfs_root[] __unused = "MFS Filesystem had better STOP here";
109#endif
110
111static int	mdrootready;
112static int	mdunits;
113static dev_t	status_dev = 0;
114
115#define CDEV_MAJOR	95
116
117static d_ioctl_t mdctlioctl;
118
119static struct cdevsw mdctl_cdevsw = {
120	.d_open =	nullopen,
121	.d_close =	nullclose,
122	.d_ioctl =	mdctlioctl,
123	.d_name =	MD_NAME,
124	.d_maj =	CDEV_MAJOR
125};
126
127
128static LIST_HEAD(, md_s) md_softc_list = LIST_HEAD_INITIALIZER(&md_softc_list);
129
130#define NINDIR	(PAGE_SIZE / sizeof(uintptr_t))
131#define NMASK	(NINDIR-1)
132static int nshift;
133
134struct indir {
135	uintptr_t	*array;
136	uint		total;
137	uint		used;
138	uint		shift;
139};
140
141struct md_s {
142	int unit;
143	LIST_ENTRY(md_s) list;
144	struct bio_queue_head bio_queue;
145	struct mtx queue_mtx;
146	struct disk disk;
147	dev_t dev;
148	enum md_types type;
149	unsigned nsect;
150	unsigned opencount;
151	unsigned secsize;
152	unsigned flags;
153	char name[20];
154	struct proc *procp;
155	struct g_geom *gp;
156	struct g_provider *pp;
157
158	/* MD_MALLOC related fields */
159	struct indir *indir;
160	uma_zone_t uma;
161
162	/* MD_PRELOAD related fields */
163	u_char *pl_ptr;
164	unsigned pl_len;
165
166	/* MD_VNODE related fields */
167	struct vnode *vnode;
168	struct ucred *cred;
169
170	/* MD_SWAP related fields */
171	vm_object_t object;
172};
173
174static int mddestroy(struct md_s *sc, struct thread *td);
175
176static struct indir *
177new_indir(uint shift)
178{
179	struct indir *ip;
180
181	ip = malloc(sizeof *ip, M_MD, M_NOWAIT | M_ZERO);
182	if (ip == NULL)
183		return (NULL);
184	ip->array = malloc(sizeof(uintptr_t) * NINDIR,
185	    M_MDSECT, M_NOWAIT | M_ZERO);
186	if (ip->array == NULL) {
187		free(ip, M_MD);
188		return (NULL);
189	}
190	ip->total = NINDIR;
191	ip->shift = shift;
192	return (ip);
193}
194
195static void
196del_indir(struct indir *ip)
197{
198
199	free(ip->array, M_MDSECT);
200	free(ip, M_MD);
201}
202
203static void
204destroy_indir(struct md_s *sc, struct indir *ip)
205{
206	int i;
207
208	for (i = 0; i < NINDIR; i++) {
209		if (!ip->array[i])
210			continue;
211		if (ip->shift)
212			destroy_indir(sc, (struct indir*)(ip->array[i]));
213		else if (ip->array[i] > 255)
214			uma_zfree(sc->uma, (void *)(ip->array[i]));
215	}
216	del_indir(ip);
217}
218
219/*
220 * This function does the math and alloctes the top level "indir" structure
221 * for a device of "size" sectors.
222 */
223
224static struct indir *
225dimension(off_t size)
226{
227	off_t rcnt;
228	struct indir *ip;
229	int i, layer;
230
231	rcnt = size;
232	layer = 0;
233	while (rcnt > NINDIR) {
234		rcnt /= NINDIR;
235		layer++;
236	}
237	/* figure out log2(NINDIR) */
238	for (i = NINDIR, nshift = -1; i; nshift++)
239		i >>= 1;
240
241	/*
242	 * XXX: the top layer is probably not fully populated, so we allocate
243	 * too much space for ip->array in here.
244	 */
245	ip = malloc(sizeof *ip, M_MD, M_WAITOK | M_ZERO);
246	ip->array = malloc(sizeof(uintptr_t) * NINDIR,
247	    M_MDSECT, M_WAITOK | M_ZERO);
248	ip->total = NINDIR;
249	ip->shift = layer * nshift;
250	return (ip);
251}
252
253/*
254 * Read a given sector
255 */
256
257static uintptr_t
258s_read(struct indir *ip, off_t offset)
259{
260	struct indir *cip;
261	int idx;
262	uintptr_t up;
263
264	if (md_debug > 1)
265		printf("s_read(%jd)\n", (intmax_t)offset);
266	up = 0;
267	for (cip = ip; cip != NULL;) {
268		if (cip->shift) {
269			idx = (offset >> cip->shift) & NMASK;
270			up = cip->array[idx];
271			cip = (struct indir *)up;
272			continue;
273		}
274		idx = offset & NMASK;
275		return (cip->array[idx]);
276	}
277	return (0);
278}
279
280/*
281 * Write a given sector, prune the tree if the value is 0
282 */
283
284static int
285s_write(struct indir *ip, off_t offset, uintptr_t ptr)
286{
287	struct indir *cip, *lip[10];
288	int idx, li;
289	uintptr_t up;
290
291	if (md_debug > 1)
292		printf("s_write(%jd, %p)\n", (intmax_t)offset, (void *)ptr);
293	up = 0;
294	li = 0;
295	cip = ip;
296	for (;;) {
297		lip[li++] = cip;
298		if (cip->shift) {
299			idx = (offset >> cip->shift) & NMASK;
300			up = cip->array[idx];
301			if (up != 0) {
302				cip = (struct indir *)up;
303				continue;
304			}
305			/* Allocate branch */
306			cip->array[idx] =
307			    (uintptr_t)new_indir(cip->shift - nshift);
308			if (cip->array[idx] == 0)
309				return (ENOSPC);
310			cip->used++;
311			up = cip->array[idx];
312			cip = (struct indir *)up;
313			continue;
314		}
315		/* leafnode */
316		idx = offset & NMASK;
317		up = cip->array[idx];
318		if (up != 0)
319			cip->used--;
320		cip->array[idx] = ptr;
321		if (ptr != 0)
322			cip->used++;
323		break;
324	}
325	if (cip->used != 0 || li == 1)
326		return (0);
327	li--;
328	while (cip->used == 0 && cip != ip) {
329		li--;
330		idx = (offset >> lip[li]->shift) & NMASK;
331		up = lip[li]->array[idx];
332		KASSERT(up == (uintptr_t)cip, ("md screwed up"));
333		del_indir(cip);
334		lip[li]->array[idx] = 0;
335		lip[li]->used--;
336		cip = lip[li];
337	}
338	return (0);
339}
340
341
342struct g_class g_md_class = {
343	.name = "MD",
344	G_CLASS_INITIALIZER
345};
346
347static int
348g_md_access(struct g_provider *pp, int r, int w, int e)
349{
350	struct md_s *sc;
351
352	sc = pp->geom->softc;
353	r += pp->acr;
354	w += pp->acw;
355	e += pp->ace;
356	if ((pp->acr + pp->acw + pp->ace) == 0 && (r + w + e) > 0) {
357		sc->opencount = 1;
358	} else if ((pp->acr + pp->acw + pp->ace) > 0 && (r + w + e) == 0) {
359		sc->opencount = 0;
360	}
361	return (0);
362}
363
364static void
365g_md_start(struct bio *bp)
366{
367	struct md_s *sc;
368
369	sc = bp->bio_to->geom->softc;
370
371	switch(bp->bio_cmd) {
372	case BIO_GETATTR:
373	case BIO_SETATTR:
374		g_io_deliver(bp, EOPNOTSUPP);
375		return;
376	}
377	bp->bio_blkno = bp->bio_offset >> DEV_BSHIFT;
378	bp->bio_pblkno = bp->bio_offset / sc->secsize;
379	bp->bio_bcount = bp->bio_length;
380	mtx_lock(&sc->queue_mtx);
381	bioqdisksort(&sc->bio_queue, bp);
382	mtx_unlock(&sc->queue_mtx);
383
384	wakeup(sc);
385}
386
387DECLARE_GEOM_CLASS(g_md_class, g_md);
388
389
390static int
391mdstart_malloc(struct md_s *sc, struct bio *bp)
392{
393	int i, error;
394	u_char *dst;
395	unsigned secno, nsec, uc;
396	uintptr_t sp, osp;
397
398	nsec = bp->bio_bcount / sc->secsize;
399	secno = bp->bio_pblkno;
400	dst = bp->bio_data;
401	error = 0;
402	while (nsec--) {
403		osp = s_read(sc->indir, secno);
404		if (bp->bio_cmd == BIO_DELETE) {
405			if (osp != 0)
406				error = s_write(sc->indir, secno, 0);
407		} else if (bp->bio_cmd == BIO_READ) {
408			if (osp == 0)
409				bzero(dst, sc->secsize);
410			else if (osp <= 255)
411				for (i = 0; i < sc->secsize; i++)
412					dst[i] = osp;
413			else
414				bcopy((void *)osp, dst, sc->secsize);
415			osp = 0;
416		} else if (bp->bio_cmd == BIO_WRITE) {
417			if (sc->flags & MD_COMPRESS) {
418				uc = dst[0];
419				for (i = 1; i < sc->secsize; i++)
420					if (dst[i] != uc)
421						break;
422			} else {
423				i = 0;
424				uc = 0;
425			}
426			if (i == sc->secsize) {
427				if (osp != uc)
428					error = s_write(sc->indir, secno, uc);
429			} else {
430				if (osp <= 255) {
431					sp = (uintptr_t) uma_zalloc(
432					    sc->uma, M_NOWAIT);
433					if (sp == 0) {
434						error = ENOSPC;
435						break;
436					}
437					bcopy(dst, (void *)sp, sc->secsize);
438					error = s_write(sc->indir, secno, sp);
439				} else {
440					bcopy(dst, (void *)osp, sc->secsize);
441					osp = 0;
442				}
443			}
444		} else {
445			error = EOPNOTSUPP;
446		}
447		if (osp > 255)
448			uma_zfree(sc->uma, (void*)osp);
449		if (error)
450			break;
451		secno++;
452		dst += sc->secsize;
453	}
454	bp->bio_resid = 0;
455	return (error);
456}
457
458static int
459mdstart_preload(struct md_s *sc, struct bio *bp)
460{
461
462	if (bp->bio_cmd == BIO_DELETE) {
463	} else if (bp->bio_cmd == BIO_READ) {
464		bcopy(sc->pl_ptr + (bp->bio_pblkno << DEV_BSHIFT), bp->bio_data, bp->bio_bcount);
465	} else {
466		bcopy(bp->bio_data, sc->pl_ptr + (bp->bio_pblkno << DEV_BSHIFT), bp->bio_bcount);
467	}
468	bp->bio_resid = 0;
469	return (0);
470}
471
472static int
473mdstart_vnode(struct md_s *sc, struct bio *bp)
474{
475	int error;
476	struct uio auio;
477	struct iovec aiov;
478	struct mount *mp;
479
480	/*
481	 * VNODE I/O
482	 *
483	 * If an error occurs, we set BIO_ERROR but we do not set
484	 * B_INVAL because (for a write anyway), the buffer is
485	 * still valid.
486	 */
487
488	bzero(&auio, sizeof(auio));
489
490	aiov.iov_base = bp->bio_data;
491	aiov.iov_len = bp->bio_bcount;
492	auio.uio_iov = &aiov;
493	auio.uio_iovcnt = 1;
494	auio.uio_offset = (vm_ooffset_t)bp->bio_pblkno * sc->secsize;
495	auio.uio_segflg = UIO_SYSSPACE;
496	if(bp->bio_cmd == BIO_READ)
497		auio.uio_rw = UIO_READ;
498	else
499		auio.uio_rw = UIO_WRITE;
500	auio.uio_resid = bp->bio_bcount;
501	auio.uio_td = curthread;
502	/*
503	 * When reading set IO_DIRECT to try to avoid double-caching
504	 * the data.  When writing IO_DIRECT is not optimal, but we
505	 * must set IO_NOWDRAIN to avoid a wdrain deadlock.
506	 */
507	if (bp->bio_cmd == BIO_READ) {
508		vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY, curthread);
509		error = VOP_READ(sc->vnode, &auio, IO_DIRECT, sc->cred);
510	} else {
511		(void) vn_start_write(sc->vnode, &mp, V_WAIT);
512		vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY, curthread);
513		error = VOP_WRITE(sc->vnode, &auio, IO_NOWDRAIN, sc->cred);
514		vn_finished_write(mp);
515	}
516	VOP_UNLOCK(sc->vnode, 0, curthread);
517	bp->bio_resid = auio.uio_resid;
518	return (error);
519}
520
521static void
522mddone_swap(struct bio *bp)
523{
524
525	bp->bio_completed = bp->bio_length - bp->bio_resid;
526	g_std_done(bp);
527}
528
529static int
530mdstart_swap(struct md_s *sc, struct bio *bp)
531{
532	{
533	struct bio *bp2;
534
535	bp2 = g_clone_bio(bp);
536	bp2->bio_done = mddone_swap;
537	bp2->bio_blkno = bp2->bio_offset >> DEV_BSHIFT;
538	bp2->bio_pblkno = bp2->bio_offset / sc->secsize;
539	bp2->bio_bcount = bp2->bio_length;
540	bp = bp2;
541	}
542
543	bp->bio_resid = 0;
544	if ((bp->bio_cmd == BIO_DELETE) && (sc->flags & MD_RESERVE))
545		biodone(bp);
546	else
547		vm_pager_strategy(sc->object, bp);
548	return (-1);
549}
550
551static void
552md_kthread(void *arg)
553{
554	struct md_s *sc;
555	struct bio *bp;
556	int error, hasgiant;
557
558	sc = arg;
559	curthread->td_base_pri = PRIBIO;
560
561	switch (sc->type) {
562	case MD_SWAP:
563	case MD_VNODE:
564		mtx_lock(&Giant);
565		hasgiant = 1;
566		break;
567	case MD_MALLOC:
568	case MD_PRELOAD:
569	default:
570		hasgiant = 0;
571		break;
572	}
573
574	for (;;) {
575		mtx_lock(&sc->queue_mtx);
576		bp = bioq_first(&sc->bio_queue);
577		if (bp)
578			bioq_remove(&sc->bio_queue, bp);
579		if (!bp) {
580			if (sc->flags & MD_SHUTDOWN) {
581				mtx_unlock(&sc->queue_mtx);
582				sc->procp = NULL;
583				wakeup(&sc->procp);
584				if (!hasgiant)
585					mtx_lock(&Giant);
586				kthread_exit(0);
587			}
588			msleep(sc, &sc->queue_mtx, PRIBIO | PDROP, "mdwait", 0);
589			continue;
590		}
591		mtx_unlock(&sc->queue_mtx);
592
593		switch (sc->type) {
594		case MD_MALLOC:
595			error = mdstart_malloc(sc, bp);
596			break;
597		case MD_PRELOAD:
598			error = mdstart_preload(sc, bp);
599			break;
600		case MD_VNODE:
601			error = mdstart_vnode(sc, bp);
602			break;
603		case MD_SWAP:
604			error = mdstart_swap(sc, bp);
605			break;
606		default:
607			panic("Impossible md(type)");
608			break;
609		}
610
611		if (error != -1) {
612			bp->bio_completed = bp->bio_length;
613			g_io_deliver(bp, error);
614		}
615	}
616}
617
618static struct md_s *
619mdfind(int unit)
620{
621	struct md_s *sc;
622
623	/* XXX: LOCK(unique unit numbers) */
624	LIST_FOREACH(sc, &md_softc_list, list) {
625		if (sc->unit == unit)
626			break;
627	}
628	/* XXX: UNLOCK(unique unit numbers) */
629	return (sc);
630}
631
632static struct md_s *
633mdnew(int unit)
634{
635	struct md_s *sc;
636	int error, max = -1;
637
638	/* XXX: LOCK(unique unit numbers) */
639	LIST_FOREACH(sc, &md_softc_list, list) {
640		if (sc->unit == unit) {
641			/* XXX: UNLOCK(unique unit numbers) */
642			return (NULL);
643		}
644		if (sc->unit > max)
645			max = sc->unit;
646	}
647	if (unit == -1)
648		unit = max + 1;
649	if (unit > 255)
650		return (NULL);
651	sc = (struct md_s *)malloc(sizeof *sc, M_MD, M_WAITOK | M_ZERO);
652	sc->unit = unit;
653	bioq_init(&sc->bio_queue);
654	mtx_init(&sc->queue_mtx, "md bio queue", NULL, MTX_DEF);
655	sprintf(sc->name, "md%d", unit);
656	error = kthread_create(md_kthread, sc, &sc->procp, 0, 0,"%s", sc->name);
657	if (error) {
658		free(sc, M_MD);
659		return (NULL);
660	}
661	LIST_INSERT_HEAD(&md_softc_list, sc, list);
662	/* XXX: UNLOCK(unique unit numbers) */
663	return (sc);
664}
665
666static void
667mdinit(struct md_s *sc)
668{
669
670	struct g_geom *gp;
671	struct g_provider *pp;
672
673	DROP_GIANT();
674	g_topology_lock();
675	gp = g_new_geomf(&g_md_class, "md%d", sc->unit);
676	gp->start = g_md_start;
677	gp->access = g_md_access;
678	gp->softc = sc;
679	pp = g_new_providerf(gp, "md%d", sc->unit);
680	pp->mediasize = (off_t)sc->nsect * sc->secsize;
681	pp->sectorsize = sc->secsize;
682	sc->gp = gp;
683	sc->pp = pp;
684	g_error_provider(pp, 0);
685	g_topology_unlock();
686	PICKUP_GIANT();
687}
688
689/*
690 * XXX: we should check that the range they feed us is mapped.
691 * XXX: we should implement read-only.
692 */
693
694static int
695mdcreate_preload(struct md_ioctl *mdio)
696{
697	struct md_s *sc;
698
699	if (mdio->md_size == 0)
700		return (EINVAL);
701	if (mdio->md_options & ~(MD_AUTOUNIT))
702		return (EINVAL);
703	if (mdio->md_options & MD_AUTOUNIT) {
704		sc = mdnew(-1);
705		if (sc == NULL)
706			return (ENOMEM);
707		mdio->md_unit = sc->unit;
708	} else {
709		sc = mdnew(mdio->md_unit);
710		if (sc == NULL)
711			return (EBUSY);
712	}
713	sc->type = MD_PRELOAD;
714	sc->secsize = DEV_BSIZE;
715	sc->nsect = mdio->md_size;
716	sc->flags = mdio->md_options & MD_FORCE;
717	/* Cast to pointer size, then to pointer to avoid warning */
718	sc->pl_ptr = (u_char *)(uintptr_t)mdio->md_base;
719	sc->pl_len = (mdio->md_size << DEV_BSHIFT);
720	mdinit(sc);
721	return (0);
722}
723
724
725static int
726mdcreate_malloc(struct md_ioctl *mdio)
727{
728	struct md_s *sc;
729	off_t u;
730	uintptr_t sp;
731	int error;
732
733	error = 0;
734	if (mdio->md_size == 0)
735		return (EINVAL);
736	if (mdio->md_options & ~(MD_AUTOUNIT | MD_COMPRESS | MD_RESERVE))
737		return (EINVAL);
738	if (mdio->md_secsize != 0 && !powerof2(mdio->md_secsize))
739		return (EINVAL);
740	/* Compression doesn't make sense if we have reserved space */
741	if (mdio->md_options & MD_RESERVE)
742		mdio->md_options &= ~MD_COMPRESS;
743	if (mdio->md_options & MD_AUTOUNIT) {
744		sc = mdnew(-1);
745		if (sc == NULL)
746			return (ENOMEM);
747		mdio->md_unit = sc->unit;
748	} else {
749		sc = mdnew(mdio->md_unit);
750		if (sc == NULL)
751			return (EBUSY);
752	}
753	sc->type = MD_MALLOC;
754	if (mdio->md_secsize != 0)
755		sc->secsize = mdio->md_secsize;
756	else
757		sc->secsize = DEV_BSIZE;
758	sc->nsect = mdio->md_size;
759	sc->nsect /= (sc->secsize / DEV_BSIZE);
760	sc->flags = mdio->md_options & (MD_COMPRESS | MD_FORCE);
761	sc->indir = dimension(sc->nsect);
762	sc->uma = uma_zcreate(sc->name, sc->secsize,
763	    NULL, NULL, NULL, NULL, 0x1ff, 0);
764	if (mdio->md_options & MD_RESERVE) {
765		for (u = 0; u < sc->nsect; u++) {
766			sp = (uintptr_t) uma_zalloc(sc->uma, M_NOWAIT | M_ZERO);
767			if (sp != 0)
768				error = s_write(sc->indir, u, sp);
769			else
770				error = ENOMEM;
771			if (error)
772				break;
773		}
774	}
775	if (error)  {
776		mddestroy(sc, NULL);
777		return (error);
778	}
779	mdinit(sc);
780	if (!(mdio->md_options & MD_RESERVE))
781		sc->pp->flags |= G_PF_CANDELETE;
782	return (0);
783}
784
785
786static int
787mdsetcred(struct md_s *sc, struct ucred *cred)
788{
789	char *tmpbuf;
790	int error = 0;
791
792	/*
793	 * Set credits in our softc
794	 */
795
796	if (sc->cred)
797		crfree(sc->cred);
798	sc->cred = crhold(cred);
799
800	/*
801	 * Horrible kludge to establish credentials for NFS  XXX.
802	 */
803
804	if (sc->vnode) {
805		struct uio auio;
806		struct iovec aiov;
807
808		tmpbuf = malloc(sc->secsize, M_TEMP, M_WAITOK);
809		bzero(&auio, sizeof(auio));
810
811		aiov.iov_base = tmpbuf;
812		aiov.iov_len = sc->secsize;
813		auio.uio_iov = &aiov;
814		auio.uio_iovcnt = 1;
815		auio.uio_offset = 0;
816		auio.uio_rw = UIO_READ;
817		auio.uio_segflg = UIO_SYSSPACE;
818		auio.uio_resid = aiov.iov_len;
819		vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY, curthread);
820		error = VOP_READ(sc->vnode, &auio, 0, sc->cred);
821		VOP_UNLOCK(sc->vnode, 0, curthread);
822		free(tmpbuf, M_TEMP);
823	}
824	return (error);
825}
826
827static int
828mdcreate_vnode(struct md_ioctl *mdio, struct thread *td)
829{
830	struct md_s *sc;
831	struct vattr vattr;
832	struct nameidata nd;
833	int error, flags;
834
835	flags = FREAD|FWRITE;
836	NDINIT(&nd, LOOKUP, FOLLOW, UIO_USERSPACE, mdio->md_file, td);
837	error = vn_open(&nd, &flags, 0);
838	if (error) {
839		if (error != EACCES && error != EPERM && error != EROFS)
840			return (error);
841		flags &= ~FWRITE;
842		NDINIT(&nd, LOOKUP, FOLLOW, UIO_USERSPACE, mdio->md_file, td);
843		error = vn_open(&nd, &flags, 0);
844		if (error)
845			return (error);
846	}
847	NDFREE(&nd, NDF_ONLY_PNBUF);
848	if (nd.ni_vp->v_type != VREG ||
849	    (error = VOP_GETATTR(nd.ni_vp, &vattr, td->td_ucred, td))) {
850		VOP_UNLOCK(nd.ni_vp, 0, td);
851		(void) vn_close(nd.ni_vp, flags, td->td_ucred, td);
852		return (error ? error : EINVAL);
853	}
854	VOP_UNLOCK(nd.ni_vp, 0, td);
855
856	if (mdio->md_options & MD_AUTOUNIT) {
857		sc = mdnew(-1);
858		mdio->md_unit = sc->unit;
859	} else {
860		sc = mdnew(mdio->md_unit);
861	}
862	if (sc == NULL) {
863		(void) vn_close(nd.ni_vp, flags, td->td_ucred, td);
864		return (EBUSY);
865	}
866
867	sc->type = MD_VNODE;
868	sc->flags = mdio->md_options & MD_FORCE;
869	if (!(flags & FWRITE))
870		sc->flags |= MD_READONLY;
871	sc->secsize = DEV_BSIZE;
872	sc->vnode = nd.ni_vp;
873
874	/*
875	 * If the size is specified, override the file attributes.
876	 */
877	if (mdio->md_size)
878		sc->nsect = mdio->md_size;
879	else
880		sc->nsect = vattr.va_size / sc->secsize; /* XXX: round up ? */
881	if (sc->nsect == 0) {
882		mddestroy(sc, td);
883		return (EINVAL);
884	}
885	error = mdsetcred(sc, td->td_ucred);
886	if (error) {
887		mddestroy(sc, td);
888		return (error);
889	}
890	mdinit(sc);
891	return (0);
892}
893
894static int
895mddestroy(struct md_s *sc, struct thread *td)
896{
897
898	GIANT_REQUIRED;
899
900	mtx_destroy(&sc->queue_mtx);
901	if (sc->gp) {
902		sc->gp->flags |= G_GEOM_WITHER;
903		sc->gp->softc = NULL;
904	}
905	if (sc->pp)
906		g_orphan_provider(sc->pp, ENXIO);
907	sc->flags |= MD_SHUTDOWN;
908	wakeup(sc);
909	while (sc->procp != NULL)
910		tsleep(&sc->procp, PRIBIO, "mddestroy", hz / 10);
911	if (sc->vnode != NULL)
912		(void)vn_close(sc->vnode, sc->flags & MD_READONLY ?
913		    FREAD : (FREAD|FWRITE), sc->cred, td);
914	if (sc->cred != NULL)
915		crfree(sc->cred);
916	if (sc->object != NULL) {
917		vm_pager_deallocate(sc->object);
918	}
919	if (sc->indir)
920		destroy_indir(sc, sc->indir);
921	if (sc->uma)
922		uma_zdestroy(sc->uma);
923
924	/* XXX: LOCK(unique unit numbers) */
925	LIST_REMOVE(sc, list);
926	/* XXX: UNLOCK(unique unit numbers) */
927	free(sc, M_MD);
928	return (0);
929}
930
931static int
932mdcreate_swap(struct md_ioctl *mdio, struct thread *td)
933{
934	int error;
935	struct md_s *sc;
936
937	GIANT_REQUIRED;
938
939	if (mdio->md_options & MD_AUTOUNIT) {
940		sc = mdnew(-1);
941		mdio->md_unit = sc->unit;
942	} else {
943		sc = mdnew(mdio->md_unit);
944	}
945	if (sc == NULL)
946		return (EBUSY);
947
948	sc->type = MD_SWAP;
949
950	/*
951	 * Range check.  Disallow negative sizes or any size less then the
952	 * size of a page.  Then round to a page.
953	 */
954
955	if (mdio->md_size == 0) {
956		mddestroy(sc, td);
957		return (EDOM);
958	}
959
960	/*
961	 * Allocate an OBJT_SWAP object.
962	 *
963	 * sc_secsize is PAGE_SIZE'd
964	 *
965	 * mdio->size is in DEV_BSIZE'd chunks.
966	 * Note the truncation.
967	 */
968
969	sc->secsize = PAGE_SIZE;
970	sc->nsect = mdio->md_size / (PAGE_SIZE / DEV_BSIZE);
971	sc->object = vm_pager_allocate(OBJT_SWAP, NULL, sc->secsize * (vm_offset_t)sc->nsect, VM_PROT_DEFAULT, 0);
972	sc->flags = mdio->md_options & MD_FORCE;
973	if (mdio->md_options & MD_RESERVE) {
974		if (swap_pager_reserve(sc->object, 0, sc->nsect) < 0) {
975			vm_pager_deallocate(sc->object);
976			sc->object = NULL;
977			mddestroy(sc, td);
978			return (EDOM);
979		}
980	}
981	error = mdsetcred(sc, td->td_ucred);
982	if (error) {
983		mddestroy(sc, td);
984		return (error);
985	}
986	mdinit(sc);
987	if (!(mdio->md_options & MD_RESERVE))
988		sc->pp->flags |= G_PF_CANDELETE;
989	return (0);
990}
991
992static int
993mddetach(int unit, struct thread *td)
994{
995	struct md_s *sc;
996
997	sc = mdfind(unit);
998	if (sc == NULL)
999		return (ENOENT);
1000	if (sc->opencount != 0 && !(sc->flags & MD_FORCE))
1001		return (EBUSY);
1002	switch(sc->type) {
1003	case MD_VNODE:
1004	case MD_SWAP:
1005	case MD_MALLOC:
1006	case MD_PRELOAD:
1007		return (mddestroy(sc, td));
1008	default:
1009		return (EOPNOTSUPP);
1010	}
1011}
1012
1013static int
1014mdctlioctl(dev_t dev, u_long cmd, caddr_t addr, int flags, struct thread *td)
1015{
1016	struct md_ioctl *mdio;
1017	struct md_s *sc;
1018	int i;
1019
1020	if (md_debug)
1021		printf("mdctlioctl(%s %lx %p %x %p)\n",
1022			devtoname(dev), cmd, addr, flags, td);
1023
1024	/*
1025	 * We assert the version number in the individual ioctl
1026	 * handlers instead of out here because (a) it is possible we
1027	 * may add another ioctl in the future which doesn't read an
1028	 * mdio, and (b) the correct return value for an unknown ioctl
1029	 * is ENOIOCTL, not EINVAL.
1030	 */
1031	mdio = (struct md_ioctl *)addr;
1032	switch (cmd) {
1033	case MDIOCATTACH:
1034		if (mdio->md_version != MDIOVERSION)
1035			return (EINVAL);
1036		switch (mdio->md_type) {
1037		case MD_MALLOC:
1038			return (mdcreate_malloc(mdio));
1039		case MD_PRELOAD:
1040			return (mdcreate_preload(mdio));
1041		case MD_VNODE:
1042			return (mdcreate_vnode(mdio, td));
1043		case MD_SWAP:
1044			return (mdcreate_swap(mdio, td));
1045		default:
1046			return (EINVAL);
1047		}
1048	case MDIOCDETACH:
1049		if (mdio->md_version != MDIOVERSION)
1050			return (EINVAL);
1051		if (mdio->md_file != NULL || mdio->md_size != 0 ||
1052		    mdio->md_options != 0)
1053			return (EINVAL);
1054		return (mddetach(mdio->md_unit, td));
1055	case MDIOCQUERY:
1056		if (mdio->md_version != MDIOVERSION)
1057			return (EINVAL);
1058		sc = mdfind(mdio->md_unit);
1059		if (sc == NULL)
1060			return (ENOENT);
1061		mdio->md_type = sc->type;
1062		mdio->md_options = sc->flags;
1063		switch (sc->type) {
1064		case MD_MALLOC:
1065			mdio->md_size = sc->nsect;
1066			break;
1067		case MD_PRELOAD:
1068			mdio->md_size = sc->nsect;
1069			mdio->md_base = (uint64_t)(intptr_t)sc->pl_ptr;
1070			break;
1071		case MD_SWAP:
1072			mdio->md_size = sc->nsect * (PAGE_SIZE / DEV_BSIZE);
1073			break;
1074		case MD_VNODE:
1075			mdio->md_size = sc->nsect;
1076			/* XXX fill this in */
1077			mdio->md_file = NULL;
1078			break;
1079		}
1080		return (0);
1081	case MDIOCLIST:
1082		i = 1;
1083		LIST_FOREACH(sc, &md_softc_list, list) {
1084			if (i == MDNPAD - 1)
1085				mdio->md_pad[i] = -1;
1086			else
1087				mdio->md_pad[i++] = sc->unit;
1088		}
1089		mdio->md_pad[0] = i - 1;
1090		return (0);
1091	default:
1092		return (ENOIOCTL);
1093	};
1094	return (ENOIOCTL);
1095}
1096
1097static void
1098md_preloaded(u_char *image, unsigned length)
1099{
1100	struct md_s *sc;
1101
1102	sc = mdnew(-1);
1103	if (sc == NULL)
1104		return;
1105	sc->type = MD_PRELOAD;
1106	sc->secsize = DEV_BSIZE;
1107	sc->nsect = length / DEV_BSIZE;
1108	sc->pl_ptr = image;
1109	sc->pl_len = length;
1110	if (sc->unit == 0)
1111		mdrootready = 1;
1112	mdinit(sc);
1113}
1114
1115static void
1116md_drvinit(void *unused)
1117{
1118
1119	caddr_t mod;
1120	caddr_t c;
1121	u_char *ptr, *name, *type;
1122	unsigned len;
1123
1124#ifdef MD_ROOT_SIZE
1125	md_preloaded(mfs_root, MD_ROOT_SIZE*1024);
1126#endif
1127	mod = NULL;
1128	while ((mod = preload_search_next_name(mod)) != NULL) {
1129		name = (char *)preload_search_info(mod, MODINFO_NAME);
1130		type = (char *)preload_search_info(mod, MODINFO_TYPE);
1131		if (name == NULL)
1132			continue;
1133		if (type == NULL)
1134			continue;
1135		if (strcmp(type, "md_image") && strcmp(type, "mfs_root"))
1136			continue;
1137		c = preload_search_info(mod, MODINFO_ADDR);
1138		ptr = *(u_char **)c;
1139		c = preload_search_info(mod, MODINFO_SIZE);
1140		len = *(size_t *)c;
1141		printf("%s%d: Preloaded image <%s> %d bytes at %p\n",
1142		    MD_NAME, mdunits, name, len, ptr);
1143		md_preloaded(ptr, len);
1144	}
1145	status_dev = make_dev(&mdctl_cdevsw, 0xffff00ff, UID_ROOT, GID_WHEEL,
1146	    0600, MDCTL_NAME);
1147}
1148
1149static int
1150md_modevent(module_t mod, int type, void *data)
1151{
1152	int error;
1153	struct md_s *sc;
1154
1155	switch (type) {
1156	case MOD_LOAD:
1157		md_drvinit(NULL);
1158		break;
1159	case MOD_UNLOAD:
1160		LIST_FOREACH(sc, &md_softc_list, list) {
1161			error = mddetach(sc->unit, curthread);
1162			if (error != 0)
1163				return (error);
1164		}
1165		if (status_dev)
1166			destroy_dev(status_dev);
1167		status_dev = 0;
1168		break;
1169	default:
1170		break;
1171	}
1172	return (0);
1173}
1174
1175static moduledata_t md_mod = {
1176	MD_NAME,
1177	md_modevent,
1178	NULL
1179};
1180DECLARE_MODULE(md, md_mod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE+CDEV_MAJOR);
1181MODULE_VERSION(md, MD_MODVER);
1182
1183
1184#ifdef MD_ROOT
1185static void
1186md_takeroot(void *junk)
1187{
1188	if (mdrootready)
1189		rootdevnames[0] = "ufs:/dev/md0";
1190}
1191
1192SYSINIT(md_root, SI_SUB_MOUNT_ROOT, SI_ORDER_FIRST, md_takeroot, NULL);
1193#endif /* MD_ROOT */
1194