geom_io.c revision 156686 
1/*-
2 * Copyright (c) 2002 Poul-Henning Kamp
3 * Copyright (c) 2002 Networks Associates Technology, Inc.
4 * All rights reserved.
5 *
6 * This software was developed for the FreeBSD Project by Poul-Henning Kamp
7 * and NAI Labs, the Security Research Division of Network Associates, Inc.
8 * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the
9 * DARPA CHATS research program.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 *    notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 *    notice, this list of conditions and the following disclaimer in the
18 *    documentation and/or other materials provided with the distribution.
19 * 3. The names of the authors may not be used to endorse or promote
20 *    products derived from this software without specific prior written
21 *    permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 * SUCH DAMAGE.
34 */
35
36#include <sys/cdefs.h>
37__FBSDID("$FreeBSD: head/sys/geom/geom_io.c 156686 2006-03-13 14:59:57Z ru $");
38
39#include <sys/param.h>
40#include <sys/systm.h>
41#include <sys/kernel.h>
42#include <sys/malloc.h>
43#include <sys/bio.h>
44#include <sys/ktr.h>
45#include <sys/proc.h>
46#include <sys/stack.h>
47
48#include <sys/errno.h>
49#include <geom/geom.h>
50#include <geom/geom_int.h>
51#include <sys/devicestat.h>
52
53#include <vm/uma.h>
54
55static struct g_bioq g_bio_run_down;
56static struct g_bioq g_bio_run_up;
57static struct g_bioq g_bio_run_task;
58
59static u_int pace;
60static uma_zone_t	biozone;
61
62#include <machine/atomic.h>
63
64static void
65g_bioq_lock(struct g_bioq *bq)
66{
67
68	mtx_lock(&bq->bio_queue_lock);
69}
70
71static void
72g_bioq_unlock(struct g_bioq *bq)
73{
74
75	mtx_unlock(&bq->bio_queue_lock);
76}
77
78#if 0
79static void
80g_bioq_destroy(struct g_bioq *bq)
81{
82
83	mtx_destroy(&bq->bio_queue_lock);
84}
85#endif
86
87static void
88g_bioq_init(struct g_bioq *bq)
89{
90
91	TAILQ_INIT(&bq->bio_queue);
92	mtx_init(&bq->bio_queue_lock, "bio queue", NULL, MTX_DEF);
93}
94
95static struct bio *
96g_bioq_first(struct g_bioq *bq)
97{
98	struct bio *bp;
99
100	bp = TAILQ_FIRST(&bq->bio_queue);
101	if (bp != NULL) {
102		KASSERT((bp->bio_flags & BIO_ONQUEUE),
103		    ("Bio not on queue bp=%p target %p", bp, bq));
104		bp->bio_flags &= ~BIO_ONQUEUE;
105		TAILQ_REMOVE(&bq->bio_queue, bp, bio_queue);
106		bq->bio_queue_length--;
107	}
108	return (bp);
109}
110
111struct bio *
112g_new_bio(void)
113{
114	struct bio *bp;
115
116	bp = uma_zalloc(biozone, M_NOWAIT | M_ZERO);
117#ifdef KTR
118	if (KTR_COMPILE & KTR_GEOM) {
119		struct stack st;
120
121		CTR1(KTR_GEOM, "g_new_bio(): %p", bp);
122		stack_save(&st);
123		CTRSTACK(KTR_GEOM, &st, 3, 0);
124	}
125#endif
126	return (bp);
127}
128
129struct bio *
130g_alloc_bio(void)
131{
132	struct bio *bp;
133
134	bp = uma_zalloc(biozone, M_WAITOK | M_ZERO);
135#ifdef KTR
136	if (KTR_COMPILE & KTR_GEOM) {
137		struct stack st;
138
139		CTR1(KTR_GEOM, "g_alloc_bio(): %p", bp);
140		stack_save(&st);
141		CTRSTACK(KTR_GEOM, &st, 3, 0);
142	}
143#endif
144	return (bp);
145}
146
147void
148g_destroy_bio(struct bio *bp)
149{
150#ifdef KTR
151	if (KTR_COMPILE & KTR_GEOM) {
152		struct stack st;
153
154		CTR1(KTR_GEOM, "g_destroy_bio(): %p", bp);
155		stack_save(&st);
156		CTRSTACK(KTR_GEOM, &st, 3, 0);
157	}
158#endif
159	uma_zfree(biozone, bp);
160}
161
162struct bio *
163g_clone_bio(struct bio *bp)
164{
165	struct bio *bp2;
166
167	bp2 = uma_zalloc(biozone, M_NOWAIT | M_ZERO);
168	if (bp2 != NULL) {
169		bp2->bio_parent = bp;
170		bp2->bio_cmd = bp->bio_cmd;
171		bp2->bio_length = bp->bio_length;
172		bp2->bio_offset = bp->bio_offset;
173		bp2->bio_data = bp->bio_data;
174		bp2->bio_attribute = bp->bio_attribute;
175		bp->bio_children++;
176	}
177#ifdef KTR
178	if (KTR_COMPILE & KTR_GEOM) {
179		struct stack st;
180
181		CTR2(KTR_GEOM, "g_clone_bio(%p): %p", bp, bp2);
182		stack_save(&st);
183		CTRSTACK(KTR_GEOM, &st, 3, 0);
184	}
185#endif
186	return(bp2);
187}
188
189void
190g_io_init()
191{
192
193	g_bioq_init(&g_bio_run_down);
194	g_bioq_init(&g_bio_run_up);
195	g_bioq_init(&g_bio_run_task);
196	biozone = uma_zcreate("g_bio", sizeof (struct bio),
197	    NULL, NULL,
198	    NULL, NULL,
199	    0, 0);
200}
201
202int
203g_io_getattr(const char *attr, struct g_consumer *cp, int *len, void *ptr)
204{
205	struct bio *bp;
206	int error;
207
208	g_trace(G_T_BIO, "bio_getattr(%s)", attr);
209	bp = g_alloc_bio();
210	bp->bio_cmd = BIO_GETATTR;
211	bp->bio_done = NULL;
212	bp->bio_attribute = attr;
213	bp->bio_length = *len;
214	bp->bio_data = ptr;
215	g_io_request(bp, cp);
216	error = biowait(bp, "ggetattr");
217	*len = bp->bio_completed;
218	g_destroy_bio(bp);
219	return (error);
220}
221
222static int
223g_io_check(struct bio *bp)
224{
225	struct g_consumer *cp;
226	struct g_provider *pp;
227
228	cp = bp->bio_from;
229	pp = bp->bio_to;
230
231	/* Fail if access counters dont allow the operation */
232	switch(bp->bio_cmd) {
233	case BIO_READ:
234	case BIO_GETATTR:
235		if (cp->acr == 0)
236			return (EPERM);
237		break;
238	case BIO_WRITE:
239	case BIO_DELETE:
240		if (cp->acw == 0)
241			return (EPERM);
242		break;
243	default:
244		return (EPERM);
245	}
246	/* if provider is marked for error, don't disturb. */
247	if (pp->error)
248		return (pp->error);
249
250	switch(bp->bio_cmd) {
251	case BIO_READ:
252	case BIO_WRITE:
253	case BIO_DELETE:
254		/* Zero sectorsize is a probably lack of media */
255		if (pp->sectorsize == 0)
256			return (ENXIO);
257		/* Reject I/O not on sector boundary */
258		if (bp->bio_offset % pp->sectorsize)
259			return (EINVAL);
260		/* Reject I/O not integral sector long */
261		if (bp->bio_length % pp->sectorsize)
262			return (EINVAL);
263		/* Reject requests before or past the end of media. */
264		if (bp->bio_offset < 0)
265			return (EIO);
266		if (bp->bio_offset > pp->mediasize)
267			return (EIO);
268		break;
269	default:
270		break;
271	}
272	return (0);
273}
274
275void
276g_io_request(struct bio *bp, struct g_consumer *cp)
277{
278	struct g_provider *pp;
279
280	KASSERT(cp != NULL, ("NULL cp in g_io_request"));
281	KASSERT(bp != NULL, ("NULL bp in g_io_request"));
282	KASSERT(bp->bio_data != NULL, ("NULL bp->data in g_io_request"));
283	pp = cp->provider;
284	KASSERT(pp != NULL, ("consumer not attached in g_io_request"));
285#ifdef DIAGNOSTIC
286	KASSERT(bp->bio_driver1 == NULL,
287	    ("bio_driver1 used by the consumer (geom %s)", cp->geom->name));
288	KASSERT(bp->bio_driver2 == NULL,
289	    ("bio_driver2 used by the consumer (geom %s)", cp->geom->name));
290	KASSERT(bp->bio_pflags == 0,
291	    ("bio_pflags used by the consumer (geom %s)", cp->geom->name));
292	/*
293	 * Remember consumer's private fields, so we can detect if they were
294	 * modified by the provider.
295	 */
296	bp->_bio_caller1 = bp->bio_caller1;
297	bp->_bio_caller2 = bp->bio_caller2;
298	bp->_bio_cflags = bp->bio_cflags;
299#endif
300
301	if (bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_DELETE)) {
302		KASSERT(bp->bio_offset % cp->provider->sectorsize == 0,
303		    ("wrong offset %jd for sectorsize %u",
304		    bp->bio_offset, cp->provider->sectorsize));
305		KASSERT(bp->bio_length % cp->provider->sectorsize == 0,
306		    ("wrong length %jd for sectorsize %u",
307		    bp->bio_length, cp->provider->sectorsize));
308	}
309
310	g_trace(G_T_BIO, "bio_request(%p) from %p(%s) to %p(%s) cmd %d",
311	    bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd);
312
313	bp->bio_from = cp;
314	bp->bio_to = pp;
315	bp->bio_error = 0;
316	bp->bio_completed = 0;
317
318	KASSERT(!(bp->bio_flags & BIO_ONQUEUE),
319	    ("Bio already on queue bp=%p", bp));
320	bp->bio_flags |= BIO_ONQUEUE;
321
322	binuptime(&bp->bio_t0);
323
324	/*
325	 * The statistics collection is lockless, as such, but we
326	 * can not update one instance of the statistics from more
327	 * than one thread at a time, so grab the lock first.
328	 */
329	g_bioq_lock(&g_bio_run_down);
330	if (g_collectstats & 1)
331		devstat_start_transaction(pp->stat, &bp->bio_t0);
332	if (g_collectstats & 2)
333		devstat_start_transaction(cp->stat, &bp->bio_t0);
334
335	pp->nstart++;
336	cp->nstart++;
337	TAILQ_INSERT_TAIL(&g_bio_run_down.bio_queue, bp, bio_queue);
338	g_bio_run_down.bio_queue_length++;
339	g_bioq_unlock(&g_bio_run_down);
340
341	/* Pass it on down. */
342	wakeup(&g_wait_down);
343}
344
345void
346g_io_deliver(struct bio *bp, int error)
347{
348	struct g_consumer *cp;
349	struct g_provider *pp;
350
351	KASSERT(bp != NULL, ("NULL bp in g_io_deliver"));
352	pp = bp->bio_to;
353	KASSERT(pp != NULL, ("NULL bio_to in g_io_deliver"));
354#ifdef DIAGNOSTIC
355	KASSERT(bp->bio_caller1 == bp->_bio_caller1,
356	    ("bio_caller1 used by the provider %s", pp->name));
357	KASSERT(bp->bio_caller2 == bp->_bio_caller2,
358	    ("bio_caller2 used by the provider %s", pp->name));
359	KASSERT(bp->bio_cflags == bp->_bio_cflags,
360	    ("bio_cflags used by the provider %s", pp->name));
361#endif
362	cp = bp->bio_from;
363	if (cp == NULL) {
364		bp->bio_error = error;
365		bp->bio_done(bp);
366		return;
367	}
368	KASSERT(cp != NULL, ("NULL bio_from in g_io_deliver"));
369	KASSERT(cp->geom != NULL, ("NULL bio_from->geom in g_io_deliver"));
370	KASSERT(bp->bio_completed >= 0, ("bio_completed can't be less than 0"));
371	KASSERT(bp->bio_completed <= bp->bio_length,
372	    ("bio_completed can't be greater than bio_length"));
373
374	g_trace(G_T_BIO,
375"g_io_deliver(%p) from %p(%s) to %p(%s) cmd %d error %d off %jd len %jd",
376	    bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd, error,
377	    (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length);
378
379	KASSERT(!(bp->bio_flags & BIO_ONQUEUE),
380	    ("Bio already on queue bp=%p", bp));
381
382	/*
383	 * XXX: next two doesn't belong here
384	 */
385	bp->bio_bcount = bp->bio_length;
386	bp->bio_resid = bp->bio_bcount - bp->bio_completed;
387
388	/*
389	 * The statistics collection is lockless, as such, but we
390	 * can not update one instance of the statistics from more
391	 * than one thread at a time, so grab the lock first.
392	 */
393	g_bioq_lock(&g_bio_run_up);
394	if (g_collectstats & 1)
395		devstat_end_transaction_bio(pp->stat, bp);
396	if (g_collectstats & 2)
397		devstat_end_transaction_bio(cp->stat, bp);
398
399	cp->nend++;
400	pp->nend++;
401	if (error != ENOMEM) {
402		bp->bio_error = error;
403		TAILQ_INSERT_TAIL(&g_bio_run_up.bio_queue, bp, bio_queue);
404		bp->bio_flags |= BIO_ONQUEUE;
405		g_bio_run_up.bio_queue_length++;
406		g_bioq_unlock(&g_bio_run_up);
407		wakeup(&g_wait_up);
408		return;
409	}
410	g_bioq_unlock(&g_bio_run_up);
411
412	if (bootverbose)
413		printf("ENOMEM %p on %p(%s)\n", bp, pp, pp->name);
414	bp->bio_children = 0;
415	bp->bio_inbed = 0;
416	g_io_request(bp, cp);
417	pace++;
418	return;
419}
420
421void
422g_io_schedule_down(struct thread *tp __unused)
423{
424	struct bio *bp;
425	off_t excess;
426	int error;
427
428	for(;;) {
429		g_bioq_lock(&g_bio_run_down);
430		bp = g_bioq_first(&g_bio_run_down);
431		if (bp == NULL) {
432			CTR0(KTR_GEOM, "g_down going to sleep");
433			msleep(&g_wait_down, &g_bio_run_down.bio_queue_lock,
434			    PRIBIO | PDROP, "-", hz/10);
435			continue;
436		}
437		CTR0(KTR_GEOM, "g_down has work to do");
438		g_bioq_unlock(&g_bio_run_down);
439		if (pace > 0) {
440			CTR1(KTR_GEOM, "g_down pacing self (pace %d)", pace);
441			msleep(&error, NULL, PRIBIO, "g_down", hz/10);
442			pace--;
443		}
444		error = g_io_check(bp);
445		if (error) {
446			CTR3(KTR_GEOM, "g_down g_io_check on bp %p provider "
447			    "%s returned %d", bp, bp->bio_to->name, error);
448			g_io_deliver(bp, error);
449			continue;
450		}
451		CTR2(KTR_GEOM, "g_down processing bp %p provider %s", bp,
452		    bp->bio_to->name);
453		switch (bp->bio_cmd) {
454		case BIO_READ:
455		case BIO_WRITE:
456		case BIO_DELETE:
457			/* Truncate requests to the end of providers media. */
458			/*
459			 * XXX: What if we truncate because of offset being
460			 * bad, not length?
461			 */
462			excess = bp->bio_offset + bp->bio_length;
463			if (excess > bp->bio_to->mediasize) {
464				excess -= bp->bio_to->mediasize;
465				bp->bio_length -= excess;
466				if (excess > 0)
467					CTR3(KTR_GEOM, "g_down truncated bio "
468					    "%p provider %s by %d", bp,
469					    bp->bio_to->name, excess);
470			}
471			/* Deliver zero length transfers right here. */
472			if (bp->bio_length == 0) {
473				g_io_deliver(bp, 0);
474				CTR2(KTR_GEOM, "g_down terminated 0-length "
475				    "bp %p provider %s", bp, bp->bio_to->name);
476				continue;
477			}
478			break;
479		default:
480			break;
481		}
482		THREAD_NO_SLEEPING();
483		CTR4(KTR_GEOM, "g_down starting bp %p provider %s off %ld "
484		    "len %ld", bp, bp->bio_to->name, bp->bio_offset,
485		    bp->bio_length);
486		bp->bio_to->geom->start(bp);
487		THREAD_SLEEPING_OK();
488	}
489}
490
491void
492bio_taskqueue(struct bio *bp, bio_task_t *func, void *arg)
493{
494	bp->bio_task = func;
495	bp->bio_task_arg = arg;
496	/*
497	 * The taskqueue is actually just a second queue off the "up"
498	 * queue, so we use the same lock.
499	 */
500	g_bioq_lock(&g_bio_run_up);
501	KASSERT(!(bp->bio_flags & BIO_ONQUEUE),
502	    ("Bio already on queue bp=%p target taskq", bp));
503	bp->bio_flags |= BIO_ONQUEUE;
504	TAILQ_INSERT_TAIL(&g_bio_run_task.bio_queue, bp, bio_queue);
505	g_bio_run_task.bio_queue_length++;
506	wakeup(&g_wait_up);
507	g_bioq_unlock(&g_bio_run_up);
508}
509
510
511void
512g_io_schedule_up(struct thread *tp __unused)
513{
514	struct bio *bp;
515	for(;;) {
516		g_bioq_lock(&g_bio_run_up);
517		bp = g_bioq_first(&g_bio_run_task);
518		if (bp != NULL) {
519			g_bioq_unlock(&g_bio_run_up);
520			THREAD_NO_SLEEPING();
521			CTR1(KTR_GEOM, "g_up processing task bp %p", bp);
522			bp->bio_task(bp->bio_task_arg);
523			THREAD_SLEEPING_OK();
524			continue;
525		}
526		bp = g_bioq_first(&g_bio_run_up);
527		if (bp != NULL) {
528			g_bioq_unlock(&g_bio_run_up);
529			THREAD_NO_SLEEPING();
530			CTR4(KTR_GEOM, "g_up biodone bp %p provider %s off "
531			    "%ld len %ld", bp, bp->bio_to->name,
532			    bp->bio_offset, bp->bio_length);
533			biodone(bp);
534			THREAD_SLEEPING_OK();
535			continue;
536		}
537		CTR0(KTR_GEOM, "g_up going to sleep");
538		msleep(&g_wait_up, &g_bio_run_up.bio_queue_lock,
539		    PRIBIO | PDROP, "-", hz/10);
540	}
541}
542
543void *
544g_read_data(struct g_consumer *cp, off_t offset, off_t length, int *error)
545{
546	struct bio *bp;
547	void *ptr;
548	int errorc;
549
550	KASSERT(length > 0 && length >= cp->provider->sectorsize &&
551	    length <= MAXPHYS, ("g_read_data(): invalid length %jd",
552	    (intmax_t)length));
553
554	bp = g_alloc_bio();
555	bp->bio_cmd = BIO_READ;
556	bp->bio_done = NULL;
557	bp->bio_offset = offset;
558	bp->bio_length = length;
559	ptr = g_malloc(length, M_WAITOK);
560	bp->bio_data = ptr;
561	g_io_request(bp, cp);
562	errorc = biowait(bp, "gread");
563	if (error != NULL)
564		*error = errorc;
565	g_destroy_bio(bp);
566	if (errorc) {
567		g_free(ptr);
568		ptr = NULL;
569	}
570	return (ptr);
571}
572
573int
574g_write_data(struct g_consumer *cp, off_t offset, void *ptr, off_t length)
575{
576	struct bio *bp;
577	int error;
578
579	KASSERT(length > 0 && length >= cp->provider->sectorsize &&
580	    length <= MAXPHYS, ("g_write_data(): invalid length %jd",
581	    (intmax_t)length));
582
583	bp = g_alloc_bio();
584	bp->bio_cmd = BIO_WRITE;
585	bp->bio_done = NULL;
586	bp->bio_offset = offset;
587	bp->bio_length = length;
588	bp->bio_data = ptr;
589	g_io_request(bp, cp);
590	error = biowait(bp, "gwrite");
591	g_destroy_bio(bp);
592	return (error);
593}
594
595void
596g_print_bio(struct bio *bp)
597{
598	const char *pname, *cmd = NULL;
599
600	if (bp->bio_to != NULL)
601		pname = bp->bio_to->name;
602	else
603		pname = "[unknown]";
604
605	switch (bp->bio_cmd) {
606	case BIO_GETATTR:
607		cmd = "GETATTR";
608		printf("%s[%s(attr=%s)]", pname, cmd, bp->bio_attribute);
609		return;
610	case BIO_READ:
611		cmd = "READ";
612	case BIO_WRITE:
613		if (cmd == NULL)
614			cmd = "WRITE";
615	case BIO_DELETE:
616		if (cmd == NULL)
617			cmd = "DELETE";
618		printf("%s[%s(offset=%jd, length=%jd)]", pname, cmd,
619		    (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length);
620		return;
621	default:
622		cmd = "UNKNOWN";
623		printf("%s[%s()]", pname, cmd);
624		return;
625	}
626	/* NOTREACHED */
627}
628