ctl_backend_block.c revision 265649
1/*-
2 * Copyright (c) 2003 Silicon Graphics International Corp.
3 * Copyright (c) 2009-2011 Spectra Logic Corporation
4 * Copyright (c) 2012 The FreeBSD Foundation
5 * All rights reserved.
6 *
7 * Portions of this software were developed by Edward Tomasz Napierala
8 * under sponsorship from the FreeBSD Foundation.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions, and the following disclaimer,
15 *    without modification.
16 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
17 *    substantially similar to the "NO WARRANTY" disclaimer below
18 *    ("Disclaimer") and any redistribution must be conditioned upon
19 *    including a substantially similar Disclaimer requirement for further
20 *    binary redistribution.
21 *
22 * NO WARRANTY
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
26 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
27 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
32 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33 * POSSIBILITY OF SUCH DAMAGES.
34 *
35 * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_backend_block.c#5 $
36 */
37/*
38 * CAM Target Layer driver backend for block devices.
39 *
40 * Author: Ken Merry <ken@FreeBSD.org>
41 */
42#include <sys/cdefs.h>
43__FBSDID("$FreeBSD: stable/9/sys/cam/ctl/ctl_backend_block.c 265649 2014-05-08 08:17:04Z mav $");
44
45#include <opt_kdtrace.h>
46
47#include <sys/param.h>
48#include <sys/systm.h>
49#include <sys/kernel.h>
50#include <sys/types.h>
51#include <sys/kthread.h>
52#include <sys/bio.h>
53#include <sys/fcntl.h>
54#include <sys/lock.h>
55#include <sys/mutex.h>
56#include <sys/condvar.h>
57#include <sys/malloc.h>
58#include <sys/conf.h>
59#include <sys/ioccom.h>
60#include <sys/queue.h>
61#include <sys/sbuf.h>
62#include <sys/endian.h>
63#include <sys/uio.h>
64#include <sys/buf.h>
65#include <sys/taskqueue.h>
66#include <sys/vnode.h>
67#include <sys/namei.h>
68#include <sys/mount.h>
69#include <sys/disk.h>
70#include <sys/fcntl.h>
71#include <sys/filedesc.h>
72#include <sys/proc.h>
73#include <sys/pcpu.h>
74#include <sys/module.h>
75#include <sys/sdt.h>
76#include <sys/devicestat.h>
77#include <sys/sysctl.h>
78
79#include <geom/geom.h>
80
81#include <cam/cam.h>
82#include <cam/scsi/scsi_all.h>
83#include <cam/scsi/scsi_da.h>
84#include <cam/ctl/ctl_io.h>
85#include <cam/ctl/ctl.h>
86#include <cam/ctl/ctl_backend.h>
87#include <cam/ctl/ctl_frontend_internal.h>
88#include <cam/ctl/ctl_ioctl.h>
89#include <cam/ctl/ctl_scsi_all.h>
90#include <cam/ctl/ctl_error.h>
91
92/*
93 * The idea here is that we'll allocate enough S/G space to hold a 16MB
94 * I/O.  If we get an I/O larger than that, we'll reject it.
95 */
96#define	CTLBLK_MAX_IO_SIZE	(16 * 1024 * 1024)
97#define	CTLBLK_MAX_SEGS		(CTLBLK_MAX_IO_SIZE / MAXPHYS) + 1
98
99#ifdef CTLBLK_DEBUG
100#define DPRINTF(fmt, args...) \
101    printf("cbb(%s:%d): " fmt, __FUNCTION__, __LINE__, ##args)
102#else
103#define DPRINTF(fmt, args...) do {} while(0)
104#endif
105
106SDT_PROVIDER_DEFINE(cbb);
107
108typedef enum {
109	CTL_BE_BLOCK_LUN_UNCONFIGURED	= 0x01,
110	CTL_BE_BLOCK_LUN_CONFIG_ERR	= 0x02,
111	CTL_BE_BLOCK_LUN_WAITING	= 0x04,
112	CTL_BE_BLOCK_LUN_MULTI_THREAD	= 0x08
113} ctl_be_block_lun_flags;
114
115typedef enum {
116	CTL_BE_BLOCK_NONE,
117	CTL_BE_BLOCK_DEV,
118	CTL_BE_BLOCK_FILE
119} ctl_be_block_type;
120
121struct ctl_be_block_devdata {
122	struct cdev *cdev;
123	struct cdevsw *csw;
124	int dev_ref;
125};
126
127struct ctl_be_block_filedata {
128	struct ucred *cred;
129};
130
131union ctl_be_block_bedata {
132	struct ctl_be_block_devdata dev;
133	struct ctl_be_block_filedata file;
134};
135
136struct ctl_be_block_io;
137struct ctl_be_block_lun;
138
139typedef void (*cbb_dispatch_t)(struct ctl_be_block_lun *be_lun,
140			       struct ctl_be_block_io *beio);
141
142/*
143 * Backend LUN structure.  There is a 1:1 mapping between a block device
144 * and a backend block LUN, and between a backend block LUN and a CTL LUN.
145 */
146struct ctl_be_block_lun {
147	struct ctl_block_disk *disk;
148	char lunname[32];
149	char *dev_path;
150	ctl_be_block_type dev_type;
151	struct vnode *vn;
152	union ctl_be_block_bedata backend;
153	cbb_dispatch_t dispatch;
154	cbb_dispatch_t lun_flush;
155	struct mtx lock;
156	uma_zone_t lun_zone;
157	uint64_t size_blocks;
158	uint64_t size_bytes;
159	uint32_t blocksize;
160	int blocksize_shift;
161	uint16_t pblockexp;
162	uint16_t pblockoff;
163	struct ctl_be_block_softc *softc;
164	struct devstat *disk_stats;
165	ctl_be_block_lun_flags flags;
166	STAILQ_ENTRY(ctl_be_block_lun) links;
167	struct ctl_be_lun ctl_be_lun;
168	struct taskqueue *io_taskqueue;
169	struct task io_task;
170	int num_threads;
171	STAILQ_HEAD(, ctl_io_hdr) input_queue;
172	STAILQ_HEAD(, ctl_io_hdr) config_write_queue;
173	STAILQ_HEAD(, ctl_io_hdr) datamove_queue;
174};
175
176/*
177 * Overall softc structure for the block backend module.
178 */
179struct ctl_be_block_softc {
180	STAILQ_HEAD(, ctl_be_block_io)   beio_free_queue;
181	struct mtx			 lock;
182	int				 prealloc_beio;
183	int				 num_disks;
184	STAILQ_HEAD(, ctl_block_disk)	 disk_list;
185	int				 num_luns;
186	STAILQ_HEAD(, ctl_be_block_lun)	 lun_list;
187};
188
189static struct ctl_be_block_softc backend_block_softc;
190
191/*
192 * Per-I/O information.
193 */
194struct ctl_be_block_io {
195	union ctl_io			*io;
196	struct ctl_sg_entry		sg_segs[CTLBLK_MAX_SEGS];
197	struct iovec			xiovecs[CTLBLK_MAX_SEGS];
198	int				bio_cmd;
199	int				bio_flags;
200	int				num_segs;
201	int				num_bios_sent;
202	int				num_bios_done;
203	int				send_complete;
204	int				num_errors;
205	struct bintime			ds_t0;
206	devstat_tag_type		ds_tag_type;
207	devstat_trans_flags		ds_trans_type;
208	uint64_t			io_len;
209	uint64_t			io_offset;
210	struct ctl_be_block_softc	*softc;
211	struct ctl_be_block_lun		*lun;
212	STAILQ_ENTRY(ctl_be_block_io)	links;
213};
214
215static int cbb_num_threads = 14;
216TUNABLE_INT("kern.cam.ctl.block.num_threads", &cbb_num_threads);
217SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, block, CTLFLAG_RD, 0,
218	    "CAM Target Layer Block Backend");
219SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RW,
220           &cbb_num_threads, 0, "Number of threads per backing file");
221
222static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc);
223static void ctl_free_beio(struct ctl_be_block_io *beio);
224static int ctl_grow_beio(struct ctl_be_block_softc *softc, int count);
225#if 0
226static void ctl_shrink_beio(struct ctl_be_block_softc *softc);
227#endif
228static void ctl_complete_beio(struct ctl_be_block_io *beio);
229static int ctl_be_block_move_done(union ctl_io *io);
230static void ctl_be_block_biodone(struct bio *bio);
231static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
232				    struct ctl_be_block_io *beio);
233static void ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
234				       struct ctl_be_block_io *beio);
235static void ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
236				   struct ctl_be_block_io *beio);
237static void ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
238				      struct ctl_be_block_io *beio);
239static void ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
240				    union ctl_io *io);
241static void ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
242				  union ctl_io *io);
243static void ctl_be_block_worker(void *context, int pending);
244static int ctl_be_block_submit(union ctl_io *io);
245static int ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
246				   int flag, struct thread *td);
247static int ctl_be_block_open_file(struct ctl_be_block_lun *be_lun,
248				  struct ctl_lun_req *req);
249static int ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun,
250				 struct ctl_lun_req *req);
251static int ctl_be_block_close(struct ctl_be_block_lun *be_lun);
252static int ctl_be_block_open(struct ctl_be_block_softc *softc,
253			     struct ctl_be_block_lun *be_lun,
254			     struct ctl_lun_req *req);
255static int ctl_be_block_create(struct ctl_be_block_softc *softc,
256			       struct ctl_lun_req *req);
257static int ctl_be_block_rm(struct ctl_be_block_softc *softc,
258			   struct ctl_lun_req *req);
259static int ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun,
260				  struct ctl_lun_req *req);
261static int ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun,
262				 struct ctl_lun_req *req);
263static int ctl_be_block_modify(struct ctl_be_block_softc *softc,
264			   struct ctl_lun_req *req);
265static void ctl_be_block_lun_shutdown(void *be_lun);
266static void ctl_be_block_lun_config_status(void *be_lun,
267					   ctl_lun_config_status status);
268static int ctl_be_block_config_write(union ctl_io *io);
269static int ctl_be_block_config_read(union ctl_io *io);
270static int ctl_be_block_lun_info(void *be_lun, struct sbuf *sb);
271int ctl_be_block_init(void);
272
273static struct ctl_backend_driver ctl_be_block_driver =
274{
275	.name = "block",
276	.flags = CTL_BE_FLAG_HAS_CONFIG,
277	.init = ctl_be_block_init,
278	.data_submit = ctl_be_block_submit,
279	.data_move_done = ctl_be_block_move_done,
280	.config_read = ctl_be_block_config_read,
281	.config_write = ctl_be_block_config_write,
282	.ioctl = ctl_be_block_ioctl,
283	.lun_info = ctl_be_block_lun_info
284};
285
286MALLOC_DEFINE(M_CTLBLK, "ctlblk", "Memory used for CTL block backend");
287CTL_BACKEND_DECLARE(cbb, ctl_be_block_driver);
288
289static struct ctl_be_block_io *
290ctl_alloc_beio(struct ctl_be_block_softc *softc)
291{
292	struct ctl_be_block_io *beio;
293	int count;
294
295	mtx_lock(&softc->lock);
296
297	beio = STAILQ_FIRST(&softc->beio_free_queue);
298	if (beio != NULL) {
299		STAILQ_REMOVE(&softc->beio_free_queue, beio,
300			      ctl_be_block_io, links);
301	}
302	mtx_unlock(&softc->lock);
303
304	if (beio != NULL) {
305		bzero(beio, sizeof(*beio));
306		beio->softc = softc;
307		return (beio);
308	}
309
310	for (;;) {
311
312		count = ctl_grow_beio(softc, /*count*/ 10);
313
314		/*
315		 * This shouldn't be possible, since ctl_grow_beio() uses a
316		 * blocking malloc.
317		 */
318		if (count == 0)
319			return (NULL);
320
321		/*
322		 * Since we have to drop the lock when we're allocating beio
323		 * structures, it's possible someone else can come along and
324		 * allocate the beio's we've just allocated.
325		 */
326		mtx_lock(&softc->lock);
327		beio = STAILQ_FIRST(&softc->beio_free_queue);
328		if (beio != NULL) {
329			STAILQ_REMOVE(&softc->beio_free_queue, beio,
330				      ctl_be_block_io, links);
331		}
332		mtx_unlock(&softc->lock);
333
334		if (beio != NULL) {
335			bzero(beio, sizeof(*beio));
336			beio->softc = softc;
337			break;
338		}
339	}
340	return (beio);
341}
342
343static void
344ctl_free_beio(struct ctl_be_block_io *beio)
345{
346	struct ctl_be_block_softc *softc;
347	int duplicate_free;
348	int i;
349
350	softc = beio->softc;
351	duplicate_free = 0;
352
353	for (i = 0; i < beio->num_segs; i++) {
354		if (beio->sg_segs[i].addr == NULL)
355			duplicate_free++;
356
357		uma_zfree(beio->lun->lun_zone, beio->sg_segs[i].addr);
358		beio->sg_segs[i].addr = NULL;
359	}
360
361	if (duplicate_free > 0) {
362		printf("%s: %d duplicate frees out of %d segments\n", __func__,
363		       duplicate_free, beio->num_segs);
364	}
365	mtx_lock(&softc->lock);
366	STAILQ_INSERT_TAIL(&softc->beio_free_queue, beio, links);
367	mtx_unlock(&softc->lock);
368}
369
370static int
371ctl_grow_beio(struct ctl_be_block_softc *softc, int count)
372{
373	int i;
374
375	for (i = 0; i < count; i++) {
376		struct ctl_be_block_io *beio;
377
378		beio = (struct ctl_be_block_io *)malloc(sizeof(*beio),
379							   M_CTLBLK,
380							   M_WAITOK | M_ZERO);
381		beio->softc = softc;
382		mtx_lock(&softc->lock);
383		STAILQ_INSERT_TAIL(&softc->beio_free_queue, beio, links);
384		mtx_unlock(&softc->lock);
385	}
386
387	return (i);
388}
389
390#if 0
391static void
392ctl_shrink_beio(struct ctl_be_block_softc *softc)
393{
394	struct ctl_be_block_io *beio, *beio_tmp;
395
396	mtx_lock(&softc->lock);
397	STAILQ_FOREACH_SAFE(beio, &softc->beio_free_queue, links, beio_tmp) {
398		STAILQ_REMOVE(&softc->beio_free_queue, beio,
399			      ctl_be_block_io, links);
400		free(beio, M_CTLBLK);
401	}
402	mtx_unlock(&softc->lock);
403}
404#endif
405
406static void
407ctl_complete_beio(struct ctl_be_block_io *beio)
408{
409	union ctl_io *io;
410	int io_len;
411
412	io = beio->io;
413
414	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)
415		io_len = beio->io_len;
416	else
417		io_len = 0;
418
419	devstat_end_transaction(beio->lun->disk_stats,
420				/*bytes*/ io_len,
421				beio->ds_tag_type,
422				beio->ds_trans_type,
423				/*now*/ NULL,
424				/*then*/&beio->ds_t0);
425
426	ctl_free_beio(beio);
427	ctl_done(io);
428}
429
430static int
431ctl_be_block_move_done(union ctl_io *io)
432{
433	struct ctl_be_block_io *beio;
434	struct ctl_be_block_lun *be_lun;
435#ifdef CTL_TIME_IO
436	struct bintime cur_bt;
437#endif
438
439	beio = (struct ctl_be_block_io *)
440		io->io_hdr.ctl_private[CTL_PRIV_BACKEND].ptr;
441
442	be_lun = beio->lun;
443
444	DPRINTF("entered\n");
445
446#ifdef CTL_TIME_IO
447	getbintime(&cur_bt);
448	bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
449	bintime_add(&io->io_hdr.dma_bt, &cur_bt);
450	io->io_hdr.num_dmas++;
451#endif
452
453	/*
454	 * We set status at this point for read commands, and write
455	 * commands with errors.
456	 */
457	if ((beio->bio_cmd == BIO_READ)
458	 && (io->io_hdr.port_status == 0)
459	 && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
460	 && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE))
461		ctl_set_success(&io->scsiio);
462	else if ((io->io_hdr.port_status != 0)
463	      && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
464	      && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
465		/*
466		 * For hardware error sense keys, the sense key
467		 * specific value is defined to be a retry count,
468		 * but we use it to pass back an internal FETD
469		 * error code.  XXX KDM  Hopefully the FETD is only
470		 * using 16 bits for an error code, since that's
471		 * all the space we have in the sks field.
472		 */
473		ctl_set_internal_failure(&io->scsiio,
474					 /*sks_valid*/ 1,
475					 /*retry_count*/
476					 io->io_hdr.port_status);
477	}
478
479	/*
480	 * If this is a read, or a write with errors, it is done.
481	 */
482	if ((beio->bio_cmd == BIO_READ)
483	 || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)
484	 || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)) {
485		ctl_complete_beio(beio);
486		return (0);
487	}
488
489	/*
490	 * At this point, we have a write and the DMA completed
491	 * successfully.  We now have to queue it to the task queue to
492	 * execute the backend I/O.  That is because we do blocking
493	 * memory allocations, and in the file backing case, blocking I/O.
494	 * This move done routine is generally called in the SIM's
495	 * interrupt context, and therefore we cannot block.
496	 */
497	mtx_lock(&be_lun->lock);
498	/*
499	 * XXX KDM make sure that links is okay to use at this point.
500	 * Otherwise, we either need to add another field to ctl_io_hdr,
501	 * or deal with resource allocation here.
502	 */
503	STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links);
504	mtx_unlock(&be_lun->lock);
505
506	taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
507
508	return (0);
509}
510
511static void
512ctl_be_block_biodone(struct bio *bio)
513{
514	struct ctl_be_block_io *beio;
515	struct ctl_be_block_lun *be_lun;
516	union ctl_io *io;
517	int error;
518
519	beio = bio->bio_caller1;
520	be_lun = beio->lun;
521	io = beio->io;
522
523	DPRINTF("entered\n");
524
525	error = bio->bio_error;
526	mtx_lock(&be_lun->lock);
527	if (error != 0)
528		beio->num_errors++;
529
530	beio->num_bios_done++;
531
532	/*
533	 * XXX KDM will this cause WITNESS to complain?  Holding a lock
534	 * during the free might cause it to complain.
535	 */
536	g_destroy_bio(bio);
537
538	/*
539	 * If the send complete bit isn't set, or we aren't the last I/O to
540	 * complete, then we're done.
541	 */
542	if ((beio->send_complete == 0)
543	 || (beio->num_bios_done < beio->num_bios_sent)) {
544		mtx_unlock(&be_lun->lock);
545		return;
546	}
547
548	/*
549	 * At this point, we've verified that we are the last I/O to
550	 * complete, so it's safe to drop the lock.
551	 */
552	mtx_unlock(&be_lun->lock);
553
554	/*
555	 * If there are any errors from the backing device, we fail the
556	 * entire I/O with a medium error.
557	 */
558	if (beio->num_errors > 0) {
559		if (error == EOPNOTSUPP) {
560			ctl_set_invalid_opcode(&io->scsiio);
561		} else if (beio->bio_cmd == BIO_FLUSH) {
562			/* XXX KDM is there is a better error here? */
563			ctl_set_internal_failure(&io->scsiio,
564						 /*sks_valid*/ 1,
565						 /*retry_count*/ 0xbad2);
566		} else
567			ctl_set_medium_error(&io->scsiio);
568		ctl_complete_beio(beio);
569		return;
570	}
571
572	/*
573	 * If this is a write or a flush, we're all done.
574	 * If this is a read, we can now send the data to the user.
575	 */
576	if ((beio->bio_cmd == BIO_WRITE)
577	 || (beio->bio_cmd == BIO_FLUSH)) {
578		ctl_set_success(&io->scsiio);
579		ctl_complete_beio(beio);
580	} else {
581		io->scsiio.be_move_done = ctl_be_block_move_done;
582		io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs;
583		io->scsiio.kern_data_len = beio->io_len;
584		io->scsiio.kern_total_len = beio->io_len;
585		io->scsiio.kern_rel_offset = 0;
586		io->scsiio.kern_data_resid = 0;
587		io->scsiio.kern_sg_entries = beio->num_segs;
588		io->io_hdr.flags |= CTL_FLAG_ALLOCATED | CTL_FLAG_KDPTR_SGLIST;
589#ifdef CTL_TIME_IO
590        	getbintime(&io->io_hdr.dma_start_bt);
591#endif
592		ctl_datamove(io);
593	}
594}
595
596static void
597ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
598			struct ctl_be_block_io *beio)
599{
600	union ctl_io *io;
601	struct mount *mountpoint;
602	int vfs_is_locked, error, lock_flags;
603
604	DPRINTF("entered\n");
605
606	io = beio->io;
607
608	vfs_is_locked = VFS_LOCK_GIANT(be_lun->vn->v_mount);
609
610       	(void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
611
612	if (MNT_SHARED_WRITES(mountpoint)
613	 || ((mountpoint == NULL)
614	  && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
615		lock_flags = LK_SHARED;
616	else
617		lock_flags = LK_EXCLUSIVE;
618
619	vn_lock(be_lun->vn, lock_flags | LK_RETRY);
620
621	binuptime(&beio->ds_t0);
622	devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
623
624	error = VOP_FSYNC(be_lun->vn, MNT_WAIT, curthread);
625	VOP_UNLOCK(be_lun->vn, 0);
626
627	vn_finished_write(mountpoint);
628
629	VFS_UNLOCK_GIANT(vfs_is_locked);
630
631	if (error == 0)
632		ctl_set_success(&io->scsiio);
633	else {
634		/* XXX KDM is there is a better error here? */
635		ctl_set_internal_failure(&io->scsiio,
636					 /*sks_valid*/ 1,
637					 /*retry_count*/ 0xbad1);
638	}
639
640	ctl_complete_beio(beio);
641}
642
643SDT_PROBE_DEFINE1(cbb, kernel, read, file_start, "uint64_t");
644SDT_PROBE_DEFINE1(cbb, kernel, write, file_start, "uint64_t");
645SDT_PROBE_DEFINE1(cbb, kernel, read, file_done,"uint64_t");
646SDT_PROBE_DEFINE1(cbb, kernel, write, file_done, "uint64_t");
647
648static void
649ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
650			   struct ctl_be_block_io *beio)
651{
652	struct ctl_be_block_filedata *file_data;
653	union ctl_io *io;
654	struct uio xuio;
655	struct iovec *xiovec;
656	int vfs_is_locked, flags;
657	int error, i;
658
659	DPRINTF("entered\n");
660
661	file_data = &be_lun->backend.file;
662	io = beio->io;
663	flags = beio->bio_flags;
664
665	if (beio->bio_cmd == BIO_READ) {
666		SDT_PROBE(cbb, kernel, read, file_start, 0, 0, 0, 0, 0);
667	} else {
668		SDT_PROBE(cbb, kernel, write, file_start, 0, 0, 0, 0, 0);
669	}
670
671	bzero(&xuio, sizeof(xuio));
672	if (beio->bio_cmd == BIO_READ)
673		xuio.uio_rw = UIO_READ;
674	else
675		xuio.uio_rw = UIO_WRITE;
676
677	xuio.uio_offset = beio->io_offset;
678	xuio.uio_resid = beio->io_len;
679	xuio.uio_segflg = UIO_SYSSPACE;
680	xuio.uio_iov = beio->xiovecs;
681	xuio.uio_iovcnt = beio->num_segs;
682	xuio.uio_td = curthread;
683
684	for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) {
685		xiovec->iov_base = beio->sg_segs[i].addr;
686		xiovec->iov_len = beio->sg_segs[i].len;
687	}
688
689	vfs_is_locked = VFS_LOCK_GIANT(be_lun->vn->v_mount);
690	if (beio->bio_cmd == BIO_READ) {
691		vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
692
693		binuptime(&beio->ds_t0);
694		devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
695
696		/*
697		 * UFS pays attention to IO_DIRECT for reads.  If the
698		 * DIRECTIO option is configured into the kernel, it calls
699		 * ffs_rawread().  But that only works for single-segment
700		 * uios with user space addresses.  In our case, with a
701		 * kernel uio, it still reads into the buffer cache, but it
702		 * will just try to release the buffer from the cache later
703		 * on in ffs_read().
704		 *
705		 * ZFS does not pay attention to IO_DIRECT for reads.
706		 *
707		 * UFS does not pay attention to IO_SYNC for reads.
708		 *
709		 * ZFS pays attention to IO_SYNC (which translates into the
710		 * Solaris define FRSYNC for zfs_read()) for reads.  It
711		 * attempts to sync the file before reading.
712		 *
713		 * So, to attempt to provide some barrier semantics in the
714		 * BIO_ORDERED case, set both IO_DIRECT and IO_SYNC.
715		 */
716		error = VOP_READ(be_lun->vn, &xuio, (flags & BIO_ORDERED) ?
717				 (IO_DIRECT|IO_SYNC) : 0, file_data->cred);
718
719		VOP_UNLOCK(be_lun->vn, 0);
720	} else {
721		struct mount *mountpoint;
722		int lock_flags;
723
724		(void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
725
726		if (MNT_SHARED_WRITES(mountpoint)
727		 || ((mountpoint == NULL)
728		  && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
729			lock_flags = LK_SHARED;
730		else
731			lock_flags = LK_EXCLUSIVE;
732
733		vn_lock(be_lun->vn, lock_flags | LK_RETRY);
734
735		binuptime(&beio->ds_t0);
736		devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
737
738		/*
739		 * UFS pays attention to IO_DIRECT for writes.  The write
740		 * is done asynchronously.  (Normally the write would just
741		 * get put into cache.
742		 *
743		 * UFS pays attention to IO_SYNC for writes.  It will
744		 * attempt to write the buffer out synchronously if that
745		 * flag is set.
746		 *
747		 * ZFS does not pay attention to IO_DIRECT for writes.
748		 *
749		 * ZFS pays attention to IO_SYNC (a.k.a. FSYNC or FRSYNC)
750		 * for writes.  It will flush the transaction from the
751		 * cache before returning.
752		 *
753		 * So if we've got the BIO_ORDERED flag set, we want
754		 * IO_SYNC in either the UFS or ZFS case.
755		 */
756		error = VOP_WRITE(be_lun->vn, &xuio, (flags & BIO_ORDERED) ?
757				  IO_SYNC : 0, file_data->cred);
758		VOP_UNLOCK(be_lun->vn, 0);
759
760		vn_finished_write(mountpoint);
761        }
762        VFS_UNLOCK_GIANT(vfs_is_locked);
763
764	/*
765	 * If we got an error, set the sense data to "MEDIUM ERROR" and
766	 * return the I/O to the user.
767	 */
768	if (error != 0) {
769		char path_str[32];
770
771		ctl_scsi_path_string(io, path_str, sizeof(path_str));
772		/*
773		 * XXX KDM ZFS returns ENOSPC when the underlying
774		 * filesystem fills up.  What kind of SCSI error should we
775		 * return for that?
776		 */
777		printf("%s%s command returned errno %d\n", path_str,
778		       (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE", error);
779		ctl_set_medium_error(&io->scsiio);
780		ctl_complete_beio(beio);
781		return;
782	}
783
784	/*
785	 * If this is a write, we're all done.
786	 * If this is a read, we can now send the data to the user.
787	 */
788	if (beio->bio_cmd == BIO_WRITE) {
789		ctl_set_success(&io->scsiio);
790		SDT_PROBE(cbb, kernel, write, file_done, 0, 0, 0, 0, 0);
791		ctl_complete_beio(beio);
792	} else {
793		SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0);
794		io->scsiio.be_move_done = ctl_be_block_move_done;
795		io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs;
796		io->scsiio.kern_data_len = beio->io_len;
797		io->scsiio.kern_total_len = beio->io_len;
798		io->scsiio.kern_rel_offset = 0;
799		io->scsiio.kern_data_resid = 0;
800		io->scsiio.kern_sg_entries = beio->num_segs;
801		io->io_hdr.flags |= CTL_FLAG_ALLOCATED | CTL_FLAG_KDPTR_SGLIST;
802#ifdef CTL_TIME_IO
803        	getbintime(&io->io_hdr.dma_start_bt);
804#endif
805		ctl_datamove(io);
806	}
807}
808
809static void
810ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
811		       struct ctl_be_block_io *beio)
812{
813	struct bio *bio;
814	union ctl_io *io;
815	struct ctl_be_block_devdata *dev_data;
816
817	dev_data = &be_lun->backend.dev;
818	io = beio->io;
819
820	DPRINTF("entered\n");
821
822	/* This can't fail, it's a blocking allocation. */
823	bio = g_alloc_bio();
824
825	bio->bio_cmd	    = BIO_FLUSH;
826	bio->bio_flags	   |= BIO_ORDERED;
827	bio->bio_dev	    = dev_data->cdev;
828	bio->bio_offset	    = 0;
829	bio->bio_data	    = 0;
830	bio->bio_done	    = ctl_be_block_biodone;
831	bio->bio_caller1    = beio;
832	bio->bio_pblkno	    = 0;
833
834	/*
835	 * We don't need to acquire the LUN lock here, because we are only
836	 * sending one bio, and so there is no other context to synchronize
837	 * with.
838	 */
839	beio->num_bios_sent = 1;
840	beio->send_complete = 1;
841
842	binuptime(&beio->ds_t0);
843	devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
844
845	(*dev_data->csw->d_strategy)(bio);
846}
847
848static void
849ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
850			  struct ctl_be_block_io *beio)
851{
852	int i;
853	struct bio *bio;
854	struct ctl_be_block_devdata *dev_data;
855	off_t cur_offset;
856	int max_iosize;
857
858	DPRINTF("entered\n");
859
860	dev_data = &be_lun->backend.dev;
861
862	/*
863	 * We have to limit our I/O size to the maximum supported by the
864	 * backend device.  Hopefully it is MAXPHYS.  If the driver doesn't
865	 * set it properly, use DFLTPHYS.
866	 */
867	max_iosize = dev_data->cdev->si_iosize_max;
868	if (max_iosize < PAGE_SIZE)
869		max_iosize = DFLTPHYS;
870
871	cur_offset = beio->io_offset;
872
873	/*
874	 * XXX KDM need to accurately reflect the number of I/Os outstanding
875	 * to a device.
876	 */
877	binuptime(&beio->ds_t0);
878	devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
879
880	for (i = 0; i < beio->num_segs; i++) {
881		size_t cur_size;
882		uint8_t *cur_ptr;
883
884		cur_size = beio->sg_segs[i].len;
885		cur_ptr = beio->sg_segs[i].addr;
886
887		while (cur_size > 0) {
888			/* This can't fail, it's a blocking allocation. */
889			bio = g_alloc_bio();
890
891			KASSERT(bio != NULL, ("g_alloc_bio() failed!\n"));
892
893			bio->bio_cmd = beio->bio_cmd;
894			bio->bio_flags |= beio->bio_flags;
895			bio->bio_dev = dev_data->cdev;
896			bio->bio_caller1 = beio;
897			bio->bio_length = min(cur_size, max_iosize);
898			bio->bio_offset = cur_offset;
899			bio->bio_data = cur_ptr;
900			bio->bio_done = ctl_be_block_biodone;
901			bio->bio_pblkno = cur_offset / be_lun->blocksize;
902
903			cur_offset += bio->bio_length;
904			cur_ptr += bio->bio_length;
905			cur_size -= bio->bio_length;
906
907			/*
908			 * Make sure we set the complete bit just before we
909			 * issue the last bio so we don't wind up with a
910			 * race.
911			 *
912			 * Use the LUN mutex here instead of a combination
913			 * of atomic variables for simplicity.
914			 *
915			 * XXX KDM we could have a per-IO lock, but that
916			 * would cause additional per-IO setup and teardown
917			 * overhead.  Hopefully there won't be too much
918			 * contention on the LUN lock.
919			 */
920			mtx_lock(&be_lun->lock);
921
922			beio->num_bios_sent++;
923
924			if ((i == beio->num_segs - 1)
925			 && (cur_size == 0))
926				beio->send_complete = 1;
927
928			mtx_unlock(&be_lun->lock);
929
930			(*dev_data->csw->d_strategy)(bio);
931		}
932	}
933}
934
935static void
936ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
937			 union ctl_io *io)
938{
939	struct ctl_be_block_io *beio;
940	struct ctl_be_block_softc *softc;
941
942	DPRINTF("entered\n");
943
944	softc = be_lun->softc;
945	beio = ctl_alloc_beio(softc);
946	if (beio == NULL) {
947		/*
948		 * This should not happen.  ctl_alloc_beio() will call
949		 * ctl_grow_beio() with a blocking malloc as needed.
950		 * A malloc with M_WAITOK should not fail.
951		 */
952		ctl_set_busy(&io->scsiio);
953		ctl_done(io);
954		return;
955	}
956
957	beio->io = io;
958	beio->softc = softc;
959	beio->lun = be_lun;
960	io->io_hdr.ctl_private[CTL_PRIV_BACKEND].ptr = beio;
961
962	switch (io->scsiio.cdb[0]) {
963	case SYNCHRONIZE_CACHE:
964	case SYNCHRONIZE_CACHE_16:
965		beio->bio_cmd = BIO_FLUSH;
966		beio->ds_trans_type = DEVSTAT_NO_DATA;
967		beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
968		beio->io_len = 0;
969		be_lun->lun_flush(be_lun, beio);
970		break;
971	default:
972		panic("Unhandled CDB type %#x", io->scsiio.cdb[0]);
973		break;
974	}
975}
976
977SDT_PROBE_DEFINE1(cbb, kernel, read, start, "uint64_t");
978SDT_PROBE_DEFINE1(cbb, kernel, write, start, "uint64_t");
979SDT_PROBE_DEFINE1(cbb, kernel, read, alloc_done, "uint64_t");
980SDT_PROBE_DEFINE1(cbb, kernel, write, alloc_done, "uint64_t");
981
982static void
983ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
984			   union ctl_io *io)
985{
986	struct ctl_be_block_io *beio;
987	struct ctl_be_block_softc *softc;
988	struct ctl_lba_len lbalen;
989	uint64_t len_left, io_size_bytes;
990	int i;
991
992	softc = be_lun->softc;
993
994	DPRINTF("entered\n");
995
996	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) {
997		SDT_PROBE(cbb, kernel, read, start, 0, 0, 0, 0, 0);
998	} else {
999		SDT_PROBE(cbb, kernel, write, start, 0, 0, 0, 0, 0);
1000	}
1001
1002	memcpy(&lbalen, io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
1003	       sizeof(lbalen));
1004
1005	io_size_bytes = lbalen.len * be_lun->blocksize;
1006
1007	/*
1008	 * XXX KDM this is temporary, until we implement chaining of beio
1009	 * structures and multiple datamove calls to move all the data in
1010	 * or out.
1011	 */
1012	if (io_size_bytes > CTLBLK_MAX_IO_SIZE) {
1013		printf("%s: IO length %ju > max io size %u\n", __func__,
1014		       io_size_bytes, CTLBLK_MAX_IO_SIZE);
1015		ctl_set_invalid_field(&io->scsiio,
1016				      /*sks_valid*/ 0,
1017				      /*command*/ 1,
1018				      /*field*/ 0,
1019				      /*bit_valid*/ 0,
1020				      /*bit*/ 0);
1021		ctl_done(io);
1022		return;
1023	}
1024
1025	beio = ctl_alloc_beio(softc);
1026	if (beio == NULL) {
1027		/*
1028		 * This should not happen.  ctl_alloc_beio() will call
1029		 * ctl_grow_beio() with a blocking malloc as needed.
1030		 * A malloc with M_WAITOK should not fail.
1031		 */
1032		ctl_set_busy(&io->scsiio);
1033		ctl_done(io);
1034		return;
1035	}
1036
1037	beio->io = io;
1038	beio->softc = softc;
1039	beio->lun = be_lun;
1040	io->io_hdr.ctl_private[CTL_PRIV_BACKEND].ptr = beio;
1041
1042	/*
1043	 * If the I/O came down with an ordered or head of queue tag, set
1044	 * the BIO_ORDERED attribute.  For head of queue tags, that's
1045	 * pretty much the best we can do.
1046	 *
1047	 * XXX KDM we don't have a great way to easily know about the FUA
1048	 * bit right now (it is decoded in ctl_read_write(), but we don't
1049	 * pass that knowledge to the backend), and in any case we would
1050	 * need to determine how to handle it.
1051	 */
1052	if ((io->scsiio.tag_type == CTL_TAG_ORDERED)
1053	 || (io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE))
1054		beio->bio_flags = BIO_ORDERED;
1055
1056	switch (io->scsiio.tag_type) {
1057	case CTL_TAG_ORDERED:
1058		beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1059		break;
1060	case CTL_TAG_HEAD_OF_QUEUE:
1061		beio->ds_tag_type = DEVSTAT_TAG_HEAD;
1062		break;
1063	case CTL_TAG_UNTAGGED:
1064	case CTL_TAG_SIMPLE:
1065	case CTL_TAG_ACA:
1066	default:
1067		beio->ds_tag_type = DEVSTAT_TAG_SIMPLE;
1068		break;
1069	}
1070
1071	/*
1072	 * This path handles read and write only.  The config write path
1073	 * handles flush operations.
1074	 */
1075	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) {
1076		beio->bio_cmd = BIO_READ;
1077		beio->ds_trans_type = DEVSTAT_READ;
1078	} else {
1079		beio->bio_cmd = BIO_WRITE;
1080		beio->ds_trans_type = DEVSTAT_WRITE;
1081	}
1082
1083	beio->io_len = lbalen.len * be_lun->blocksize;
1084	beio->io_offset = lbalen.lba * be_lun->blocksize;
1085
1086	DPRINTF("%s at LBA %jx len %u\n",
1087	       (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE",
1088	       (uintmax_t)lbalen.lba, lbalen.len);
1089
1090	for (i = 0, len_left = io_size_bytes; i < CTLBLK_MAX_SEGS &&
1091	     len_left > 0; i++) {
1092
1093		/*
1094		 * Setup the S/G entry for this chunk.
1095		 */
1096		beio->sg_segs[i].len = min(MAXPHYS, len_left);
1097		beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK);
1098
1099		DPRINTF("segment %d addr %p len %zd\n", i,
1100			beio->sg_segs[i].addr, beio->sg_segs[i].len);
1101
1102		beio->num_segs++;
1103		len_left -= beio->sg_segs[i].len;
1104	}
1105
1106	/*
1107	 * For the read case, we need to read the data into our buffers and
1108	 * then we can send it back to the user.  For the write case, we
1109	 * need to get the data from the user first.
1110	 */
1111	if (beio->bio_cmd == BIO_READ) {
1112		SDT_PROBE(cbb, kernel, read, alloc_done, 0, 0, 0, 0, 0);
1113		be_lun->dispatch(be_lun, beio);
1114	} else {
1115		SDT_PROBE(cbb, kernel, write, alloc_done, 0, 0, 0, 0, 0);
1116		io->scsiio.be_move_done = ctl_be_block_move_done;
1117		io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs;
1118		io->scsiio.kern_data_len = beio->io_len;
1119		io->scsiio.kern_total_len = beio->io_len;
1120		io->scsiio.kern_rel_offset = 0;
1121		io->scsiio.kern_data_resid = 0;
1122		io->scsiio.kern_sg_entries = beio->num_segs;
1123		io->io_hdr.flags |= CTL_FLAG_ALLOCATED | CTL_FLAG_KDPTR_SGLIST;
1124#ifdef CTL_TIME_IO
1125        	getbintime(&io->io_hdr.dma_start_bt);
1126#endif
1127		ctl_datamove(io);
1128	}
1129}
1130
1131static void
1132ctl_be_block_worker(void *context, int pending)
1133{
1134	struct ctl_be_block_lun *be_lun;
1135	struct ctl_be_block_softc *softc;
1136	union ctl_io *io;
1137
1138	be_lun = (struct ctl_be_block_lun *)context;
1139	softc = be_lun->softc;
1140
1141	DPRINTF("entered\n");
1142
1143	mtx_lock(&be_lun->lock);
1144	for (;;) {
1145		io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue);
1146		if (io != NULL) {
1147			struct ctl_be_block_io *beio;
1148
1149			DPRINTF("datamove queue\n");
1150
1151			STAILQ_REMOVE(&be_lun->datamove_queue, &io->io_hdr,
1152				      ctl_io_hdr, links);
1153
1154			mtx_unlock(&be_lun->lock);
1155
1156			beio = (struct ctl_be_block_io *)
1157			    io->io_hdr.ctl_private[CTL_PRIV_BACKEND].ptr;
1158
1159			be_lun->dispatch(be_lun, beio);
1160
1161			mtx_lock(&be_lun->lock);
1162			continue;
1163		}
1164		io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue);
1165		if (io != NULL) {
1166
1167			DPRINTF("config write queue\n");
1168
1169			STAILQ_REMOVE(&be_lun->config_write_queue, &io->io_hdr,
1170				      ctl_io_hdr, links);
1171
1172			mtx_unlock(&be_lun->lock);
1173
1174			ctl_be_block_cw_dispatch(be_lun, io);
1175
1176			mtx_lock(&be_lun->lock);
1177			continue;
1178		}
1179		io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue);
1180		if (io != NULL) {
1181			DPRINTF("input queue\n");
1182
1183			STAILQ_REMOVE(&be_lun->input_queue, &io->io_hdr,
1184				      ctl_io_hdr, links);
1185			mtx_unlock(&be_lun->lock);
1186
1187			/*
1188			 * We must drop the lock, since this routine and
1189			 * its children may sleep.
1190			 */
1191			ctl_be_block_dispatch(be_lun, io);
1192
1193			mtx_lock(&be_lun->lock);
1194			continue;
1195		}
1196
1197		/*
1198		 * If we get here, there is no work left in the queues, so
1199		 * just break out and let the task queue go to sleep.
1200		 */
1201		break;
1202	}
1203	mtx_unlock(&be_lun->lock);
1204}
1205
1206/*
1207 * Entry point from CTL to the backend for I/O.  We queue everything to a
1208 * work thread, so this just puts the I/O on a queue and wakes up the
1209 * thread.
1210 */
1211static int
1212ctl_be_block_submit(union ctl_io *io)
1213{
1214	struct ctl_be_block_lun *be_lun;
1215	struct ctl_be_lun *ctl_be_lun;
1216	int retval;
1217
1218	DPRINTF("entered\n");
1219
1220	retval = CTL_RETVAL_COMPLETE;
1221
1222	ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[
1223		CTL_PRIV_BACKEND_LUN].ptr;
1224	be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun;
1225
1226	/*
1227	 * Make sure we only get SCSI I/O.
1228	 */
1229	KASSERT(io->io_hdr.io_type == CTL_IO_SCSI, ("Non-SCSI I/O (type "
1230		"%#x) encountered", io->io_hdr.io_type));
1231
1232	mtx_lock(&be_lun->lock);
1233	/*
1234	 * XXX KDM make sure that links is okay to use at this point.
1235	 * Otherwise, we either need to add another field to ctl_io_hdr,
1236	 * or deal with resource allocation here.
1237	 */
1238	STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);
1239	mtx_unlock(&be_lun->lock);
1240
1241	taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
1242
1243	return (retval);
1244}
1245
1246static int
1247ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
1248			int flag, struct thread *td)
1249{
1250	struct ctl_be_block_softc *softc;
1251	int error;
1252
1253	softc = &backend_block_softc;
1254
1255	error = 0;
1256
1257	switch (cmd) {
1258	case CTL_LUN_REQ: {
1259		struct ctl_lun_req *lun_req;
1260
1261		lun_req = (struct ctl_lun_req *)addr;
1262
1263		switch (lun_req->reqtype) {
1264		case CTL_LUNREQ_CREATE:
1265			error = ctl_be_block_create(softc, lun_req);
1266			break;
1267		case CTL_LUNREQ_RM:
1268			error = ctl_be_block_rm(softc, lun_req);
1269			break;
1270		case CTL_LUNREQ_MODIFY:
1271			error = ctl_be_block_modify(softc, lun_req);
1272			break;
1273		default:
1274			lun_req->status = CTL_LUN_ERROR;
1275			snprintf(lun_req->error_str, sizeof(lun_req->error_str),
1276				 "%s: invalid LUN request type %d", __func__,
1277				 lun_req->reqtype);
1278			break;
1279		}
1280		break;
1281	}
1282	default:
1283		error = ENOTTY;
1284		break;
1285	}
1286
1287	return (error);
1288}
1289
1290static int
1291ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1292{
1293	struct ctl_be_block_filedata *file_data;
1294	struct ctl_lun_create_params *params;
1295	struct vattr		      vattr;
1296	int			      error;
1297
1298	error = 0;
1299	file_data = &be_lun->backend.file;
1300	params = &req->reqdata.create;
1301
1302	be_lun->dev_type = CTL_BE_BLOCK_FILE;
1303	be_lun->dispatch = ctl_be_block_dispatch_file;
1304	be_lun->lun_flush = ctl_be_block_flush_file;
1305
1306	error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
1307	if (error != 0) {
1308		snprintf(req->error_str, sizeof(req->error_str),
1309			 "error calling VOP_GETATTR() for file %s",
1310			 be_lun->dev_path);
1311		return (error);
1312	}
1313
1314	/*
1315	 * Verify that we have the ability to upgrade to exclusive
1316	 * access on this file so we can trap errors at open instead
1317	 * of reporting them during first access.
1318	 */
1319	if (VOP_ISLOCKED(be_lun->vn) != LK_EXCLUSIVE) {
1320		vn_lock(be_lun->vn, LK_UPGRADE | LK_RETRY);
1321		if (be_lun->vn->v_iflag & VI_DOOMED) {
1322			error = EBADF;
1323			snprintf(req->error_str, sizeof(req->error_str),
1324				 "error locking file %s", be_lun->dev_path);
1325			return (error);
1326		}
1327	}
1328
1329
1330	file_data->cred = crhold(curthread->td_ucred);
1331	if (params->lun_size_bytes != 0)
1332		be_lun->size_bytes = params->lun_size_bytes;
1333	else
1334		be_lun->size_bytes = vattr.va_size;
1335	/*
1336	 * We set the multi thread flag for file operations because all
1337	 * filesystems (in theory) are capable of allowing multiple readers
1338	 * of a file at once.  So we want to get the maximum possible
1339	 * concurrency.
1340	 */
1341	be_lun->flags |= CTL_BE_BLOCK_LUN_MULTI_THREAD;
1342
1343	/*
1344	 * XXX KDM vattr.va_blocksize may be larger than 512 bytes here.
1345	 * With ZFS, it is 131072 bytes.  Block sizes that large don't work
1346	 * with disklabel and UFS on FreeBSD at least.  Large block sizes
1347	 * may not work with other OSes as well.  So just export a sector
1348	 * size of 512 bytes, which should work with any OS or
1349	 * application.  Since our backing is a file, any block size will
1350	 * work fine for the backing store.
1351	 */
1352#if 0
1353	be_lun->blocksize= vattr.va_blocksize;
1354#endif
1355	if (params->blocksize_bytes != 0)
1356		be_lun->blocksize = params->blocksize_bytes;
1357	else
1358		be_lun->blocksize = 512;
1359
1360	/*
1361	 * Sanity check.  The media size has to be at least one
1362	 * sector long.
1363	 */
1364	if (be_lun->size_bytes < be_lun->blocksize) {
1365		error = EINVAL;
1366		snprintf(req->error_str, sizeof(req->error_str),
1367			 "file %s size %ju < block size %u", be_lun->dev_path,
1368			 (uintmax_t)be_lun->size_bytes, be_lun->blocksize);
1369	}
1370	return (error);
1371}
1372
1373static int
1374ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1375{
1376	struct ctl_lun_create_params *params;
1377	struct vattr		      vattr;
1378	struct cdev		     *dev;
1379	struct cdevsw		     *devsw;
1380	int			      error;
1381	off_t			      ps, pss, po, pos;
1382
1383	params = &req->reqdata.create;
1384
1385	be_lun->dev_type = CTL_BE_BLOCK_DEV;
1386	be_lun->dispatch = ctl_be_block_dispatch_dev;
1387	be_lun->lun_flush = ctl_be_block_flush_dev;
1388	be_lun->backend.dev.cdev = be_lun->vn->v_rdev;
1389	be_lun->backend.dev.csw = dev_refthread(be_lun->backend.dev.cdev,
1390					     &be_lun->backend.dev.dev_ref);
1391	if (be_lun->backend.dev.csw == NULL)
1392		panic("Unable to retrieve device switch");
1393
1394	error = VOP_GETATTR(be_lun->vn, &vattr, NOCRED);
1395	if (error) {
1396		snprintf(req->error_str, sizeof(req->error_str),
1397			 "%s: error getting vnode attributes for device %s",
1398			 __func__, be_lun->dev_path);
1399		return (error);
1400	}
1401
1402	dev = be_lun->vn->v_rdev;
1403	devsw = dev->si_devsw;
1404	if (!devsw->d_ioctl) {
1405		snprintf(req->error_str, sizeof(req->error_str),
1406			 "%s: no d_ioctl for device %s!", __func__,
1407			 be_lun->dev_path);
1408		return (ENODEV);
1409	}
1410
1411	error = devsw->d_ioctl(dev, DIOCGSECTORSIZE,
1412			       (caddr_t)&be_lun->blocksize, FREAD,
1413			       curthread);
1414	if (error) {
1415		snprintf(req->error_str, sizeof(req->error_str),
1416			 "%s: error %d returned for DIOCGSECTORSIZE ioctl "
1417			 "on %s!", __func__, error, be_lun->dev_path);
1418		return (error);
1419	}
1420
1421	/*
1422	 * If the user has asked for a blocksize that is greater than the
1423	 * backing device's blocksize, we can do it only if the blocksize
1424	 * the user is asking for is an even multiple of the underlying
1425	 * device's blocksize.
1426	 */
1427	if ((params->blocksize_bytes != 0)
1428	 && (params->blocksize_bytes > be_lun->blocksize)) {
1429		uint32_t bs_multiple, tmp_blocksize;
1430
1431		bs_multiple = params->blocksize_bytes / be_lun->blocksize;
1432
1433		tmp_blocksize = bs_multiple * be_lun->blocksize;
1434
1435		if (tmp_blocksize == params->blocksize_bytes) {
1436			be_lun->blocksize = params->blocksize_bytes;
1437		} else {
1438			snprintf(req->error_str, sizeof(req->error_str),
1439				 "%s: requested blocksize %u is not an even "
1440				 "multiple of backing device blocksize %u",
1441				 __func__, params->blocksize_bytes,
1442				 be_lun->blocksize);
1443			return (EINVAL);
1444
1445		}
1446	} else if ((params->blocksize_bytes != 0)
1447		&& (params->blocksize_bytes != be_lun->blocksize)) {
1448		snprintf(req->error_str, sizeof(req->error_str),
1449			 "%s: requested blocksize %u < backing device "
1450			 "blocksize %u", __func__, params->blocksize_bytes,
1451			 be_lun->blocksize);
1452		return (EINVAL);
1453	}
1454
1455	error = devsw->d_ioctl(dev, DIOCGMEDIASIZE,
1456			       (caddr_t)&be_lun->size_bytes, FREAD,
1457			       curthread);
1458	if (error) {
1459		snprintf(req->error_str, sizeof(req->error_str),
1460			 "%s: error %d returned for DIOCGMEDIASIZE "
1461			 " ioctl on %s!", __func__, error,
1462			 be_lun->dev_path);
1463		return (error);
1464	}
1465
1466	if (params->lun_size_bytes != 0) {
1467		if (params->lun_size_bytes > be_lun->size_bytes) {
1468			snprintf(req->error_str, sizeof(req->error_str),
1469				 "%s: requested LUN size %ju > backing device "
1470				 "size %ju", __func__,
1471				 (uintmax_t)params->lun_size_bytes,
1472				 (uintmax_t)be_lun->size_bytes);
1473			return (EINVAL);
1474		}
1475
1476		be_lun->size_bytes = params->lun_size_bytes;
1477	}
1478
1479	error = devsw->d_ioctl(dev, DIOCGSTRIPESIZE,
1480			       (caddr_t)&ps, FREAD, curthread);
1481	if (error)
1482		ps = po = 0;
1483	else {
1484		error = devsw->d_ioctl(dev, DIOCGSTRIPEOFFSET,
1485				       (caddr_t)&po, FREAD, curthread);
1486		if (error)
1487			po = 0;
1488	}
1489	pss = ps / be_lun->blocksize;
1490	pos = po / be_lun->blocksize;
1491	if ((pss > 0) && (pss * be_lun->blocksize == ps) && (pss >= pos) &&
1492	    ((pss & (pss - 1)) == 0) && (pos * be_lun->blocksize == po)) {
1493		be_lun->pblockexp = fls(pss) - 1;
1494		be_lun->pblockoff = (pss - pos) % pss;
1495	}
1496
1497	return (0);
1498}
1499
1500static int
1501ctl_be_block_close(struct ctl_be_block_lun *be_lun)
1502{
1503	DROP_GIANT();
1504	if (be_lun->vn) {
1505		int flags = FREAD | FWRITE;
1506		int vfs_is_locked = 0;
1507
1508		switch (be_lun->dev_type) {
1509		case CTL_BE_BLOCK_DEV:
1510			if (be_lun->backend.dev.csw) {
1511				dev_relthread(be_lun->backend.dev.cdev,
1512					      be_lun->backend.dev.dev_ref);
1513				be_lun->backend.dev.csw  = NULL;
1514				be_lun->backend.dev.cdev = NULL;
1515			}
1516			break;
1517		case CTL_BE_BLOCK_FILE:
1518			vfs_is_locked = VFS_LOCK_GIANT(be_lun->vn->v_mount);
1519			break;
1520		case CTL_BE_BLOCK_NONE:
1521			break;
1522		default:
1523			panic("Unexpected backend type.");
1524			break;
1525		}
1526
1527		(void)vn_close(be_lun->vn, flags, NOCRED, curthread);
1528		be_lun->vn = NULL;
1529
1530		switch (be_lun->dev_type) {
1531		case CTL_BE_BLOCK_DEV:
1532			break;
1533		case CTL_BE_BLOCK_FILE:
1534			VFS_UNLOCK_GIANT(vfs_is_locked);
1535			if (be_lun->backend.file.cred != NULL) {
1536				crfree(be_lun->backend.file.cred);
1537				be_lun->backend.file.cred = NULL;
1538			}
1539			break;
1540		case CTL_BE_BLOCK_NONE:
1541			break;
1542		default:
1543			panic("Unexpected backend type.");
1544			break;
1545		}
1546	}
1547	PICKUP_GIANT();
1548
1549	return (0);
1550}
1551
1552static int
1553ctl_be_block_open(struct ctl_be_block_softc *softc,
1554		       struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1555{
1556	struct nameidata nd;
1557	int		 flags;
1558	int		 error;
1559	int		 vfs_is_locked;
1560
1561	/*
1562	 * XXX KDM allow a read-only option?
1563	 */
1564	flags = FREAD | FWRITE;
1565	error = 0;
1566
1567	if (rootvnode == NULL) {
1568		snprintf(req->error_str, sizeof(req->error_str),
1569			 "%s: Root filesystem is not mounted", __func__);
1570		return (1);
1571	}
1572
1573	if (!curthread->td_proc->p_fd->fd_cdir) {
1574		curthread->td_proc->p_fd->fd_cdir = rootvnode;
1575		VREF(rootvnode);
1576	}
1577	if (!curthread->td_proc->p_fd->fd_rdir) {
1578		curthread->td_proc->p_fd->fd_rdir = rootvnode;
1579		VREF(rootvnode);
1580	}
1581	if (!curthread->td_proc->p_fd->fd_jdir) {
1582		curthread->td_proc->p_fd->fd_jdir = rootvnode;
1583		VREF(rootvnode);
1584	}
1585
1586 again:
1587	NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, be_lun->dev_path, curthread);
1588	error = vn_open(&nd, &flags, 0, NULL);
1589	if (error) {
1590		/*
1591		 * This is the only reasonable guess we can make as far as
1592		 * path if the user doesn't give us a fully qualified path.
1593		 * If they want to specify a file, they need to specify the
1594		 * full path.
1595		 */
1596		if (be_lun->dev_path[0] != '/') {
1597			char *dev_path = "/dev/";
1598			char *dev_name;
1599
1600			/* Try adding device path at beginning of name */
1601			dev_name = malloc(strlen(be_lun->dev_path)
1602					+ strlen(dev_path) + 1,
1603					  M_CTLBLK, M_WAITOK);
1604			if (dev_name) {
1605				sprintf(dev_name, "%s%s", dev_path,
1606					be_lun->dev_path);
1607				free(be_lun->dev_path, M_CTLBLK);
1608				be_lun->dev_path = dev_name;
1609				goto again;
1610			}
1611		}
1612		snprintf(req->error_str, sizeof(req->error_str),
1613			 "%s: error opening %s", __func__, be_lun->dev_path);
1614		return (error);
1615	}
1616
1617	vfs_is_locked = NDHASGIANT(&nd);
1618
1619	NDFREE(&nd, NDF_ONLY_PNBUF);
1620
1621	be_lun->vn = nd.ni_vp;
1622
1623	/* We only support disks and files. */
1624	if (vn_isdisk(be_lun->vn, &error)) {
1625		error = ctl_be_block_open_dev(be_lun, req);
1626	} else if (be_lun->vn->v_type == VREG) {
1627		error = ctl_be_block_open_file(be_lun, req);
1628	} else {
1629		error = EINVAL;
1630		snprintf(req->error_str, sizeof(req->error_str),
1631			 "%s is not a disk or plain file", be_lun->dev_path);
1632	}
1633	VOP_UNLOCK(be_lun->vn, 0);
1634	VFS_UNLOCK_GIANT(vfs_is_locked);
1635
1636	if (error != 0) {
1637		ctl_be_block_close(be_lun);
1638		return (error);
1639	}
1640
1641	be_lun->blocksize_shift = fls(be_lun->blocksize) - 1;
1642	be_lun->size_blocks = be_lun->size_bytes >> be_lun->blocksize_shift;
1643
1644	return (0);
1645}
1646
1647static int
1648ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
1649{
1650	struct ctl_be_block_lun *be_lun;
1651	struct ctl_lun_create_params *params;
1652	struct ctl_be_arg *file_arg;
1653	char tmpstr[32];
1654	int retval, num_threads;
1655	int i;
1656
1657	params = &req->reqdata.create;
1658	retval = 0;
1659
1660	num_threads = cbb_num_threads;
1661
1662	file_arg = NULL;
1663
1664	be_lun = malloc(sizeof(*be_lun), M_CTLBLK, M_ZERO | M_WAITOK);
1665
1666	be_lun->softc = softc;
1667	STAILQ_INIT(&be_lun->input_queue);
1668	STAILQ_INIT(&be_lun->config_write_queue);
1669	STAILQ_INIT(&be_lun->datamove_queue);
1670	sprintf(be_lun->lunname, "cblk%d", softc->num_luns);
1671	mtx_init(&be_lun->lock, be_lun->lunname, NULL, MTX_DEF);
1672
1673	be_lun->lun_zone = uma_zcreate(be_lun->lunname, MAXPHYS,
1674	    NULL, NULL, NULL, NULL, /*align*/ 0, /*flags*/0);
1675
1676	if (be_lun->lun_zone == NULL) {
1677		snprintf(req->error_str, sizeof(req->error_str),
1678			 "%s: error allocating UMA zone", __func__);
1679		goto bailout_error;
1680	}
1681
1682	if (params->flags & CTL_LUN_FLAG_DEV_TYPE)
1683		be_lun->ctl_be_lun.lun_type = params->device_type;
1684	else
1685		be_lun->ctl_be_lun.lun_type = T_DIRECT;
1686
1687	if (be_lun->ctl_be_lun.lun_type == T_DIRECT) {
1688		for (i = 0; i < req->num_be_args; i++) {
1689			if (strcmp(req->kern_be_args[i].kname, "file") == 0) {
1690				file_arg = &req->kern_be_args[i];
1691				break;
1692			}
1693		}
1694
1695		if (file_arg == NULL) {
1696			snprintf(req->error_str, sizeof(req->error_str),
1697				 "%s: no file argument specified", __func__);
1698			goto bailout_error;
1699		}
1700
1701		be_lun->dev_path = malloc(file_arg->vallen, M_CTLBLK,
1702					  M_WAITOK | M_ZERO);
1703
1704		strlcpy(be_lun->dev_path, (char *)file_arg->kvalue,
1705			file_arg->vallen);
1706
1707		retval = ctl_be_block_open(softc, be_lun, req);
1708		if (retval != 0) {
1709			retval = 0;
1710			goto bailout_error;
1711		}
1712
1713		/*
1714		 * Tell the user the size of the file/device.
1715		 */
1716		params->lun_size_bytes = be_lun->size_bytes;
1717
1718		/*
1719		 * The maximum LBA is the size - 1.
1720		 */
1721		be_lun->ctl_be_lun.maxlba = be_lun->size_blocks - 1;
1722	} else {
1723		/*
1724		 * For processor devices, we don't have any size.
1725		 */
1726		be_lun->blocksize = 0;
1727		be_lun->pblockexp = 0;
1728		be_lun->pblockoff = 0;
1729		be_lun->size_blocks = 0;
1730		be_lun->size_bytes = 0;
1731		be_lun->ctl_be_lun.maxlba = 0;
1732		params->lun_size_bytes = 0;
1733
1734		/*
1735		 * Default to just 1 thread for processor devices.
1736		 */
1737		num_threads = 1;
1738	}
1739
1740	/*
1741	 * XXX This searching loop might be refactored to be combined with
1742	 * the loop above,
1743	 */
1744	for (i = 0; i < req->num_be_args; i++) {
1745		if (strcmp(req->kern_be_args[i].kname, "num_threads") == 0) {
1746			struct ctl_be_arg *thread_arg;
1747			char num_thread_str[16];
1748			int tmp_num_threads;
1749
1750
1751			thread_arg = &req->kern_be_args[i];
1752
1753			strlcpy(num_thread_str, (char *)thread_arg->kvalue,
1754				min(thread_arg->vallen,
1755				sizeof(num_thread_str)));
1756
1757			tmp_num_threads = strtol(num_thread_str, NULL, 0);
1758
1759			/*
1760			 * We don't let the user specify less than one
1761			 * thread, but hope he's clueful enough not to
1762			 * specify 1000 threads.
1763			 */
1764			if (tmp_num_threads < 1) {
1765				snprintf(req->error_str, sizeof(req->error_str),
1766					 "%s: invalid number of threads %s",
1767				         __func__, num_thread_str);
1768				goto bailout_error;
1769			}
1770
1771			num_threads = tmp_num_threads;
1772		}
1773	}
1774
1775	be_lun->flags = CTL_BE_BLOCK_LUN_UNCONFIGURED;
1776	be_lun->ctl_be_lun.flags = CTL_LUN_FLAG_PRIMARY;
1777	be_lun->ctl_be_lun.be_lun = be_lun;
1778	be_lun->ctl_be_lun.blocksize = be_lun->blocksize;
1779	be_lun->ctl_be_lun.pblockexp = be_lun->pblockexp;
1780	be_lun->ctl_be_lun.pblockoff = be_lun->pblockoff;
1781	/* Tell the user the blocksize we ended up using */
1782	params->blocksize_bytes = be_lun->blocksize;
1783	if (params->flags & CTL_LUN_FLAG_ID_REQ) {
1784		be_lun->ctl_be_lun.req_lun_id = params->req_lun_id;
1785		be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_ID_REQ;
1786	} else
1787		be_lun->ctl_be_lun.req_lun_id = 0;
1788
1789	be_lun->ctl_be_lun.lun_shutdown = ctl_be_block_lun_shutdown;
1790	be_lun->ctl_be_lun.lun_config_status =
1791		ctl_be_block_lun_config_status;
1792	be_lun->ctl_be_lun.be = &ctl_be_block_driver;
1793
1794	if ((params->flags & CTL_LUN_FLAG_SERIAL_NUM) == 0) {
1795		snprintf(tmpstr, sizeof(tmpstr), "MYSERIAL%4d",
1796			 softc->num_luns);
1797		strncpy((char *)be_lun->ctl_be_lun.serial_num, tmpstr,
1798			ctl_min(sizeof(be_lun->ctl_be_lun.serial_num),
1799			sizeof(tmpstr)));
1800
1801		/* Tell the user what we used for a serial number */
1802		strncpy((char *)params->serial_num, tmpstr,
1803			ctl_min(sizeof(params->serial_num), sizeof(tmpstr)));
1804	} else {
1805		strncpy((char *)be_lun->ctl_be_lun.serial_num,
1806			params->serial_num,
1807			ctl_min(sizeof(be_lun->ctl_be_lun.serial_num),
1808			sizeof(params->serial_num)));
1809	}
1810	if ((params->flags & CTL_LUN_FLAG_DEVID) == 0) {
1811		snprintf(tmpstr, sizeof(tmpstr), "MYDEVID%4d", softc->num_luns);
1812		strncpy((char *)be_lun->ctl_be_lun.device_id, tmpstr,
1813			ctl_min(sizeof(be_lun->ctl_be_lun.device_id),
1814			sizeof(tmpstr)));
1815
1816		/* Tell the user what we used for a device ID */
1817		strncpy((char *)params->device_id, tmpstr,
1818			ctl_min(sizeof(params->device_id), sizeof(tmpstr)));
1819	} else {
1820		strncpy((char *)be_lun->ctl_be_lun.device_id,
1821			params->device_id,
1822			ctl_min(sizeof(be_lun->ctl_be_lun.device_id),
1823				sizeof(params->device_id)));
1824	}
1825
1826	TASK_INIT(&be_lun->io_task, /*priority*/0, ctl_be_block_worker, be_lun);
1827
1828	be_lun->io_taskqueue = taskqueue_create(be_lun->lunname, M_WAITOK,
1829	    taskqueue_thread_enqueue, /*context*/&be_lun->io_taskqueue);
1830
1831	if (be_lun->io_taskqueue == NULL) {
1832		snprintf(req->error_str, sizeof(req->error_str),
1833			 "%s: Unable to create taskqueue", __func__);
1834		goto bailout_error;
1835	}
1836
1837	/*
1838	 * Note that we start the same number of threads by default for
1839	 * both the file case and the block device case.  For the file
1840	 * case, we need multiple threads to allow concurrency, because the
1841	 * vnode interface is designed to be a blocking interface.  For the
1842	 * block device case, ZFS zvols at least will block the caller's
1843	 * context in many instances, and so we need multiple threads to
1844	 * overcome that problem.  Other block devices don't need as many
1845	 * threads, but they shouldn't cause too many problems.
1846	 *
1847	 * If the user wants to just have a single thread for a block
1848	 * device, he can specify that when the LUN is created, or change
1849	 * the tunable/sysctl to alter the default number of threads.
1850	 */
1851	retval = taskqueue_start_threads(&be_lun->io_taskqueue,
1852					 /*num threads*/num_threads,
1853					 /*priority*/PWAIT,
1854					 /*thread name*/
1855					 "%s taskq", be_lun->lunname);
1856
1857	if (retval != 0)
1858		goto bailout_error;
1859
1860	be_lun->num_threads = num_threads;
1861
1862	mtx_lock(&softc->lock);
1863	softc->num_luns++;
1864	STAILQ_INSERT_TAIL(&softc->lun_list, be_lun, links);
1865
1866	mtx_unlock(&softc->lock);
1867
1868	retval = ctl_add_lun(&be_lun->ctl_be_lun);
1869	if (retval != 0) {
1870		mtx_lock(&softc->lock);
1871		STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun,
1872			      links);
1873		softc->num_luns--;
1874		mtx_unlock(&softc->lock);
1875		snprintf(req->error_str, sizeof(req->error_str),
1876			 "%s: ctl_add_lun() returned error %d, see dmesg for "
1877			"details", __func__, retval);
1878		retval = 0;
1879		goto bailout_error;
1880	}
1881
1882	mtx_lock(&softc->lock);
1883
1884	/*
1885	 * Tell the config_status routine that we're waiting so it won't
1886	 * clean up the LUN in the event of an error.
1887	 */
1888	be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING;
1889
1890	while (be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) {
1891		retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0);
1892		if (retval == EINTR)
1893			break;
1894	}
1895	be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;
1896
1897	if (be_lun->flags & CTL_BE_BLOCK_LUN_CONFIG_ERR) {
1898		snprintf(req->error_str, sizeof(req->error_str),
1899			 "%s: LUN configuration error, see dmesg for details",
1900			 __func__);
1901		STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun,
1902			      links);
1903		softc->num_luns--;
1904		mtx_unlock(&softc->lock);
1905		goto bailout_error;
1906	} else {
1907		params->req_lun_id = be_lun->ctl_be_lun.lun_id;
1908	}
1909
1910	mtx_unlock(&softc->lock);
1911
1912	be_lun->disk_stats = devstat_new_entry("cbb", params->req_lun_id,
1913					       be_lun->blocksize,
1914					       DEVSTAT_ALL_SUPPORTED,
1915					       be_lun->ctl_be_lun.lun_type
1916					       | DEVSTAT_TYPE_IF_OTHER,
1917					       DEVSTAT_PRIORITY_OTHER);
1918
1919
1920	req->status = CTL_LUN_OK;
1921
1922	return (retval);
1923
1924bailout_error:
1925	req->status = CTL_LUN_ERROR;
1926
1927	ctl_be_block_close(be_lun);
1928
1929	free(be_lun->dev_path, M_CTLBLK);
1930	free(be_lun, M_CTLBLK);
1931
1932	return (retval);
1933}
1934
1935static int
1936ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
1937{
1938	struct ctl_lun_rm_params *params;
1939	struct ctl_be_block_lun *be_lun;
1940	int retval;
1941
1942	params = &req->reqdata.rm;
1943
1944	mtx_lock(&softc->lock);
1945
1946	be_lun = NULL;
1947
1948	STAILQ_FOREACH(be_lun, &softc->lun_list, links) {
1949		if (be_lun->ctl_be_lun.lun_id == params->lun_id)
1950			break;
1951	}
1952	mtx_unlock(&softc->lock);
1953
1954	if (be_lun == NULL) {
1955		snprintf(req->error_str, sizeof(req->error_str),
1956			 "%s: LUN %u is not managed by the block backend",
1957			 __func__, params->lun_id);
1958		goto bailout_error;
1959	}
1960
1961	retval = ctl_disable_lun(&be_lun->ctl_be_lun);
1962
1963	if (retval != 0) {
1964		snprintf(req->error_str, sizeof(req->error_str),
1965			 "%s: error %d returned from ctl_disable_lun() for "
1966			 "LUN %d", __func__, retval, params->lun_id);
1967		goto bailout_error;
1968
1969	}
1970
1971	retval = ctl_invalidate_lun(&be_lun->ctl_be_lun);
1972	if (retval != 0) {
1973		snprintf(req->error_str, sizeof(req->error_str),
1974			 "%s: error %d returned from ctl_invalidate_lun() for "
1975			 "LUN %d", __func__, retval, params->lun_id);
1976		goto bailout_error;
1977	}
1978
1979	mtx_lock(&softc->lock);
1980
1981	be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING;
1982
1983	while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) {
1984                retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0);
1985                if (retval == EINTR)
1986                        break;
1987        }
1988
1989	be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;
1990
1991	if ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) {
1992		snprintf(req->error_str, sizeof(req->error_str),
1993			 "%s: interrupted waiting for LUN to be freed",
1994			 __func__);
1995		mtx_unlock(&softc->lock);
1996		goto bailout_error;
1997	}
1998
1999	STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, links);
2000
2001	softc->num_luns--;
2002	mtx_unlock(&softc->lock);
2003
2004	taskqueue_drain(be_lun->io_taskqueue, &be_lun->io_task);
2005
2006	taskqueue_free(be_lun->io_taskqueue);
2007
2008	ctl_be_block_close(be_lun);
2009
2010	if (be_lun->disk_stats != NULL)
2011		devstat_remove_entry(be_lun->disk_stats);
2012
2013	uma_zdestroy(be_lun->lun_zone);
2014
2015	free(be_lun->dev_path, M_CTLBLK);
2016
2017	free(be_lun, M_CTLBLK);
2018
2019	req->status = CTL_LUN_OK;
2020
2021	return (0);
2022
2023bailout_error:
2024
2025	req->status = CTL_LUN_ERROR;
2026
2027	return (0);
2028}
2029
2030static int
2031ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun,
2032			 struct ctl_lun_req *req)
2033{
2034	struct vattr vattr;
2035	int error;
2036	struct ctl_lun_modify_params *params;
2037
2038	params = &req->reqdata.modify;
2039
2040	if (params->lun_size_bytes != 0) {
2041		be_lun->size_bytes = params->lun_size_bytes;
2042	} else  {
2043		error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
2044		if (error != 0) {
2045			snprintf(req->error_str, sizeof(req->error_str),
2046				 "error calling VOP_GETATTR() for file %s",
2047				 be_lun->dev_path);
2048			return (error);
2049		}
2050
2051		be_lun->size_bytes = vattr.va_size;
2052	}
2053
2054	return (0);
2055}
2056
2057static int
2058ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun,
2059			struct ctl_lun_req *req)
2060{
2061	struct cdev *dev;
2062	struct cdevsw *devsw;
2063	int error;
2064	struct ctl_lun_modify_params *params;
2065	uint64_t size_bytes;
2066
2067	params = &req->reqdata.modify;
2068
2069	dev = be_lun->vn->v_rdev;
2070	devsw = dev->si_devsw;
2071	if (!devsw->d_ioctl) {
2072		snprintf(req->error_str, sizeof(req->error_str),
2073			 "%s: no d_ioctl for device %s!", __func__,
2074			 be_lun->dev_path);
2075		return (ENODEV);
2076	}
2077
2078	error = devsw->d_ioctl(dev, DIOCGMEDIASIZE,
2079			       (caddr_t)&size_bytes, FREAD,
2080			       curthread);
2081	if (error) {
2082		snprintf(req->error_str, sizeof(req->error_str),
2083			 "%s: error %d returned for DIOCGMEDIASIZE ioctl "
2084			 "on %s!", __func__, error, be_lun->dev_path);
2085		return (error);
2086	}
2087
2088	if (params->lun_size_bytes != 0) {
2089		if (params->lun_size_bytes > size_bytes) {
2090			snprintf(req->error_str, sizeof(req->error_str),
2091				 "%s: requested LUN size %ju > backing device "
2092				 "size %ju", __func__,
2093				 (uintmax_t)params->lun_size_bytes,
2094				 (uintmax_t)size_bytes);
2095			return (EINVAL);
2096		}
2097
2098		be_lun->size_bytes = params->lun_size_bytes;
2099	} else {
2100		be_lun->size_bytes = size_bytes;
2101	}
2102
2103	return (0);
2104}
2105
2106static int
2107ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2108{
2109	struct ctl_lun_modify_params *params;
2110	struct ctl_be_block_lun *be_lun;
2111	int vfs_is_locked, error;
2112
2113	params = &req->reqdata.modify;
2114
2115	mtx_lock(&softc->lock);
2116
2117	be_lun = NULL;
2118
2119	STAILQ_FOREACH(be_lun, &softc->lun_list, links) {
2120		if (be_lun->ctl_be_lun.lun_id == params->lun_id)
2121			break;
2122	}
2123	mtx_unlock(&softc->lock);
2124
2125	if (be_lun == NULL) {
2126		snprintf(req->error_str, sizeof(req->error_str),
2127			 "%s: LUN %u is not managed by the block backend",
2128			 __func__, params->lun_id);
2129		goto bailout_error;
2130	}
2131
2132	if (params->lun_size_bytes != 0) {
2133		if (params->lun_size_bytes < be_lun->blocksize) {
2134			snprintf(req->error_str, sizeof(req->error_str),
2135				"%s: LUN size %ju < blocksize %u", __func__,
2136				params->lun_size_bytes, be_lun->blocksize);
2137			goto bailout_error;
2138		}
2139	}
2140
2141	vfs_is_locked = VFS_LOCK_GIANT(be_lun->vn->v_mount);
2142	vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
2143
2144	if (be_lun->vn->v_type == VREG)
2145		error = ctl_be_block_modify_file(be_lun, req);
2146	else
2147		error = ctl_be_block_modify_dev(be_lun, req);
2148
2149	VOP_UNLOCK(be_lun->vn, 0);
2150	VFS_UNLOCK_GIANT(vfs_is_locked);
2151
2152	if (error != 0)
2153		goto bailout_error;
2154
2155	be_lun->size_blocks = be_lun->size_bytes >> be_lun->blocksize_shift;
2156
2157	/*
2158	 * The maximum LBA is the size - 1.
2159	 *
2160	 * XXX: Note that this field is being updated without locking,
2161	 * 	which might cause problems on 32-bit architectures.
2162	 */
2163	be_lun->ctl_be_lun.maxlba = be_lun->size_blocks - 1;
2164	ctl_lun_capacity_changed(&be_lun->ctl_be_lun);
2165
2166	/* Tell the user the exact size we ended up using */
2167	params->lun_size_bytes = be_lun->size_bytes;
2168
2169	req->status = CTL_LUN_OK;
2170
2171	return (0);
2172
2173bailout_error:
2174	req->status = CTL_LUN_ERROR;
2175
2176	return (0);
2177}
2178
2179static void
2180ctl_be_block_lun_shutdown(void *be_lun)
2181{
2182	struct ctl_be_block_lun *lun;
2183	struct ctl_be_block_softc *softc;
2184
2185	lun = (struct ctl_be_block_lun *)be_lun;
2186
2187	softc = lun->softc;
2188
2189	mtx_lock(&softc->lock);
2190	lun->flags |= CTL_BE_BLOCK_LUN_UNCONFIGURED;
2191	if (lun->flags & CTL_BE_BLOCK_LUN_WAITING)
2192		wakeup(lun);
2193	mtx_unlock(&softc->lock);
2194
2195}
2196
2197static void
2198ctl_be_block_lun_config_status(void *be_lun, ctl_lun_config_status status)
2199{
2200	struct ctl_be_block_lun *lun;
2201	struct ctl_be_block_softc *softc;
2202
2203	lun = (struct ctl_be_block_lun *)be_lun;
2204	softc = lun->softc;
2205
2206	if (status == CTL_LUN_CONFIG_OK) {
2207		mtx_lock(&softc->lock);
2208		lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED;
2209		if (lun->flags & CTL_BE_BLOCK_LUN_WAITING)
2210			wakeup(lun);
2211		mtx_unlock(&softc->lock);
2212
2213		/*
2214		 * We successfully added the LUN, attempt to enable it.
2215		 */
2216		if (ctl_enable_lun(&lun->ctl_be_lun) != 0) {
2217			printf("%s: ctl_enable_lun() failed!\n", __func__);
2218			if (ctl_invalidate_lun(&lun->ctl_be_lun) != 0) {
2219				printf("%s: ctl_invalidate_lun() failed!\n",
2220				       __func__);
2221			}
2222		}
2223
2224		return;
2225	}
2226
2227
2228	mtx_lock(&softc->lock);
2229	lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED;
2230	lun->flags |= CTL_BE_BLOCK_LUN_CONFIG_ERR;
2231	wakeup(lun);
2232	mtx_unlock(&softc->lock);
2233}
2234
2235
2236static int
2237ctl_be_block_config_write(union ctl_io *io)
2238{
2239	struct ctl_be_block_lun *be_lun;
2240	struct ctl_be_lun *ctl_be_lun;
2241	int retval;
2242
2243	retval = 0;
2244
2245	DPRINTF("entered\n");
2246
2247	ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[
2248		CTL_PRIV_BACKEND_LUN].ptr;
2249	be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun;
2250
2251	switch (io->scsiio.cdb[0]) {
2252	case SYNCHRONIZE_CACHE:
2253	case SYNCHRONIZE_CACHE_16:
2254		/*
2255		 * The upper level CTL code will filter out any CDBs with
2256		 * the immediate bit set and return the proper error.
2257		 *
2258		 * We don't really need to worry about what LBA range the
2259		 * user asked to be synced out.  When they issue a sync
2260		 * cache command, we'll sync out the whole thing.
2261		 */
2262		mtx_lock(&be_lun->lock);
2263		STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr,
2264				   links);
2265		mtx_unlock(&be_lun->lock);
2266		taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
2267		break;
2268	case START_STOP_UNIT: {
2269		struct scsi_start_stop_unit *cdb;
2270
2271		cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb;
2272
2273		if (cdb->how & SSS_START)
2274			retval = ctl_start_lun(ctl_be_lun);
2275		else {
2276			retval = ctl_stop_lun(ctl_be_lun);
2277			/*
2278			 * XXX KDM Copan-specific offline behavior.
2279			 * Figure out a reasonable way to port this?
2280			 */
2281#ifdef NEEDTOPORT
2282			if ((retval == 0)
2283			 && (cdb->byte2 & SSS_ONOFFLINE))
2284				retval = ctl_lun_offline(ctl_be_lun);
2285#endif
2286		}
2287
2288		/*
2289		 * In general, the above routines should not fail.  They
2290		 * just set state for the LUN.  So we've got something
2291		 * pretty wrong here if we can't start or stop the LUN.
2292		 */
2293		if (retval != 0) {
2294			ctl_set_internal_failure(&io->scsiio,
2295						 /*sks_valid*/ 1,
2296						 /*retry_count*/ 0xf051);
2297			retval = CTL_RETVAL_COMPLETE;
2298		} else {
2299			ctl_set_success(&io->scsiio);
2300		}
2301		ctl_config_write_done(io);
2302		break;
2303	}
2304	default:
2305		ctl_set_invalid_opcode(&io->scsiio);
2306		ctl_config_write_done(io);
2307		retval = CTL_RETVAL_COMPLETE;
2308		break;
2309	}
2310
2311	return (retval);
2312
2313}
2314
2315static int
2316ctl_be_block_config_read(union ctl_io *io)
2317{
2318	return (0);
2319}
2320
2321static int
2322ctl_be_block_lun_info(void *be_lun, struct sbuf *sb)
2323{
2324	struct ctl_be_block_lun *lun;
2325	int retval;
2326
2327	lun = (struct ctl_be_block_lun *)be_lun;
2328	retval = 0;
2329
2330	retval = sbuf_printf(sb, "<num_threads>");
2331
2332	if (retval != 0)
2333		goto bailout;
2334
2335	retval = sbuf_printf(sb, "%d", lun->num_threads);
2336
2337	if (retval != 0)
2338		goto bailout;
2339
2340	retval = sbuf_printf(sb, "</num_threads>");
2341
2342	/*
2343	 * For processor devices, we don't have a path variable.
2344	 */
2345	if ((retval != 0)
2346	 || (lun->dev_path == NULL))
2347		goto bailout;
2348
2349	retval = sbuf_printf(sb, "<file>");
2350
2351	if (retval != 0)
2352		goto bailout;
2353
2354	retval = ctl_sbuf_printf_esc(sb, lun->dev_path);
2355
2356	if (retval != 0)
2357		goto bailout;
2358
2359	retval = sbuf_printf(sb, "</file>\n");
2360
2361bailout:
2362
2363	return (retval);
2364}
2365
2366int
2367ctl_be_block_init(void)
2368{
2369	struct ctl_be_block_softc *softc;
2370	int retval;
2371
2372	softc = &backend_block_softc;
2373	retval = 0;
2374
2375	mtx_init(&softc->lock, "ctlblk", NULL, MTX_DEF);
2376	STAILQ_INIT(&softc->beio_free_queue);
2377	STAILQ_INIT(&softc->disk_list);
2378	STAILQ_INIT(&softc->lun_list);
2379	ctl_grow_beio(softc, 200);
2380
2381	return (retval);
2382}
2383