scsi_target.c revision 237601 
1/*
2 * SCSI Disk Emulator
3 *
4 * Copyright (c) 2002 Nate Lawson.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions, and the following disclaimer,
12 *    without modification, immediately at the beginning of the file.
13 * 2. The name of the author may not be used to endorse or promote products
14 *    derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * $FreeBSD: head/share/examples/scsi_target/scsi_target.c 237601 2012-06-26 14:51:35Z ken $
29 */
30
31#include <sys/types.h>
32#include <ctype.h>
33#include <errno.h>
34#include <err.h>
35#include <fcntl.h>
36#include <signal.h>
37#include <stddef.h>
38#include <stdio.h>
39#include <stdlib.h>
40#include <string.h>
41#include <sysexits.h>
42#include <unistd.h>
43#include <aio.h>
44#include <assert.h>
45#include <sys/stat.h>
46#include <sys/queue.h>
47#include <sys/event.h>
48#include <sys/param.h>
49#include <sys/disk.h>
50#include <cam/cam_queue.h>
51#include <cam/scsi/scsi_all.h>
52#include <cam/scsi/scsi_targetio.h>
53#include <cam/scsi/scsi_message.h>
54#include "scsi_target.h"
55
56/* Maximum amount to transfer per CTIO */
57#define MAX_XFER	MAXPHYS
58/* Maximum number of allocated CTIOs */
59#define MAX_CTIOS	64
60/* Maximum sector size for emulated volume */
61#define MAX_SECTOR	32768
62
63/* Global variables */
64int		debug;
65int		notaio = 0;
66off_t		volume_size;
67u_int		sector_size;
68size_t		buf_size;
69
70/* Local variables */
71static int    targ_fd;
72static int    kq_fd;
73static int    file_fd;
74static int    num_ctios;
75static struct ccb_queue		pending_queue;
76static struct ccb_queue		work_queue;
77static struct ioc_enable_lun	ioc_enlun = {
78	CAM_BUS_WILDCARD,
79	CAM_TARGET_WILDCARD,
80	CAM_LUN_WILDCARD
81};
82
83/* Local functions */
84static void		cleanup(void);
85static int		init_ccbs(void);
86static void		request_loop(void);
87static void		handle_read(void);
88/* static int		work_atio(struct ccb_accept_tio *); */
89static void		queue_io(struct ccb_scsiio *);
90static int		run_queue(struct ccb_accept_tio *);
91static int		work_inot(struct ccb_immediate_notify *);
92static struct ccb_scsiio *
93			get_ctio(void);
94/* static void		free_ccb(union ccb *); */
95static cam_status	get_sim_flags(u_int16_t *);
96static void		rel_simq(void);
97static void		abort_all_pending(void);
98static void		usage(void);
99
100int
101main(int argc, char *argv[])
102{
103	int ch;
104	char *file_name;
105	u_int16_t req_flags, sim_flags;
106	off_t user_size;
107
108	/* Initialize */
109	debug = 0;
110	req_flags = sim_flags = 0;
111	user_size = 0;
112	targ_fd = file_fd = kq_fd = -1;
113	num_ctios = 0;
114	sector_size = SECTOR_SIZE;
115	buf_size = DFLTPHYS;
116
117	/* Prepare resource pools */
118	TAILQ_INIT(&pending_queue);
119	TAILQ_INIT(&work_queue);
120
121	while ((ch = getopt(argc, argv, "AdSTYb:c:s:W:")) != -1) {
122		switch(ch) {
123		case 'A':
124			req_flags |= SID_Addr16;
125			break;
126		case 'd':
127			debug = 1;
128			break;
129		case 'S':
130			req_flags |= SID_Sync;
131			break;
132		case 'T':
133			req_flags |= SID_CmdQue;
134			break;
135		case 'b':
136			buf_size = atoi(optarg);
137			if (buf_size < 256 || buf_size > MAX_XFER)
138				errx(1, "Unreasonable buf size: %s", optarg);
139			break;
140		case 'c':
141			sector_size = atoi(optarg);
142			if (sector_size < 512 || sector_size > MAX_SECTOR)
143				errx(1, "Unreasonable sector size: %s", optarg);
144			break;
145		case 's':
146		{
147			int last, shift = 0;
148
149			last = strlen(optarg) - 1;
150			if (last > 0) {
151				switch (tolower(optarg[last])) {
152				case 'e':
153					shift += 10;
154					/* FALLTHROUGH */
155				case 'p':
156					shift += 10;
157					/* FALLTHROUGH */
158				case 't':
159					shift += 10;
160					/* FALLTHROUGH */
161				case 'g':
162					shift += 10;
163					/* FALLTHROUGH */
164				case 'm':
165					shift += 10;
166					/* FALLTHROUGH */
167				case 'k':
168					shift += 10;
169					optarg[last] = 0;
170					break;
171				}
172			}
173			user_size = strtoll(optarg, (char **)NULL, /*base*/10);
174			user_size <<= shift;
175			if (user_size < 0)
176				errx(1, "Unreasonable volume size: %s", optarg);
177			break;
178		}
179		case 'W':
180			req_flags &= ~(SID_WBus16 | SID_WBus32);
181			switch (atoi(optarg)) {
182			case 8:
183				/* Leave req_flags zeroed */
184				break;
185			case 16:
186				req_flags |= SID_WBus16;
187				break;
188			case 32:
189				req_flags |= SID_WBus32;
190				break;
191			default:
192				warnx("Width %s not supported", optarg);
193				usage();
194				/* NOTREACHED */
195			}
196			break;
197		case 'Y':
198			notaio = 1;
199			break;
200		default:
201			usage();
202			/* NOTREACHED */
203		}
204	}
205	argc -= optind;
206	argv += optind;
207
208	if (argc != 2)
209		usage();
210
211	sscanf(argv[0], "%u:%u:%u", &ioc_enlun.path_id, &ioc_enlun.target_id,
212	       &ioc_enlun.lun_id);
213	file_name = argv[1];
214
215	if (ioc_enlun.path_id == CAM_BUS_WILDCARD ||
216	    ioc_enlun.target_id == CAM_TARGET_WILDCARD ||
217	    ioc_enlun.lun_id == CAM_LUN_WILDCARD) {
218		warnx("Incomplete target path specified");
219		usage();
220		/* NOTREACHED */
221	}
222	/* We don't support any vendor-specific commands */
223	ioc_enlun.grp6_len = 0;
224	ioc_enlun.grp7_len = 0;
225
226	/* Open backing store for IO */
227	file_fd = open(file_name, O_RDWR);
228	if (file_fd < 0)
229		errx(EX_NOINPUT, "open backing store file");
230
231	/* Check backing store size or use the size user gave us */
232	if (user_size == 0) {
233		struct stat st;
234
235		if (fstat(file_fd, &st) < 0)
236			err(1, "fstat file");
237#if __FreeBSD_version >= 500000
238		if ((st.st_mode & S_IFCHR) != 0) {
239			/* raw device */
240			off_t mediasize;
241			if (ioctl(file_fd, DIOCGMEDIASIZE, &mediasize) < 0)
242				err(1, "DIOCGMEDIASIZE");
243
244			/* XXX get sector size by ioctl()?? */
245			volume_size = mediasize / sector_size;
246		} else
247#endif
248			volume_size = st.st_size / sector_size;
249	} else {
250		volume_size = user_size / sector_size;
251	}
252	if (debug)
253		warnx("volume_size: %d bytes x " OFF_FMT " sectors",
254		    sector_size, volume_size);
255
256	if (volume_size <= 0)
257		errx(1, "volume must be larger than %d", sector_size);
258
259	if (notaio == 0) {
260		struct aiocb aio, *aiop;
261
262		/* See if we have we have working AIO support */
263		memset(&aio, 0, sizeof(aio));
264		aio.aio_buf = malloc(sector_size);
265		if (aio.aio_buf == NULL)
266			err(1, "malloc");
267		aio.aio_fildes = file_fd;
268		aio.aio_offset = 0;
269		aio.aio_nbytes = sector_size;
270		signal(SIGSYS, SIG_IGN);
271		if (aio_read(&aio) != 0) {
272			printf("AIO support is not available- switchin to"
273			       " single-threaded mode.\n");
274			notaio = 1;
275		} else {
276			if (aio_waitcomplete(&aiop, NULL) != sector_size)
277				err(1, "aio_waitcomplete");
278			assert(aiop == &aio);
279			signal(SIGSYS, SIG_DFL);
280		}
281		free((void *)aio.aio_buf);
282		if (debug && notaio == 0)
283			warnx("aio support tested ok");
284	}
285
286	targ_fd = open("/dev/targ", O_RDWR);
287	if (targ_fd < 0)
288    	    err(1, "/dev/targ");
289	else
290	    warnx("opened /dev/targ");
291
292	/* The first three are handled by kevent() later */
293	signal(SIGHUP, SIG_IGN);
294	signal(SIGINT, SIG_IGN);
295	signal(SIGTERM, SIG_IGN);
296	signal(SIGPROF, SIG_IGN);
297	signal(SIGALRM, SIG_IGN);
298	signal(SIGSTOP, SIG_IGN);
299	signal(SIGTSTP, SIG_IGN);
300
301	/* Register a cleanup handler to run when exiting */
302	atexit(cleanup);
303
304	/* Enable listening on the specified LUN */
305	if (ioctl(targ_fd, TARGIOCENABLE, &ioc_enlun) != 0)
306		err(1, "TARGIOCENABLE");
307
308	/* Enable debugging if requested */
309	if (debug) {
310		if (ioctl(targ_fd, TARGIOCDEBUG, &debug) != 0)
311			warnx("TARGIOCDEBUG");
312	}
313
314	/* Set up inquiry data according to what SIM supports */
315	if (get_sim_flags(&sim_flags) != CAM_REQ_CMP)
316		errx(1, "get_sim_flags");
317
318	if (tcmd_init(req_flags, sim_flags) != 0)
319		errx(1, "Initializing tcmd subsystem failed");
320
321	/* Queue ATIOs and INOTs on descriptor */
322	if (init_ccbs() != 0)
323		errx(1, "init_ccbs failed");
324
325	if (debug)
326		warnx("main loop beginning");
327
328	request_loop();
329
330	exit(0);
331}
332
333static void
334cleanup()
335{
336	struct ccb_hdr *ccb_h;
337
338	if (debug) {
339		warnx("cleanup called");
340		debug = 0;
341		ioctl(targ_fd, TARGIOCDEBUG, &debug);
342	}
343	ioctl(targ_fd, TARGIOCDISABLE, NULL);
344	close(targ_fd);
345
346	while ((ccb_h = TAILQ_FIRST(&pending_queue)) != NULL) {
347		TAILQ_REMOVE(&pending_queue, ccb_h, periph_links.tqe);
348		free_ccb((union ccb *)ccb_h);
349	}
350	while ((ccb_h = TAILQ_FIRST(&work_queue)) != NULL) {
351		TAILQ_REMOVE(&work_queue, ccb_h, periph_links.tqe);
352		free_ccb((union ccb *)ccb_h);
353	}
354
355	if (kq_fd != -1)
356		close(kq_fd);
357}
358
359/* Allocate ATIOs/INOTs and queue on HBA */
360static int
361init_ccbs()
362{
363	int i;
364
365	for (i = 0; i < MAX_INITIATORS; i++) {
366		struct ccb_accept_tio *atio;
367		struct atio_descr *a_descr;
368		struct ccb_immed_notify *inot;
369
370		atio = (struct ccb_accept_tio *)malloc(sizeof(*atio));
371		if (atio == NULL) {
372			warn("malloc ATIO");
373			return (-1);
374		}
375		a_descr = (struct atio_descr *)malloc(sizeof(*a_descr));
376		if (a_descr == NULL) {
377			free(atio);
378			warn("malloc atio_descr");
379			return (-1);
380		}
381		atio->ccb_h.func_code = XPT_ACCEPT_TARGET_IO;
382		atio->ccb_h.targ_descr = a_descr;
383		send_ccb((union ccb *)atio, /*priority*/1);
384
385		inot = (struct ccb_immed_notify *)malloc(sizeof(*inot));
386		if (inot == NULL) {
387			warn("malloc INOT");
388			return (-1);
389		}
390		inot->ccb_h.func_code = XPT_IMMEDIATE_NOTIFY;
391		send_ccb((union ccb *)inot, /*priority*/1);
392	}
393
394	return (0);
395}
396
397static void
398request_loop()
399{
400	struct kevent events[MAX_EVENTS];
401	struct timespec ts, *tptr;
402	int quit;
403
404	/* Register kqueue for event notification */
405	if ((kq_fd = kqueue()) < 0)
406		err(1, "init kqueue");
407
408	/* Set up some default events */
409	EV_SET(&events[0], SIGHUP, EVFILT_SIGNAL, EV_ADD|EV_ENABLE, 0, 0, 0);
410	EV_SET(&events[1], SIGINT, EVFILT_SIGNAL, EV_ADD|EV_ENABLE, 0, 0, 0);
411	EV_SET(&events[2], SIGTERM, EVFILT_SIGNAL, EV_ADD|EV_ENABLE, 0, 0, 0);
412	EV_SET(&events[3], targ_fd, EVFILT_READ, EV_ADD|EV_ENABLE, 0, 0, 0);
413	if (kevent(kq_fd, events, 4, NULL, 0, NULL) < 0)
414		err(1, "kevent signal registration");
415
416	ts.tv_sec = 0;
417	ts.tv_nsec = 0;
418	tptr = NULL;
419	quit = 0;
420
421	/* Loop until user signal */
422	while (quit == 0) {
423		int retval, i, oo;
424		struct ccb_hdr *ccb_h;
425
426		/* Check for the next signal, read ready, or AIO completion */
427		retval = kevent(kq_fd, NULL, 0, events, MAX_EVENTS, tptr);
428		if (retval < 0) {
429			if (errno == EINTR) {
430				if (debug)
431					warnx("EINTR, looping");
432				continue;
433            		}
434			else {
435				err(1, "kevent failed");
436			}
437		} else if (retval > MAX_EVENTS) {
438			errx(1, "kevent returned more events than allocated?");
439		}
440
441		/* Process all received events. */
442		for (oo = i = 0; i < retval; i++) {
443			if ((events[i].flags & EV_ERROR) != 0)
444				errx(1, "kevent registration failed");
445
446			switch (events[i].filter) {
447			case EVFILT_READ:
448				if (debug)
449					warnx("read ready");
450				handle_read();
451				break;
452			case EVFILT_AIO:
453			{
454				struct ccb_scsiio *ctio;
455				struct ctio_descr *c_descr;
456				if (debug)
457					warnx("aio ready");
458
459				ctio = (struct ccb_scsiio *)events[i].udata;
460				c_descr = (struct ctio_descr *)
461					  ctio->ccb_h.targ_descr;
462				c_descr->event = AIO_DONE;
463				/* Queue on the appropriate ATIO */
464				queue_io(ctio);
465				/* Process any queued completions. */
466				oo += run_queue(c_descr->atio);
467				break;
468			}
469			case EVFILT_SIGNAL:
470				if (debug)
471					warnx("signal ready, setting quit");
472				quit = 1;
473				break;
474			default:
475				warnx("unknown event %d", events[i].filter);
476				break;
477			}
478
479			if (debug)
480				warnx("event %d done", events[i].filter);
481		}
482
483		if (oo) {
484			tptr = &ts;
485			continue;
486		}
487
488		/* Grab the first CCB and perform one work unit. */
489		if ((ccb_h = TAILQ_FIRST(&work_queue)) != NULL) {
490			union ccb *ccb;
491
492			ccb = (union ccb *)ccb_h;
493			switch (ccb_h->func_code) {
494			case XPT_ACCEPT_TARGET_IO:
495				/* Start one more transfer. */
496				retval = work_atio(&ccb->atio);
497				break;
498			case XPT_IMMEDIATE_NOTIFY:
499				retval = work_inot(&ccb->cin1);
500				break;
501			default:
502				warnx("Unhandled ccb type %#x on workq",
503				      ccb_h->func_code);
504				abort();
505				/* NOTREACHED */
506			}
507
508			/* Assume work function handled the exception */
509			if ((ccb_h->status & CAM_DEV_QFRZN) != 0) {
510				if (debug) {
511					warnx("Queue frozen receiving CCB, "
512					      "releasing");
513				}
514				rel_simq();
515			}
516
517			/* No more work needed for this command. */
518			if (retval == 0) {
519				TAILQ_REMOVE(&work_queue, ccb_h,
520					     periph_links.tqe);
521			}
522		}
523
524		/*
525		 * Poll for new events (i.e. completions) while we
526		 * are processing CCBs on the work_queue. Once it's
527		 * empty, use an infinite wait.
528		 */
529		if (!TAILQ_EMPTY(&work_queue))
530			tptr = &ts;
531		else
532			tptr = NULL;
533	}
534}
535
536/* CCBs are ready from the kernel */
537static void
538handle_read()
539{
540	union ccb *ccb_array[MAX_INITIATORS], *ccb;
541	int ccb_count, i, oo;
542
543	ccb_count = read(targ_fd, ccb_array, sizeof(ccb_array));
544	if (ccb_count <= 0) {
545		warn("read ccb ptrs");
546		return;
547	}
548	ccb_count /= sizeof(union ccb *);
549	if (ccb_count < 1) {
550		warnx("truncated read ccb ptr?");
551		return;
552	}
553
554	for (i = 0; i < ccb_count; i++) {
555		ccb = ccb_array[i];
556		TAILQ_REMOVE(&pending_queue, &ccb->ccb_h, periph_links.tqe);
557
558		switch (ccb->ccb_h.func_code) {
559		case XPT_ACCEPT_TARGET_IO:
560		{
561			struct ccb_accept_tio *atio;
562			struct atio_descr *a_descr;
563
564			/* Initialize ATIO descr for this transaction */
565			atio = &ccb->atio;
566			a_descr = (struct atio_descr *)atio->ccb_h.targ_descr;
567			bzero(a_descr, sizeof(*a_descr));
568			TAILQ_INIT(&a_descr->cmplt_io);
569			a_descr->flags = atio->ccb_h.flags &
570				(CAM_DIS_DISCONNECT | CAM_TAG_ACTION_VALID);
571			/* XXX add a_descr->priority */
572			if ((atio->ccb_h.flags & CAM_CDB_POINTER) == 0)
573				a_descr->cdb = atio->cdb_io.cdb_bytes;
574			else
575				a_descr->cdb = atio->cdb_io.cdb_ptr;
576
577			/* ATIOs are processed in FIFO order */
578			TAILQ_INSERT_TAIL(&work_queue, &ccb->ccb_h,
579					  periph_links.tqe);
580			break;
581		}
582		case XPT_CONT_TARGET_IO:
583		{
584			struct ccb_scsiio *ctio;
585			struct ctio_descr *c_descr;
586
587			ctio = &ccb->ctio;
588			c_descr = (struct ctio_descr *)ctio->ccb_h.targ_descr;
589			c_descr->event = CTIO_DONE;
590			/* Queue on the appropriate ATIO */
591			queue_io(ctio);
592			/* Process any queued completions. */
593			oo += run_queue(c_descr->atio);
594			break;
595		}
596		case XPT_IMMED_NOTIFY:
597			/* INOTs are handled with priority */
598			TAILQ_INSERT_HEAD(&work_queue, &ccb->ccb_h,
599					  periph_links.tqe);
600			break;
601		default:
602			warnx("Unhandled ccb type %#x in handle_read",
603			      ccb->ccb_h.func_code);
604			break;
605		}
606	}
607}
608
609/* Process an ATIO CCB from the kernel */
610int
611work_atio(struct ccb_accept_tio *atio)
612{
613	struct ccb_scsiio *ctio;
614	struct atio_descr *a_descr;
615	struct ctio_descr *c_descr;
616	cam_status status;
617	int ret;
618
619	if (debug)
620		warnx("Working on ATIO %p", atio);
621
622	a_descr = (struct atio_descr *)atio->ccb_h.targ_descr;
623
624	/* Get a CTIO and initialize it according to our known parameters */
625	ctio = get_ctio();
626	if (ctio == NULL) {
627		return (1);
628	}
629	ret = 0;
630	ctio->ccb_h.flags = a_descr->flags;
631	ctio->tag_id = atio->tag_id;
632	ctio->init_id = atio->init_id;
633	/* XXX priority needs to be added to a_descr */
634	c_descr = (struct ctio_descr *)ctio->ccb_h.targ_descr;
635	c_descr->atio = atio;
636	if ((a_descr->flags & CAM_DIR_IN) != 0)
637		c_descr->offset = a_descr->base_off + a_descr->targ_req;
638	else if ((a_descr->flags & CAM_DIR_MASK) == CAM_DIR_OUT)
639		c_descr->offset = a_descr->base_off + a_descr->init_req;
640	else
641		c_descr->offset = a_descr->base_off;
642
643	/*
644	 * Return a check condition if there was an error while
645	 * receiving this ATIO.
646	 */
647	if (atio->sense_len != 0) {
648		struct scsi_sense_data_fixed *sense;
649
650		if (debug) {
651			warnx("ATIO with %u bytes sense received",
652			      atio->sense_len);
653		}
654		sense = (struct scsi_sense_data_fixed *)&atio->sense_data;
655		tcmd_sense(ctio->init_id, ctio, sense->flags,
656			   sense->add_sense_code, sense->add_sense_code_qual);
657		send_ccb((union ccb *)ctio, /*priority*/1);
658		return (0);
659	}
660
661	status = atio->ccb_h.status & CAM_STATUS_MASK;
662	switch (status) {
663	case CAM_CDB_RECVD:
664		ret = tcmd_handle(atio, ctio, ATIO_WORK);
665		break;
666	case CAM_REQ_ABORTED:
667		warn("ATIO %p aborted", a_descr);
668		/* Requeue on HBA */
669		TAILQ_REMOVE(&work_queue, &atio->ccb_h, periph_links.tqe);
670		send_ccb((union ccb *)atio, /*priority*/1);
671		ret = 1;
672		break;
673	default:
674		warnx("ATIO completed with unhandled status %#x", status);
675		abort();
676		/* NOTREACHED */
677		break;
678	}
679
680	return (ret);
681}
682
683static void
684queue_io(struct ccb_scsiio *ctio)
685{
686	struct ccb_hdr *ccb_h;
687	struct io_queue *ioq;
688	struct ctio_descr *c_descr;
689
690	c_descr = (struct ctio_descr *)ctio->ccb_h.targ_descr;
691	if (c_descr->atio == NULL) {
692		errx(1, "CTIO %p has NULL ATIO", ctio);
693	}
694	ioq = &((struct atio_descr *)c_descr->atio->ccb_h.targ_descr)->cmplt_io;
695
696	if (TAILQ_EMPTY(ioq)) {
697		TAILQ_INSERT_HEAD(ioq, &ctio->ccb_h, periph_links.tqe);
698		return;
699	}
700
701	TAILQ_FOREACH_REVERSE(ccb_h, ioq, io_queue, periph_links.tqe) {
702		struct ctio_descr *curr_descr =
703		    (struct ctio_descr *)ccb_h->targ_descr;
704		if (curr_descr->offset <= c_descr->offset) {
705			break;
706		}
707	}
708
709	if (ccb_h) {
710		TAILQ_INSERT_AFTER(ioq, ccb_h, &ctio->ccb_h, periph_links.tqe);
711	} else {
712		TAILQ_INSERT_HEAD(ioq, &ctio->ccb_h, periph_links.tqe);
713	}
714}
715
716/*
717 * Go through all completed AIO/CTIOs for a given ATIO and advance data
718 * counts, start continuation IO, etc.
719 */
720static int
721run_queue(struct ccb_accept_tio *atio)
722{
723	struct atio_descr *a_descr;
724	struct ccb_hdr *ccb_h;
725	int sent_status, event;
726
727	if (atio == NULL)
728		return (0);
729
730	a_descr = (struct atio_descr *)atio->ccb_h.targ_descr;
731
732	while ((ccb_h = TAILQ_FIRST(&a_descr->cmplt_io)) != NULL) {
733		struct ccb_scsiio *ctio;
734		struct ctio_descr *c_descr;
735
736		ctio = (struct ccb_scsiio *)ccb_h;
737		c_descr = (struct ctio_descr *)ctio->ccb_h.targ_descr;
738
739		if (ctio->ccb_h.status == CAM_REQ_ABORTED) {
740			TAILQ_REMOVE(&a_descr->cmplt_io, ccb_h,
741				     periph_links.tqe);
742			free_ccb((union ccb *)ctio);
743			send_ccb((union ccb *)atio, /*priority*/1);
744			continue;
745		}
746
747		/* If completed item is in range, call handler */
748		if ((c_descr->event == AIO_DONE &&
749		    c_descr->offset == a_descr->base_off + a_descr->targ_ack)
750		 || (c_descr->event == CTIO_DONE &&
751		    c_descr->offset == a_descr->base_off + a_descr->init_ack)) {
752			sent_status = (ccb_h->flags & CAM_SEND_STATUS) != 0;
753			event = c_descr->event;
754
755			TAILQ_REMOVE(&a_descr->cmplt_io, ccb_h,
756				     periph_links.tqe);
757			tcmd_handle(atio, ctio, c_descr->event);
758
759			/* If entire transfer complete, send back ATIO */
760			if (sent_status != 0 && event == CTIO_DONE)
761				send_ccb((union ccb *)atio, /*priority*/1);
762		} else {
763			/* Gap in offsets so wait until later callback */
764			if (/* debug */ 1)
765				warnx("IO %p:%p out of order %s",  ccb_h,
766				    a_descr, c_descr->event == AIO_DONE?
767				    "aio" : "ctio");
768			return (1);
769		}
770	}
771	return (0);
772}
773
774static int
775work_inot(struct ccb_immediate_notify *inot)
776{
777	cam_status status;
778
779	if (debug)
780		warnx("Working on INOT %p", inot);
781
782	status = inot->ccb_h.status;
783	status &= CAM_STATUS_MASK;
784
785	switch (status) {
786	case CAM_SCSI_BUS_RESET:
787		tcmd_ua(CAM_TARGET_WILDCARD, UA_BUS_RESET);
788		abort_all_pending();
789		break;
790	case CAM_BDR_SENT:
791		tcmd_ua(CAM_TARGET_WILDCARD, UA_BDR);
792		abort_all_pending();
793		break;
794	case CAM_MESSAGE_RECV:
795		switch (inot->arg) {
796		case MSG_TASK_COMPLETE:
797		case MSG_INITIATOR_DET_ERR:
798		case MSG_ABORT_TASK_SET:
799		case MSG_MESSAGE_REJECT:
800		case MSG_NOOP:
801		case MSG_PARITY_ERROR:
802		case MSG_TARGET_RESET:
803		case MSG_ABORT_TASK:
804		case MSG_CLEAR_TASK_SET:
805		default:
806			warnx("INOT message %#x", inot->arg);
807			break;
808		}
809		break;
810	case CAM_REQ_ABORTED:
811		warnx("INOT %p aborted", inot);
812		break;
813	default:
814		warnx("Unhandled INOT status %#x", status);
815		break;
816	}
817
818	/* Requeue on SIM */
819	TAILQ_REMOVE(&work_queue, &inot->ccb_h, periph_links.tqe);
820	send_ccb((union ccb *)inot, /*priority*/1);
821
822	return (1);
823}
824
825void
826send_ccb(union ccb *ccb, int priority)
827{
828	if (debug)
829		warnx("sending ccb (%#x)", ccb->ccb_h.func_code);
830	ccb->ccb_h.pinfo.priority = priority;
831	if (XPT_FC_IS_QUEUED(ccb)) {
832		TAILQ_INSERT_TAIL(&pending_queue, &ccb->ccb_h,
833				  periph_links.tqe);
834	}
835	if (write(targ_fd, &ccb, sizeof(ccb)) != sizeof(ccb)) {
836		warn("write ccb");
837		ccb->ccb_h.status = CAM_PROVIDE_FAIL;
838	}
839}
840
841/* Return a CTIO/descr/buf combo from the freelist or malloc one */
842static struct ccb_scsiio *
843get_ctio()
844{
845	struct ccb_scsiio *ctio;
846	struct ctio_descr *c_descr;
847	struct sigevent *se;
848
849	if (num_ctios == MAX_CTIOS) {
850		warnx("at CTIO max");
851		return (NULL);
852	}
853
854	ctio = (struct ccb_scsiio *)malloc(sizeof(*ctio));
855	if (ctio == NULL) {
856		warn("malloc CTIO");
857		return (NULL);
858	}
859	c_descr = (struct ctio_descr *)malloc(sizeof(*c_descr));
860	if (c_descr == NULL) {
861		free(ctio);
862		warn("malloc ctio_descr");
863		return (NULL);
864	}
865	c_descr->buf = malloc(buf_size);
866	if (c_descr->buf == NULL) {
867		free(c_descr);
868		free(ctio);
869		warn("malloc backing store");
870		return (NULL);
871	}
872	num_ctios++;
873
874	/* Initialize CTIO, CTIO descr, and AIO */
875	ctio->ccb_h.func_code = XPT_CONT_TARGET_IO;
876	ctio->ccb_h.retry_count = 2;
877	ctio->ccb_h.timeout = CAM_TIME_INFINITY;
878	ctio->data_ptr = c_descr->buf;
879	ctio->ccb_h.targ_descr = c_descr;
880	c_descr->aiocb.aio_buf = c_descr->buf;
881	c_descr->aiocb.aio_fildes = file_fd;
882	se = &c_descr->aiocb.aio_sigevent;
883	se->sigev_notify = SIGEV_KEVENT;
884	se->sigev_notify_kqueue = kq_fd;
885	se->sigev_value.sival_ptr = ctio;
886
887	return (ctio);
888}
889
890void
891free_ccb(union ccb *ccb)
892{
893	switch (ccb->ccb_h.func_code) {
894	case XPT_CONT_TARGET_IO:
895	{
896		struct ctio_descr *c_descr;
897
898		c_descr = (struct ctio_descr *)ccb->ccb_h.targ_descr;
899		free(c_descr->buf);
900		num_ctios--;
901		/* FALLTHROUGH */
902	}
903	case XPT_ACCEPT_TARGET_IO:
904		free(ccb->ccb_h.targ_descr);
905		/* FALLTHROUGH */
906	case XPT_IMMED_NOTIFY:
907	default:
908		free(ccb);
909		break;
910	}
911}
912
913static cam_status
914get_sim_flags(u_int16_t *flags)
915{
916	struct ccb_pathinq cpi;
917	cam_status status;
918
919	/* Find SIM capabilities */
920	bzero(&cpi, sizeof(cpi));
921	cpi.ccb_h.func_code = XPT_PATH_INQ;
922	send_ccb((union ccb *)&cpi, /*priority*/1);
923	status = cpi.ccb_h.status & CAM_STATUS_MASK;
924	if (status != CAM_REQ_CMP) {
925		fprintf(stderr, "CPI failed, status %#x\n", status);
926		return (status);
927	}
928
929	/* Can only enable on controllers that support target mode */
930	if ((cpi.target_sprt & PIT_PROCESSOR) == 0) {
931		fprintf(stderr, "HBA does not support target mode\n");
932		status = CAM_PATH_INVALID;
933		return (status);
934	}
935
936	*flags = cpi.hba_inquiry;
937	return (status);
938}
939
940static void
941rel_simq()
942{
943	struct ccb_relsim crs;
944
945	bzero(&crs, sizeof(crs));
946	crs.ccb_h.func_code = XPT_REL_SIMQ;
947	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
948	crs.openings = 0;
949	crs.release_timeout = 0;
950	crs.qfrozen_cnt = 0;
951	send_ccb((union ccb *)&crs, /*priority*/0);
952}
953
954/* Cancel all pending CCBs. */
955static void
956abort_all_pending()
957{
958	struct ccb_abort	 cab;
959	struct ccb_hdr		*ccb_h;
960
961	if (debug)
962		  warnx("abort_all_pending");
963
964	bzero(&cab, sizeof(cab));
965	cab.ccb_h.func_code = XPT_ABORT;
966	TAILQ_FOREACH(ccb_h, &pending_queue, periph_links.tqe) {
967		if (debug)
968			  warnx("Aborting pending CCB %p\n", ccb_h);
969		cab.abort_ccb = (union ccb *)ccb_h;
970		send_ccb((union ccb *)&cab, /*priority*/1);
971		if (cab.ccb_h.status != CAM_REQ_CMP) {
972			warnx("Unable to abort CCB, status %#x\n",
973			       cab.ccb_h.status);
974		}
975	}
976}
977
978static void
979usage()
980{
981	fprintf(stderr,
982		"Usage: scsi_target [-AdSTY] [-b bufsize] [-c sectorsize]\n"
983		"\t\t[-r numbufs] [-s volsize] [-W 8,16,32]\n"
984		"\t\tbus:target:lun filename\n");
985	exit(1);
986}
987