cam_xpt.c revision 260387 
1/*-
2 * Implementation of the Common Access Method Transport (XPT) layer.
3 *
4 * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs.
5 * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions, and the following disclaimer,
13 *    without modification, immediately at the beginning of the file.
14 * 2. The name of the author may not be used to endorse or promote products
15 *    derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
21 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: stable/10/sys/cam/cam_xpt.c 260387 2014-01-07 01:51:48Z scottl $");
32
33#include <sys/param.h>
34#include <sys/bus.h>
35#include <sys/systm.h>
36#include <sys/types.h>
37#include <sys/malloc.h>
38#include <sys/kernel.h>
39#include <sys/time.h>
40#include <sys/conf.h>
41#include <sys/fcntl.h>
42#include <sys/interrupt.h>
43#include <sys/proc.h>
44#include <sys/sbuf.h>
45#include <sys/smp.h>
46#include <sys/taskqueue.h>
47
48#include <sys/lock.h>
49#include <sys/mutex.h>
50#include <sys/sysctl.h>
51#include <sys/kthread.h>
52
53#include <cam/cam.h>
54#include <cam/cam_ccb.h>
55#include <cam/cam_periph.h>
56#include <cam/cam_queue.h>
57#include <cam/cam_sim.h>
58#include <cam/cam_xpt.h>
59#include <cam/cam_xpt_sim.h>
60#include <cam/cam_xpt_periph.h>
61#include <cam/cam_xpt_internal.h>
62#include <cam/cam_debug.h>
63#include <cam/cam_compat.h>
64
65#include <cam/scsi/scsi_all.h>
66#include <cam/scsi/scsi_message.h>
67#include <cam/scsi/scsi_pass.h>
68
69#include <machine/md_var.h>	/* geometry translation */
70#include <machine/stdarg.h>	/* for xpt_print below */
71
72#include "opt_cam.h"
73
74/*
75 * This is the maximum number of high powered commands (e.g. start unit)
76 * that can be outstanding at a particular time.
77 */
78#ifndef CAM_MAX_HIGHPOWER
79#define CAM_MAX_HIGHPOWER  4
80#endif
81
82/* Datastructures internal to the xpt layer */
83MALLOC_DEFINE(M_CAMXPT, "CAM XPT", "CAM XPT buffers");
84MALLOC_DEFINE(M_CAMDEV, "CAM DEV", "CAM devices");
85MALLOC_DEFINE(M_CAMCCB, "CAM CCB", "CAM CCBs");
86MALLOC_DEFINE(M_CAMPATH, "CAM path", "CAM paths");
87
88/* Object for defering XPT actions to a taskqueue */
89struct xpt_task {
90	struct task	task;
91	void		*data1;
92	uintptr_t	data2;
93};
94
95struct xpt_softc {
96	/* number of high powered commands that can go through right now */
97	struct mtx		xpt_highpower_lock;
98	STAILQ_HEAD(highpowerlist, cam_ed)	highpowerq;
99	int			num_highpower;
100
101	/* queue for handling async rescan requests. */
102	TAILQ_HEAD(, ccb_hdr) ccb_scanq;
103	int buses_to_config;
104	int buses_config_done;
105
106	/* Registered busses */
107	TAILQ_HEAD(,cam_eb)	xpt_busses;
108	u_int			bus_generation;
109
110	struct intr_config_hook	*xpt_config_hook;
111
112	int			boot_delay;
113	struct callout 		boot_callout;
114
115	struct mtx		xpt_topo_lock;
116	struct mtx		xpt_lock;
117	struct taskqueue	*xpt_taskq;
118};
119
120typedef enum {
121	DM_RET_COPY		= 0x01,
122	DM_RET_FLAG_MASK	= 0x0f,
123	DM_RET_NONE		= 0x00,
124	DM_RET_STOP		= 0x10,
125	DM_RET_DESCEND		= 0x20,
126	DM_RET_ERROR		= 0x30,
127	DM_RET_ACTION_MASK	= 0xf0
128} dev_match_ret;
129
130typedef enum {
131	XPT_DEPTH_BUS,
132	XPT_DEPTH_TARGET,
133	XPT_DEPTH_DEVICE,
134	XPT_DEPTH_PERIPH
135} xpt_traverse_depth;
136
137struct xpt_traverse_config {
138	xpt_traverse_depth	depth;
139	void			*tr_func;
140	void			*tr_arg;
141};
142
143typedef	int	xpt_busfunc_t (struct cam_eb *bus, void *arg);
144typedef	int	xpt_targetfunc_t (struct cam_et *target, void *arg);
145typedef	int	xpt_devicefunc_t (struct cam_ed *device, void *arg);
146typedef	int	xpt_periphfunc_t (struct cam_periph *periph, void *arg);
147typedef int	xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
148
149/* Transport layer configuration information */
150static struct xpt_softc xsoftc;
151
152TUNABLE_INT("kern.cam.boot_delay", &xsoftc.boot_delay);
153SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
154           &xsoftc.boot_delay, 0, "Bus registration wait time");
155
156struct cam_doneq {
157	struct mtx_padalign	cam_doneq_mtx;
158	STAILQ_HEAD(, ccb_hdr)	cam_doneq;
159	int			cam_doneq_sleep;
160};
161
162static struct cam_doneq cam_doneqs[MAXCPU];
163static int cam_num_doneqs;
164static struct proc *cam_proc;
165
166TUNABLE_INT("kern.cam.num_doneqs", &cam_num_doneqs);
167SYSCTL_INT(_kern_cam, OID_AUTO, num_doneqs, CTLFLAG_RDTUN,
168           &cam_num_doneqs, 0, "Number of completion queues/threads");
169
170struct cam_periph *xpt_periph;
171
172static periph_init_t xpt_periph_init;
173
174static struct periph_driver xpt_driver =
175{
176	xpt_periph_init, "xpt",
177	TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
178	CAM_PERIPH_DRV_EARLY
179};
180
181PERIPHDRIVER_DECLARE(xpt, xpt_driver);
182
183static d_open_t xptopen;
184static d_close_t xptclose;
185static d_ioctl_t xptioctl;
186static d_ioctl_t xptdoioctl;
187
188static struct cdevsw xpt_cdevsw = {
189	.d_version =	D_VERSION,
190	.d_flags =	0,
191	.d_open =	xptopen,
192	.d_close =	xptclose,
193	.d_ioctl =	xptioctl,
194	.d_name =	"xpt",
195};
196
197/* Storage for debugging datastructures */
198struct cam_path *cam_dpath;
199u_int32_t cam_dflags = CAM_DEBUG_FLAGS;
200TUNABLE_INT("kern.cam.dflags", &cam_dflags);
201SYSCTL_UINT(_kern_cam, OID_AUTO, dflags, CTLFLAG_RW,
202	&cam_dflags, 0, "Enabled debug flags");
203u_int32_t cam_debug_delay = CAM_DEBUG_DELAY;
204TUNABLE_INT("kern.cam.debug_delay", &cam_debug_delay);
205SYSCTL_UINT(_kern_cam, OID_AUTO, debug_delay, CTLFLAG_RW,
206	&cam_debug_delay, 0, "Delay in us after each debug message");
207
208/* Our boot-time initialization hook */
209static int cam_module_event_handler(module_t, int /*modeventtype_t*/, void *);
210
211static moduledata_t cam_moduledata = {
212	"cam",
213	cam_module_event_handler,
214	NULL
215};
216
217static int	xpt_init(void *);
218
219DECLARE_MODULE(cam, cam_moduledata, SI_SUB_CONFIGURE, SI_ORDER_SECOND);
220MODULE_VERSION(cam, 1);
221
222
223static void		xpt_async_bcast(struct async_list *async_head,
224					u_int32_t async_code,
225					struct cam_path *path,
226					void *async_arg);
227static path_id_t xptnextfreepathid(void);
228static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
229static union ccb *xpt_get_ccb(struct cam_periph *periph);
230static union ccb *xpt_get_ccb_nowait(struct cam_periph *periph);
231static void	 xpt_run_allocq(struct cam_periph *periph, int sleep);
232static void	 xpt_run_allocq_task(void *context, int pending);
233static void	 xpt_run_devq(struct cam_devq *devq);
234static timeout_t xpt_release_devq_timeout;
235static void	 xpt_release_simq_timeout(void *arg) __unused;
236static void	 xpt_acquire_bus(struct cam_eb *bus);
237static void	 xpt_release_bus(struct cam_eb *bus);
238static uint32_t	 xpt_freeze_devq_device(struct cam_ed *dev, u_int count);
239static int	 xpt_release_devq_device(struct cam_ed *dev, u_int count,
240		    int run_queue);
241static struct cam_et*
242		 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
243static void	 xpt_acquire_target(struct cam_et *target);
244static void	 xpt_release_target(struct cam_et *target);
245static struct cam_eb*
246		 xpt_find_bus(path_id_t path_id);
247static struct cam_et*
248		 xpt_find_target(struct cam_eb *bus, target_id_t target_id);
249static struct cam_ed*
250		 xpt_find_device(struct cam_et *target, lun_id_t lun_id);
251static void	 xpt_config(void *arg);
252static int	 xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
253				 u_int32_t new_priority);
254static xpt_devicefunc_t xptpassannouncefunc;
255static void	 xptaction(struct cam_sim *sim, union ccb *work_ccb);
256static void	 xptpoll(struct cam_sim *sim);
257static void	 camisr_runqueue(void);
258static void	 xpt_done_process(struct ccb_hdr *ccb_h);
259static void	 xpt_done_td(void *);
260static dev_match_ret	xptbusmatch(struct dev_match_pattern *patterns,
261				    u_int num_patterns, struct cam_eb *bus);
262static dev_match_ret	xptdevicematch(struct dev_match_pattern *patterns,
263				       u_int num_patterns,
264				       struct cam_ed *device);
265static dev_match_ret	xptperiphmatch(struct dev_match_pattern *patterns,
266				       u_int num_patterns,
267				       struct cam_periph *periph);
268static xpt_busfunc_t	xptedtbusfunc;
269static xpt_targetfunc_t	xptedttargetfunc;
270static xpt_devicefunc_t	xptedtdevicefunc;
271static xpt_periphfunc_t	xptedtperiphfunc;
272static xpt_pdrvfunc_t	xptplistpdrvfunc;
273static xpt_periphfunc_t	xptplistperiphfunc;
274static int		xptedtmatch(struct ccb_dev_match *cdm);
275static int		xptperiphlistmatch(struct ccb_dev_match *cdm);
276static int		xptbustraverse(struct cam_eb *start_bus,
277				       xpt_busfunc_t *tr_func, void *arg);
278static int		xpttargettraverse(struct cam_eb *bus,
279					  struct cam_et *start_target,
280					  xpt_targetfunc_t *tr_func, void *arg);
281static int		xptdevicetraverse(struct cam_et *target,
282					  struct cam_ed *start_device,
283					  xpt_devicefunc_t *tr_func, void *arg);
284static int		xptperiphtraverse(struct cam_ed *device,
285					  struct cam_periph *start_periph,
286					  xpt_periphfunc_t *tr_func, void *arg);
287static int		xptpdrvtraverse(struct periph_driver **start_pdrv,
288					xpt_pdrvfunc_t *tr_func, void *arg);
289static int		xptpdperiphtraverse(struct periph_driver **pdrv,
290					    struct cam_periph *start_periph,
291					    xpt_periphfunc_t *tr_func,
292					    void *arg);
293static xpt_busfunc_t	xptdefbusfunc;
294static xpt_targetfunc_t	xptdeftargetfunc;
295static xpt_devicefunc_t	xptdefdevicefunc;
296static xpt_periphfunc_t	xptdefperiphfunc;
297static void		xpt_finishconfig_task(void *context, int pending);
298static void		xpt_dev_async_default(u_int32_t async_code,
299					      struct cam_eb *bus,
300					      struct cam_et *target,
301					      struct cam_ed *device,
302					      void *async_arg);
303static struct cam_ed *	xpt_alloc_device_default(struct cam_eb *bus,
304						 struct cam_et *target,
305						 lun_id_t lun_id);
306static xpt_devicefunc_t	xptsetasyncfunc;
307static xpt_busfunc_t	xptsetasyncbusfunc;
308static cam_status	xptregister(struct cam_periph *periph,
309				    void *arg);
310static __inline int device_is_queued(struct cam_ed *device);
311
312static __inline int
313xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev)
314{
315	int	retval;
316
317	mtx_assert(&devq->send_mtx, MA_OWNED);
318	if ((dev->ccbq.queue.entries > 0) &&
319	    (dev->ccbq.dev_openings > 0) &&
320	    (dev->ccbq.queue.qfrozen_cnt == 0)) {
321		/*
322		 * The priority of a device waiting for controller
323		 * resources is that of the highest priority CCB
324		 * enqueued.
325		 */
326		retval =
327		    xpt_schedule_dev(&devq->send_queue,
328				     &dev->devq_entry,
329				     CAMQ_GET_PRIO(&dev->ccbq.queue));
330	} else {
331		retval = 0;
332	}
333	return (retval);
334}
335
336static __inline int
337device_is_queued(struct cam_ed *device)
338{
339	return (device->devq_entry.index != CAM_UNQUEUED_INDEX);
340}
341
342static void
343xpt_periph_init()
344{
345	make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0");
346}
347
348static int
349xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
350{
351
352	/*
353	 * Only allow read-write access.
354	 */
355	if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0))
356		return(EPERM);
357
358	/*
359	 * We don't allow nonblocking access.
360	 */
361	if ((flags & O_NONBLOCK) != 0) {
362		printf("%s: can't do nonblocking access\n", devtoname(dev));
363		return(ENODEV);
364	}
365
366	return(0);
367}
368
369static int
370xptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
371{
372
373	return(0);
374}
375
376/*
377 * Don't automatically grab the xpt softc lock here even though this is going
378 * through the xpt device.  The xpt device is really just a back door for
379 * accessing other devices and SIMs, so the right thing to do is to grab
380 * the appropriate SIM lock once the bus/SIM is located.
381 */
382static int
383xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
384{
385	int error;
386
387	if ((error = xptdoioctl(dev, cmd, addr, flag, td)) == ENOTTY) {
388		error = cam_compat_ioctl(dev, cmd, addr, flag, td, xptdoioctl);
389	}
390	return (error);
391}
392
393static int
394xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
395{
396	int error;
397
398	error = 0;
399
400	switch(cmd) {
401	/*
402	 * For the transport layer CAMIOCOMMAND ioctl, we really only want
403	 * to accept CCB types that don't quite make sense to send through a
404	 * passthrough driver. XPT_PATH_INQ is an exception to this, as stated
405	 * in the CAM spec.
406	 */
407	case CAMIOCOMMAND: {
408		union ccb *ccb;
409		union ccb *inccb;
410		struct cam_eb *bus;
411
412		inccb = (union ccb *)addr;
413
414		bus = xpt_find_bus(inccb->ccb_h.path_id);
415		if (bus == NULL)
416			return (EINVAL);
417
418		switch (inccb->ccb_h.func_code) {
419		case XPT_SCAN_BUS:
420		case XPT_RESET_BUS:
421			if (inccb->ccb_h.target_id != CAM_TARGET_WILDCARD ||
422			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
423				xpt_release_bus(bus);
424				return (EINVAL);
425			}
426			break;
427		case XPT_SCAN_TGT:
428			if (inccb->ccb_h.target_id == CAM_TARGET_WILDCARD ||
429			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
430				xpt_release_bus(bus);
431				return (EINVAL);
432			}
433			break;
434		default:
435			break;
436		}
437
438		switch(inccb->ccb_h.func_code) {
439		case XPT_SCAN_BUS:
440		case XPT_RESET_BUS:
441		case XPT_PATH_INQ:
442		case XPT_ENG_INQ:
443		case XPT_SCAN_LUN:
444		case XPT_SCAN_TGT:
445
446			ccb = xpt_alloc_ccb();
447
448			/*
449			 * Create a path using the bus, target, and lun the
450			 * user passed in.
451			 */
452			if (xpt_create_path(&ccb->ccb_h.path, NULL,
453					    inccb->ccb_h.path_id,
454					    inccb->ccb_h.target_id,
455					    inccb->ccb_h.target_lun) !=
456					    CAM_REQ_CMP){
457				error = EINVAL;
458				xpt_free_ccb(ccb);
459				break;
460			}
461			/* Ensure all of our fields are correct */
462			xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path,
463				      inccb->ccb_h.pinfo.priority);
464			xpt_merge_ccb(ccb, inccb);
465			xpt_path_lock(ccb->ccb_h.path);
466			cam_periph_runccb(ccb, NULL, 0, 0, NULL);
467			xpt_path_unlock(ccb->ccb_h.path);
468			bcopy(ccb, inccb, sizeof(union ccb));
469			xpt_free_path(ccb->ccb_h.path);
470			xpt_free_ccb(ccb);
471			break;
472
473		case XPT_DEBUG: {
474			union ccb ccb;
475
476			/*
477			 * This is an immediate CCB, so it's okay to
478			 * allocate it on the stack.
479			 */
480
481			/*
482			 * Create a path using the bus, target, and lun the
483			 * user passed in.
484			 */
485			if (xpt_create_path(&ccb.ccb_h.path, NULL,
486					    inccb->ccb_h.path_id,
487					    inccb->ccb_h.target_id,
488					    inccb->ccb_h.target_lun) !=
489					    CAM_REQ_CMP){
490				error = EINVAL;
491				break;
492			}
493			/* Ensure all of our fields are correct */
494			xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path,
495				      inccb->ccb_h.pinfo.priority);
496			xpt_merge_ccb(&ccb, inccb);
497			xpt_action(&ccb);
498			bcopy(&ccb, inccb, sizeof(union ccb));
499			xpt_free_path(ccb.ccb_h.path);
500			break;
501
502		}
503		case XPT_DEV_MATCH: {
504			struct cam_periph_map_info mapinfo;
505			struct cam_path *old_path;
506
507			/*
508			 * We can't deal with physical addresses for this
509			 * type of transaction.
510			 */
511			if ((inccb->ccb_h.flags & CAM_DATA_MASK) !=
512			    CAM_DATA_VADDR) {
513				error = EINVAL;
514				break;
515			}
516
517			/*
518			 * Save this in case the caller had it set to
519			 * something in particular.
520			 */
521			old_path = inccb->ccb_h.path;
522
523			/*
524			 * We really don't need a path for the matching
525			 * code.  The path is needed because of the
526			 * debugging statements in xpt_action().  They
527			 * assume that the CCB has a valid path.
528			 */
529			inccb->ccb_h.path = xpt_periph->path;
530
531			bzero(&mapinfo, sizeof(mapinfo));
532
533			/*
534			 * Map the pattern and match buffers into kernel
535			 * virtual address space.
536			 */
537			error = cam_periph_mapmem(inccb, &mapinfo);
538
539			if (error) {
540				inccb->ccb_h.path = old_path;
541				break;
542			}
543
544			/*
545			 * This is an immediate CCB, we can send it on directly.
546			 */
547			xpt_action(inccb);
548
549			/*
550			 * Map the buffers back into user space.
551			 */
552			cam_periph_unmapmem(inccb, &mapinfo);
553
554			inccb->ccb_h.path = old_path;
555
556			error = 0;
557			break;
558		}
559		default:
560			error = ENOTSUP;
561			break;
562		}
563		xpt_release_bus(bus);
564		break;
565	}
566	/*
567	 * This is the getpassthru ioctl. It takes a XPT_GDEVLIST ccb as input,
568	 * with the periphal driver name and unit name filled in.  The other
569	 * fields don't really matter as input.  The passthrough driver name
570	 * ("pass"), and unit number are passed back in the ccb.  The current
571	 * device generation number, and the index into the device peripheral
572	 * driver list, and the status are also passed back.  Note that
573	 * since we do everything in one pass, unlike the XPT_GDEVLIST ccb,
574	 * we never return a status of CAM_GDEVLIST_LIST_CHANGED.  It is
575	 * (or rather should be) impossible for the device peripheral driver
576	 * list to change since we look at the whole thing in one pass, and
577	 * we do it with lock protection.
578	 *
579	 */
580	case CAMGETPASSTHRU: {
581		union ccb *ccb;
582		struct cam_periph *periph;
583		struct periph_driver **p_drv;
584		char   *name;
585		u_int unit;
586		int base_periph_found;
587
588		ccb = (union ccb *)addr;
589		unit = ccb->cgdl.unit_number;
590		name = ccb->cgdl.periph_name;
591		base_periph_found = 0;
592
593		/*
594		 * Sanity check -- make sure we don't get a null peripheral
595		 * driver name.
596		 */
597		if (*ccb->cgdl.periph_name == '\0') {
598			error = EINVAL;
599			break;
600		}
601
602		/* Keep the list from changing while we traverse it */
603		xpt_lock_buses();
604
605		/* first find our driver in the list of drivers */
606		for (p_drv = periph_drivers; *p_drv != NULL; p_drv++)
607			if (strcmp((*p_drv)->driver_name, name) == 0)
608				break;
609
610		if (*p_drv == NULL) {
611			xpt_unlock_buses();
612			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
613			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
614			*ccb->cgdl.periph_name = '\0';
615			ccb->cgdl.unit_number = 0;
616			error = ENOENT;
617			break;
618		}
619
620		/*
621		 * Run through every peripheral instance of this driver
622		 * and check to see whether it matches the unit passed
623		 * in by the user.  If it does, get out of the loops and
624		 * find the passthrough driver associated with that
625		 * peripheral driver.
626		 */
627		for (periph = TAILQ_FIRST(&(*p_drv)->units); periph != NULL;
628		     periph = TAILQ_NEXT(periph, unit_links)) {
629
630			if (periph->unit_number == unit)
631				break;
632		}
633		/*
634		 * If we found the peripheral driver that the user passed
635		 * in, go through all of the peripheral drivers for that
636		 * particular device and look for a passthrough driver.
637		 */
638		if (periph != NULL) {
639			struct cam_ed *device;
640			int i;
641
642			base_periph_found = 1;
643			device = periph->path->device;
644			for (i = 0, periph = SLIST_FIRST(&device->periphs);
645			     periph != NULL;
646			     periph = SLIST_NEXT(periph, periph_links), i++) {
647				/*
648				 * Check to see whether we have a
649				 * passthrough device or not.
650				 */
651				if (strcmp(periph->periph_name, "pass") == 0) {
652					/*
653					 * Fill in the getdevlist fields.
654					 */
655					strcpy(ccb->cgdl.periph_name,
656					       periph->periph_name);
657					ccb->cgdl.unit_number =
658						periph->unit_number;
659					if (SLIST_NEXT(periph, periph_links))
660						ccb->cgdl.status =
661							CAM_GDEVLIST_MORE_DEVS;
662					else
663						ccb->cgdl.status =
664						       CAM_GDEVLIST_LAST_DEVICE;
665					ccb->cgdl.generation =
666						device->generation;
667					ccb->cgdl.index = i;
668					/*
669					 * Fill in some CCB header fields
670					 * that the user may want.
671					 */
672					ccb->ccb_h.path_id =
673						periph->path->bus->path_id;
674					ccb->ccb_h.target_id =
675						periph->path->target->target_id;
676					ccb->ccb_h.target_lun =
677						periph->path->device->lun_id;
678					ccb->ccb_h.status = CAM_REQ_CMP;
679					break;
680				}
681			}
682		}
683
684		/*
685		 * If the periph is null here, one of two things has
686		 * happened.  The first possibility is that we couldn't
687		 * find the unit number of the particular peripheral driver
688		 * that the user is asking about.  e.g. the user asks for
689		 * the passthrough driver for "da11".  We find the list of
690		 * "da" peripherals all right, but there is no unit 11.
691		 * The other possibility is that we went through the list
692		 * of peripheral drivers attached to the device structure,
693		 * but didn't find one with the name "pass".  Either way,
694		 * we return ENOENT, since we couldn't find something.
695		 */
696		if (periph == NULL) {
697			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
698			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
699			*ccb->cgdl.periph_name = '\0';
700			ccb->cgdl.unit_number = 0;
701			error = ENOENT;
702			/*
703			 * It is unfortunate that this is even necessary,
704			 * but there are many, many clueless users out there.
705			 * If this is true, the user is looking for the
706			 * passthrough driver, but doesn't have one in his
707			 * kernel.
708			 */
709			if (base_periph_found == 1) {
710				printf("xptioctl: pass driver is not in the "
711				       "kernel\n");
712				printf("xptioctl: put \"device pass\" in "
713				       "your kernel config file\n");
714			}
715		}
716		xpt_unlock_buses();
717		break;
718		}
719	default:
720		error = ENOTTY;
721		break;
722	}
723
724	return(error);
725}
726
727static int
728cam_module_event_handler(module_t mod, int what, void *arg)
729{
730	int error;
731
732	switch (what) {
733	case MOD_LOAD:
734		if ((error = xpt_init(NULL)) != 0)
735			return (error);
736		break;
737	case MOD_UNLOAD:
738		return EBUSY;
739	default:
740		return EOPNOTSUPP;
741	}
742
743	return 0;
744}
745
746static void
747xpt_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
748{
749
750	if (done_ccb->ccb_h.ppriv_ptr1 == NULL) {
751		xpt_free_path(done_ccb->ccb_h.path);
752		xpt_free_ccb(done_ccb);
753	} else {
754		done_ccb->ccb_h.cbfcnp = done_ccb->ccb_h.ppriv_ptr1;
755		(*done_ccb->ccb_h.cbfcnp)(periph, done_ccb);
756	}
757	xpt_release_boot();
758}
759
760/* thread to handle bus rescans */
761static void
762xpt_scanner_thread(void *dummy)
763{
764	union ccb	*ccb;
765	struct cam_path	 path;
766
767	xpt_lock_buses();
768	for (;;) {
769		if (TAILQ_EMPTY(&xsoftc.ccb_scanq))
770			msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
771			       "ccb_scanq", 0);
772		if ((ccb = (union ccb *)TAILQ_FIRST(&xsoftc.ccb_scanq)) != NULL) {
773			TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
774			xpt_unlock_buses();
775
776			/*
777			 * Since lock can be dropped inside and path freed
778			 * by completion callback even before return here,
779			 * take our own path copy for reference.
780			 */
781			xpt_copy_path(&path, ccb->ccb_h.path);
782			xpt_path_lock(&path);
783			xpt_action(ccb);
784			xpt_path_unlock(&path);
785			xpt_release_path(&path);
786
787			xpt_lock_buses();
788		}
789	}
790}
791
792void
793xpt_rescan(union ccb *ccb)
794{
795	struct ccb_hdr *hdr;
796
797	/* Prepare request */
798	if (ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD &&
799	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
800		ccb->ccb_h.func_code = XPT_SCAN_BUS;
801	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
802	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
803		ccb->ccb_h.func_code = XPT_SCAN_TGT;
804	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
805	    ccb->ccb_h.path->device->lun_id != CAM_LUN_WILDCARD)
806		ccb->ccb_h.func_code = XPT_SCAN_LUN;
807	else {
808		xpt_print(ccb->ccb_h.path, "illegal scan path\n");
809		xpt_free_path(ccb->ccb_h.path);
810		xpt_free_ccb(ccb);
811		return;
812	}
813	ccb->ccb_h.ppriv_ptr1 = ccb->ccb_h.cbfcnp;
814	ccb->ccb_h.cbfcnp = xpt_rescan_done;
815	xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_XPT);
816	/* Don't make duplicate entries for the same paths. */
817	xpt_lock_buses();
818	if (ccb->ccb_h.ppriv_ptr1 == NULL) {
819		TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
820			if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
821				wakeup(&xsoftc.ccb_scanq);
822				xpt_unlock_buses();
823				xpt_print(ccb->ccb_h.path, "rescan already queued\n");
824				xpt_free_path(ccb->ccb_h.path);
825				xpt_free_ccb(ccb);
826				return;
827			}
828		}
829	}
830	TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
831	xsoftc.buses_to_config++;
832	wakeup(&xsoftc.ccb_scanq);
833	xpt_unlock_buses();
834}
835
836/* Functions accessed by the peripheral drivers */
837static int
838xpt_init(void *dummy)
839{
840	struct cam_sim *xpt_sim;
841	struct cam_path *path;
842	struct cam_devq *devq;
843	cam_status status;
844	int error, i;
845
846	TAILQ_INIT(&xsoftc.xpt_busses);
847	TAILQ_INIT(&xsoftc.ccb_scanq);
848	STAILQ_INIT(&xsoftc.highpowerq);
849	xsoftc.num_highpower = CAM_MAX_HIGHPOWER;
850
851	mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF);
852	mtx_init(&xsoftc.xpt_highpower_lock, "XPT highpower lock", NULL, MTX_DEF);
853	mtx_init(&xsoftc.xpt_topo_lock, "XPT topology lock", NULL, MTX_DEF);
854	xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK,
855	    taskqueue_thread_enqueue, /*context*/&xsoftc.xpt_taskq);
856
857#ifdef CAM_BOOT_DELAY
858	/*
859	 * Override this value at compile time to assist our users
860	 * who don't use loader to boot a kernel.
861	 */
862	xsoftc.boot_delay = CAM_BOOT_DELAY;
863#endif
864	/*
865	 * The xpt layer is, itself, the equivelent of a SIM.
866	 * Allow 16 ccbs in the ccb pool for it.  This should
867	 * give decent parallelism when we probe busses and
868	 * perform other XPT functions.
869	 */
870	devq = cam_simq_alloc(16);
871	xpt_sim = cam_sim_alloc(xptaction,
872				xptpoll,
873				"xpt",
874				/*softc*/NULL,
875				/*unit*/0,
876				/*mtx*/&xsoftc.xpt_lock,
877				/*max_dev_transactions*/0,
878				/*max_tagged_dev_transactions*/0,
879				devq);
880	if (xpt_sim == NULL)
881		return (ENOMEM);
882
883	mtx_lock(&xsoftc.xpt_lock);
884	if ((status = xpt_bus_register(xpt_sim, NULL, 0)) != CAM_SUCCESS) {
885		mtx_unlock(&xsoftc.xpt_lock);
886		printf("xpt_init: xpt_bus_register failed with status %#x,"
887		       " failing attach\n", status);
888		return (EINVAL);
889	}
890	mtx_unlock(&xsoftc.xpt_lock);
891
892	/*
893	 * Looking at the XPT from the SIM layer, the XPT is
894	 * the equivelent of a peripheral driver.  Allocate
895	 * a peripheral driver entry for us.
896	 */
897	if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
898				      CAM_TARGET_WILDCARD,
899				      CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
900		mtx_unlock(&xsoftc.xpt_lock);
901		printf("xpt_init: xpt_create_path failed with status %#x,"
902		       " failing attach\n", status);
903		return (EINVAL);
904	}
905	xpt_path_lock(path);
906	cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
907			 path, NULL, 0, xpt_sim);
908	xpt_path_unlock(path);
909	xpt_free_path(path);
910
911	if (cam_num_doneqs < 1)
912		cam_num_doneqs = 1 + mp_ncpus / 6;
913	else if (cam_num_doneqs > MAXCPU)
914		cam_num_doneqs = MAXCPU;
915	for (i = 0; i < cam_num_doneqs; i++) {
916		mtx_init(&cam_doneqs[i].cam_doneq_mtx, "CAM doneq", NULL,
917		    MTX_DEF);
918		STAILQ_INIT(&cam_doneqs[i].cam_doneq);
919		error = kproc_kthread_add(xpt_done_td, &cam_doneqs[i],
920		    &cam_proc, NULL, 0, 0, "cam", "doneq%d", i);
921		if (error != 0) {
922			cam_num_doneqs = i;
923			break;
924		}
925	}
926	if (cam_num_doneqs < 1) {
927		printf("xpt_init: Cannot init completion queues "
928		       "- failing attach\n");
929		return (ENOMEM);
930	}
931	/*
932	 * Register a callback for when interrupts are enabled.
933	 */
934	xsoftc.xpt_config_hook =
935	    (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
936					      M_CAMXPT, M_NOWAIT | M_ZERO);
937	if (xsoftc.xpt_config_hook == NULL) {
938		printf("xpt_init: Cannot malloc config hook "
939		       "- failing attach\n");
940		return (ENOMEM);
941	}
942	xsoftc.xpt_config_hook->ich_func = xpt_config;
943	if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
944		free (xsoftc.xpt_config_hook, M_CAMXPT);
945		printf("xpt_init: config_intrhook_establish failed "
946		       "- failing attach\n");
947	}
948
949	return (0);
950}
951
952static cam_status
953xptregister(struct cam_periph *periph, void *arg)
954{
955	struct cam_sim *xpt_sim;
956
957	if (periph == NULL) {
958		printf("xptregister: periph was NULL!!\n");
959		return(CAM_REQ_CMP_ERR);
960	}
961
962	xpt_sim = (struct cam_sim *)arg;
963	xpt_sim->softc = periph;
964	xpt_periph = periph;
965	periph->softc = NULL;
966
967	return(CAM_REQ_CMP);
968}
969
970int32_t
971xpt_add_periph(struct cam_periph *periph)
972{
973	struct cam_ed *device;
974	int32_t	 status;
975
976	TASK_INIT(&periph->periph_run_task, 0, xpt_run_allocq_task, periph);
977	device = periph->path->device;
978	status = CAM_REQ_CMP;
979	if (device != NULL) {
980		mtx_lock(&device->target->bus->eb_mtx);
981		device->generation++;
982		SLIST_INSERT_HEAD(&device->periphs, periph, periph_links);
983		mtx_unlock(&device->target->bus->eb_mtx);
984	}
985
986	return (status);
987}
988
989void
990xpt_remove_periph(struct cam_periph *periph)
991{
992	struct cam_ed *device;
993
994	device = periph->path->device;
995	if (device != NULL) {
996		mtx_lock(&device->target->bus->eb_mtx);
997		device->generation++;
998		SLIST_REMOVE(&device->periphs, periph, cam_periph, periph_links);
999		mtx_unlock(&device->target->bus->eb_mtx);
1000	}
1001}
1002
1003
1004void
1005xpt_announce_periph(struct cam_periph *periph, char *announce_string)
1006{
1007	struct	cam_path *path = periph->path;
1008
1009	cam_periph_assert(periph, MA_OWNED);
1010	periph->flags |= CAM_PERIPH_ANNOUNCED;
1011
1012	printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1013	       periph->periph_name, periph->unit_number,
1014	       path->bus->sim->sim_name,
1015	       path->bus->sim->unit_number,
1016	       path->bus->sim->bus_id,
1017	       path->bus->path_id,
1018	       path->target->target_id,
1019	       (uintmax_t)path->device->lun_id);
1020	printf("%s%d: ", periph->periph_name, periph->unit_number);
1021	if (path->device->protocol == PROTO_SCSI)
1022		scsi_print_inquiry(&path->device->inq_data);
1023	else if (path->device->protocol == PROTO_ATA ||
1024	    path->device->protocol == PROTO_SATAPM)
1025		ata_print_ident(&path->device->ident_data);
1026	else if (path->device->protocol == PROTO_SEMB)
1027		semb_print_ident(
1028		    (struct sep_identify_data *)&path->device->ident_data);
1029	else
1030		printf("Unknown protocol device\n");
1031	if (path->device->serial_num_len > 0) {
1032		/* Don't wrap the screen  - print only the first 60 chars */
1033		printf("%s%d: Serial Number %.60s\n", periph->periph_name,
1034		       periph->unit_number, path->device->serial_num);
1035	}
1036	/* Announce transport details. */
1037	(*(path->bus->xport->announce))(periph);
1038	/* Announce command queueing. */
1039	if (path->device->inq_flags & SID_CmdQue
1040	 || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1041		printf("%s%d: Command Queueing enabled\n",
1042		       periph->periph_name, periph->unit_number);
1043	}
1044	/* Announce caller's details if they've passed in. */
1045	if (announce_string != NULL)
1046		printf("%s%d: %s\n", periph->periph_name,
1047		       periph->unit_number, announce_string);
1048}
1049
1050void
1051xpt_announce_quirks(struct cam_periph *periph, int quirks, char *bit_string)
1052{
1053	if (quirks != 0) {
1054		printf("%s%d: quirks=0x%b\n", periph->periph_name,
1055		    periph->unit_number, quirks, bit_string);
1056	}
1057}
1058
1059void
1060xpt_denounce_periph(struct cam_periph *periph)
1061{
1062	struct	cam_path *path = periph->path;
1063
1064	cam_periph_assert(periph, MA_OWNED);
1065	printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1066	       periph->periph_name, periph->unit_number,
1067	       path->bus->sim->sim_name,
1068	       path->bus->sim->unit_number,
1069	       path->bus->sim->bus_id,
1070	       path->bus->path_id,
1071	       path->target->target_id,
1072	       (uintmax_t)path->device->lun_id);
1073	printf("%s%d: ", periph->periph_name, periph->unit_number);
1074	if (path->device->protocol == PROTO_SCSI)
1075		scsi_print_inquiry_short(&path->device->inq_data);
1076	else if (path->device->protocol == PROTO_ATA ||
1077	    path->device->protocol == PROTO_SATAPM)
1078		ata_print_ident_short(&path->device->ident_data);
1079	else if (path->device->protocol == PROTO_SEMB)
1080		semb_print_ident_short(
1081		    (struct sep_identify_data *)&path->device->ident_data);
1082	else
1083		printf("Unknown protocol device");
1084	if (path->device->serial_num_len > 0)
1085		printf(" s/n %.60s", path->device->serial_num);
1086	printf(" detached\n");
1087}
1088
1089
1090int
1091xpt_getattr(char *buf, size_t len, const char *attr, struct cam_path *path)
1092{
1093	int ret = -1, l;
1094	struct ccb_dev_advinfo cdai;
1095	struct scsi_vpd_id_descriptor *idd;
1096
1097	xpt_path_assert(path, MA_OWNED);
1098
1099	memset(&cdai, 0, sizeof(cdai));
1100	xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
1101	cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
1102	cdai.bufsiz = len;
1103
1104	if (!strcmp(attr, "GEOM::ident"))
1105		cdai.buftype = CDAI_TYPE_SERIAL_NUM;
1106	else if (!strcmp(attr, "GEOM::physpath"))
1107		cdai.buftype = CDAI_TYPE_PHYS_PATH;
1108	else if (strcmp(attr, "GEOM::lunid") == 0 ||
1109		 strcmp(attr, "GEOM::lunname") == 0) {
1110		cdai.buftype = CDAI_TYPE_SCSI_DEVID;
1111		cdai.bufsiz = CAM_SCSI_DEVID_MAXLEN;
1112	} else
1113		goto out;
1114
1115	cdai.buf = malloc(cdai.bufsiz, M_CAMXPT, M_NOWAIT|M_ZERO);
1116	if (cdai.buf == NULL) {
1117		ret = ENOMEM;
1118		goto out;
1119	}
1120	xpt_action((union ccb *)&cdai); /* can only be synchronous */
1121	if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
1122		cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
1123	if (cdai.provsiz == 0)
1124		goto out;
1125	if (cdai.buftype == CDAI_TYPE_SCSI_DEVID) {
1126		if (strcmp(attr, "GEOM::lunid") == 0) {
1127			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1128			    cdai.provsiz, scsi_devid_is_lun_naa);
1129			if (idd == NULL)
1130				idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1131				    cdai.provsiz, scsi_devid_is_lun_eui64);
1132		} else
1133			idd = NULL;
1134		if (idd == NULL)
1135			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1136			    cdai.provsiz, scsi_devid_is_lun_t10);
1137		if (idd == NULL)
1138			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1139			    cdai.provsiz, scsi_devid_is_lun_name);
1140		if (idd == NULL)
1141			goto out;
1142		ret = 0;
1143		if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_ASCII ||
1144		    (idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_UTF8) {
1145			l = strnlen(idd->identifier, idd->length);
1146			if (l < len) {
1147				bcopy(idd->identifier, buf, l);
1148				buf[l] = 0;
1149			} else
1150				ret = EFAULT;
1151		} else {
1152			if (idd->length * 2 < len) {
1153				for (l = 0; l < idd->length; l++)
1154					sprintf(buf + l * 2, "%02x",
1155					    idd->identifier[l]);
1156			} else
1157				ret = EFAULT;
1158		}
1159	} else {
1160		ret = 0;
1161		if (strlcpy(buf, cdai.buf, len) >= len)
1162			ret = EFAULT;
1163	}
1164
1165out:
1166	if (cdai.buf != NULL)
1167		free(cdai.buf, M_CAMXPT);
1168	return ret;
1169}
1170
1171static dev_match_ret
1172xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1173	    struct cam_eb *bus)
1174{
1175	dev_match_ret retval;
1176	int i;
1177
1178	retval = DM_RET_NONE;
1179
1180	/*
1181	 * If we aren't given something to match against, that's an error.
1182	 */
1183	if (bus == NULL)
1184		return(DM_RET_ERROR);
1185
1186	/*
1187	 * If there are no match entries, then this bus matches no
1188	 * matter what.
1189	 */
1190	if ((patterns == NULL) || (num_patterns == 0))
1191		return(DM_RET_DESCEND | DM_RET_COPY);
1192
1193	for (i = 0; i < num_patterns; i++) {
1194		struct bus_match_pattern *cur_pattern;
1195
1196		/*
1197		 * If the pattern in question isn't for a bus node, we
1198		 * aren't interested.  However, we do indicate to the
1199		 * calling routine that we should continue descending the
1200		 * tree, since the user wants to match against lower-level
1201		 * EDT elements.
1202		 */
1203		if (patterns[i].type != DEV_MATCH_BUS) {
1204			if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1205				retval |= DM_RET_DESCEND;
1206			continue;
1207		}
1208
1209		cur_pattern = &patterns[i].pattern.bus_pattern;
1210
1211		/*
1212		 * If they want to match any bus node, we give them any
1213		 * device node.
1214		 */
1215		if (cur_pattern->flags == BUS_MATCH_ANY) {
1216			/* set the copy flag */
1217			retval |= DM_RET_COPY;
1218
1219			/*
1220			 * If we've already decided on an action, go ahead
1221			 * and return.
1222			 */
1223			if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1224				return(retval);
1225		}
1226
1227		/*
1228		 * Not sure why someone would do this...
1229		 */
1230		if (cur_pattern->flags == BUS_MATCH_NONE)
1231			continue;
1232
1233		if (((cur_pattern->flags & BUS_MATCH_PATH) != 0)
1234		 && (cur_pattern->path_id != bus->path_id))
1235			continue;
1236
1237		if (((cur_pattern->flags & BUS_MATCH_BUS_ID) != 0)
1238		 && (cur_pattern->bus_id != bus->sim->bus_id))
1239			continue;
1240
1241		if (((cur_pattern->flags & BUS_MATCH_UNIT) != 0)
1242		 && (cur_pattern->unit_number != bus->sim->unit_number))
1243			continue;
1244
1245		if (((cur_pattern->flags & BUS_MATCH_NAME) != 0)
1246		 && (strncmp(cur_pattern->dev_name, bus->sim->sim_name,
1247			     DEV_IDLEN) != 0))
1248			continue;
1249
1250		/*
1251		 * If we get to this point, the user definitely wants
1252		 * information on this bus.  So tell the caller to copy the
1253		 * data out.
1254		 */
1255		retval |= DM_RET_COPY;
1256
1257		/*
1258		 * If the return action has been set to descend, then we
1259		 * know that we've already seen a non-bus matching
1260		 * expression, therefore we need to further descend the tree.
1261		 * This won't change by continuing around the loop, so we
1262		 * go ahead and return.  If we haven't seen a non-bus
1263		 * matching expression, we keep going around the loop until
1264		 * we exhaust the matching expressions.  We'll set the stop
1265		 * flag once we fall out of the loop.
1266		 */
1267		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1268			return(retval);
1269	}
1270
1271	/*
1272	 * If the return action hasn't been set to descend yet, that means
1273	 * we haven't seen anything other than bus matching patterns.  So
1274	 * tell the caller to stop descending the tree -- the user doesn't
1275	 * want to match against lower level tree elements.
1276	 */
1277	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1278		retval |= DM_RET_STOP;
1279
1280	return(retval);
1281}
1282
1283static dev_match_ret
1284xptdevicematch(struct dev_match_pattern *patterns, u_int num_patterns,
1285	       struct cam_ed *device)
1286{
1287	dev_match_ret retval;
1288	int i;
1289
1290	retval = DM_RET_NONE;
1291
1292	/*
1293	 * If we aren't given something to match against, that's an error.
1294	 */
1295	if (device == NULL)
1296		return(DM_RET_ERROR);
1297
1298	/*
1299	 * If there are no match entries, then this device matches no
1300	 * matter what.
1301	 */
1302	if ((patterns == NULL) || (num_patterns == 0))
1303		return(DM_RET_DESCEND | DM_RET_COPY);
1304
1305	for (i = 0; i < num_patterns; i++) {
1306		struct device_match_pattern *cur_pattern;
1307		struct scsi_vpd_device_id *device_id_page;
1308
1309		/*
1310		 * If the pattern in question isn't for a device node, we
1311		 * aren't interested.
1312		 */
1313		if (patterns[i].type != DEV_MATCH_DEVICE) {
1314			if ((patterns[i].type == DEV_MATCH_PERIPH)
1315			 && ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE))
1316				retval |= DM_RET_DESCEND;
1317			continue;
1318		}
1319
1320		cur_pattern = &patterns[i].pattern.device_pattern;
1321
1322		/* Error out if mutually exclusive options are specified. */
1323		if ((cur_pattern->flags & (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1324		 == (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1325			return(DM_RET_ERROR);
1326
1327		/*
1328		 * If they want to match any device node, we give them any
1329		 * device node.
1330		 */
1331		if (cur_pattern->flags == DEV_MATCH_ANY)
1332			goto copy_dev_node;
1333
1334		/*
1335		 * Not sure why someone would do this...
1336		 */
1337		if (cur_pattern->flags == DEV_MATCH_NONE)
1338			continue;
1339
1340		if (((cur_pattern->flags & DEV_MATCH_PATH) != 0)
1341		 && (cur_pattern->path_id != device->target->bus->path_id))
1342			continue;
1343
1344		if (((cur_pattern->flags & DEV_MATCH_TARGET) != 0)
1345		 && (cur_pattern->target_id != device->target->target_id))
1346			continue;
1347
1348		if (((cur_pattern->flags & DEV_MATCH_LUN) != 0)
1349		 && (cur_pattern->target_lun != device->lun_id))
1350			continue;
1351
1352		if (((cur_pattern->flags & DEV_MATCH_INQUIRY) != 0)
1353		 && (cam_quirkmatch((caddr_t)&device->inq_data,
1354				    (caddr_t)&cur_pattern->data.inq_pat,
1355				    1, sizeof(cur_pattern->data.inq_pat),
1356				    scsi_static_inquiry_match) == NULL))
1357			continue;
1358
1359		device_id_page = (struct scsi_vpd_device_id *)device->device_id;
1360		if (((cur_pattern->flags & DEV_MATCH_DEVID) != 0)
1361		 && (device->device_id_len < SVPD_DEVICE_ID_HDR_LEN
1362		  || scsi_devid_match((uint8_t *)device_id_page->desc_list,
1363				      device->device_id_len
1364				    - SVPD_DEVICE_ID_HDR_LEN,
1365				      cur_pattern->data.devid_pat.id,
1366				      cur_pattern->data.devid_pat.id_len) != 0))
1367			continue;
1368
1369copy_dev_node:
1370		/*
1371		 * If we get to this point, the user definitely wants
1372		 * information on this device.  So tell the caller to copy
1373		 * the data out.
1374		 */
1375		retval |= DM_RET_COPY;
1376
1377		/*
1378		 * If the return action has been set to descend, then we
1379		 * know that we've already seen a peripheral matching
1380		 * expression, therefore we need to further descend the tree.
1381		 * This won't change by continuing around the loop, so we
1382		 * go ahead and return.  If we haven't seen a peripheral
1383		 * matching expression, we keep going around the loop until
1384		 * we exhaust the matching expressions.  We'll set the stop
1385		 * flag once we fall out of the loop.
1386		 */
1387		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1388			return(retval);
1389	}
1390
1391	/*
1392	 * If the return action hasn't been set to descend yet, that means
1393	 * we haven't seen any peripheral matching patterns.  So tell the
1394	 * caller to stop descending the tree -- the user doesn't want to
1395	 * match against lower level tree elements.
1396	 */
1397	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1398		retval |= DM_RET_STOP;
1399
1400	return(retval);
1401}
1402
1403/*
1404 * Match a single peripheral against any number of match patterns.
1405 */
1406static dev_match_ret
1407xptperiphmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1408	       struct cam_periph *periph)
1409{
1410	dev_match_ret retval;
1411	int i;
1412
1413	/*
1414	 * If we aren't given something to match against, that's an error.
1415	 */
1416	if (periph == NULL)
1417		return(DM_RET_ERROR);
1418
1419	/*
1420	 * If there are no match entries, then this peripheral matches no
1421	 * matter what.
1422	 */
1423	if ((patterns == NULL) || (num_patterns == 0))
1424		return(DM_RET_STOP | DM_RET_COPY);
1425
1426	/*
1427	 * There aren't any nodes below a peripheral node, so there's no
1428	 * reason to descend the tree any further.
1429	 */
1430	retval = DM_RET_STOP;
1431
1432	for (i = 0; i < num_patterns; i++) {
1433		struct periph_match_pattern *cur_pattern;
1434
1435		/*
1436		 * If the pattern in question isn't for a peripheral, we
1437		 * aren't interested.
1438		 */
1439		if (patterns[i].type != DEV_MATCH_PERIPH)
1440			continue;
1441
1442		cur_pattern = &patterns[i].pattern.periph_pattern;
1443
1444		/*
1445		 * If they want to match on anything, then we will do so.
1446		 */
1447		if (cur_pattern->flags == PERIPH_MATCH_ANY) {
1448			/* set the copy flag */
1449			retval |= DM_RET_COPY;
1450
1451			/*
1452			 * We've already set the return action to stop,
1453			 * since there are no nodes below peripherals in
1454			 * the tree.
1455			 */
1456			return(retval);
1457		}
1458
1459		/*
1460		 * Not sure why someone would do this...
1461		 */
1462		if (cur_pattern->flags == PERIPH_MATCH_NONE)
1463			continue;
1464
1465		if (((cur_pattern->flags & PERIPH_MATCH_PATH) != 0)
1466		 && (cur_pattern->path_id != periph->path->bus->path_id))
1467			continue;
1468
1469		/*
1470		 * For the target and lun id's, we have to make sure the
1471		 * target and lun pointers aren't NULL.  The xpt peripheral
1472		 * has a wildcard target and device.
1473		 */
1474		if (((cur_pattern->flags & PERIPH_MATCH_TARGET) != 0)
1475		 && ((periph->path->target == NULL)
1476		 ||(cur_pattern->target_id != periph->path->target->target_id)))
1477			continue;
1478
1479		if (((cur_pattern->flags & PERIPH_MATCH_LUN) != 0)
1480		 && ((periph->path->device == NULL)
1481		 || (cur_pattern->target_lun != periph->path->device->lun_id)))
1482			continue;
1483
1484		if (((cur_pattern->flags & PERIPH_MATCH_UNIT) != 0)
1485		 && (cur_pattern->unit_number != periph->unit_number))
1486			continue;
1487
1488		if (((cur_pattern->flags & PERIPH_MATCH_NAME) != 0)
1489		 && (strncmp(cur_pattern->periph_name, periph->periph_name,
1490			     DEV_IDLEN) != 0))
1491			continue;
1492
1493		/*
1494		 * If we get to this point, the user definitely wants
1495		 * information on this peripheral.  So tell the caller to
1496		 * copy the data out.
1497		 */
1498		retval |= DM_RET_COPY;
1499
1500		/*
1501		 * The return action has already been set to stop, since
1502		 * peripherals don't have any nodes below them in the EDT.
1503		 */
1504		return(retval);
1505	}
1506
1507	/*
1508	 * If we get to this point, the peripheral that was passed in
1509	 * doesn't match any of the patterns.
1510	 */
1511	return(retval);
1512}
1513
1514static int
1515xptedtbusfunc(struct cam_eb *bus, void *arg)
1516{
1517	struct ccb_dev_match *cdm;
1518	struct cam_et *target;
1519	dev_match_ret retval;
1520
1521	cdm = (struct ccb_dev_match *)arg;
1522
1523	/*
1524	 * If our position is for something deeper in the tree, that means
1525	 * that we've already seen this node.  So, we keep going down.
1526	 */
1527	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1528	 && (cdm->pos.cookie.bus == bus)
1529	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1530	 && (cdm->pos.cookie.target != NULL))
1531		retval = DM_RET_DESCEND;
1532	else
1533		retval = xptbusmatch(cdm->patterns, cdm->num_patterns, bus);
1534
1535	/*
1536	 * If we got an error, bail out of the search.
1537	 */
1538	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1539		cdm->status = CAM_DEV_MATCH_ERROR;
1540		return(0);
1541	}
1542
1543	/*
1544	 * If the copy flag is set, copy this bus out.
1545	 */
1546	if (retval & DM_RET_COPY) {
1547		int spaceleft, j;
1548
1549		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1550			sizeof(struct dev_match_result));
1551
1552		/*
1553		 * If we don't have enough space to put in another
1554		 * match result, save our position and tell the
1555		 * user there are more devices to check.
1556		 */
1557		if (spaceleft < sizeof(struct dev_match_result)) {
1558			bzero(&cdm->pos, sizeof(cdm->pos));
1559			cdm->pos.position_type =
1560				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS;
1561
1562			cdm->pos.cookie.bus = bus;
1563			cdm->pos.generations[CAM_BUS_GENERATION]=
1564				xsoftc.bus_generation;
1565			cdm->status = CAM_DEV_MATCH_MORE;
1566			return(0);
1567		}
1568		j = cdm->num_matches;
1569		cdm->num_matches++;
1570		cdm->matches[j].type = DEV_MATCH_BUS;
1571		cdm->matches[j].result.bus_result.path_id = bus->path_id;
1572		cdm->matches[j].result.bus_result.bus_id = bus->sim->bus_id;
1573		cdm->matches[j].result.bus_result.unit_number =
1574			bus->sim->unit_number;
1575		strncpy(cdm->matches[j].result.bus_result.dev_name,
1576			bus->sim->sim_name, DEV_IDLEN);
1577	}
1578
1579	/*
1580	 * If the user is only interested in busses, there's no
1581	 * reason to descend to the next level in the tree.
1582	 */
1583	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1584		return(1);
1585
1586	/*
1587	 * If there is a target generation recorded, check it to
1588	 * make sure the target list hasn't changed.
1589	 */
1590	mtx_lock(&bus->eb_mtx);
1591	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1592	 && (cdm->pos.cookie.bus == bus)
1593	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1594	 && (cdm->pos.cookie.target != NULL)) {
1595		if ((cdm->pos.generations[CAM_TARGET_GENERATION] !=
1596		    bus->generation)) {
1597			mtx_unlock(&bus->eb_mtx);
1598			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1599			return (0);
1600		}
1601		target = (struct cam_et *)cdm->pos.cookie.target;
1602		target->refcount++;
1603	} else
1604		target = NULL;
1605	mtx_unlock(&bus->eb_mtx);
1606
1607	return (xpttargettraverse(bus, target, xptedttargetfunc, arg));
1608}
1609
1610static int
1611xptedttargetfunc(struct cam_et *target, void *arg)
1612{
1613	struct ccb_dev_match *cdm;
1614	struct cam_eb *bus;
1615	struct cam_ed *device;
1616
1617	cdm = (struct ccb_dev_match *)arg;
1618	bus = target->bus;
1619
1620	/*
1621	 * If there is a device list generation recorded, check it to
1622	 * make sure the device list hasn't changed.
1623	 */
1624	mtx_lock(&bus->eb_mtx);
1625	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1626	 && (cdm->pos.cookie.bus == bus)
1627	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1628	 && (cdm->pos.cookie.target == target)
1629	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1630	 && (cdm->pos.cookie.device != NULL)) {
1631		if (cdm->pos.generations[CAM_DEV_GENERATION] !=
1632		    target->generation) {
1633			mtx_unlock(&bus->eb_mtx);
1634			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1635			return(0);
1636		}
1637		device = (struct cam_ed *)cdm->pos.cookie.device;
1638		device->refcount++;
1639	} else
1640		device = NULL;
1641	mtx_unlock(&bus->eb_mtx);
1642
1643	return (xptdevicetraverse(target, device, xptedtdevicefunc, arg));
1644}
1645
1646static int
1647xptedtdevicefunc(struct cam_ed *device, void *arg)
1648{
1649	struct cam_eb *bus;
1650	struct cam_periph *periph;
1651	struct ccb_dev_match *cdm;
1652	dev_match_ret retval;
1653
1654	cdm = (struct ccb_dev_match *)arg;
1655	bus = device->target->bus;
1656
1657	/*
1658	 * If our position is for something deeper in the tree, that means
1659	 * that we've already seen this node.  So, we keep going down.
1660	 */
1661	if ((cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1662	 && (cdm->pos.cookie.device == device)
1663	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1664	 && (cdm->pos.cookie.periph != NULL))
1665		retval = DM_RET_DESCEND;
1666	else
1667		retval = xptdevicematch(cdm->patterns, cdm->num_patterns,
1668					device);
1669
1670	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1671		cdm->status = CAM_DEV_MATCH_ERROR;
1672		return(0);
1673	}
1674
1675	/*
1676	 * If the copy flag is set, copy this device out.
1677	 */
1678	if (retval & DM_RET_COPY) {
1679		int spaceleft, j;
1680
1681		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1682			sizeof(struct dev_match_result));
1683
1684		/*
1685		 * If we don't have enough space to put in another
1686		 * match result, save our position and tell the
1687		 * user there are more devices to check.
1688		 */
1689		if (spaceleft < sizeof(struct dev_match_result)) {
1690			bzero(&cdm->pos, sizeof(cdm->pos));
1691			cdm->pos.position_type =
1692				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1693				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE;
1694
1695			cdm->pos.cookie.bus = device->target->bus;
1696			cdm->pos.generations[CAM_BUS_GENERATION]=
1697				xsoftc.bus_generation;
1698			cdm->pos.cookie.target = device->target;
1699			cdm->pos.generations[CAM_TARGET_GENERATION] =
1700				device->target->bus->generation;
1701			cdm->pos.cookie.device = device;
1702			cdm->pos.generations[CAM_DEV_GENERATION] =
1703				device->target->generation;
1704			cdm->status = CAM_DEV_MATCH_MORE;
1705			return(0);
1706		}
1707		j = cdm->num_matches;
1708		cdm->num_matches++;
1709		cdm->matches[j].type = DEV_MATCH_DEVICE;
1710		cdm->matches[j].result.device_result.path_id =
1711			device->target->bus->path_id;
1712		cdm->matches[j].result.device_result.target_id =
1713			device->target->target_id;
1714		cdm->matches[j].result.device_result.target_lun =
1715			device->lun_id;
1716		cdm->matches[j].result.device_result.protocol =
1717			device->protocol;
1718		bcopy(&device->inq_data,
1719		      &cdm->matches[j].result.device_result.inq_data,
1720		      sizeof(struct scsi_inquiry_data));
1721		bcopy(&device->ident_data,
1722		      &cdm->matches[j].result.device_result.ident_data,
1723		      sizeof(struct ata_params));
1724
1725		/* Let the user know whether this device is unconfigured */
1726		if (device->flags & CAM_DEV_UNCONFIGURED)
1727			cdm->matches[j].result.device_result.flags =
1728				DEV_RESULT_UNCONFIGURED;
1729		else
1730			cdm->matches[j].result.device_result.flags =
1731				DEV_RESULT_NOFLAG;
1732	}
1733
1734	/*
1735	 * If the user isn't interested in peripherals, don't descend
1736	 * the tree any further.
1737	 */
1738	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1739		return(1);
1740
1741	/*
1742	 * If there is a peripheral list generation recorded, make sure
1743	 * it hasn't changed.
1744	 */
1745	xpt_lock_buses();
1746	mtx_lock(&bus->eb_mtx);
1747	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1748	 && (cdm->pos.cookie.bus == bus)
1749	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1750	 && (cdm->pos.cookie.target == device->target)
1751	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1752	 && (cdm->pos.cookie.device == device)
1753	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1754	 && (cdm->pos.cookie.periph != NULL)) {
1755		if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1756		    device->generation) {
1757			mtx_unlock(&bus->eb_mtx);
1758			xpt_unlock_buses();
1759			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1760			return(0);
1761		}
1762		periph = (struct cam_periph *)cdm->pos.cookie.periph;
1763		periph->refcount++;
1764	} else
1765		periph = NULL;
1766	mtx_unlock(&bus->eb_mtx);
1767	xpt_unlock_buses();
1768
1769	return (xptperiphtraverse(device, periph, xptedtperiphfunc, arg));
1770}
1771
1772static int
1773xptedtperiphfunc(struct cam_periph *periph, void *arg)
1774{
1775	struct ccb_dev_match *cdm;
1776	dev_match_ret retval;
1777
1778	cdm = (struct ccb_dev_match *)arg;
1779
1780	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1781
1782	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1783		cdm->status = CAM_DEV_MATCH_ERROR;
1784		return(0);
1785	}
1786
1787	/*
1788	 * If the copy flag is set, copy this peripheral out.
1789	 */
1790	if (retval & DM_RET_COPY) {
1791		int spaceleft, j;
1792
1793		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1794			sizeof(struct dev_match_result));
1795
1796		/*
1797		 * If we don't have enough space to put in another
1798		 * match result, save our position and tell the
1799		 * user there are more devices to check.
1800		 */
1801		if (spaceleft < sizeof(struct dev_match_result)) {
1802			bzero(&cdm->pos, sizeof(cdm->pos));
1803			cdm->pos.position_type =
1804				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1805				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE |
1806				CAM_DEV_POS_PERIPH;
1807
1808			cdm->pos.cookie.bus = periph->path->bus;
1809			cdm->pos.generations[CAM_BUS_GENERATION]=
1810				xsoftc.bus_generation;
1811			cdm->pos.cookie.target = periph->path->target;
1812			cdm->pos.generations[CAM_TARGET_GENERATION] =
1813				periph->path->bus->generation;
1814			cdm->pos.cookie.device = periph->path->device;
1815			cdm->pos.generations[CAM_DEV_GENERATION] =
1816				periph->path->target->generation;
1817			cdm->pos.cookie.periph = periph;
1818			cdm->pos.generations[CAM_PERIPH_GENERATION] =
1819				periph->path->device->generation;
1820			cdm->status = CAM_DEV_MATCH_MORE;
1821			return(0);
1822		}
1823
1824		j = cdm->num_matches;
1825		cdm->num_matches++;
1826		cdm->matches[j].type = DEV_MATCH_PERIPH;
1827		cdm->matches[j].result.periph_result.path_id =
1828			periph->path->bus->path_id;
1829		cdm->matches[j].result.periph_result.target_id =
1830			periph->path->target->target_id;
1831		cdm->matches[j].result.periph_result.target_lun =
1832			periph->path->device->lun_id;
1833		cdm->matches[j].result.periph_result.unit_number =
1834			periph->unit_number;
1835		strncpy(cdm->matches[j].result.periph_result.periph_name,
1836			periph->periph_name, DEV_IDLEN);
1837	}
1838
1839	return(1);
1840}
1841
1842static int
1843xptedtmatch(struct ccb_dev_match *cdm)
1844{
1845	struct cam_eb *bus;
1846	int ret;
1847
1848	cdm->num_matches = 0;
1849
1850	/*
1851	 * Check the bus list generation.  If it has changed, the user
1852	 * needs to reset everything and start over.
1853	 */
1854	xpt_lock_buses();
1855	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1856	 && (cdm->pos.cookie.bus != NULL)) {
1857		if (cdm->pos.generations[CAM_BUS_GENERATION] !=
1858		    xsoftc.bus_generation) {
1859			xpt_unlock_buses();
1860			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1861			return(0);
1862		}
1863		bus = (struct cam_eb *)cdm->pos.cookie.bus;
1864		bus->refcount++;
1865	} else
1866		bus = NULL;
1867	xpt_unlock_buses();
1868
1869	ret = xptbustraverse(bus, xptedtbusfunc, cdm);
1870
1871	/*
1872	 * If we get back 0, that means that we had to stop before fully
1873	 * traversing the EDT.  It also means that one of the subroutines
1874	 * has set the status field to the proper value.  If we get back 1,
1875	 * we've fully traversed the EDT and copied out any matching entries.
1876	 */
1877	if (ret == 1)
1878		cdm->status = CAM_DEV_MATCH_LAST;
1879
1880	return(ret);
1881}
1882
1883static int
1884xptplistpdrvfunc(struct periph_driver **pdrv, void *arg)
1885{
1886	struct cam_periph *periph;
1887	struct ccb_dev_match *cdm;
1888
1889	cdm = (struct ccb_dev_match *)arg;
1890
1891	xpt_lock_buses();
1892	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
1893	 && (cdm->pos.cookie.pdrv == pdrv)
1894	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1895	 && (cdm->pos.cookie.periph != NULL)) {
1896		if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1897		    (*pdrv)->generation) {
1898			xpt_unlock_buses();
1899			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1900			return(0);
1901		}
1902		periph = (struct cam_periph *)cdm->pos.cookie.periph;
1903		periph->refcount++;
1904	} else
1905		periph = NULL;
1906	xpt_unlock_buses();
1907
1908	return (xptpdperiphtraverse(pdrv, periph, xptplistperiphfunc, arg));
1909}
1910
1911static int
1912xptplistperiphfunc(struct cam_periph *periph, void *arg)
1913{
1914	struct ccb_dev_match *cdm;
1915	dev_match_ret retval;
1916
1917	cdm = (struct ccb_dev_match *)arg;
1918
1919	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1920
1921	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1922		cdm->status = CAM_DEV_MATCH_ERROR;
1923		return(0);
1924	}
1925
1926	/*
1927	 * If the copy flag is set, copy this peripheral out.
1928	 */
1929	if (retval & DM_RET_COPY) {
1930		int spaceleft, j;
1931
1932		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1933			sizeof(struct dev_match_result));
1934
1935		/*
1936		 * If we don't have enough space to put in another
1937		 * match result, save our position and tell the
1938		 * user there are more devices to check.
1939		 */
1940		if (spaceleft < sizeof(struct dev_match_result)) {
1941			struct periph_driver **pdrv;
1942
1943			pdrv = NULL;
1944			bzero(&cdm->pos, sizeof(cdm->pos));
1945			cdm->pos.position_type =
1946				CAM_DEV_POS_PDRV | CAM_DEV_POS_PDPTR |
1947				CAM_DEV_POS_PERIPH;
1948
1949			/*
1950			 * This may look a bit non-sensical, but it is
1951			 * actually quite logical.  There are very few
1952			 * peripheral drivers, and bloating every peripheral
1953			 * structure with a pointer back to its parent
1954			 * peripheral driver linker set entry would cost
1955			 * more in the long run than doing this quick lookup.
1956			 */
1957			for (pdrv = periph_drivers; *pdrv != NULL; pdrv++) {
1958				if (strcmp((*pdrv)->driver_name,
1959				    periph->periph_name) == 0)
1960					break;
1961			}
1962
1963			if (*pdrv == NULL) {
1964				cdm->status = CAM_DEV_MATCH_ERROR;
1965				return(0);
1966			}
1967
1968			cdm->pos.cookie.pdrv = pdrv;
1969			/*
1970			 * The periph generation slot does double duty, as
1971			 * does the periph pointer slot.  They are used for
1972			 * both edt and pdrv lookups and positioning.
1973			 */
1974			cdm->pos.cookie.periph = periph;
1975			cdm->pos.generations[CAM_PERIPH_GENERATION] =
1976				(*pdrv)->generation;
1977			cdm->status = CAM_DEV_MATCH_MORE;
1978			return(0);
1979		}
1980
1981		j = cdm->num_matches;
1982		cdm->num_matches++;
1983		cdm->matches[j].type = DEV_MATCH_PERIPH;
1984		cdm->matches[j].result.periph_result.path_id =
1985			periph->path->bus->path_id;
1986
1987		/*
1988		 * The transport layer peripheral doesn't have a target or
1989		 * lun.
1990		 */
1991		if (periph->path->target)
1992			cdm->matches[j].result.periph_result.target_id =
1993				periph->path->target->target_id;
1994		else
1995			cdm->matches[j].result.periph_result.target_id = -1;
1996
1997		if (periph->path->device)
1998			cdm->matches[j].result.periph_result.target_lun =
1999				periph->path->device->lun_id;
2000		else
2001			cdm->matches[j].result.periph_result.target_lun = -1;
2002
2003		cdm->matches[j].result.periph_result.unit_number =
2004			periph->unit_number;
2005		strncpy(cdm->matches[j].result.periph_result.periph_name,
2006			periph->periph_name, DEV_IDLEN);
2007	}
2008
2009	return(1);
2010}
2011
2012static int
2013xptperiphlistmatch(struct ccb_dev_match *cdm)
2014{
2015	int ret;
2016
2017	cdm->num_matches = 0;
2018
2019	/*
2020	 * At this point in the edt traversal function, we check the bus
2021	 * list generation to make sure that no busses have been added or
2022	 * removed since the user last sent a XPT_DEV_MATCH ccb through.
2023	 * For the peripheral driver list traversal function, however, we
2024	 * don't have to worry about new peripheral driver types coming or
2025	 * going; they're in a linker set, and therefore can't change
2026	 * without a recompile.
2027	 */
2028
2029	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2030	 && (cdm->pos.cookie.pdrv != NULL))
2031		ret = xptpdrvtraverse(
2032				(struct periph_driver **)cdm->pos.cookie.pdrv,
2033				xptplistpdrvfunc, cdm);
2034	else
2035		ret = xptpdrvtraverse(NULL, xptplistpdrvfunc, cdm);
2036
2037	/*
2038	 * If we get back 0, that means that we had to stop before fully
2039	 * traversing the peripheral driver tree.  It also means that one of
2040	 * the subroutines has set the status field to the proper value.  If
2041	 * we get back 1, we've fully traversed the EDT and copied out any
2042	 * matching entries.
2043	 */
2044	if (ret == 1)
2045		cdm->status = CAM_DEV_MATCH_LAST;
2046
2047	return(ret);
2048}
2049
2050static int
2051xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
2052{
2053	struct cam_eb *bus, *next_bus;
2054	int retval;
2055
2056	retval = 1;
2057	if (start_bus)
2058		bus = start_bus;
2059	else {
2060		xpt_lock_buses();
2061		bus = TAILQ_FIRST(&xsoftc.xpt_busses);
2062		if (bus == NULL) {
2063			xpt_unlock_buses();
2064			return (retval);
2065		}
2066		bus->refcount++;
2067		xpt_unlock_buses();
2068	}
2069	for (; bus != NULL; bus = next_bus) {
2070		retval = tr_func(bus, arg);
2071		if (retval == 0) {
2072			xpt_release_bus(bus);
2073			break;
2074		}
2075		xpt_lock_buses();
2076		next_bus = TAILQ_NEXT(bus, links);
2077		if (next_bus)
2078			next_bus->refcount++;
2079		xpt_unlock_buses();
2080		xpt_release_bus(bus);
2081	}
2082	return(retval);
2083}
2084
2085static int
2086xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target,
2087		  xpt_targetfunc_t *tr_func, void *arg)
2088{
2089	struct cam_et *target, *next_target;
2090	int retval;
2091
2092	retval = 1;
2093	if (start_target)
2094		target = start_target;
2095	else {
2096		mtx_lock(&bus->eb_mtx);
2097		target = TAILQ_FIRST(&bus->et_entries);
2098		if (target == NULL) {
2099			mtx_unlock(&bus->eb_mtx);
2100			return (retval);
2101		}
2102		target->refcount++;
2103		mtx_unlock(&bus->eb_mtx);
2104	}
2105	for (; target != NULL; target = next_target) {
2106		retval = tr_func(target, arg);
2107		if (retval == 0) {
2108			xpt_release_target(target);
2109			break;
2110		}
2111		mtx_lock(&bus->eb_mtx);
2112		next_target = TAILQ_NEXT(target, links);
2113		if (next_target)
2114			next_target->refcount++;
2115		mtx_unlock(&bus->eb_mtx);
2116		xpt_release_target(target);
2117	}
2118	return(retval);
2119}
2120
2121static int
2122xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device,
2123		  xpt_devicefunc_t *tr_func, void *arg)
2124{
2125	struct cam_eb *bus;
2126	struct cam_ed *device, *next_device;
2127	int retval;
2128
2129	retval = 1;
2130	bus = target->bus;
2131	if (start_device)
2132		device = start_device;
2133	else {
2134		mtx_lock(&bus->eb_mtx);
2135		device = TAILQ_FIRST(&target->ed_entries);
2136		if (device == NULL) {
2137			mtx_unlock(&bus->eb_mtx);
2138			return (retval);
2139		}
2140		device->refcount++;
2141		mtx_unlock(&bus->eb_mtx);
2142	}
2143	for (; device != NULL; device = next_device) {
2144		mtx_lock(&device->device_mtx);
2145		retval = tr_func(device, arg);
2146		mtx_unlock(&device->device_mtx);
2147		if (retval == 0) {
2148			xpt_release_device(device);
2149			break;
2150		}
2151		mtx_lock(&bus->eb_mtx);
2152		next_device = TAILQ_NEXT(device, links);
2153		if (next_device)
2154			next_device->refcount++;
2155		mtx_unlock(&bus->eb_mtx);
2156		xpt_release_device(device);
2157	}
2158	return(retval);
2159}
2160
2161static int
2162xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph,
2163		  xpt_periphfunc_t *tr_func, void *arg)
2164{
2165	struct cam_eb *bus;
2166	struct cam_periph *periph, *next_periph;
2167	int retval;
2168
2169	retval = 1;
2170
2171	bus = device->target->bus;
2172	if (start_periph)
2173		periph = start_periph;
2174	else {
2175		xpt_lock_buses();
2176		mtx_lock(&bus->eb_mtx);
2177		periph = SLIST_FIRST(&device->periphs);
2178		while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2179			periph = SLIST_NEXT(periph, periph_links);
2180		if (periph == NULL) {
2181			mtx_unlock(&bus->eb_mtx);
2182			xpt_unlock_buses();
2183			return (retval);
2184		}
2185		periph->refcount++;
2186		mtx_unlock(&bus->eb_mtx);
2187		xpt_unlock_buses();
2188	}
2189	for (; periph != NULL; periph = next_periph) {
2190		retval = tr_func(periph, arg);
2191		if (retval == 0) {
2192			cam_periph_release_locked(periph);
2193			break;
2194		}
2195		xpt_lock_buses();
2196		mtx_lock(&bus->eb_mtx);
2197		next_periph = SLIST_NEXT(periph, periph_links);
2198		while (next_periph != NULL &&
2199		    (next_periph->flags & CAM_PERIPH_FREE) != 0)
2200			next_periph = SLIST_NEXT(periph, periph_links);
2201		if (next_periph)
2202			next_periph->refcount++;
2203		mtx_unlock(&bus->eb_mtx);
2204		xpt_unlock_buses();
2205		cam_periph_release_locked(periph);
2206	}
2207	return(retval);
2208}
2209
2210static int
2211xptpdrvtraverse(struct periph_driver **start_pdrv,
2212		xpt_pdrvfunc_t *tr_func, void *arg)
2213{
2214	struct periph_driver **pdrv;
2215	int retval;
2216
2217	retval = 1;
2218
2219	/*
2220	 * We don't traverse the peripheral driver list like we do the
2221	 * other lists, because it is a linker set, and therefore cannot be
2222	 * changed during runtime.  If the peripheral driver list is ever
2223	 * re-done to be something other than a linker set (i.e. it can
2224	 * change while the system is running), the list traversal should
2225	 * be modified to work like the other traversal functions.
2226	 */
2227	for (pdrv = (start_pdrv ? start_pdrv : periph_drivers);
2228	     *pdrv != NULL; pdrv++) {
2229		retval = tr_func(pdrv, arg);
2230
2231		if (retval == 0)
2232			return(retval);
2233	}
2234
2235	return(retval);
2236}
2237
2238static int
2239xptpdperiphtraverse(struct periph_driver **pdrv,
2240		    struct cam_periph *start_periph,
2241		    xpt_periphfunc_t *tr_func, void *arg)
2242{
2243	struct cam_periph *periph, *next_periph;
2244	int retval;
2245
2246	retval = 1;
2247
2248	if (start_periph)
2249		periph = start_periph;
2250	else {
2251		xpt_lock_buses();
2252		periph = TAILQ_FIRST(&(*pdrv)->units);
2253		while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2254			periph = TAILQ_NEXT(periph, unit_links);
2255		if (periph == NULL) {
2256			xpt_unlock_buses();
2257			return (retval);
2258		}
2259		periph->refcount++;
2260		xpt_unlock_buses();
2261	}
2262	for (; periph != NULL; periph = next_periph) {
2263		cam_periph_lock(periph);
2264		retval = tr_func(periph, arg);
2265		cam_periph_unlock(periph);
2266		if (retval == 0) {
2267			cam_periph_release(periph);
2268			break;
2269		}
2270		xpt_lock_buses();
2271		next_periph = TAILQ_NEXT(periph, unit_links);
2272		while (next_periph != NULL &&
2273		    (next_periph->flags & CAM_PERIPH_FREE) != 0)
2274			next_periph = TAILQ_NEXT(periph, unit_links);
2275		if (next_periph)
2276			next_periph->refcount++;
2277		xpt_unlock_buses();
2278		cam_periph_release(periph);
2279	}
2280	return(retval);
2281}
2282
2283static int
2284xptdefbusfunc(struct cam_eb *bus, void *arg)
2285{
2286	struct xpt_traverse_config *tr_config;
2287
2288	tr_config = (struct xpt_traverse_config *)arg;
2289
2290	if (tr_config->depth == XPT_DEPTH_BUS) {
2291		xpt_busfunc_t *tr_func;
2292
2293		tr_func = (xpt_busfunc_t *)tr_config->tr_func;
2294
2295		return(tr_func(bus, tr_config->tr_arg));
2296	} else
2297		return(xpttargettraverse(bus, NULL, xptdeftargetfunc, arg));
2298}
2299
2300static int
2301xptdeftargetfunc(struct cam_et *target, void *arg)
2302{
2303	struct xpt_traverse_config *tr_config;
2304
2305	tr_config = (struct xpt_traverse_config *)arg;
2306
2307	if (tr_config->depth == XPT_DEPTH_TARGET) {
2308		xpt_targetfunc_t *tr_func;
2309
2310		tr_func = (xpt_targetfunc_t *)tr_config->tr_func;
2311
2312		return(tr_func(target, tr_config->tr_arg));
2313	} else
2314		return(xptdevicetraverse(target, NULL, xptdefdevicefunc, arg));
2315}
2316
2317static int
2318xptdefdevicefunc(struct cam_ed *device, void *arg)
2319{
2320	struct xpt_traverse_config *tr_config;
2321
2322	tr_config = (struct xpt_traverse_config *)arg;
2323
2324	if (tr_config->depth == XPT_DEPTH_DEVICE) {
2325		xpt_devicefunc_t *tr_func;
2326
2327		tr_func = (xpt_devicefunc_t *)tr_config->tr_func;
2328
2329		return(tr_func(device, tr_config->tr_arg));
2330	} else
2331		return(xptperiphtraverse(device, NULL, xptdefperiphfunc, arg));
2332}
2333
2334static int
2335xptdefperiphfunc(struct cam_periph *periph, void *arg)
2336{
2337	struct xpt_traverse_config *tr_config;
2338	xpt_periphfunc_t *tr_func;
2339
2340	tr_config = (struct xpt_traverse_config *)arg;
2341
2342	tr_func = (xpt_periphfunc_t *)tr_config->tr_func;
2343
2344	/*
2345	 * Unlike the other default functions, we don't check for depth
2346	 * here.  The peripheral driver level is the last level in the EDT,
2347	 * so if we're here, we should execute the function in question.
2348	 */
2349	return(tr_func(periph, tr_config->tr_arg));
2350}
2351
2352/*
2353 * Execute the given function for every bus in the EDT.
2354 */
2355static int
2356xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
2357{
2358	struct xpt_traverse_config tr_config;
2359
2360	tr_config.depth = XPT_DEPTH_BUS;
2361	tr_config.tr_func = tr_func;
2362	tr_config.tr_arg = arg;
2363
2364	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2365}
2366
2367/*
2368 * Execute the given function for every device in the EDT.
2369 */
2370static int
2371xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
2372{
2373	struct xpt_traverse_config tr_config;
2374
2375	tr_config.depth = XPT_DEPTH_DEVICE;
2376	tr_config.tr_func = tr_func;
2377	tr_config.tr_arg = arg;
2378
2379	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2380}
2381
2382static int
2383xptsetasyncfunc(struct cam_ed *device, void *arg)
2384{
2385	struct cam_path path;
2386	struct ccb_getdev cgd;
2387	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2388
2389	/*
2390	 * Don't report unconfigured devices (Wildcard devs,
2391	 * devices only for target mode, device instances
2392	 * that have been invalidated but are waiting for
2393	 * their last reference count to be released).
2394	 */
2395	if ((device->flags & CAM_DEV_UNCONFIGURED) != 0)
2396		return (1);
2397
2398	xpt_compile_path(&path,
2399			 NULL,
2400			 device->target->bus->path_id,
2401			 device->target->target_id,
2402			 device->lun_id);
2403	xpt_setup_ccb(&cgd.ccb_h, &path, CAM_PRIORITY_NORMAL);
2404	cgd.ccb_h.func_code = XPT_GDEV_TYPE;
2405	xpt_action((union ccb *)&cgd);
2406	csa->callback(csa->callback_arg,
2407			    AC_FOUND_DEVICE,
2408			    &path, &cgd);
2409	xpt_release_path(&path);
2410
2411	return(1);
2412}
2413
2414static int
2415xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
2416{
2417	struct cam_path path;
2418	struct ccb_pathinq cpi;
2419	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2420
2421	xpt_compile_path(&path, /*periph*/NULL,
2422			 bus->path_id,
2423			 CAM_TARGET_WILDCARD,
2424			 CAM_LUN_WILDCARD);
2425	xpt_path_lock(&path);
2426	xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL);
2427	cpi.ccb_h.func_code = XPT_PATH_INQ;
2428	xpt_action((union ccb *)&cpi);
2429	csa->callback(csa->callback_arg,
2430			    AC_PATH_REGISTERED,
2431			    &path, &cpi);
2432	xpt_path_unlock(&path);
2433	xpt_release_path(&path);
2434
2435	return(1);
2436}
2437
2438void
2439xpt_action(union ccb *start_ccb)
2440{
2441
2442	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_action\n"));
2443
2444	start_ccb->ccb_h.status = CAM_REQ_INPROG;
2445	(*(start_ccb->ccb_h.path->bus->xport->action))(start_ccb);
2446}
2447
2448void
2449xpt_action_default(union ccb *start_ccb)
2450{
2451	struct cam_path *path;
2452	struct cam_sim *sim;
2453	int lock;
2454
2455	path = start_ccb->ccb_h.path;
2456	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_action_default\n"));
2457
2458	switch (start_ccb->ccb_h.func_code) {
2459	case XPT_SCSI_IO:
2460	{
2461		struct cam_ed *device;
2462
2463		/*
2464		 * For the sake of compatibility with SCSI-1
2465		 * devices that may not understand the identify
2466		 * message, we include lun information in the
2467		 * second byte of all commands.  SCSI-1 specifies
2468		 * that luns are a 3 bit value and reserves only 3
2469		 * bits for lun information in the CDB.  Later
2470		 * revisions of the SCSI spec allow for more than 8
2471		 * luns, but have deprecated lun information in the
2472		 * CDB.  So, if the lun won't fit, we must omit.
2473		 *
2474		 * Also be aware that during initial probing for devices,
2475		 * the inquiry information is unknown but initialized to 0.
2476		 * This means that this code will be exercised while probing
2477		 * devices with an ANSI revision greater than 2.
2478		 */
2479		device = path->device;
2480		if (device->protocol_version <= SCSI_REV_2
2481		 && start_ccb->ccb_h.target_lun < 8
2482		 && (start_ccb->ccb_h.flags & CAM_CDB_POINTER) == 0) {
2483
2484			start_ccb->csio.cdb_io.cdb_bytes[1] |=
2485			    start_ccb->ccb_h.target_lun << 5;
2486		}
2487		start_ccb->csio.scsi_status = SCSI_STATUS_OK;
2488	}
2489	/* FALLTHROUGH */
2490	case XPT_TARGET_IO:
2491	case XPT_CONT_TARGET_IO:
2492		start_ccb->csio.sense_resid = 0;
2493		start_ccb->csio.resid = 0;
2494		/* FALLTHROUGH */
2495	case XPT_ATA_IO:
2496		if (start_ccb->ccb_h.func_code == XPT_ATA_IO)
2497			start_ccb->ataio.resid = 0;
2498		/* FALLTHROUGH */
2499	case XPT_RESET_DEV:
2500	case XPT_ENG_EXEC:
2501	case XPT_SMP_IO:
2502	{
2503		struct cam_devq *devq;
2504
2505		devq = path->bus->sim->devq;
2506		mtx_lock(&devq->send_mtx);
2507		cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
2508		if (xpt_schedule_devq(devq, path->device) != 0)
2509			xpt_run_devq(devq);
2510		mtx_unlock(&devq->send_mtx);
2511		break;
2512	}
2513	case XPT_CALC_GEOMETRY:
2514		/* Filter out garbage */
2515		if (start_ccb->ccg.block_size == 0
2516		 || start_ccb->ccg.volume_size == 0) {
2517			start_ccb->ccg.cylinders = 0;
2518			start_ccb->ccg.heads = 0;
2519			start_ccb->ccg.secs_per_track = 0;
2520			start_ccb->ccb_h.status = CAM_REQ_CMP;
2521			break;
2522		}
2523#if defined(PC98) || defined(__sparc64__)
2524		/*
2525		 * In a PC-98 system, geometry translation depens on
2526		 * the "real" device geometry obtained from mode page 4.
2527		 * SCSI geometry translation is performed in the
2528		 * initialization routine of the SCSI BIOS and the result
2529		 * stored in host memory.  If the translation is available
2530		 * in host memory, use it.  If not, rely on the default
2531		 * translation the device driver performs.
2532		 * For sparc64, we may need adjust the geometry of large
2533		 * disks in order to fit the limitations of the 16-bit
2534		 * fields of the VTOC8 disk label.
2535		 */
2536		if (scsi_da_bios_params(&start_ccb->ccg) != 0) {
2537			start_ccb->ccb_h.status = CAM_REQ_CMP;
2538			break;
2539		}
2540#endif
2541		goto call_sim;
2542	case XPT_ABORT:
2543	{
2544		union ccb* abort_ccb;
2545
2546		abort_ccb = start_ccb->cab.abort_ccb;
2547		if (XPT_FC_IS_DEV_QUEUED(abort_ccb)) {
2548
2549			if (abort_ccb->ccb_h.pinfo.index >= 0) {
2550				struct cam_ccbq *ccbq;
2551				struct cam_ed *device;
2552
2553				device = abort_ccb->ccb_h.path->device;
2554				ccbq = &device->ccbq;
2555				cam_ccbq_remove_ccb(ccbq, abort_ccb);
2556				abort_ccb->ccb_h.status =
2557				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2558				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2559				xpt_done(abort_ccb);
2560				start_ccb->ccb_h.status = CAM_REQ_CMP;
2561				break;
2562			}
2563			if (abort_ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX
2564			 && (abort_ccb->ccb_h.status & CAM_SIM_QUEUED) == 0) {
2565				/*
2566				 * We've caught this ccb en route to
2567				 * the SIM.  Flag it for abort and the
2568				 * SIM will do so just before starting
2569				 * real work on the CCB.
2570				 */
2571				abort_ccb->ccb_h.status =
2572				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2573				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2574				start_ccb->ccb_h.status = CAM_REQ_CMP;
2575				break;
2576			}
2577		}
2578		if (XPT_FC_IS_QUEUED(abort_ccb)
2579		 && (abort_ccb->ccb_h.pinfo.index == CAM_DONEQ_INDEX)) {
2580			/*
2581			 * It's already completed but waiting
2582			 * for our SWI to get to it.
2583			 */
2584			start_ccb->ccb_h.status = CAM_UA_ABORT;
2585			break;
2586		}
2587		/*
2588		 * If we weren't able to take care of the abort request
2589		 * in the XPT, pass the request down to the SIM for processing.
2590		 */
2591	}
2592	/* FALLTHROUGH */
2593	case XPT_ACCEPT_TARGET_IO:
2594	case XPT_EN_LUN:
2595	case XPT_IMMED_NOTIFY:
2596	case XPT_NOTIFY_ACK:
2597	case XPT_RESET_BUS:
2598	case XPT_IMMEDIATE_NOTIFY:
2599	case XPT_NOTIFY_ACKNOWLEDGE:
2600	case XPT_GET_SIM_KNOB:
2601	case XPT_SET_SIM_KNOB:
2602	case XPT_GET_TRAN_SETTINGS:
2603	case XPT_SET_TRAN_SETTINGS:
2604	case XPT_PATH_INQ:
2605call_sim:
2606		sim = path->bus->sim;
2607		lock = (mtx_owned(sim->mtx) == 0);
2608		if (lock)
2609			CAM_SIM_LOCK(sim);
2610		(*(sim->sim_action))(sim, start_ccb);
2611		if (lock)
2612			CAM_SIM_UNLOCK(sim);
2613		break;
2614	case XPT_PATH_STATS:
2615		start_ccb->cpis.last_reset = path->bus->last_reset;
2616		start_ccb->ccb_h.status = CAM_REQ_CMP;
2617		break;
2618	case XPT_GDEV_TYPE:
2619	{
2620		struct cam_ed *dev;
2621
2622		dev = path->device;
2623		if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2624			start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2625		} else {
2626			struct ccb_getdev *cgd;
2627
2628			cgd = &start_ccb->cgd;
2629			cgd->protocol = dev->protocol;
2630			cgd->inq_data = dev->inq_data;
2631			cgd->ident_data = dev->ident_data;
2632			cgd->inq_flags = dev->inq_flags;
2633			cgd->ccb_h.status = CAM_REQ_CMP;
2634			cgd->serial_num_len = dev->serial_num_len;
2635			if ((dev->serial_num_len > 0)
2636			 && (dev->serial_num != NULL))
2637				bcopy(dev->serial_num, cgd->serial_num,
2638				      dev->serial_num_len);
2639		}
2640		break;
2641	}
2642	case XPT_GDEV_STATS:
2643	{
2644		struct cam_ed *dev;
2645
2646		dev = path->device;
2647		if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2648			start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2649		} else {
2650			struct ccb_getdevstats *cgds;
2651			struct cam_eb *bus;
2652			struct cam_et *tar;
2653
2654			cgds = &start_ccb->cgds;
2655			bus = path->bus;
2656			tar = path->target;
2657			cgds->dev_openings = dev->ccbq.dev_openings;
2658			cgds->dev_active = dev->ccbq.dev_active;
2659			cgds->devq_openings = dev->ccbq.devq_openings;
2660			cgds->devq_queued = cam_ccbq_pending_ccb_count(&dev->ccbq);
2661			cgds->held = dev->ccbq.held;
2662			cgds->last_reset = tar->last_reset;
2663			cgds->maxtags = dev->maxtags;
2664			cgds->mintags = dev->mintags;
2665			if (timevalcmp(&tar->last_reset, &bus->last_reset, <))
2666				cgds->last_reset = bus->last_reset;
2667			cgds->ccb_h.status = CAM_REQ_CMP;
2668		}
2669		break;
2670	}
2671	case XPT_GDEVLIST:
2672	{
2673		struct cam_periph	*nperiph;
2674		struct periph_list	*periph_head;
2675		struct ccb_getdevlist	*cgdl;
2676		u_int			i;
2677		struct cam_ed		*device;
2678		int			found;
2679
2680
2681		found = 0;
2682
2683		/*
2684		 * Don't want anyone mucking with our data.
2685		 */
2686		device = path->device;
2687		periph_head = &device->periphs;
2688		cgdl = &start_ccb->cgdl;
2689
2690		/*
2691		 * Check and see if the list has changed since the user
2692		 * last requested a list member.  If so, tell them that the
2693		 * list has changed, and therefore they need to start over
2694		 * from the beginning.
2695		 */
2696		if ((cgdl->index != 0) &&
2697		    (cgdl->generation != device->generation)) {
2698			cgdl->status = CAM_GDEVLIST_LIST_CHANGED;
2699			break;
2700		}
2701
2702		/*
2703		 * Traverse the list of peripherals and attempt to find
2704		 * the requested peripheral.
2705		 */
2706		for (nperiph = SLIST_FIRST(periph_head), i = 0;
2707		     (nperiph != NULL) && (i <= cgdl->index);
2708		     nperiph = SLIST_NEXT(nperiph, periph_links), i++) {
2709			if (i == cgdl->index) {
2710				strncpy(cgdl->periph_name,
2711					nperiph->periph_name,
2712					DEV_IDLEN);
2713				cgdl->unit_number = nperiph->unit_number;
2714				found = 1;
2715			}
2716		}
2717		if (found == 0) {
2718			cgdl->status = CAM_GDEVLIST_ERROR;
2719			break;
2720		}
2721
2722		if (nperiph == NULL)
2723			cgdl->status = CAM_GDEVLIST_LAST_DEVICE;
2724		else
2725			cgdl->status = CAM_GDEVLIST_MORE_DEVS;
2726
2727		cgdl->index++;
2728		cgdl->generation = device->generation;
2729
2730		cgdl->ccb_h.status = CAM_REQ_CMP;
2731		break;
2732	}
2733	case XPT_DEV_MATCH:
2734	{
2735		dev_pos_type position_type;
2736		struct ccb_dev_match *cdm;
2737
2738		cdm = &start_ccb->cdm;
2739
2740		/*
2741		 * There are two ways of getting at information in the EDT.
2742		 * The first way is via the primary EDT tree.  It starts
2743		 * with a list of busses, then a list of targets on a bus,
2744		 * then devices/luns on a target, and then peripherals on a
2745		 * device/lun.  The "other" way is by the peripheral driver
2746		 * lists.  The peripheral driver lists are organized by
2747		 * peripheral driver.  (obviously)  So it makes sense to
2748		 * use the peripheral driver list if the user is looking
2749		 * for something like "da1", or all "da" devices.  If the
2750		 * user is looking for something on a particular bus/target
2751		 * or lun, it's generally better to go through the EDT tree.
2752		 */
2753
2754		if (cdm->pos.position_type != CAM_DEV_POS_NONE)
2755			position_type = cdm->pos.position_type;
2756		else {
2757			u_int i;
2758
2759			position_type = CAM_DEV_POS_NONE;
2760
2761			for (i = 0; i < cdm->num_patterns; i++) {
2762				if ((cdm->patterns[i].type == DEV_MATCH_BUS)
2763				 ||(cdm->patterns[i].type == DEV_MATCH_DEVICE)){
2764					position_type = CAM_DEV_POS_EDT;
2765					break;
2766				}
2767			}
2768
2769			if (cdm->num_patterns == 0)
2770				position_type = CAM_DEV_POS_EDT;
2771			else if (position_type == CAM_DEV_POS_NONE)
2772				position_type = CAM_DEV_POS_PDRV;
2773		}
2774
2775		switch(position_type & CAM_DEV_POS_TYPEMASK) {
2776		case CAM_DEV_POS_EDT:
2777			xptedtmatch(cdm);
2778			break;
2779		case CAM_DEV_POS_PDRV:
2780			xptperiphlistmatch(cdm);
2781			break;
2782		default:
2783			cdm->status = CAM_DEV_MATCH_ERROR;
2784			break;
2785		}
2786
2787		if (cdm->status == CAM_DEV_MATCH_ERROR)
2788			start_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2789		else
2790			start_ccb->ccb_h.status = CAM_REQ_CMP;
2791
2792		break;
2793	}
2794	case XPT_SASYNC_CB:
2795	{
2796		struct ccb_setasync *csa;
2797		struct async_node *cur_entry;
2798		struct async_list *async_head;
2799		u_int32_t added;
2800
2801		csa = &start_ccb->csa;
2802		added = csa->event_enable;
2803		async_head = &path->device->asyncs;
2804
2805		/*
2806		 * If there is already an entry for us, simply
2807		 * update it.
2808		 */
2809		cur_entry = SLIST_FIRST(async_head);
2810		while (cur_entry != NULL) {
2811			if ((cur_entry->callback_arg == csa->callback_arg)
2812			 && (cur_entry->callback == csa->callback))
2813				break;
2814			cur_entry = SLIST_NEXT(cur_entry, links);
2815		}
2816
2817		if (cur_entry != NULL) {
2818		 	/*
2819			 * If the request has no flags set,
2820			 * remove the entry.
2821			 */
2822			added &= ~cur_entry->event_enable;
2823			if (csa->event_enable == 0) {
2824				SLIST_REMOVE(async_head, cur_entry,
2825					     async_node, links);
2826				xpt_release_device(path->device);
2827				free(cur_entry, M_CAMXPT);
2828			} else {
2829				cur_entry->event_enable = csa->event_enable;
2830			}
2831			csa->event_enable = added;
2832		} else {
2833			cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
2834					   M_NOWAIT);
2835			if (cur_entry == NULL) {
2836				csa->ccb_h.status = CAM_RESRC_UNAVAIL;
2837				break;
2838			}
2839			cur_entry->event_enable = csa->event_enable;
2840			cur_entry->event_lock =
2841			    mtx_owned(path->bus->sim->mtx) ? 1 : 0;
2842			cur_entry->callback_arg = csa->callback_arg;
2843			cur_entry->callback = csa->callback;
2844			SLIST_INSERT_HEAD(async_head, cur_entry, links);
2845			xpt_acquire_device(path->device);
2846		}
2847		start_ccb->ccb_h.status = CAM_REQ_CMP;
2848		break;
2849	}
2850	case XPT_REL_SIMQ:
2851	{
2852		struct ccb_relsim *crs;
2853		struct cam_ed *dev;
2854
2855		crs = &start_ccb->crs;
2856		dev = path->device;
2857		if (dev == NULL) {
2858
2859			crs->ccb_h.status = CAM_DEV_NOT_THERE;
2860			break;
2861		}
2862
2863		if ((crs->release_flags & RELSIM_ADJUST_OPENINGS) != 0) {
2864
2865			/* Don't ever go below one opening */
2866			if (crs->openings > 0) {
2867				xpt_dev_ccbq_resize(path, crs->openings);
2868				if (bootverbose) {
2869					xpt_print(path,
2870					    "number of openings is now %d\n",
2871					    crs->openings);
2872				}
2873			}
2874		}
2875
2876		mtx_lock(&dev->sim->devq->send_mtx);
2877		if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) {
2878
2879			if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
2880
2881				/*
2882				 * Just extend the old timeout and decrement
2883				 * the freeze count so that a single timeout
2884				 * is sufficient for releasing the queue.
2885				 */
2886				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2887				callout_stop(&dev->callout);
2888			} else {
2889
2890				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2891			}
2892
2893			callout_reset(&dev->callout,
2894			    (crs->release_timeout * hz) / 1000,
2895			    xpt_release_devq_timeout, dev);
2896
2897			dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
2898
2899		}
2900
2901		if ((crs->release_flags & RELSIM_RELEASE_AFTER_CMDCMPLT) != 0) {
2902
2903			if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0) {
2904				/*
2905				 * Decrement the freeze count so that a single
2906				 * completion is still sufficient to unfreeze
2907				 * the queue.
2908				 */
2909				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2910			} else {
2911
2912				dev->flags |= CAM_DEV_REL_ON_COMPLETE;
2913				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2914			}
2915		}
2916
2917		if ((crs->release_flags & RELSIM_RELEASE_AFTER_QEMPTY) != 0) {
2918
2919			if ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
2920			 || (dev->ccbq.dev_active == 0)) {
2921
2922				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2923			} else {
2924
2925				dev->flags |= CAM_DEV_REL_ON_QUEUE_EMPTY;
2926				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2927			}
2928		}
2929		mtx_unlock(&dev->sim->devq->send_mtx);
2930
2931		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0)
2932			xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
2933		start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt;
2934		start_ccb->ccb_h.status = CAM_REQ_CMP;
2935		break;
2936	}
2937	case XPT_DEBUG: {
2938		struct cam_path *oldpath;
2939
2940		/* Check that all request bits are supported. */
2941		if (start_ccb->cdbg.flags & ~(CAM_DEBUG_COMPILE)) {
2942			start_ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2943			break;
2944		}
2945
2946		cam_dflags = CAM_DEBUG_NONE;
2947		if (cam_dpath != NULL) {
2948			oldpath = cam_dpath;
2949			cam_dpath = NULL;
2950			xpt_free_path(oldpath);
2951		}
2952		if (start_ccb->cdbg.flags != CAM_DEBUG_NONE) {
2953			if (xpt_create_path(&cam_dpath, NULL,
2954					    start_ccb->ccb_h.path_id,
2955					    start_ccb->ccb_h.target_id,
2956					    start_ccb->ccb_h.target_lun) !=
2957					    CAM_REQ_CMP) {
2958				start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2959			} else {
2960				cam_dflags = start_ccb->cdbg.flags;
2961				start_ccb->ccb_h.status = CAM_REQ_CMP;
2962				xpt_print(cam_dpath, "debugging flags now %x\n",
2963				    cam_dflags);
2964			}
2965		} else
2966			start_ccb->ccb_h.status = CAM_REQ_CMP;
2967		break;
2968	}
2969	case XPT_NOOP:
2970		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
2971			xpt_freeze_devq(path, 1);
2972		start_ccb->ccb_h.status = CAM_REQ_CMP;
2973		break;
2974	default:
2975	case XPT_SDEV_TYPE:
2976	case XPT_TERM_IO:
2977	case XPT_ENG_INQ:
2978		/* XXX Implement */
2979		printf("%s: CCB type %#x not supported\n", __func__,
2980		       start_ccb->ccb_h.func_code);
2981		start_ccb->ccb_h.status = CAM_PROVIDE_FAIL;
2982		if (start_ccb->ccb_h.func_code & XPT_FC_DEV_QUEUED) {
2983			xpt_done(start_ccb);
2984		}
2985		break;
2986	}
2987}
2988
2989void
2990xpt_polled_action(union ccb *start_ccb)
2991{
2992	u_int32_t timeout;
2993	struct	  cam_sim *sim;
2994	struct	  cam_devq *devq;
2995	struct	  cam_ed *dev;
2996
2997	timeout = start_ccb->ccb_h.timeout * 10;
2998	sim = start_ccb->ccb_h.path->bus->sim;
2999	devq = sim->devq;
3000	dev = start_ccb->ccb_h.path->device;
3001
3002	mtx_unlock(&dev->device_mtx);
3003
3004	/*
3005	 * Steal an opening so that no other queued requests
3006	 * can get it before us while we simulate interrupts.
3007	 */
3008	mtx_lock(&devq->send_mtx);
3009	dev->ccbq.devq_openings--;
3010	dev->ccbq.dev_openings--;
3011	while((devq->send_openings <= 0 || dev->ccbq.dev_openings < 0) &&
3012	    (--timeout > 0)) {
3013		mtx_unlock(&devq->send_mtx);
3014		DELAY(100);
3015		CAM_SIM_LOCK(sim);
3016		(*(sim->sim_poll))(sim);
3017		CAM_SIM_UNLOCK(sim);
3018		camisr_runqueue();
3019		mtx_lock(&devq->send_mtx);
3020	}
3021	dev->ccbq.devq_openings++;
3022	dev->ccbq.dev_openings++;
3023	mtx_unlock(&devq->send_mtx);
3024
3025	if (timeout != 0) {
3026		xpt_action(start_ccb);
3027		while(--timeout > 0) {
3028			CAM_SIM_LOCK(sim);
3029			(*(sim->sim_poll))(sim);
3030			CAM_SIM_UNLOCK(sim);
3031			camisr_runqueue();
3032			if ((start_ccb->ccb_h.status  & CAM_STATUS_MASK)
3033			    != CAM_REQ_INPROG)
3034				break;
3035			DELAY(100);
3036		}
3037		if (timeout == 0) {
3038			/*
3039			 * XXX Is it worth adding a sim_timeout entry
3040			 * point so we can attempt recovery?  If
3041			 * this is only used for dumps, I don't think
3042			 * it is.
3043			 */
3044			start_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
3045		}
3046	} else {
3047		start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3048	}
3049
3050	mtx_lock(&dev->device_mtx);
3051}
3052
3053/*
3054 * Schedule a peripheral driver to receive a ccb when it's
3055 * target device has space for more transactions.
3056 */
3057void
3058xpt_schedule(struct cam_periph *periph, u_int32_t new_priority)
3059{
3060
3061	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
3062	cam_periph_assert(periph, MA_OWNED);
3063	if (new_priority < periph->scheduled_priority) {
3064		periph->scheduled_priority = new_priority;
3065		xpt_run_allocq(periph, 0);
3066	}
3067}
3068
3069
3070/*
3071 * Schedule a device to run on a given queue.
3072 * If the device was inserted as a new entry on the queue,
3073 * return 1 meaning the device queue should be run. If we
3074 * were already queued, implying someone else has already
3075 * started the queue, return 0 so the caller doesn't attempt
3076 * to run the queue.
3077 */
3078static int
3079xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
3080		 u_int32_t new_priority)
3081{
3082	int retval;
3083	u_int32_t old_priority;
3084
3085	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n"));
3086
3087	old_priority = pinfo->priority;
3088
3089	/*
3090	 * Are we already queued?
3091	 */
3092	if (pinfo->index != CAM_UNQUEUED_INDEX) {
3093		/* Simply reorder based on new priority */
3094		if (new_priority < old_priority) {
3095			camq_change_priority(queue, pinfo->index,
3096					     new_priority);
3097			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3098					("changed priority to %d\n",
3099					 new_priority));
3100			retval = 1;
3101		} else
3102			retval = 0;
3103	} else {
3104		/* New entry on the queue */
3105		if (new_priority < old_priority)
3106			pinfo->priority = new_priority;
3107
3108		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3109				("Inserting onto queue\n"));
3110		pinfo->generation = ++queue->generation;
3111		camq_insert(queue, pinfo);
3112		retval = 1;
3113	}
3114	return (retval);
3115}
3116
3117static void
3118xpt_run_allocq_task(void *context, int pending)
3119{
3120	struct cam_periph *periph = context;
3121
3122	cam_periph_lock(periph);
3123	periph->flags &= ~CAM_PERIPH_RUN_TASK;
3124	xpt_run_allocq(periph, 1);
3125	cam_periph_unlock(periph);
3126	cam_periph_release(periph);
3127}
3128
3129static void
3130xpt_run_allocq(struct cam_periph *periph, int sleep)
3131{
3132	struct cam_ed	*device;
3133	union ccb	*ccb;
3134	uint32_t	 prio;
3135
3136	cam_periph_assert(periph, MA_OWNED);
3137	if (periph->periph_allocating)
3138		return;
3139	periph->periph_allocating = 1;
3140	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_allocq(%p)\n", periph));
3141	device = periph->path->device;
3142	ccb = NULL;
3143restart:
3144	while ((prio = min(periph->scheduled_priority,
3145	    periph->immediate_priority)) != CAM_PRIORITY_NONE &&
3146	    (periph->periph_allocated - (ccb != NULL ? 1 : 0) <
3147	     device->ccbq.total_openings || prio <= CAM_PRIORITY_OOB)) {
3148
3149		if (ccb == NULL &&
3150		    (ccb = xpt_get_ccb_nowait(periph)) == NULL) {
3151			if (sleep) {
3152				ccb = xpt_get_ccb(periph);
3153				goto restart;
3154			}
3155			if (periph->flags & CAM_PERIPH_RUN_TASK)
3156				break;
3157			xpt_lock_buses();
3158			periph->refcount++;	/* Unconditionally acquire */
3159			xpt_unlock_buses();
3160			periph->flags |= CAM_PERIPH_RUN_TASK;
3161			taskqueue_enqueue(xsoftc.xpt_taskq,
3162			    &periph->periph_run_task);
3163			break;
3164		}
3165		xpt_setup_ccb(&ccb->ccb_h, periph->path, prio);
3166		if (prio == periph->immediate_priority) {
3167			periph->immediate_priority = CAM_PRIORITY_NONE;
3168			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3169					("waking cam_periph_getccb()\n"));
3170			SLIST_INSERT_HEAD(&periph->ccb_list, &ccb->ccb_h,
3171					  periph_links.sle);
3172			wakeup(&periph->ccb_list);
3173		} else {
3174			periph->scheduled_priority = CAM_PRIORITY_NONE;
3175			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3176					("calling periph_start()\n"));
3177			periph->periph_start(periph, ccb);
3178		}
3179		ccb = NULL;
3180	}
3181	if (ccb != NULL)
3182		xpt_release_ccb(ccb);
3183	periph->periph_allocating = 0;
3184}
3185
3186static void
3187xpt_run_devq(struct cam_devq *devq)
3188{
3189	char cdb_str[(SCSI_MAX_CDBLEN * 3) + 1];
3190	int lock;
3191
3192	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_devq\n"));
3193
3194	devq->send_queue.qfrozen_cnt++;
3195	while ((devq->send_queue.entries > 0)
3196	    && (devq->send_openings > 0)
3197	    && (devq->send_queue.qfrozen_cnt <= 1)) {
3198		struct	cam_ed *device;
3199		union ccb *work_ccb;
3200		struct	cam_sim *sim;
3201
3202		device = (struct cam_ed *)camq_remove(&devq->send_queue,
3203							   CAMQ_HEAD);
3204		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3205				("running device %p\n", device));
3206
3207		work_ccb = cam_ccbq_peek_ccb(&device->ccbq, CAMQ_HEAD);
3208		if (work_ccb == NULL) {
3209			printf("device on run queue with no ccbs???\n");
3210			continue;
3211		}
3212
3213		if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
3214
3215			mtx_lock(&xsoftc.xpt_highpower_lock);
3216		 	if (xsoftc.num_highpower <= 0) {
3217				/*
3218				 * We got a high power command, but we
3219				 * don't have any available slots.  Freeze
3220				 * the device queue until we have a slot
3221				 * available.
3222				 */
3223				xpt_freeze_devq_device(device, 1);
3224				STAILQ_INSERT_TAIL(&xsoftc.highpowerq, device,
3225						   highpowerq_entry);
3226
3227				mtx_unlock(&xsoftc.xpt_highpower_lock);
3228				continue;
3229			} else {
3230				/*
3231				 * Consume a high power slot while
3232				 * this ccb runs.
3233				 */
3234				xsoftc.num_highpower--;
3235			}
3236			mtx_unlock(&xsoftc.xpt_highpower_lock);
3237		}
3238		cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
3239		cam_ccbq_send_ccb(&device->ccbq, work_ccb);
3240		devq->send_openings--;
3241		devq->send_active++;
3242		xpt_schedule_devq(devq, device);
3243		mtx_unlock(&devq->send_mtx);
3244
3245		if ((work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0) {
3246			/*
3247			 * The client wants to freeze the queue
3248			 * after this CCB is sent.
3249			 */
3250			xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3251		}
3252
3253		/* In Target mode, the peripheral driver knows best... */
3254		if (work_ccb->ccb_h.func_code == XPT_SCSI_IO) {
3255			if ((device->inq_flags & SID_CmdQue) != 0
3256			 && work_ccb->csio.tag_action != CAM_TAG_ACTION_NONE)
3257				work_ccb->ccb_h.flags |= CAM_TAG_ACTION_VALID;
3258			else
3259				/*
3260				 * Clear this in case of a retried CCB that
3261				 * failed due to a rejected tag.
3262				 */
3263				work_ccb->ccb_h.flags &= ~CAM_TAG_ACTION_VALID;
3264		}
3265
3266		switch (work_ccb->ccb_h.func_code) {
3267		case XPT_SCSI_IO:
3268			CAM_DEBUG(work_ccb->ccb_h.path,
3269			    CAM_DEBUG_CDB,("%s. CDB: %s\n",
3270			     scsi_op_desc(work_ccb->csio.cdb_io.cdb_bytes[0],
3271					  &device->inq_data),
3272			     scsi_cdb_string(work_ccb->csio.cdb_io.cdb_bytes,
3273					     cdb_str, sizeof(cdb_str))));
3274			break;
3275		case XPT_ATA_IO:
3276			CAM_DEBUG(work_ccb->ccb_h.path,
3277			    CAM_DEBUG_CDB,("%s. ACB: %s\n",
3278			     ata_op_string(&work_ccb->ataio.cmd),
3279			     ata_cmd_string(&work_ccb->ataio.cmd,
3280					    cdb_str, sizeof(cdb_str))));
3281			break;
3282		default:
3283			break;
3284		}
3285
3286		/*
3287		 * Device queues can be shared among multiple SIM instances
3288		 * that reside on different busses.  Use the SIM from the
3289		 * queued device, rather than the one from the calling bus.
3290		 */
3291		sim = device->sim;
3292		lock = (mtx_owned(sim->mtx) == 0);
3293		if (lock)
3294			CAM_SIM_LOCK(sim);
3295		(*(sim->sim_action))(sim, work_ccb);
3296		if (lock)
3297			CAM_SIM_UNLOCK(sim);
3298		mtx_lock(&devq->send_mtx);
3299	}
3300	devq->send_queue.qfrozen_cnt--;
3301}
3302
3303/*
3304 * This function merges stuff from the slave ccb into the master ccb, while
3305 * keeping important fields in the master ccb constant.
3306 */
3307void
3308xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
3309{
3310
3311	/*
3312	 * Pull fields that are valid for peripheral drivers to set
3313	 * into the master CCB along with the CCB "payload".
3314	 */
3315	master_ccb->ccb_h.retry_count = slave_ccb->ccb_h.retry_count;
3316	master_ccb->ccb_h.func_code = slave_ccb->ccb_h.func_code;
3317	master_ccb->ccb_h.timeout = slave_ccb->ccb_h.timeout;
3318	master_ccb->ccb_h.flags = slave_ccb->ccb_h.flags;
3319	bcopy(&(&slave_ccb->ccb_h)[1], &(&master_ccb->ccb_h)[1],
3320	      sizeof(union ccb) - sizeof(struct ccb_hdr));
3321}
3322
3323void
3324xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
3325{
3326
3327	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
3328	ccb_h->pinfo.priority = priority;
3329	ccb_h->path = path;
3330	ccb_h->path_id = path->bus->path_id;
3331	if (path->target)
3332		ccb_h->target_id = path->target->target_id;
3333	else
3334		ccb_h->target_id = CAM_TARGET_WILDCARD;
3335	if (path->device) {
3336		ccb_h->target_lun = path->device->lun_id;
3337		ccb_h->pinfo.generation = ++path->device->ccbq.queue.generation;
3338	} else {
3339		ccb_h->target_lun = CAM_TARGET_WILDCARD;
3340	}
3341	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
3342	ccb_h->flags = 0;
3343	ccb_h->xflags = 0;
3344}
3345
3346/* Path manipulation functions */
3347cam_status
3348xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
3349		path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3350{
3351	struct	   cam_path *path;
3352	cam_status status;
3353
3354	path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3355
3356	if (path == NULL) {
3357		status = CAM_RESRC_UNAVAIL;
3358		return(status);
3359	}
3360	status = xpt_compile_path(path, perph, path_id, target_id, lun_id);
3361	if (status != CAM_REQ_CMP) {
3362		free(path, M_CAMPATH);
3363		path = NULL;
3364	}
3365	*new_path_ptr = path;
3366	return (status);
3367}
3368
3369cam_status
3370xpt_create_path_unlocked(struct cam_path **new_path_ptr,
3371			 struct cam_periph *periph, path_id_t path_id,
3372			 target_id_t target_id, lun_id_t lun_id)
3373{
3374
3375	return (xpt_create_path(new_path_ptr, periph, path_id, target_id,
3376	    lun_id));
3377}
3378
3379cam_status
3380xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
3381		 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3382{
3383	struct	     cam_eb *bus;
3384	struct	     cam_et *target;
3385	struct	     cam_ed *device;
3386	cam_status   status;
3387
3388	status = CAM_REQ_CMP;	/* Completed without error */
3389	target = NULL;		/* Wildcarded */
3390	device = NULL;		/* Wildcarded */
3391
3392	/*
3393	 * We will potentially modify the EDT, so block interrupts
3394	 * that may attempt to create cam paths.
3395	 */
3396	bus = xpt_find_bus(path_id);
3397	if (bus == NULL) {
3398		status = CAM_PATH_INVALID;
3399	} else {
3400		xpt_lock_buses();
3401		mtx_lock(&bus->eb_mtx);
3402		target = xpt_find_target(bus, target_id);
3403		if (target == NULL) {
3404			/* Create one */
3405			struct cam_et *new_target;
3406
3407			new_target = xpt_alloc_target(bus, target_id);
3408			if (new_target == NULL) {
3409				status = CAM_RESRC_UNAVAIL;
3410			} else {
3411				target = new_target;
3412			}
3413		}
3414		xpt_unlock_buses();
3415		if (target != NULL) {
3416			device = xpt_find_device(target, lun_id);
3417			if (device == NULL) {
3418				/* Create one */
3419				struct cam_ed *new_device;
3420
3421				new_device =
3422				    (*(bus->xport->alloc_device))(bus,
3423								      target,
3424								      lun_id);
3425				if (new_device == NULL) {
3426					status = CAM_RESRC_UNAVAIL;
3427				} else {
3428					device = new_device;
3429				}
3430			}
3431		}
3432		mtx_unlock(&bus->eb_mtx);
3433	}
3434
3435	/*
3436	 * Only touch the user's data if we are successful.
3437	 */
3438	if (status == CAM_REQ_CMP) {
3439		new_path->periph = perph;
3440		new_path->bus = bus;
3441		new_path->target = target;
3442		new_path->device = device;
3443		CAM_DEBUG(new_path, CAM_DEBUG_TRACE, ("xpt_compile_path\n"));
3444	} else {
3445		if (device != NULL)
3446			xpt_release_device(device);
3447		if (target != NULL)
3448			xpt_release_target(target);
3449		if (bus != NULL)
3450			xpt_release_bus(bus);
3451	}
3452	return (status);
3453}
3454
3455cam_status
3456xpt_clone_path(struct cam_path **new_path_ptr, struct cam_path *path)
3457{
3458	struct	   cam_path *new_path;
3459
3460	new_path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3461	if (new_path == NULL)
3462		return(CAM_RESRC_UNAVAIL);
3463	xpt_copy_path(new_path, path);
3464	*new_path_ptr = new_path;
3465	return (CAM_REQ_CMP);
3466}
3467
3468void
3469xpt_copy_path(struct cam_path *new_path, struct cam_path *path)
3470{
3471
3472	*new_path = *path;
3473	if (path->bus != NULL)
3474		xpt_acquire_bus(path->bus);
3475	if (path->target != NULL)
3476		xpt_acquire_target(path->target);
3477	if (path->device != NULL)
3478		xpt_acquire_device(path->device);
3479}
3480
3481void
3482xpt_release_path(struct cam_path *path)
3483{
3484	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_path\n"));
3485	if (path->device != NULL) {
3486		xpt_release_device(path->device);
3487		path->device = NULL;
3488	}
3489	if (path->target != NULL) {
3490		xpt_release_target(path->target);
3491		path->target = NULL;
3492	}
3493	if (path->bus != NULL) {
3494		xpt_release_bus(path->bus);
3495		path->bus = NULL;
3496	}
3497}
3498
3499void
3500xpt_free_path(struct cam_path *path)
3501{
3502
3503	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
3504	xpt_release_path(path);
3505	free(path, M_CAMPATH);
3506}
3507
3508void
3509xpt_path_counts(struct cam_path *path, uint32_t *bus_ref,
3510    uint32_t *periph_ref, uint32_t *target_ref, uint32_t *device_ref)
3511{
3512
3513	xpt_lock_buses();
3514	if (bus_ref) {
3515		if (path->bus)
3516			*bus_ref = path->bus->refcount;
3517		else
3518			*bus_ref = 0;
3519	}
3520	if (periph_ref) {
3521		if (path->periph)
3522			*periph_ref = path->periph->refcount;
3523		else
3524			*periph_ref = 0;
3525	}
3526	xpt_unlock_buses();
3527	if (target_ref) {
3528		if (path->target)
3529			*target_ref = path->target->refcount;
3530		else
3531			*target_ref = 0;
3532	}
3533	if (device_ref) {
3534		if (path->device)
3535			*device_ref = path->device->refcount;
3536		else
3537			*device_ref = 0;
3538	}
3539}
3540
3541/*
3542 * Return -1 for failure, 0 for exact match, 1 for match with wildcards
3543 * in path1, 2 for match with wildcards in path2.
3544 */
3545int
3546xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
3547{
3548	int retval = 0;
3549
3550	if (path1->bus != path2->bus) {
3551		if (path1->bus->path_id == CAM_BUS_WILDCARD)
3552			retval = 1;
3553		else if (path2->bus->path_id == CAM_BUS_WILDCARD)
3554			retval = 2;
3555		else
3556			return (-1);
3557	}
3558	if (path1->target != path2->target) {
3559		if (path1->target->target_id == CAM_TARGET_WILDCARD) {
3560			if (retval == 0)
3561				retval = 1;
3562		} else if (path2->target->target_id == CAM_TARGET_WILDCARD)
3563			retval = 2;
3564		else
3565			return (-1);
3566	}
3567	if (path1->device != path2->device) {
3568		if (path1->device->lun_id == CAM_LUN_WILDCARD) {
3569			if (retval == 0)
3570				retval = 1;
3571		} else if (path2->device->lun_id == CAM_LUN_WILDCARD)
3572			retval = 2;
3573		else
3574			return (-1);
3575	}
3576	return (retval);
3577}
3578
3579int
3580xpt_path_comp_dev(struct cam_path *path, struct cam_ed *dev)
3581{
3582	int retval = 0;
3583
3584	if (path->bus != dev->target->bus) {
3585		if (path->bus->path_id == CAM_BUS_WILDCARD)
3586			retval = 1;
3587		else if (dev->target->bus->path_id == CAM_BUS_WILDCARD)
3588			retval = 2;
3589		else
3590			return (-1);
3591	}
3592	if (path->target != dev->target) {
3593		if (path->target->target_id == CAM_TARGET_WILDCARD) {
3594			if (retval == 0)
3595				retval = 1;
3596		} else if (dev->target->target_id == CAM_TARGET_WILDCARD)
3597			retval = 2;
3598		else
3599			return (-1);
3600	}
3601	if (path->device != dev) {
3602		if (path->device->lun_id == CAM_LUN_WILDCARD) {
3603			if (retval == 0)
3604				retval = 1;
3605		} else if (dev->lun_id == CAM_LUN_WILDCARD)
3606			retval = 2;
3607		else
3608			return (-1);
3609	}
3610	return (retval);
3611}
3612
3613void
3614xpt_print_path(struct cam_path *path)
3615{
3616
3617	if (path == NULL)
3618		printf("(nopath): ");
3619	else {
3620		if (path->periph != NULL)
3621			printf("(%s%d:", path->periph->periph_name,
3622			       path->periph->unit_number);
3623		else
3624			printf("(noperiph:");
3625
3626		if (path->bus != NULL)
3627			printf("%s%d:%d:", path->bus->sim->sim_name,
3628			       path->bus->sim->unit_number,
3629			       path->bus->sim->bus_id);
3630		else
3631			printf("nobus:");
3632
3633		if (path->target != NULL)
3634			printf("%d:", path->target->target_id);
3635		else
3636			printf("X:");
3637
3638		if (path->device != NULL)
3639			printf("%jx): ", (uintmax_t)path->device->lun_id);
3640		else
3641			printf("X): ");
3642	}
3643}
3644
3645void
3646xpt_print_device(struct cam_ed *device)
3647{
3648
3649	if (device == NULL)
3650		printf("(nopath): ");
3651	else {
3652		printf("(noperiph:%s%d:%d:%d:%jx): ", device->sim->sim_name,
3653		       device->sim->unit_number,
3654		       device->sim->bus_id,
3655		       device->target->target_id,
3656		       (uintmax_t)device->lun_id);
3657	}
3658}
3659
3660void
3661xpt_print(struct cam_path *path, const char *fmt, ...)
3662{
3663	va_list ap;
3664	xpt_print_path(path);
3665	va_start(ap, fmt);
3666	vprintf(fmt, ap);
3667	va_end(ap);
3668}
3669
3670int
3671xpt_path_string(struct cam_path *path, char *str, size_t str_len)
3672{
3673	struct sbuf sb;
3674
3675	sbuf_new(&sb, str, str_len, 0);
3676
3677	if (path == NULL)
3678		sbuf_printf(&sb, "(nopath): ");
3679	else {
3680		if (path->periph != NULL)
3681			sbuf_printf(&sb, "(%s%d:", path->periph->periph_name,
3682				    path->periph->unit_number);
3683		else
3684			sbuf_printf(&sb, "(noperiph:");
3685
3686		if (path->bus != NULL)
3687			sbuf_printf(&sb, "%s%d:%d:", path->bus->sim->sim_name,
3688				    path->bus->sim->unit_number,
3689				    path->bus->sim->bus_id);
3690		else
3691			sbuf_printf(&sb, "nobus:");
3692
3693		if (path->target != NULL)
3694			sbuf_printf(&sb, "%d:", path->target->target_id);
3695		else
3696			sbuf_printf(&sb, "X:");
3697
3698		if (path->device != NULL)
3699			sbuf_printf(&sb, "%jx): ",
3700			    (uintmax_t)path->device->lun_id);
3701		else
3702			sbuf_printf(&sb, "X): ");
3703	}
3704	sbuf_finish(&sb);
3705
3706	return(sbuf_len(&sb));
3707}
3708
3709path_id_t
3710xpt_path_path_id(struct cam_path *path)
3711{
3712	return(path->bus->path_id);
3713}
3714
3715target_id_t
3716xpt_path_target_id(struct cam_path *path)
3717{
3718	if (path->target != NULL)
3719		return (path->target->target_id);
3720	else
3721		return (CAM_TARGET_WILDCARD);
3722}
3723
3724lun_id_t
3725xpt_path_lun_id(struct cam_path *path)
3726{
3727	if (path->device != NULL)
3728		return (path->device->lun_id);
3729	else
3730		return (CAM_LUN_WILDCARD);
3731}
3732
3733struct cam_sim *
3734xpt_path_sim(struct cam_path *path)
3735{
3736
3737	return (path->bus->sim);
3738}
3739
3740struct cam_periph*
3741xpt_path_periph(struct cam_path *path)
3742{
3743
3744	return (path->periph);
3745}
3746
3747int
3748xpt_path_legacy_ata_id(struct cam_path *path)
3749{
3750	struct cam_eb *bus;
3751	int bus_id;
3752
3753	if ((strcmp(path->bus->sim->sim_name, "ata") != 0) &&
3754	    strcmp(path->bus->sim->sim_name, "ahcich") != 0 &&
3755	    strcmp(path->bus->sim->sim_name, "mvsch") != 0 &&
3756	    strcmp(path->bus->sim->sim_name, "siisch") != 0)
3757		return (-1);
3758
3759	if (strcmp(path->bus->sim->sim_name, "ata") == 0 &&
3760	    path->bus->sim->unit_number < 2) {
3761		bus_id = path->bus->sim->unit_number;
3762	} else {
3763		bus_id = 2;
3764		xpt_lock_buses();
3765		TAILQ_FOREACH(bus, &xsoftc.xpt_busses, links) {
3766			if (bus == path->bus)
3767				break;
3768			if ((strcmp(bus->sim->sim_name, "ata") == 0 &&
3769			     bus->sim->unit_number >= 2) ||
3770			    strcmp(bus->sim->sim_name, "ahcich") == 0 ||
3771			    strcmp(bus->sim->sim_name, "mvsch") == 0 ||
3772			    strcmp(bus->sim->sim_name, "siisch") == 0)
3773				bus_id++;
3774		}
3775		xpt_unlock_buses();
3776	}
3777	if (path->target != NULL) {
3778		if (path->target->target_id < 2)
3779			return (bus_id * 2 + path->target->target_id);
3780		else
3781			return (-1);
3782	} else
3783		return (bus_id * 2);
3784}
3785
3786/*
3787 * Release a CAM control block for the caller.  Remit the cost of the structure
3788 * to the device referenced by the path.  If the this device had no 'credits'
3789 * and peripheral drivers have registered async callbacks for this notification
3790 * call them now.
3791 */
3792void
3793xpt_release_ccb(union ccb *free_ccb)
3794{
3795	struct	 cam_ed *device;
3796	struct	 cam_periph *periph;
3797
3798	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
3799	xpt_path_assert(free_ccb->ccb_h.path, MA_OWNED);
3800	device = free_ccb->ccb_h.path->device;
3801	periph = free_ccb->ccb_h.path->periph;
3802
3803	xpt_free_ccb(free_ccb);
3804	periph->periph_allocated--;
3805	cam_ccbq_release_opening(&device->ccbq);
3806	xpt_run_allocq(periph, 0);
3807}
3808
3809/* Functions accessed by SIM drivers */
3810
3811static struct xpt_xport xport_default = {
3812	.alloc_device = xpt_alloc_device_default,
3813	.action = xpt_action_default,
3814	.async = xpt_dev_async_default,
3815};
3816
3817/*
3818 * A sim structure, listing the SIM entry points and instance
3819 * identification info is passed to xpt_bus_register to hook the SIM
3820 * into the CAM framework.  xpt_bus_register creates a cam_eb entry
3821 * for this new bus and places it in the array of busses and assigns
3822 * it a path_id.  The path_id may be influenced by "hard wiring"
3823 * information specified by the user.  Once interrupt services are
3824 * available, the bus will be probed.
3825 */
3826int32_t
3827xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
3828{
3829	struct cam_eb *new_bus;
3830	struct cam_eb *old_bus;
3831	struct ccb_pathinq cpi;
3832	struct cam_path *path;
3833	cam_status status;
3834
3835	mtx_assert(sim->mtx, MA_OWNED);
3836
3837	sim->bus_id = bus;
3838	new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
3839					  M_CAMXPT, M_NOWAIT|M_ZERO);
3840	if (new_bus == NULL) {
3841		/* Couldn't satisfy request */
3842		return (CAM_RESRC_UNAVAIL);
3843	}
3844
3845	mtx_init(&new_bus->eb_mtx, "CAM bus lock", NULL, MTX_DEF);
3846	TAILQ_INIT(&new_bus->et_entries);
3847	cam_sim_hold(sim);
3848	new_bus->sim = sim;
3849	timevalclear(&new_bus->last_reset);
3850	new_bus->flags = 0;
3851	new_bus->refcount = 1;	/* Held until a bus_deregister event */
3852	new_bus->generation = 0;
3853
3854	xpt_lock_buses();
3855	sim->path_id = new_bus->path_id =
3856	    xptpathid(sim->sim_name, sim->unit_number, sim->bus_id);
3857	old_bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3858	while (old_bus != NULL
3859	    && old_bus->path_id < new_bus->path_id)
3860		old_bus = TAILQ_NEXT(old_bus, links);
3861	if (old_bus != NULL)
3862		TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
3863	else
3864		TAILQ_INSERT_TAIL(&xsoftc.xpt_busses, new_bus, links);
3865	xsoftc.bus_generation++;
3866	xpt_unlock_buses();
3867
3868	/*
3869	 * Set a default transport so that a PATH_INQ can be issued to
3870	 * the SIM.  This will then allow for probing and attaching of
3871	 * a more appropriate transport.
3872	 */
3873	new_bus->xport = &xport_default;
3874
3875	status = xpt_create_path(&path, /*periph*/NULL, sim->path_id,
3876				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3877	if (status != CAM_REQ_CMP) {
3878		xpt_release_bus(new_bus);
3879		free(path, M_CAMXPT);
3880		return (CAM_RESRC_UNAVAIL);
3881	}
3882
3883	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
3884	cpi.ccb_h.func_code = XPT_PATH_INQ;
3885	xpt_action((union ccb *)&cpi);
3886
3887	if (cpi.ccb_h.status == CAM_REQ_CMP) {
3888		switch (cpi.transport) {
3889		case XPORT_SPI:
3890		case XPORT_SAS:
3891		case XPORT_FC:
3892		case XPORT_USB:
3893		case XPORT_ISCSI:
3894		case XPORT_SRP:
3895		case XPORT_PPB:
3896			new_bus->xport = scsi_get_xport();
3897			break;
3898		case XPORT_ATA:
3899		case XPORT_SATA:
3900			new_bus->xport = ata_get_xport();
3901			break;
3902		default:
3903			new_bus->xport = &xport_default;
3904			break;
3905		}
3906	}
3907
3908	/* Notify interested parties */
3909	if (sim->path_id != CAM_XPT_PATH_ID) {
3910
3911		xpt_async(AC_PATH_REGISTERED, path, &cpi);
3912		if ((cpi.hba_misc & PIM_NOSCAN) == 0) {
3913			union	ccb *scan_ccb;
3914
3915			/* Initiate bus rescan. */
3916			scan_ccb = xpt_alloc_ccb_nowait();
3917			if (scan_ccb != NULL) {
3918				scan_ccb->ccb_h.path = path;
3919				scan_ccb->ccb_h.func_code = XPT_SCAN_BUS;
3920				scan_ccb->crcn.flags = 0;
3921				xpt_rescan(scan_ccb);
3922			} else
3923				xpt_print(path,
3924					  "Can't allocate CCB to scan bus\n");
3925		} else
3926			xpt_free_path(path);
3927	} else
3928		xpt_free_path(path);
3929	return (CAM_SUCCESS);
3930}
3931
3932int32_t
3933xpt_bus_deregister(path_id_t pathid)
3934{
3935	struct cam_path bus_path;
3936	cam_status status;
3937
3938	status = xpt_compile_path(&bus_path, NULL, pathid,
3939				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3940	if (status != CAM_REQ_CMP)
3941		return (status);
3942
3943	xpt_async(AC_LOST_DEVICE, &bus_path, NULL);
3944	xpt_async(AC_PATH_DEREGISTERED, &bus_path, NULL);
3945
3946	/* Release the reference count held while registered. */
3947	xpt_release_bus(bus_path.bus);
3948	xpt_release_path(&bus_path);
3949
3950	return (CAM_REQ_CMP);
3951}
3952
3953static path_id_t
3954xptnextfreepathid(void)
3955{
3956	struct cam_eb *bus;
3957	path_id_t pathid;
3958	const char *strval;
3959
3960	mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
3961	pathid = 0;
3962	bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3963retry:
3964	/* Find an unoccupied pathid */
3965	while (bus != NULL && bus->path_id <= pathid) {
3966		if (bus->path_id == pathid)
3967			pathid++;
3968		bus = TAILQ_NEXT(bus, links);
3969	}
3970
3971	/*
3972	 * Ensure that this pathid is not reserved for
3973	 * a bus that may be registered in the future.
3974	 */
3975	if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
3976		++pathid;
3977		/* Start the search over */
3978		goto retry;
3979	}
3980	return (pathid);
3981}
3982
3983static path_id_t
3984xptpathid(const char *sim_name, int sim_unit, int sim_bus)
3985{
3986	path_id_t pathid;
3987	int i, dunit, val;
3988	char buf[32];
3989	const char *dname;
3990
3991	pathid = CAM_XPT_PATH_ID;
3992	snprintf(buf, sizeof(buf), "%s%d", sim_name, sim_unit);
3993	if (strcmp(buf, "xpt0") == 0 && sim_bus == 0)
3994		return (pathid);
3995	i = 0;
3996	while ((resource_find_match(&i, &dname, &dunit, "at", buf)) == 0) {
3997		if (strcmp(dname, "scbus")) {
3998			/* Avoid a bit of foot shooting. */
3999			continue;
4000		}
4001		if (dunit < 0)		/* unwired?! */
4002			continue;
4003		if (resource_int_value("scbus", dunit, "bus", &val) == 0) {
4004			if (sim_bus == val) {
4005				pathid = dunit;
4006				break;
4007			}
4008		} else if (sim_bus == 0) {
4009			/* Unspecified matches bus 0 */
4010			pathid = dunit;
4011			break;
4012		} else {
4013			printf("Ambiguous scbus configuration for %s%d "
4014			       "bus %d, cannot wire down.  The kernel "
4015			       "config entry for scbus%d should "
4016			       "specify a controller bus.\n"
4017			       "Scbus will be assigned dynamically.\n",
4018			       sim_name, sim_unit, sim_bus, dunit);
4019			break;
4020		}
4021	}
4022
4023	if (pathid == CAM_XPT_PATH_ID)
4024		pathid = xptnextfreepathid();
4025	return (pathid);
4026}
4027
4028static const char *
4029xpt_async_string(u_int32_t async_code)
4030{
4031
4032	switch (async_code) {
4033	case AC_BUS_RESET: return ("AC_BUS_RESET");
4034	case AC_UNSOL_RESEL: return ("AC_UNSOL_RESEL");
4035	case AC_SCSI_AEN: return ("AC_SCSI_AEN");
4036	case AC_SENT_BDR: return ("AC_SENT_BDR");
4037	case AC_PATH_REGISTERED: return ("AC_PATH_REGISTERED");
4038	case AC_PATH_DEREGISTERED: return ("AC_PATH_DEREGISTERED");
4039	case AC_FOUND_DEVICE: return ("AC_FOUND_DEVICE");
4040	case AC_LOST_DEVICE: return ("AC_LOST_DEVICE");
4041	case AC_TRANSFER_NEG: return ("AC_TRANSFER_NEG");
4042	case AC_INQ_CHANGED: return ("AC_INQ_CHANGED");
4043	case AC_GETDEV_CHANGED: return ("AC_GETDEV_CHANGED");
4044	case AC_CONTRACT: return ("AC_CONTRACT");
4045	case AC_ADVINFO_CHANGED: return ("AC_ADVINFO_CHANGED");
4046	case AC_UNIT_ATTENTION: return ("AC_UNIT_ATTENTION");
4047	}
4048	return ("AC_UNKNOWN");
4049}
4050
4051static int
4052xpt_async_size(u_int32_t async_code)
4053{
4054
4055	switch (async_code) {
4056	case AC_BUS_RESET: return (0);
4057	case AC_UNSOL_RESEL: return (0);
4058	case AC_SCSI_AEN: return (0);
4059	case AC_SENT_BDR: return (0);
4060	case AC_PATH_REGISTERED: return (sizeof(struct ccb_pathinq));
4061	case AC_PATH_DEREGISTERED: return (0);
4062	case AC_FOUND_DEVICE: return (sizeof(struct ccb_getdev));
4063	case AC_LOST_DEVICE: return (0);
4064	case AC_TRANSFER_NEG: return (sizeof(struct ccb_trans_settings));
4065	case AC_INQ_CHANGED: return (0);
4066	case AC_GETDEV_CHANGED: return (0);
4067	case AC_CONTRACT: return (sizeof(struct ac_contract));
4068	case AC_ADVINFO_CHANGED: return (-1);
4069	case AC_UNIT_ATTENTION: return (sizeof(struct ccb_scsiio));
4070	}
4071	return (0);
4072}
4073
4074static int
4075xpt_async_process_dev(struct cam_ed *device, void *arg)
4076{
4077	union ccb *ccb = arg;
4078	struct cam_path *path = ccb->ccb_h.path;
4079	void *async_arg = ccb->casync.async_arg_ptr;
4080	u_int32_t async_code = ccb->casync.async_code;
4081	int relock;
4082
4083	if (path->device != device
4084	 && path->device->lun_id != CAM_LUN_WILDCARD
4085	 && device->lun_id != CAM_LUN_WILDCARD)
4086		return (1);
4087
4088	/*
4089	 * The async callback could free the device.
4090	 * If it is a broadcast async, it doesn't hold
4091	 * device reference, so take our own reference.
4092	 */
4093	xpt_acquire_device(device);
4094
4095	/*
4096	 * If async for specific device is to be delivered to
4097	 * the wildcard client, take the specific device lock.
4098	 * XXX: We may need a way for client to specify it.
4099	 */
4100	if ((device->lun_id == CAM_LUN_WILDCARD &&
4101	     path->device->lun_id != CAM_LUN_WILDCARD) ||
4102	    (device->target->target_id == CAM_TARGET_WILDCARD &&
4103	     path->target->target_id != CAM_TARGET_WILDCARD) ||
4104	    (device->target->bus->path_id == CAM_BUS_WILDCARD &&
4105	     path->target->bus->path_id != CAM_BUS_WILDCARD)) {
4106		mtx_unlock(&device->device_mtx);
4107		xpt_path_lock(path);
4108		relock = 1;
4109	} else
4110		relock = 0;
4111
4112	(*(device->target->bus->xport->async))(async_code,
4113	    device->target->bus, device->target, device, async_arg);
4114	xpt_async_bcast(&device->asyncs, async_code, path, async_arg);
4115
4116	if (relock) {
4117		xpt_path_unlock(path);
4118		mtx_lock(&device->device_mtx);
4119	}
4120	xpt_release_device(device);
4121	return (1);
4122}
4123
4124static int
4125xpt_async_process_tgt(struct cam_et *target, void *arg)
4126{
4127	union ccb *ccb = arg;
4128	struct cam_path *path = ccb->ccb_h.path;
4129
4130	if (path->target != target
4131	 && path->target->target_id != CAM_TARGET_WILDCARD
4132	 && target->target_id != CAM_TARGET_WILDCARD)
4133		return (1);
4134
4135	if (ccb->casync.async_code == AC_SENT_BDR) {
4136		/* Update our notion of when the last reset occurred */
4137		microtime(&target->last_reset);
4138	}
4139
4140	return (xptdevicetraverse(target, NULL, xpt_async_process_dev, ccb));
4141}
4142
4143static void
4144xpt_async_process(struct cam_periph *periph, union ccb *ccb)
4145{
4146	struct cam_eb *bus;
4147	struct cam_path *path;
4148	void *async_arg;
4149	u_int32_t async_code;
4150
4151	path = ccb->ccb_h.path;
4152	async_code = ccb->casync.async_code;
4153	async_arg = ccb->casync.async_arg_ptr;
4154	CAM_DEBUG(path, CAM_DEBUG_TRACE | CAM_DEBUG_INFO,
4155	    ("xpt_async(%s)\n", xpt_async_string(async_code)));
4156	bus = path->bus;
4157
4158	if (async_code == AC_BUS_RESET) {
4159		/* Update our notion of when the last reset occurred */
4160		microtime(&bus->last_reset);
4161	}
4162
4163	xpttargettraverse(bus, NULL, xpt_async_process_tgt, ccb);
4164
4165	/*
4166	 * If this wasn't a fully wildcarded async, tell all
4167	 * clients that want all async events.
4168	 */
4169	if (bus != xpt_periph->path->bus) {
4170		xpt_path_lock(xpt_periph->path);
4171		xpt_async_process_dev(xpt_periph->path->device, ccb);
4172		xpt_path_unlock(xpt_periph->path);
4173	}
4174
4175	if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4176		xpt_release_devq(path, 1, TRUE);
4177	else
4178		xpt_release_simq(path->bus->sim, TRUE);
4179	if (ccb->casync.async_arg_size > 0)
4180		free(async_arg, M_CAMXPT);
4181	xpt_free_path(path);
4182	xpt_free_ccb(ccb);
4183}
4184
4185static void
4186xpt_async_bcast(struct async_list *async_head,
4187		u_int32_t async_code,
4188		struct cam_path *path, void *async_arg)
4189{
4190	struct async_node *cur_entry;
4191	int lock;
4192
4193	cur_entry = SLIST_FIRST(async_head);
4194	while (cur_entry != NULL) {
4195		struct async_node *next_entry;
4196		/*
4197		 * Grab the next list entry before we call the current
4198		 * entry's callback.  This is because the callback function
4199		 * can delete its async callback entry.
4200		 */
4201		next_entry = SLIST_NEXT(cur_entry, links);
4202		if ((cur_entry->event_enable & async_code) != 0) {
4203			lock = cur_entry->event_lock;
4204			if (lock)
4205				CAM_SIM_LOCK(path->device->sim);
4206			cur_entry->callback(cur_entry->callback_arg,
4207					    async_code, path,
4208					    async_arg);
4209			if (lock)
4210				CAM_SIM_UNLOCK(path->device->sim);
4211		}
4212		cur_entry = next_entry;
4213	}
4214}
4215
4216void
4217xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
4218{
4219	union ccb *ccb;
4220	int size;
4221
4222	ccb = xpt_alloc_ccb_nowait();
4223	if (ccb == NULL) {
4224		xpt_print(path, "Can't allocate CCB to send %s\n",
4225		    xpt_async_string(async_code));
4226		return;
4227	}
4228
4229	if (xpt_clone_path(&ccb->ccb_h.path, path) != CAM_REQ_CMP) {
4230		xpt_print(path, "Can't allocate path to send %s\n",
4231		    xpt_async_string(async_code));
4232		xpt_free_ccb(ccb);
4233		return;
4234	}
4235	ccb->ccb_h.path->periph = NULL;
4236	ccb->ccb_h.func_code = XPT_ASYNC;
4237	ccb->ccb_h.cbfcnp = xpt_async_process;
4238	ccb->ccb_h.flags |= CAM_UNLOCKED;
4239	ccb->casync.async_code = async_code;
4240	ccb->casync.async_arg_size = 0;
4241	size = xpt_async_size(async_code);
4242	if (size > 0 && async_arg != NULL) {
4243		ccb->casync.async_arg_ptr = malloc(size, M_CAMXPT, M_NOWAIT);
4244		if (ccb->casync.async_arg_ptr == NULL) {
4245			xpt_print(path, "Can't allocate argument to send %s\n",
4246			    xpt_async_string(async_code));
4247			xpt_free_path(ccb->ccb_h.path);
4248			xpt_free_ccb(ccb);
4249			return;
4250		}
4251		memcpy(ccb->casync.async_arg_ptr, async_arg, size);
4252		ccb->casync.async_arg_size = size;
4253	} else if (size < 0)
4254		ccb->casync.async_arg_size = size;
4255	if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4256		xpt_freeze_devq(path, 1);
4257	else
4258		xpt_freeze_simq(path->bus->sim, 1);
4259	xpt_done(ccb);
4260}
4261
4262static void
4263xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus,
4264		      struct cam_et *target, struct cam_ed *device,
4265		      void *async_arg)
4266{
4267
4268	/*
4269	 * We only need to handle events for real devices.
4270	 */
4271	if (target->target_id == CAM_TARGET_WILDCARD
4272	 || device->lun_id == CAM_LUN_WILDCARD)
4273		return;
4274
4275	printf("%s called\n", __func__);
4276}
4277
4278static uint32_t
4279xpt_freeze_devq_device(struct cam_ed *dev, u_int count)
4280{
4281	struct cam_devq	*devq;
4282	uint32_t freeze;
4283
4284	devq = dev->sim->devq;
4285	mtx_assert(&devq->send_mtx, MA_OWNED);
4286	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4287	    ("xpt_freeze_devq_device(%d) %u->%u\n", count,
4288	    dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt + count));
4289	freeze = (dev->ccbq.queue.qfrozen_cnt += count);
4290	/* Remove frozen device from sendq. */
4291	if (device_is_queued(dev))
4292		camq_remove(&devq->send_queue, dev->devq_entry.index);
4293	return (freeze);
4294}
4295
4296u_int32_t
4297xpt_freeze_devq(struct cam_path *path, u_int count)
4298{
4299	struct cam_ed	*dev = path->device;
4300	struct cam_devq	*devq;
4301	uint32_t	 freeze;
4302
4303	devq = dev->sim->devq;
4304	mtx_lock(&devq->send_mtx);
4305	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_freeze_devq(%d)\n", count));
4306	freeze = xpt_freeze_devq_device(dev, count);
4307	mtx_unlock(&devq->send_mtx);
4308	return (freeze);
4309}
4310
4311u_int32_t
4312xpt_freeze_simq(struct cam_sim *sim, u_int count)
4313{
4314	struct cam_devq	*devq;
4315	uint32_t	 freeze;
4316
4317	devq = sim->devq;
4318	mtx_lock(&devq->send_mtx);
4319	freeze = (devq->send_queue.qfrozen_cnt += count);
4320	mtx_unlock(&devq->send_mtx);
4321	return (freeze);
4322}
4323
4324static void
4325xpt_release_devq_timeout(void *arg)
4326{
4327	struct cam_ed *dev;
4328	struct cam_devq *devq;
4329
4330	dev = (struct cam_ed *)arg;
4331	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, ("xpt_release_devq_timeout\n"));
4332	devq = dev->sim->devq;
4333	mtx_assert(&devq->send_mtx, MA_OWNED);
4334	if (xpt_release_devq_device(dev, /*count*/1, /*run_queue*/TRUE))
4335		xpt_run_devq(devq);
4336}
4337
4338void
4339xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
4340{
4341	struct cam_ed *dev;
4342	struct cam_devq *devq;
4343
4344	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_devq(%d, %d)\n",
4345	    count, run_queue));
4346	dev = path->device;
4347	devq = dev->sim->devq;
4348	mtx_lock(&devq->send_mtx);
4349	if (xpt_release_devq_device(dev, count, run_queue))
4350		xpt_run_devq(dev->sim->devq);
4351	mtx_unlock(&devq->send_mtx);
4352}
4353
4354static int
4355xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
4356{
4357
4358	mtx_assert(&dev->sim->devq->send_mtx, MA_OWNED);
4359	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4360	    ("xpt_release_devq_device(%d, %d) %u->%u\n", count, run_queue,
4361	    dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt - count));
4362	if (count > dev->ccbq.queue.qfrozen_cnt) {
4363#ifdef INVARIANTS
4364		printf("xpt_release_devq(): requested %u > present %u\n",
4365		    count, dev->ccbq.queue.qfrozen_cnt);
4366#endif
4367		count = dev->ccbq.queue.qfrozen_cnt;
4368	}
4369	dev->ccbq.queue.qfrozen_cnt -= count;
4370	if (dev->ccbq.queue.qfrozen_cnt == 0) {
4371		/*
4372		 * No longer need to wait for a successful
4373		 * command completion.
4374		 */
4375		dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
4376		/*
4377		 * Remove any timeouts that might be scheduled
4378		 * to release this queue.
4379		 */
4380		if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
4381			callout_stop(&dev->callout);
4382			dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
4383		}
4384		/*
4385		 * Now that we are unfrozen schedule the
4386		 * device so any pending transactions are
4387		 * run.
4388		 */
4389		xpt_schedule_devq(dev->sim->devq, dev);
4390	} else
4391		run_queue = 0;
4392	return (run_queue);
4393}
4394
4395void
4396xpt_release_simq(struct cam_sim *sim, int run_queue)
4397{
4398	struct cam_devq	*devq;
4399
4400	devq = sim->devq;
4401	mtx_lock(&devq->send_mtx);
4402	if (devq->send_queue.qfrozen_cnt <= 0) {
4403#ifdef INVARIANTS
4404		printf("xpt_release_simq: requested 1 > present %u\n",
4405		    devq->send_queue.qfrozen_cnt);
4406#endif
4407	} else
4408		devq->send_queue.qfrozen_cnt--;
4409	if (devq->send_queue.qfrozen_cnt == 0) {
4410		/*
4411		 * If there is a timeout scheduled to release this
4412		 * sim queue, remove it.  The queue frozen count is
4413		 * already at 0.
4414		 */
4415		if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
4416			callout_stop(&sim->callout);
4417			sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
4418		}
4419		if (run_queue) {
4420			/*
4421			 * Now that we are unfrozen run the send queue.
4422			 */
4423			xpt_run_devq(sim->devq);
4424		}
4425	}
4426	mtx_unlock(&devq->send_mtx);
4427}
4428
4429/*
4430 * XXX Appears to be unused.
4431 */
4432static void
4433xpt_release_simq_timeout(void *arg)
4434{
4435	struct cam_sim *sim;
4436
4437	sim = (struct cam_sim *)arg;
4438	xpt_release_simq(sim, /* run_queue */ TRUE);
4439}
4440
4441void
4442xpt_done(union ccb *done_ccb)
4443{
4444	struct cam_doneq *queue;
4445	int	run, hash;
4446
4447	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done\n"));
4448	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4449		return;
4450
4451	hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id +
4452	    done_ccb->ccb_h.target_lun) % cam_num_doneqs;
4453	queue = &cam_doneqs[hash];
4454	mtx_lock(&queue->cam_doneq_mtx);
4455	run = (queue->cam_doneq_sleep && STAILQ_EMPTY(&queue->cam_doneq));
4456	STAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.stqe);
4457	done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4458	mtx_unlock(&queue->cam_doneq_mtx);
4459	if (run)
4460		wakeup(&queue->cam_doneq);
4461}
4462
4463void
4464xpt_done_direct(union ccb *done_ccb)
4465{
4466
4467	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done_direct\n"));
4468	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4469		return;
4470
4471	xpt_done_process(&done_ccb->ccb_h);
4472}
4473
4474union ccb *
4475xpt_alloc_ccb()
4476{
4477	union ccb *new_ccb;
4478
4479	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4480	return (new_ccb);
4481}
4482
4483union ccb *
4484xpt_alloc_ccb_nowait()
4485{
4486	union ccb *new_ccb;
4487
4488	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4489	return (new_ccb);
4490}
4491
4492void
4493xpt_free_ccb(union ccb *free_ccb)
4494{
4495	free(free_ccb, M_CAMCCB);
4496}
4497
4498
4499
4500/* Private XPT functions */
4501
4502/*
4503 * Get a CAM control block for the caller. Charge the structure to the device
4504 * referenced by the path.  If we don't have sufficient resources to allocate
4505 * more ccbs, we return NULL.
4506 */
4507static union ccb *
4508xpt_get_ccb_nowait(struct cam_periph *periph)
4509{
4510	union ccb *new_ccb;
4511
4512	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_NOWAIT);
4513	if (new_ccb == NULL)
4514		return (NULL);
4515	periph->periph_allocated++;
4516	cam_ccbq_take_opening(&periph->path->device->ccbq);
4517	return (new_ccb);
4518}
4519
4520static union ccb *
4521xpt_get_ccb(struct cam_periph *periph)
4522{
4523	union ccb *new_ccb;
4524
4525	cam_periph_unlock(periph);
4526	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_WAITOK);
4527	cam_periph_lock(periph);
4528	periph->periph_allocated++;
4529	cam_ccbq_take_opening(&periph->path->device->ccbq);
4530	return (new_ccb);
4531}
4532
4533union ccb *
4534cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
4535{
4536	struct ccb_hdr *ccb_h;
4537
4538	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("cam_periph_getccb\n"));
4539	cam_periph_assert(periph, MA_OWNED);
4540	while ((ccb_h = SLIST_FIRST(&periph->ccb_list)) == NULL ||
4541	    ccb_h->pinfo.priority != priority) {
4542		if (priority < periph->immediate_priority) {
4543			periph->immediate_priority = priority;
4544			xpt_run_allocq(periph, 0);
4545		} else
4546			cam_periph_sleep(periph, &periph->ccb_list, PRIBIO,
4547			    "cgticb", 0);
4548	}
4549	SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle);
4550	return ((union ccb *)ccb_h);
4551}
4552
4553static void
4554xpt_acquire_bus(struct cam_eb *bus)
4555{
4556
4557	xpt_lock_buses();
4558	bus->refcount++;
4559	xpt_unlock_buses();
4560}
4561
4562static void
4563xpt_release_bus(struct cam_eb *bus)
4564{
4565
4566	xpt_lock_buses();
4567	KASSERT(bus->refcount >= 1, ("bus->refcount >= 1"));
4568	if (--bus->refcount > 0) {
4569		xpt_unlock_buses();
4570		return;
4571	}
4572	TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
4573	xsoftc.bus_generation++;
4574	xpt_unlock_buses();
4575	KASSERT(TAILQ_EMPTY(&bus->et_entries),
4576	    ("destroying bus, but target list is not empty"));
4577	cam_sim_release(bus->sim);
4578	mtx_destroy(&bus->eb_mtx);
4579	free(bus, M_CAMXPT);
4580}
4581
4582static struct cam_et *
4583xpt_alloc_target(struct cam_eb *bus, target_id_t target_id)
4584{
4585	struct cam_et *cur_target, *target;
4586
4587	mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4588	mtx_assert(&bus->eb_mtx, MA_OWNED);
4589	target = (struct cam_et *)malloc(sizeof(*target), M_CAMXPT,
4590					 M_NOWAIT|M_ZERO);
4591	if (target == NULL)
4592		return (NULL);
4593
4594	TAILQ_INIT(&target->ed_entries);
4595	target->bus = bus;
4596	target->target_id = target_id;
4597	target->refcount = 1;
4598	target->generation = 0;
4599	target->luns = NULL;
4600	mtx_init(&target->luns_mtx, "CAM LUNs lock", NULL, MTX_DEF);
4601	timevalclear(&target->last_reset);
4602	/*
4603	 * Hold a reference to our parent bus so it
4604	 * will not go away before we do.
4605	 */
4606	bus->refcount++;
4607
4608	/* Insertion sort into our bus's target list */
4609	cur_target = TAILQ_FIRST(&bus->et_entries);
4610	while (cur_target != NULL && cur_target->target_id < target_id)
4611		cur_target = TAILQ_NEXT(cur_target, links);
4612	if (cur_target != NULL) {
4613		TAILQ_INSERT_BEFORE(cur_target, target, links);
4614	} else {
4615		TAILQ_INSERT_TAIL(&bus->et_entries, target, links);
4616	}
4617	bus->generation++;
4618	return (target);
4619}
4620
4621static void
4622xpt_acquire_target(struct cam_et *target)
4623{
4624	struct cam_eb *bus = target->bus;
4625
4626	mtx_lock(&bus->eb_mtx);
4627	target->refcount++;
4628	mtx_unlock(&bus->eb_mtx);
4629}
4630
4631static void
4632xpt_release_target(struct cam_et *target)
4633{
4634	struct cam_eb *bus = target->bus;
4635
4636	mtx_lock(&bus->eb_mtx);
4637	if (--target->refcount > 0) {
4638		mtx_unlock(&bus->eb_mtx);
4639		return;
4640	}
4641	TAILQ_REMOVE(&bus->et_entries, target, links);
4642	bus->generation++;
4643	mtx_unlock(&bus->eb_mtx);
4644	KASSERT(TAILQ_EMPTY(&target->ed_entries),
4645	    ("destroying target, but device list is not empty"));
4646	xpt_release_bus(bus);
4647	mtx_destroy(&target->luns_mtx);
4648	if (target->luns)
4649		free(target->luns, M_CAMXPT);
4650	free(target, M_CAMXPT);
4651}
4652
4653static struct cam_ed *
4654xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target,
4655			 lun_id_t lun_id)
4656{
4657	struct cam_ed *device;
4658
4659	device = xpt_alloc_device(bus, target, lun_id);
4660	if (device == NULL)
4661		return (NULL);
4662
4663	device->mintags = 1;
4664	device->maxtags = 1;
4665	return (device);
4666}
4667
4668static void
4669xpt_destroy_device(void *context, int pending)
4670{
4671	struct cam_ed	*device = context;
4672
4673	mtx_lock(&device->device_mtx);
4674	mtx_destroy(&device->device_mtx);
4675	free(device, M_CAMDEV);
4676}
4677
4678struct cam_ed *
4679xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
4680{
4681	struct cam_ed	*cur_device, *device;
4682	struct cam_devq	*devq;
4683	cam_status status;
4684
4685	mtx_assert(&bus->eb_mtx, MA_OWNED);
4686	/* Make space for us in the device queue on our bus */
4687	devq = bus->sim->devq;
4688	mtx_lock(&devq->send_mtx);
4689	status = cam_devq_resize(devq, devq->send_queue.array_size + 1);
4690	mtx_unlock(&devq->send_mtx);
4691	if (status != CAM_REQ_CMP)
4692		return (NULL);
4693
4694	device = (struct cam_ed *)malloc(sizeof(*device),
4695					 M_CAMDEV, M_NOWAIT|M_ZERO);
4696	if (device == NULL)
4697		return (NULL);
4698
4699	cam_init_pinfo(&device->devq_entry);
4700	device->target = target;
4701	device->lun_id = lun_id;
4702	device->sim = bus->sim;
4703	if (cam_ccbq_init(&device->ccbq,
4704			  bus->sim->max_dev_openings) != 0) {
4705		free(device, M_CAMDEV);
4706		return (NULL);
4707	}
4708	SLIST_INIT(&device->asyncs);
4709	SLIST_INIT(&device->periphs);
4710	device->generation = 0;
4711	device->flags = CAM_DEV_UNCONFIGURED;
4712	device->tag_delay_count = 0;
4713	device->tag_saved_openings = 0;
4714	device->refcount = 1;
4715	mtx_init(&device->device_mtx, "CAM device lock", NULL, MTX_DEF);
4716	callout_init_mtx(&device->callout, &devq->send_mtx, 0);
4717	TASK_INIT(&device->device_destroy_task, 0, xpt_destroy_device, device);
4718	/*
4719	 * Hold a reference to our parent bus so it
4720	 * will not go away before we do.
4721	 */
4722	target->refcount++;
4723
4724	cur_device = TAILQ_FIRST(&target->ed_entries);
4725	while (cur_device != NULL && cur_device->lun_id < lun_id)
4726		cur_device = TAILQ_NEXT(cur_device, links);
4727	if (cur_device != NULL)
4728		TAILQ_INSERT_BEFORE(cur_device, device, links);
4729	else
4730		TAILQ_INSERT_TAIL(&target->ed_entries, device, links);
4731	target->generation++;
4732	return (device);
4733}
4734
4735void
4736xpt_acquire_device(struct cam_ed *device)
4737{
4738	struct cam_eb *bus = device->target->bus;
4739
4740	mtx_lock(&bus->eb_mtx);
4741	device->refcount++;
4742	mtx_unlock(&bus->eb_mtx);
4743}
4744
4745void
4746xpt_release_device(struct cam_ed *device)
4747{
4748	struct cam_eb *bus = device->target->bus;
4749	struct cam_devq *devq;
4750
4751	mtx_lock(&bus->eb_mtx);
4752	if (--device->refcount > 0) {
4753		mtx_unlock(&bus->eb_mtx);
4754		return;
4755	}
4756
4757	TAILQ_REMOVE(&device->target->ed_entries, device,links);
4758	device->target->generation++;
4759	mtx_unlock(&bus->eb_mtx);
4760
4761	/* Release our slot in the devq */
4762	devq = bus->sim->devq;
4763	mtx_lock(&devq->send_mtx);
4764	cam_devq_resize(devq, devq->send_queue.array_size - 1);
4765	mtx_unlock(&devq->send_mtx);
4766
4767	KASSERT(SLIST_EMPTY(&device->periphs),
4768	    ("destroying device, but periphs list is not empty"));
4769	KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX,
4770	    ("destroying device while still queued for ccbs"));
4771
4772	if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
4773		callout_stop(&device->callout);
4774
4775	xpt_release_target(device->target);
4776
4777	cam_ccbq_fini(&device->ccbq);
4778	/*
4779	 * Free allocated memory.  free(9) does nothing if the
4780	 * supplied pointer is NULL, so it is safe to call without
4781	 * checking.
4782	 */
4783	free(device->supported_vpds, M_CAMXPT);
4784	free(device->device_id, M_CAMXPT);
4785	free(device->physpath, M_CAMXPT);
4786	free(device->rcap_buf, M_CAMXPT);
4787	free(device->serial_num, M_CAMXPT);
4788	taskqueue_enqueue(xsoftc.xpt_taskq, &device->device_destroy_task);
4789}
4790
4791u_int32_t
4792xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
4793{
4794	int	result;
4795	struct	cam_ed *dev;
4796
4797	dev = path->device;
4798	mtx_lock(&dev->sim->devq->send_mtx);
4799	result = cam_ccbq_resize(&dev->ccbq, newopenings);
4800	mtx_unlock(&dev->sim->devq->send_mtx);
4801	if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
4802	 || (dev->inq_flags & SID_CmdQue) != 0)
4803		dev->tag_saved_openings = newopenings;
4804	return (result);
4805}
4806
4807static struct cam_eb *
4808xpt_find_bus(path_id_t path_id)
4809{
4810	struct cam_eb *bus;
4811
4812	xpt_lock_buses();
4813	for (bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4814	     bus != NULL;
4815	     bus = TAILQ_NEXT(bus, links)) {
4816		if (bus->path_id == path_id) {
4817			bus->refcount++;
4818			break;
4819		}
4820	}
4821	xpt_unlock_buses();
4822	return (bus);
4823}
4824
4825static struct cam_et *
4826xpt_find_target(struct cam_eb *bus, target_id_t	target_id)
4827{
4828	struct cam_et *target;
4829
4830	mtx_assert(&bus->eb_mtx, MA_OWNED);
4831	for (target = TAILQ_FIRST(&bus->et_entries);
4832	     target != NULL;
4833	     target = TAILQ_NEXT(target, links)) {
4834		if (target->target_id == target_id) {
4835			target->refcount++;
4836			break;
4837		}
4838	}
4839	return (target);
4840}
4841
4842static struct cam_ed *
4843xpt_find_device(struct cam_et *target, lun_id_t lun_id)
4844{
4845	struct cam_ed *device;
4846
4847	mtx_assert(&target->bus->eb_mtx, MA_OWNED);
4848	for (device = TAILQ_FIRST(&target->ed_entries);
4849	     device != NULL;
4850	     device = TAILQ_NEXT(device, links)) {
4851		if (device->lun_id == lun_id) {
4852			device->refcount++;
4853			break;
4854		}
4855	}
4856	return (device);
4857}
4858
4859void
4860xpt_start_tags(struct cam_path *path)
4861{
4862	struct ccb_relsim crs;
4863	struct cam_ed *device;
4864	struct cam_sim *sim;
4865	int    newopenings;
4866
4867	device = path->device;
4868	sim = path->bus->sim;
4869	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4870	xpt_freeze_devq(path, /*count*/1);
4871	device->inq_flags |= SID_CmdQue;
4872	if (device->tag_saved_openings != 0)
4873		newopenings = device->tag_saved_openings;
4874	else
4875		newopenings = min(device->maxtags,
4876				  sim->max_tagged_dev_openings);
4877	xpt_dev_ccbq_resize(path, newopenings);
4878	xpt_async(AC_GETDEV_CHANGED, path, NULL);
4879	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4880	crs.ccb_h.func_code = XPT_REL_SIMQ;
4881	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4882	crs.openings
4883	    = crs.release_timeout
4884	    = crs.qfrozen_cnt
4885	    = 0;
4886	xpt_action((union ccb *)&crs);
4887}
4888
4889void
4890xpt_stop_tags(struct cam_path *path)
4891{
4892	struct ccb_relsim crs;
4893	struct cam_ed *device;
4894	struct cam_sim *sim;
4895
4896	device = path->device;
4897	sim = path->bus->sim;
4898	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4899	device->tag_delay_count = 0;
4900	xpt_freeze_devq(path, /*count*/1);
4901	device->inq_flags &= ~SID_CmdQue;
4902	xpt_dev_ccbq_resize(path, sim->max_dev_openings);
4903	xpt_async(AC_GETDEV_CHANGED, path, NULL);
4904	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4905	crs.ccb_h.func_code = XPT_REL_SIMQ;
4906	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4907	crs.openings
4908	    = crs.release_timeout
4909	    = crs.qfrozen_cnt
4910	    = 0;
4911	xpt_action((union ccb *)&crs);
4912}
4913
4914static void
4915xpt_boot_delay(void *arg)
4916{
4917
4918	xpt_release_boot();
4919}
4920
4921static void
4922xpt_config(void *arg)
4923{
4924	/*
4925	 * Now that interrupts are enabled, go find our devices
4926	 */
4927	if (taskqueue_start_threads(&xsoftc.xpt_taskq, 1, PRIBIO, "CAM taskq"))
4928		printf("xpt_config: failed to create taskqueue thread.\n");
4929
4930	/* Setup debugging path */
4931	if (cam_dflags != CAM_DEBUG_NONE) {
4932		if (xpt_create_path(&cam_dpath, NULL,
4933				    CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
4934				    CAM_DEBUG_LUN) != CAM_REQ_CMP) {
4935			printf("xpt_config: xpt_create_path() failed for debug"
4936			       " target %d:%d:%d, debugging disabled\n",
4937			       CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN);
4938			cam_dflags = CAM_DEBUG_NONE;
4939		}
4940	} else
4941		cam_dpath = NULL;
4942
4943	periphdriver_init(1);
4944	xpt_hold_boot();
4945	callout_init(&xsoftc.boot_callout, 1);
4946	callout_reset(&xsoftc.boot_callout, hz * xsoftc.boot_delay / 1000,
4947	    xpt_boot_delay, NULL);
4948	/* Fire up rescan thread. */
4949	if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0,
4950	    "cam", "scanner")) {
4951		printf("xpt_config: failed to create rescan thread.\n");
4952	}
4953}
4954
4955void
4956xpt_hold_boot(void)
4957{
4958	xpt_lock_buses();
4959	xsoftc.buses_to_config++;
4960	xpt_unlock_buses();
4961}
4962
4963void
4964xpt_release_boot(void)
4965{
4966	xpt_lock_buses();
4967	xsoftc.buses_to_config--;
4968	if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
4969		struct	xpt_task *task;
4970
4971		xsoftc.buses_config_done = 1;
4972		xpt_unlock_buses();
4973		/* Call manually because we don't have any busses */
4974		task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
4975		if (task != NULL) {
4976			TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
4977			taskqueue_enqueue(taskqueue_thread, &task->task);
4978		}
4979	} else
4980		xpt_unlock_buses();
4981}
4982
4983/*
4984 * If the given device only has one peripheral attached to it, and if that
4985 * peripheral is the passthrough driver, announce it.  This insures that the
4986 * user sees some sort of announcement for every peripheral in their system.
4987 */
4988static int
4989xptpassannouncefunc(struct cam_ed *device, void *arg)
4990{
4991	struct cam_periph *periph;
4992	int i;
4993
4994	for (periph = SLIST_FIRST(&device->periphs), i = 0; periph != NULL;
4995	     periph = SLIST_NEXT(periph, periph_links), i++);
4996
4997	periph = SLIST_FIRST(&device->periphs);
4998	if ((i == 1)
4999	 && (strncmp(periph->periph_name, "pass", 4) == 0))
5000		xpt_announce_periph(periph, NULL);
5001
5002	return(1);
5003}
5004
5005static void
5006xpt_finishconfig_task(void *context, int pending)
5007{
5008
5009	periphdriver_init(2);
5010	/*
5011	 * Check for devices with no "standard" peripheral driver
5012	 * attached.  For any devices like that, announce the
5013	 * passthrough driver so the user will see something.
5014	 */
5015	if (!bootverbose)
5016		xpt_for_all_devices(xptpassannouncefunc, NULL);
5017
5018	/* Release our hook so that the boot can continue. */
5019	config_intrhook_disestablish(xsoftc.xpt_config_hook);
5020	free(xsoftc.xpt_config_hook, M_CAMXPT);
5021	xsoftc.xpt_config_hook = NULL;
5022
5023	free(context, M_CAMXPT);
5024}
5025
5026cam_status
5027xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
5028		   struct cam_path *path)
5029{
5030	struct ccb_setasync csa;
5031	cam_status status;
5032	int xptpath = 0;
5033
5034	if (path == NULL) {
5035		status = xpt_create_path(&path, /*periph*/NULL, CAM_XPT_PATH_ID,
5036					 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
5037		if (status != CAM_REQ_CMP)
5038			return (status);
5039		xpt_path_lock(path);
5040		xptpath = 1;
5041	}
5042
5043	xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL);
5044	csa.ccb_h.func_code = XPT_SASYNC_CB;
5045	csa.event_enable = event;
5046	csa.callback = cbfunc;
5047	csa.callback_arg = cbarg;
5048	xpt_action((union ccb *)&csa);
5049	status = csa.ccb_h.status;
5050
5051	if (xptpath) {
5052		xpt_path_unlock(path);
5053		xpt_free_path(path);
5054	}
5055
5056	if ((status == CAM_REQ_CMP) &&
5057	    (csa.event_enable & AC_FOUND_DEVICE)) {
5058		/*
5059		 * Get this peripheral up to date with all
5060		 * the currently existing devices.
5061		 */
5062		xpt_for_all_devices(xptsetasyncfunc, &csa);
5063	}
5064	if ((status == CAM_REQ_CMP) &&
5065	    (csa.event_enable & AC_PATH_REGISTERED)) {
5066		/*
5067		 * Get this peripheral up to date with all
5068		 * the currently existing busses.
5069		 */
5070		xpt_for_all_busses(xptsetasyncbusfunc, &csa);
5071	}
5072
5073	return (status);
5074}
5075
5076static void
5077xptaction(struct cam_sim *sim, union ccb *work_ccb)
5078{
5079	CAM_DEBUG(work_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xptaction\n"));
5080
5081	switch (work_ccb->ccb_h.func_code) {
5082	/* Common cases first */
5083	case XPT_PATH_INQ:		/* Path routing inquiry */
5084	{
5085		struct ccb_pathinq *cpi;
5086
5087		cpi = &work_ccb->cpi;
5088		cpi->version_num = 1; /* XXX??? */
5089		cpi->hba_inquiry = 0;
5090		cpi->target_sprt = 0;
5091		cpi->hba_misc = 0;
5092		cpi->hba_eng_cnt = 0;
5093		cpi->max_target = 0;
5094		cpi->max_lun = 0;
5095		cpi->initiator_id = 0;
5096		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
5097		strncpy(cpi->hba_vid, "", HBA_IDLEN);
5098		strncpy(cpi->dev_name, sim->sim_name, DEV_IDLEN);
5099		cpi->unit_number = sim->unit_number;
5100		cpi->bus_id = sim->bus_id;
5101		cpi->base_transfer_speed = 0;
5102		cpi->protocol = PROTO_UNSPECIFIED;
5103		cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
5104		cpi->transport = XPORT_UNSPECIFIED;
5105		cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
5106		cpi->ccb_h.status = CAM_REQ_CMP;
5107		xpt_done(work_ccb);
5108		break;
5109	}
5110	default:
5111		work_ccb->ccb_h.status = CAM_REQ_INVALID;
5112		xpt_done(work_ccb);
5113		break;
5114	}
5115}
5116
5117/*
5118 * The xpt as a "controller" has no interrupt sources, so polling
5119 * is a no-op.
5120 */
5121static void
5122xptpoll(struct cam_sim *sim)
5123{
5124}
5125
5126void
5127xpt_lock_buses(void)
5128{
5129	mtx_lock(&xsoftc.xpt_topo_lock);
5130}
5131
5132void
5133xpt_unlock_buses(void)
5134{
5135	mtx_unlock(&xsoftc.xpt_topo_lock);
5136}
5137
5138struct mtx *
5139xpt_path_mtx(struct cam_path *path)
5140{
5141
5142	return (&path->device->device_mtx);
5143}
5144
5145static void
5146xpt_done_process(struct ccb_hdr *ccb_h)
5147{
5148	struct cam_sim *sim;
5149	struct cam_devq *devq;
5150	struct mtx *mtx = NULL;
5151
5152	if (ccb_h->flags & CAM_HIGH_POWER) {
5153		struct highpowerlist	*hphead;
5154		struct cam_ed		*device;
5155
5156		mtx_lock(&xsoftc.xpt_highpower_lock);
5157		hphead = &xsoftc.highpowerq;
5158
5159		device = STAILQ_FIRST(hphead);
5160
5161		/*
5162		 * Increment the count since this command is done.
5163		 */
5164		xsoftc.num_highpower++;
5165
5166		/*
5167		 * Any high powered commands queued up?
5168		 */
5169		if (device != NULL) {
5170
5171			STAILQ_REMOVE_HEAD(hphead, highpowerq_entry);
5172			mtx_unlock(&xsoftc.xpt_highpower_lock);
5173
5174			mtx_lock(&device->sim->devq->send_mtx);
5175			xpt_release_devq_device(device,
5176					 /*count*/1, /*runqueue*/TRUE);
5177			mtx_unlock(&device->sim->devq->send_mtx);
5178		} else
5179			mtx_unlock(&xsoftc.xpt_highpower_lock);
5180	}
5181
5182	sim = ccb_h->path->bus->sim;
5183
5184	if (ccb_h->status & CAM_RELEASE_SIMQ) {
5185		xpt_release_simq(sim, /*run_queue*/FALSE);
5186		ccb_h->status &= ~CAM_RELEASE_SIMQ;
5187	}
5188
5189	if ((ccb_h->flags & CAM_DEV_QFRZDIS)
5190	 && (ccb_h->status & CAM_DEV_QFRZN)) {
5191		xpt_release_devq(ccb_h->path, /*count*/1,
5192				 /*run_queue*/FALSE);
5193		ccb_h->status &= ~CAM_DEV_QFRZN;
5194	}
5195
5196	devq = sim->devq;
5197	if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
5198		struct cam_ed *dev = ccb_h->path->device;
5199
5200		mtx_lock(&devq->send_mtx);
5201		devq->send_active--;
5202		devq->send_openings++;
5203		cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
5204
5205		if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
5206		  && (dev->ccbq.dev_active == 0))) {
5207			dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY;
5208			xpt_release_devq_device(dev, /*count*/1,
5209					 /*run_queue*/FALSE);
5210		}
5211
5212		if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
5213		  && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) {
5214			dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
5215			xpt_release_devq_device(dev, /*count*/1,
5216					 /*run_queue*/FALSE);
5217		}
5218
5219		if (!device_is_queued(dev))
5220			(void)xpt_schedule_devq(devq, dev);
5221		mtx_unlock(&devq->send_mtx);
5222
5223		if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0) {
5224			mtx = xpt_path_mtx(ccb_h->path);
5225			mtx_lock(mtx);
5226
5227			if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5228			 && (--dev->tag_delay_count == 0))
5229				xpt_start_tags(ccb_h->path);
5230		}
5231	}
5232
5233	if ((ccb_h->flags & CAM_UNLOCKED) == 0) {
5234		if (mtx == NULL) {
5235			mtx = xpt_path_mtx(ccb_h->path);
5236			mtx_lock(mtx);
5237		}
5238	} else {
5239		if (mtx != NULL) {
5240			mtx_unlock(mtx);
5241			mtx = NULL;
5242		}
5243	}
5244
5245	/* Call the peripheral driver's callback */
5246	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
5247	(*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
5248	if (mtx != NULL)
5249		mtx_unlock(mtx);
5250
5251	mtx_lock(&devq->send_mtx);
5252	xpt_run_devq(devq);
5253	mtx_unlock(&devq->send_mtx);
5254}
5255
5256void
5257xpt_done_td(void *arg)
5258{
5259	struct cam_doneq *queue = arg;
5260	struct ccb_hdr *ccb_h;
5261	STAILQ_HEAD(, ccb_hdr)	doneq;
5262
5263	STAILQ_INIT(&doneq);
5264	mtx_lock(&queue->cam_doneq_mtx);
5265	while (1) {
5266		while (STAILQ_EMPTY(&queue->cam_doneq)) {
5267			queue->cam_doneq_sleep = 1;
5268			msleep(&queue->cam_doneq, &queue->cam_doneq_mtx,
5269			    PRIBIO, "-", 0);
5270			queue->cam_doneq_sleep = 0;
5271		}
5272		STAILQ_CONCAT(&doneq, &queue->cam_doneq);
5273		mtx_unlock(&queue->cam_doneq_mtx);
5274
5275		THREAD_NO_SLEEPING();
5276		while ((ccb_h = STAILQ_FIRST(&doneq)) != NULL) {
5277			STAILQ_REMOVE_HEAD(&doneq, sim_links.stqe);
5278			xpt_done_process(ccb_h);
5279		}
5280		THREAD_SLEEPING_OK();
5281
5282		mtx_lock(&queue->cam_doneq_mtx);
5283	}
5284}
5285
5286static void
5287camisr_runqueue(void)
5288{
5289	struct	ccb_hdr *ccb_h;
5290	struct cam_doneq *queue;
5291	int i;
5292
5293	/* Process global queues. */
5294	for (i = 0; i < cam_num_doneqs; i++) {
5295		queue = &cam_doneqs[i];
5296		mtx_lock(&queue->cam_doneq_mtx);
5297		while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) {
5298			STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe);
5299			mtx_unlock(&queue->cam_doneq_mtx);
5300			xpt_done_process(ccb_h);
5301			mtx_lock(&queue->cam_doneq_mtx);
5302		}
5303		mtx_unlock(&queue->cam_doneq_mtx);
5304	}
5305}
5306