cam_xpt.c revision 292348 
1/*-
2 * Implementation of the Common Access Method Transport (XPT) layer.
3 *
4 * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs.
5 * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions, and the following disclaimer,
13 *    without modification, immediately at the beginning of the file.
14 * 2. The name of the author may not be used to endorse or promote products
15 *    derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
21 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: stable/10/sys/cam/cam_xpt.c 292348 2015-12-16 19:01:14Z ken $");
32
33#include <sys/param.h>
34#include <sys/bus.h>
35#include <sys/systm.h>
36#include <sys/types.h>
37#include <sys/malloc.h>
38#include <sys/kernel.h>
39#include <sys/time.h>
40#include <sys/conf.h>
41#include <sys/fcntl.h>
42#include <sys/interrupt.h>
43#include <sys/proc.h>
44#include <sys/sbuf.h>
45#include <sys/smp.h>
46#include <sys/taskqueue.h>
47
48#include <sys/lock.h>
49#include <sys/mutex.h>
50#include <sys/sysctl.h>
51#include <sys/kthread.h>
52
53#include <cam/cam.h>
54#include <cam/cam_ccb.h>
55#include <cam/cam_periph.h>
56#include <cam/cam_queue.h>
57#include <cam/cam_sim.h>
58#include <cam/cam_xpt.h>
59#include <cam/cam_xpt_sim.h>
60#include <cam/cam_xpt_periph.h>
61#include <cam/cam_xpt_internal.h>
62#include <cam/cam_debug.h>
63#include <cam/cam_compat.h>
64
65#include <cam/scsi/scsi_all.h>
66#include <cam/scsi/scsi_message.h>
67#include <cam/scsi/scsi_pass.h>
68
69#include <machine/md_var.h>	/* geometry translation */
70#include <machine/stdarg.h>	/* for xpt_print below */
71
72#include "opt_cam.h"
73
74/*
75 * This is the maximum number of high powered commands (e.g. start unit)
76 * that can be outstanding at a particular time.
77 */
78#ifndef CAM_MAX_HIGHPOWER
79#define CAM_MAX_HIGHPOWER  4
80#endif
81
82/* Datastructures internal to the xpt layer */
83MALLOC_DEFINE(M_CAMXPT, "CAM XPT", "CAM XPT buffers");
84MALLOC_DEFINE(M_CAMDEV, "CAM DEV", "CAM devices");
85MALLOC_DEFINE(M_CAMCCB, "CAM CCB", "CAM CCBs");
86MALLOC_DEFINE(M_CAMPATH, "CAM path", "CAM paths");
87
88/* Object for defering XPT actions to a taskqueue */
89struct xpt_task {
90	struct task	task;
91	void		*data1;
92	uintptr_t	data2;
93};
94
95struct xpt_softc {
96	uint32_t		xpt_generation;
97
98	/* number of high powered commands that can go through right now */
99	struct mtx		xpt_highpower_lock;
100	STAILQ_HEAD(highpowerlist, cam_ed)	highpowerq;
101	int			num_highpower;
102
103	/* queue for handling async rescan requests. */
104	TAILQ_HEAD(, ccb_hdr) ccb_scanq;
105	int buses_to_config;
106	int buses_config_done;
107
108	/* Registered busses */
109	TAILQ_HEAD(,cam_eb)	xpt_busses;
110	u_int			bus_generation;
111
112	struct intr_config_hook	*xpt_config_hook;
113
114	int			boot_delay;
115	struct callout 		boot_callout;
116
117	struct mtx		xpt_topo_lock;
118	struct mtx		xpt_lock;
119	struct taskqueue	*xpt_taskq;
120};
121
122typedef enum {
123	DM_RET_COPY		= 0x01,
124	DM_RET_FLAG_MASK	= 0x0f,
125	DM_RET_NONE		= 0x00,
126	DM_RET_STOP		= 0x10,
127	DM_RET_DESCEND		= 0x20,
128	DM_RET_ERROR		= 0x30,
129	DM_RET_ACTION_MASK	= 0xf0
130} dev_match_ret;
131
132typedef enum {
133	XPT_DEPTH_BUS,
134	XPT_DEPTH_TARGET,
135	XPT_DEPTH_DEVICE,
136	XPT_DEPTH_PERIPH
137} xpt_traverse_depth;
138
139struct xpt_traverse_config {
140	xpt_traverse_depth	depth;
141	void			*tr_func;
142	void			*tr_arg;
143};
144
145typedef	int	xpt_busfunc_t (struct cam_eb *bus, void *arg);
146typedef	int	xpt_targetfunc_t (struct cam_et *target, void *arg);
147typedef	int	xpt_devicefunc_t (struct cam_ed *device, void *arg);
148typedef	int	xpt_periphfunc_t (struct cam_periph *periph, void *arg);
149typedef int	xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
150
151/* Transport layer configuration information */
152static struct xpt_softc xsoftc;
153
154MTX_SYSINIT(xpt_topo_init, &xsoftc.xpt_topo_lock, "XPT topology lock", MTX_DEF);
155
156TUNABLE_INT("kern.cam.boot_delay", &xsoftc.boot_delay);
157SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
158           &xsoftc.boot_delay, 0, "Bus registration wait time");
159SYSCTL_UINT(_kern_cam, OID_AUTO, xpt_generation, CTLFLAG_RD,
160	    &xsoftc.xpt_generation, 0, "CAM peripheral generation count");
161
162struct cam_doneq {
163	struct mtx_padalign	cam_doneq_mtx;
164	STAILQ_HEAD(, ccb_hdr)	cam_doneq;
165	int			cam_doneq_sleep;
166};
167
168static struct cam_doneq cam_doneqs[MAXCPU];
169static int cam_num_doneqs;
170static struct proc *cam_proc;
171
172TUNABLE_INT("kern.cam.num_doneqs", &cam_num_doneqs);
173SYSCTL_INT(_kern_cam, OID_AUTO, num_doneqs, CTLFLAG_RDTUN,
174           &cam_num_doneqs, 0, "Number of completion queues/threads");
175
176struct cam_periph *xpt_periph;
177
178static periph_init_t xpt_periph_init;
179
180static struct periph_driver xpt_driver =
181{
182	xpt_periph_init, "xpt",
183	TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
184	CAM_PERIPH_DRV_EARLY
185};
186
187PERIPHDRIVER_DECLARE(xpt, xpt_driver);
188
189static d_open_t xptopen;
190static d_close_t xptclose;
191static d_ioctl_t xptioctl;
192static d_ioctl_t xptdoioctl;
193
194static struct cdevsw xpt_cdevsw = {
195	.d_version =	D_VERSION,
196	.d_flags =	0,
197	.d_open =	xptopen,
198	.d_close =	xptclose,
199	.d_ioctl =	xptioctl,
200	.d_name =	"xpt",
201};
202
203/* Storage for debugging datastructures */
204struct cam_path *cam_dpath;
205u_int32_t cam_dflags = CAM_DEBUG_FLAGS;
206TUNABLE_INT("kern.cam.dflags", &cam_dflags);
207SYSCTL_UINT(_kern_cam, OID_AUTO, dflags, CTLFLAG_RW,
208	&cam_dflags, 0, "Enabled debug flags");
209u_int32_t cam_debug_delay = CAM_DEBUG_DELAY;
210TUNABLE_INT("kern.cam.debug_delay", &cam_debug_delay);
211SYSCTL_UINT(_kern_cam, OID_AUTO, debug_delay, CTLFLAG_RW,
212	&cam_debug_delay, 0, "Delay in us after each debug message");
213
214/* Our boot-time initialization hook */
215static int cam_module_event_handler(module_t, int /*modeventtype_t*/, void *);
216
217static moduledata_t cam_moduledata = {
218	"cam",
219	cam_module_event_handler,
220	NULL
221};
222
223static int	xpt_init(void *);
224
225DECLARE_MODULE(cam, cam_moduledata, SI_SUB_CONFIGURE, SI_ORDER_SECOND);
226MODULE_VERSION(cam, 1);
227
228
229static void		xpt_async_bcast(struct async_list *async_head,
230					u_int32_t async_code,
231					struct cam_path *path,
232					void *async_arg);
233static path_id_t xptnextfreepathid(void);
234static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
235static union ccb *xpt_get_ccb(struct cam_periph *periph);
236static union ccb *xpt_get_ccb_nowait(struct cam_periph *periph);
237static void	 xpt_run_allocq(struct cam_periph *periph, int sleep);
238static void	 xpt_run_allocq_task(void *context, int pending);
239static void	 xpt_run_devq(struct cam_devq *devq);
240static timeout_t xpt_release_devq_timeout;
241static void	 xpt_release_simq_timeout(void *arg) __unused;
242static void	 xpt_acquire_bus(struct cam_eb *bus);
243static void	 xpt_release_bus(struct cam_eb *bus);
244static uint32_t	 xpt_freeze_devq_device(struct cam_ed *dev, u_int count);
245static int	 xpt_release_devq_device(struct cam_ed *dev, u_int count,
246		    int run_queue);
247static struct cam_et*
248		 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
249static void	 xpt_acquire_target(struct cam_et *target);
250static void	 xpt_release_target(struct cam_et *target);
251static struct cam_eb*
252		 xpt_find_bus(path_id_t path_id);
253static struct cam_et*
254		 xpt_find_target(struct cam_eb *bus, target_id_t target_id);
255static struct cam_ed*
256		 xpt_find_device(struct cam_et *target, lun_id_t lun_id);
257static void	 xpt_config(void *arg);
258static int	 xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
259				 u_int32_t new_priority);
260static xpt_devicefunc_t xptpassannouncefunc;
261static void	 xptaction(struct cam_sim *sim, union ccb *work_ccb);
262static void	 xptpoll(struct cam_sim *sim);
263static void	 camisr_runqueue(void);
264static void	 xpt_done_process(struct ccb_hdr *ccb_h);
265static void	 xpt_done_td(void *);
266static dev_match_ret	xptbusmatch(struct dev_match_pattern *patterns,
267				    u_int num_patterns, struct cam_eb *bus);
268static dev_match_ret	xptdevicematch(struct dev_match_pattern *patterns,
269				       u_int num_patterns,
270				       struct cam_ed *device);
271static dev_match_ret	xptperiphmatch(struct dev_match_pattern *patterns,
272				       u_int num_patterns,
273				       struct cam_periph *periph);
274static xpt_busfunc_t	xptedtbusfunc;
275static xpt_targetfunc_t	xptedttargetfunc;
276static xpt_devicefunc_t	xptedtdevicefunc;
277static xpt_periphfunc_t	xptedtperiphfunc;
278static xpt_pdrvfunc_t	xptplistpdrvfunc;
279static xpt_periphfunc_t	xptplistperiphfunc;
280static int		xptedtmatch(struct ccb_dev_match *cdm);
281static int		xptperiphlistmatch(struct ccb_dev_match *cdm);
282static int		xptbustraverse(struct cam_eb *start_bus,
283				       xpt_busfunc_t *tr_func, void *arg);
284static int		xpttargettraverse(struct cam_eb *bus,
285					  struct cam_et *start_target,
286					  xpt_targetfunc_t *tr_func, void *arg);
287static int		xptdevicetraverse(struct cam_et *target,
288					  struct cam_ed *start_device,
289					  xpt_devicefunc_t *tr_func, void *arg);
290static int		xptperiphtraverse(struct cam_ed *device,
291					  struct cam_periph *start_periph,
292					  xpt_periphfunc_t *tr_func, void *arg);
293static int		xptpdrvtraverse(struct periph_driver **start_pdrv,
294					xpt_pdrvfunc_t *tr_func, void *arg);
295static int		xptpdperiphtraverse(struct periph_driver **pdrv,
296					    struct cam_periph *start_periph,
297					    xpt_periphfunc_t *tr_func,
298					    void *arg);
299static xpt_busfunc_t	xptdefbusfunc;
300static xpt_targetfunc_t	xptdeftargetfunc;
301static xpt_devicefunc_t	xptdefdevicefunc;
302static xpt_periphfunc_t	xptdefperiphfunc;
303static void		xpt_finishconfig_task(void *context, int pending);
304static void		xpt_dev_async_default(u_int32_t async_code,
305					      struct cam_eb *bus,
306					      struct cam_et *target,
307					      struct cam_ed *device,
308					      void *async_arg);
309static struct cam_ed *	xpt_alloc_device_default(struct cam_eb *bus,
310						 struct cam_et *target,
311						 lun_id_t lun_id);
312static xpt_devicefunc_t	xptsetasyncfunc;
313static xpt_busfunc_t	xptsetasyncbusfunc;
314static cam_status	xptregister(struct cam_periph *periph,
315				    void *arg);
316static __inline int device_is_queued(struct cam_ed *device);
317
318static __inline int
319xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev)
320{
321	int	retval;
322
323	mtx_assert(&devq->send_mtx, MA_OWNED);
324	if ((dev->ccbq.queue.entries > 0) &&
325	    (dev->ccbq.dev_openings > 0) &&
326	    (dev->ccbq.queue.qfrozen_cnt == 0)) {
327		/*
328		 * The priority of a device waiting for controller
329		 * resources is that of the highest priority CCB
330		 * enqueued.
331		 */
332		retval =
333		    xpt_schedule_dev(&devq->send_queue,
334				     &dev->devq_entry,
335				     CAMQ_GET_PRIO(&dev->ccbq.queue));
336	} else {
337		retval = 0;
338	}
339	return (retval);
340}
341
342static __inline int
343device_is_queued(struct cam_ed *device)
344{
345	return (device->devq_entry.index != CAM_UNQUEUED_INDEX);
346}
347
348static void
349xpt_periph_init()
350{
351	make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0");
352}
353
354static int
355xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
356{
357
358	/*
359	 * Only allow read-write access.
360	 */
361	if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0))
362		return(EPERM);
363
364	/*
365	 * We don't allow nonblocking access.
366	 */
367	if ((flags & O_NONBLOCK) != 0) {
368		printf("%s: can't do nonblocking access\n", devtoname(dev));
369		return(ENODEV);
370	}
371
372	return(0);
373}
374
375static int
376xptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
377{
378
379	return(0);
380}
381
382/*
383 * Don't automatically grab the xpt softc lock here even though this is going
384 * through the xpt device.  The xpt device is really just a back door for
385 * accessing other devices and SIMs, so the right thing to do is to grab
386 * the appropriate SIM lock once the bus/SIM is located.
387 */
388static int
389xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
390{
391	int error;
392
393	if ((error = xptdoioctl(dev, cmd, addr, flag, td)) == ENOTTY) {
394		error = cam_compat_ioctl(dev, cmd, addr, flag, td, xptdoioctl);
395	}
396	return (error);
397}
398
399static int
400xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
401{
402	int error;
403
404	error = 0;
405
406	switch(cmd) {
407	/*
408	 * For the transport layer CAMIOCOMMAND ioctl, we really only want
409	 * to accept CCB types that don't quite make sense to send through a
410	 * passthrough driver. XPT_PATH_INQ is an exception to this, as stated
411	 * in the CAM spec.
412	 */
413	case CAMIOCOMMAND: {
414		union ccb *ccb;
415		union ccb *inccb;
416		struct cam_eb *bus;
417
418		inccb = (union ccb *)addr;
419
420		bus = xpt_find_bus(inccb->ccb_h.path_id);
421		if (bus == NULL)
422			return (EINVAL);
423
424		switch (inccb->ccb_h.func_code) {
425		case XPT_SCAN_BUS:
426		case XPT_RESET_BUS:
427			if (inccb->ccb_h.target_id != CAM_TARGET_WILDCARD ||
428			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
429				xpt_release_bus(bus);
430				return (EINVAL);
431			}
432			break;
433		case XPT_SCAN_TGT:
434			if (inccb->ccb_h.target_id == CAM_TARGET_WILDCARD ||
435			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
436				xpt_release_bus(bus);
437				return (EINVAL);
438			}
439			break;
440		default:
441			break;
442		}
443
444		switch(inccb->ccb_h.func_code) {
445		case XPT_SCAN_BUS:
446		case XPT_RESET_BUS:
447		case XPT_PATH_INQ:
448		case XPT_ENG_INQ:
449		case XPT_SCAN_LUN:
450		case XPT_SCAN_TGT:
451
452			ccb = xpt_alloc_ccb();
453
454			/*
455			 * Create a path using the bus, target, and lun the
456			 * user passed in.
457			 */
458			if (xpt_create_path(&ccb->ccb_h.path, NULL,
459					    inccb->ccb_h.path_id,
460					    inccb->ccb_h.target_id,
461					    inccb->ccb_h.target_lun) !=
462					    CAM_REQ_CMP){
463				error = EINVAL;
464				xpt_free_ccb(ccb);
465				break;
466			}
467			/* Ensure all of our fields are correct */
468			xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path,
469				      inccb->ccb_h.pinfo.priority);
470			xpt_merge_ccb(ccb, inccb);
471			xpt_path_lock(ccb->ccb_h.path);
472			cam_periph_runccb(ccb, NULL, 0, 0, NULL);
473			xpt_path_unlock(ccb->ccb_h.path);
474			bcopy(ccb, inccb, sizeof(union ccb));
475			xpt_free_path(ccb->ccb_h.path);
476			xpt_free_ccb(ccb);
477			break;
478
479		case XPT_DEBUG: {
480			union ccb ccb;
481
482			/*
483			 * This is an immediate CCB, so it's okay to
484			 * allocate it on the stack.
485			 */
486
487			/*
488			 * Create a path using the bus, target, and lun the
489			 * user passed in.
490			 */
491			if (xpt_create_path(&ccb.ccb_h.path, NULL,
492					    inccb->ccb_h.path_id,
493					    inccb->ccb_h.target_id,
494					    inccb->ccb_h.target_lun) !=
495					    CAM_REQ_CMP){
496				error = EINVAL;
497				break;
498			}
499			/* Ensure all of our fields are correct */
500			xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path,
501				      inccb->ccb_h.pinfo.priority);
502			xpt_merge_ccb(&ccb, inccb);
503			xpt_action(&ccb);
504			bcopy(&ccb, inccb, sizeof(union ccb));
505			xpt_free_path(ccb.ccb_h.path);
506			break;
507
508		}
509		case XPT_DEV_MATCH: {
510			struct cam_periph_map_info mapinfo;
511			struct cam_path *old_path;
512
513			/*
514			 * We can't deal with physical addresses for this
515			 * type of transaction.
516			 */
517			if ((inccb->ccb_h.flags & CAM_DATA_MASK) !=
518			    CAM_DATA_VADDR) {
519				error = EINVAL;
520				break;
521			}
522
523			/*
524			 * Save this in case the caller had it set to
525			 * something in particular.
526			 */
527			old_path = inccb->ccb_h.path;
528
529			/*
530			 * We really don't need a path for the matching
531			 * code.  The path is needed because of the
532			 * debugging statements in xpt_action().  They
533			 * assume that the CCB has a valid path.
534			 */
535			inccb->ccb_h.path = xpt_periph->path;
536
537			bzero(&mapinfo, sizeof(mapinfo));
538
539			/*
540			 * Map the pattern and match buffers into kernel
541			 * virtual address space.
542			 */
543			error = cam_periph_mapmem(inccb, &mapinfo, MAXPHYS);
544
545			if (error) {
546				inccb->ccb_h.path = old_path;
547				break;
548			}
549
550			/*
551			 * This is an immediate CCB, we can send it on directly.
552			 */
553			xpt_action(inccb);
554
555			/*
556			 * Map the buffers back into user space.
557			 */
558			cam_periph_unmapmem(inccb, &mapinfo);
559
560			inccb->ccb_h.path = old_path;
561
562			error = 0;
563			break;
564		}
565		default:
566			error = ENOTSUP;
567			break;
568		}
569		xpt_release_bus(bus);
570		break;
571	}
572	/*
573	 * This is the getpassthru ioctl. It takes a XPT_GDEVLIST ccb as input,
574	 * with the periphal driver name and unit name filled in.  The other
575	 * fields don't really matter as input.  The passthrough driver name
576	 * ("pass"), and unit number are passed back in the ccb.  The current
577	 * device generation number, and the index into the device peripheral
578	 * driver list, and the status are also passed back.  Note that
579	 * since we do everything in one pass, unlike the XPT_GDEVLIST ccb,
580	 * we never return a status of CAM_GDEVLIST_LIST_CHANGED.  It is
581	 * (or rather should be) impossible for the device peripheral driver
582	 * list to change since we look at the whole thing in one pass, and
583	 * we do it with lock protection.
584	 *
585	 */
586	case CAMGETPASSTHRU: {
587		union ccb *ccb;
588		struct cam_periph *periph;
589		struct periph_driver **p_drv;
590		char   *name;
591		u_int unit;
592		int base_periph_found;
593
594		ccb = (union ccb *)addr;
595		unit = ccb->cgdl.unit_number;
596		name = ccb->cgdl.periph_name;
597		base_periph_found = 0;
598
599		/*
600		 * Sanity check -- make sure we don't get a null peripheral
601		 * driver name.
602		 */
603		if (*ccb->cgdl.periph_name == '\0') {
604			error = EINVAL;
605			break;
606		}
607
608		/* Keep the list from changing while we traverse it */
609		xpt_lock_buses();
610
611		/* first find our driver in the list of drivers */
612		for (p_drv = periph_drivers; *p_drv != NULL; p_drv++)
613			if (strcmp((*p_drv)->driver_name, name) == 0)
614				break;
615
616		if (*p_drv == NULL) {
617			xpt_unlock_buses();
618			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
619			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
620			*ccb->cgdl.periph_name = '\0';
621			ccb->cgdl.unit_number = 0;
622			error = ENOENT;
623			break;
624		}
625
626		/*
627		 * Run through every peripheral instance of this driver
628		 * and check to see whether it matches the unit passed
629		 * in by the user.  If it does, get out of the loops and
630		 * find the passthrough driver associated with that
631		 * peripheral driver.
632		 */
633		for (periph = TAILQ_FIRST(&(*p_drv)->units); periph != NULL;
634		     periph = TAILQ_NEXT(periph, unit_links)) {
635
636			if (periph->unit_number == unit)
637				break;
638		}
639		/*
640		 * If we found the peripheral driver that the user passed
641		 * in, go through all of the peripheral drivers for that
642		 * particular device and look for a passthrough driver.
643		 */
644		if (periph != NULL) {
645			struct cam_ed *device;
646			int i;
647
648			base_periph_found = 1;
649			device = periph->path->device;
650			for (i = 0, periph = SLIST_FIRST(&device->periphs);
651			     periph != NULL;
652			     periph = SLIST_NEXT(periph, periph_links), i++) {
653				/*
654				 * Check to see whether we have a
655				 * passthrough device or not.
656				 */
657				if (strcmp(periph->periph_name, "pass") == 0) {
658					/*
659					 * Fill in the getdevlist fields.
660					 */
661					strcpy(ccb->cgdl.periph_name,
662					       periph->periph_name);
663					ccb->cgdl.unit_number =
664						periph->unit_number;
665					if (SLIST_NEXT(periph, periph_links))
666						ccb->cgdl.status =
667							CAM_GDEVLIST_MORE_DEVS;
668					else
669						ccb->cgdl.status =
670						       CAM_GDEVLIST_LAST_DEVICE;
671					ccb->cgdl.generation =
672						device->generation;
673					ccb->cgdl.index = i;
674					/*
675					 * Fill in some CCB header fields
676					 * that the user may want.
677					 */
678					ccb->ccb_h.path_id =
679						periph->path->bus->path_id;
680					ccb->ccb_h.target_id =
681						periph->path->target->target_id;
682					ccb->ccb_h.target_lun =
683						periph->path->device->lun_id;
684					ccb->ccb_h.status = CAM_REQ_CMP;
685					break;
686				}
687			}
688		}
689
690		/*
691		 * If the periph is null here, one of two things has
692		 * happened.  The first possibility is that we couldn't
693		 * find the unit number of the particular peripheral driver
694		 * that the user is asking about.  e.g. the user asks for
695		 * the passthrough driver for "da11".  We find the list of
696		 * "da" peripherals all right, but there is no unit 11.
697		 * The other possibility is that we went through the list
698		 * of peripheral drivers attached to the device structure,
699		 * but didn't find one with the name "pass".  Either way,
700		 * we return ENOENT, since we couldn't find something.
701		 */
702		if (periph == NULL) {
703			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
704			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
705			*ccb->cgdl.periph_name = '\0';
706			ccb->cgdl.unit_number = 0;
707			error = ENOENT;
708			/*
709			 * It is unfortunate that this is even necessary,
710			 * but there are many, many clueless users out there.
711			 * If this is true, the user is looking for the
712			 * passthrough driver, but doesn't have one in his
713			 * kernel.
714			 */
715			if (base_periph_found == 1) {
716				printf("xptioctl: pass driver is not in the "
717				       "kernel\n");
718				printf("xptioctl: put \"device pass\" in "
719				       "your kernel config file\n");
720			}
721		}
722		xpt_unlock_buses();
723		break;
724		}
725	default:
726		error = ENOTTY;
727		break;
728	}
729
730	return(error);
731}
732
733static int
734cam_module_event_handler(module_t mod, int what, void *arg)
735{
736	int error;
737
738	switch (what) {
739	case MOD_LOAD:
740		if ((error = xpt_init(NULL)) != 0)
741			return (error);
742		break;
743	case MOD_UNLOAD:
744		return EBUSY;
745	default:
746		return EOPNOTSUPP;
747	}
748
749	return 0;
750}
751
752static void
753xpt_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
754{
755
756	if (done_ccb->ccb_h.ppriv_ptr1 == NULL) {
757		xpt_free_path(done_ccb->ccb_h.path);
758		xpt_free_ccb(done_ccb);
759	} else {
760		done_ccb->ccb_h.cbfcnp = done_ccb->ccb_h.ppriv_ptr1;
761		(*done_ccb->ccb_h.cbfcnp)(periph, done_ccb);
762	}
763	xpt_release_boot();
764}
765
766/* thread to handle bus rescans */
767static void
768xpt_scanner_thread(void *dummy)
769{
770	union ccb	*ccb;
771	struct cam_path	 path;
772
773	xpt_lock_buses();
774	for (;;) {
775		if (TAILQ_EMPTY(&xsoftc.ccb_scanq))
776			msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
777			       "-", 0);
778		if ((ccb = (union ccb *)TAILQ_FIRST(&xsoftc.ccb_scanq)) != NULL) {
779			TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
780			xpt_unlock_buses();
781
782			/*
783			 * Since lock can be dropped inside and path freed
784			 * by completion callback even before return here,
785			 * take our own path copy for reference.
786			 */
787			xpt_copy_path(&path, ccb->ccb_h.path);
788			xpt_path_lock(&path);
789			xpt_action(ccb);
790			xpt_path_unlock(&path);
791			xpt_release_path(&path);
792
793			xpt_lock_buses();
794		}
795	}
796}
797
798void
799xpt_rescan(union ccb *ccb)
800{
801	struct ccb_hdr *hdr;
802
803	/* Prepare request */
804	if (ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD &&
805	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
806		ccb->ccb_h.func_code = XPT_SCAN_BUS;
807	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
808	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
809		ccb->ccb_h.func_code = XPT_SCAN_TGT;
810	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
811	    ccb->ccb_h.path->device->lun_id != CAM_LUN_WILDCARD)
812		ccb->ccb_h.func_code = XPT_SCAN_LUN;
813	else {
814		xpt_print(ccb->ccb_h.path, "illegal scan path\n");
815		xpt_free_path(ccb->ccb_h.path);
816		xpt_free_ccb(ccb);
817		return;
818	}
819	ccb->ccb_h.ppriv_ptr1 = ccb->ccb_h.cbfcnp;
820	ccb->ccb_h.cbfcnp = xpt_rescan_done;
821	xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_XPT);
822	/* Don't make duplicate entries for the same paths. */
823	xpt_lock_buses();
824	if (ccb->ccb_h.ppriv_ptr1 == NULL) {
825		TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
826			if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
827				wakeup(&xsoftc.ccb_scanq);
828				xpt_unlock_buses();
829				xpt_print(ccb->ccb_h.path, "rescan already queued\n");
830				xpt_free_path(ccb->ccb_h.path);
831				xpt_free_ccb(ccb);
832				return;
833			}
834		}
835	}
836	TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
837	xsoftc.buses_to_config++;
838	wakeup(&xsoftc.ccb_scanq);
839	xpt_unlock_buses();
840}
841
842/* Functions accessed by the peripheral drivers */
843static int
844xpt_init(void *dummy)
845{
846	struct cam_sim *xpt_sim;
847	struct cam_path *path;
848	struct cam_devq *devq;
849	cam_status status;
850	int error, i;
851
852	TAILQ_INIT(&xsoftc.xpt_busses);
853	TAILQ_INIT(&xsoftc.ccb_scanq);
854	STAILQ_INIT(&xsoftc.highpowerq);
855	xsoftc.num_highpower = CAM_MAX_HIGHPOWER;
856
857	mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF);
858	mtx_init(&xsoftc.xpt_highpower_lock, "XPT highpower lock", NULL, MTX_DEF);
859	xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK,
860	    taskqueue_thread_enqueue, /*context*/&xsoftc.xpt_taskq);
861
862#ifdef CAM_BOOT_DELAY
863	/*
864	 * Override this value at compile time to assist our users
865	 * who don't use loader to boot a kernel.
866	 */
867	xsoftc.boot_delay = CAM_BOOT_DELAY;
868#endif
869	/*
870	 * The xpt layer is, itself, the equivelent of a SIM.
871	 * Allow 16 ccbs in the ccb pool for it.  This should
872	 * give decent parallelism when we probe busses and
873	 * perform other XPT functions.
874	 */
875	devq = cam_simq_alloc(16);
876	xpt_sim = cam_sim_alloc(xptaction,
877				xptpoll,
878				"xpt",
879				/*softc*/NULL,
880				/*unit*/0,
881				/*mtx*/&xsoftc.xpt_lock,
882				/*max_dev_transactions*/0,
883				/*max_tagged_dev_transactions*/0,
884				devq);
885	if (xpt_sim == NULL)
886		return (ENOMEM);
887
888	mtx_lock(&xsoftc.xpt_lock);
889	if ((status = xpt_bus_register(xpt_sim, NULL, 0)) != CAM_SUCCESS) {
890		mtx_unlock(&xsoftc.xpt_lock);
891		printf("xpt_init: xpt_bus_register failed with status %#x,"
892		       " failing attach\n", status);
893		return (EINVAL);
894	}
895	mtx_unlock(&xsoftc.xpt_lock);
896
897	/*
898	 * Looking at the XPT from the SIM layer, the XPT is
899	 * the equivelent of a peripheral driver.  Allocate
900	 * a peripheral driver entry for us.
901	 */
902	if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
903				      CAM_TARGET_WILDCARD,
904				      CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
905		printf("xpt_init: xpt_create_path failed with status %#x,"
906		       " failing attach\n", status);
907		return (EINVAL);
908	}
909	xpt_path_lock(path);
910	cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
911			 path, NULL, 0, xpt_sim);
912	xpt_path_unlock(path);
913	xpt_free_path(path);
914
915	if (cam_num_doneqs < 1)
916		cam_num_doneqs = 1 + mp_ncpus / 6;
917	else if (cam_num_doneqs > MAXCPU)
918		cam_num_doneqs = MAXCPU;
919	for (i = 0; i < cam_num_doneqs; i++) {
920		mtx_init(&cam_doneqs[i].cam_doneq_mtx, "CAM doneq", NULL,
921		    MTX_DEF);
922		STAILQ_INIT(&cam_doneqs[i].cam_doneq);
923		error = kproc_kthread_add(xpt_done_td, &cam_doneqs[i],
924		    &cam_proc, NULL, 0, 0, "cam", "doneq%d", i);
925		if (error != 0) {
926			cam_num_doneqs = i;
927			break;
928		}
929	}
930	if (cam_num_doneqs < 1) {
931		printf("xpt_init: Cannot init completion queues "
932		       "- failing attach\n");
933		return (ENOMEM);
934	}
935	/*
936	 * Register a callback for when interrupts are enabled.
937	 */
938	xsoftc.xpt_config_hook =
939	    (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
940					      M_CAMXPT, M_NOWAIT | M_ZERO);
941	if (xsoftc.xpt_config_hook == NULL) {
942		printf("xpt_init: Cannot malloc config hook "
943		       "- failing attach\n");
944		return (ENOMEM);
945	}
946	xsoftc.xpt_config_hook->ich_func = xpt_config;
947	if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
948		free (xsoftc.xpt_config_hook, M_CAMXPT);
949		printf("xpt_init: config_intrhook_establish failed "
950		       "- failing attach\n");
951	}
952
953	return (0);
954}
955
956static cam_status
957xptregister(struct cam_periph *periph, void *arg)
958{
959	struct cam_sim *xpt_sim;
960
961	if (periph == NULL) {
962		printf("xptregister: periph was NULL!!\n");
963		return(CAM_REQ_CMP_ERR);
964	}
965
966	xpt_sim = (struct cam_sim *)arg;
967	xpt_sim->softc = periph;
968	xpt_periph = periph;
969	periph->softc = NULL;
970
971	return(CAM_REQ_CMP);
972}
973
974int32_t
975xpt_add_periph(struct cam_periph *periph)
976{
977	struct cam_ed *device;
978	int32_t	 status;
979
980	TASK_INIT(&periph->periph_run_task, 0, xpt_run_allocq_task, periph);
981	device = periph->path->device;
982	status = CAM_REQ_CMP;
983	if (device != NULL) {
984		mtx_lock(&device->target->bus->eb_mtx);
985		device->generation++;
986		SLIST_INSERT_HEAD(&device->periphs, periph, periph_links);
987		mtx_unlock(&device->target->bus->eb_mtx);
988		atomic_add_32(&xsoftc.xpt_generation, 1);
989	}
990
991	return (status);
992}
993
994void
995xpt_remove_periph(struct cam_periph *periph)
996{
997	struct cam_ed *device;
998
999	device = periph->path->device;
1000	if (device != NULL) {
1001		mtx_lock(&device->target->bus->eb_mtx);
1002		device->generation++;
1003		SLIST_REMOVE(&device->periphs, periph, cam_periph, periph_links);
1004		mtx_unlock(&device->target->bus->eb_mtx);
1005		atomic_add_32(&xsoftc.xpt_generation, 1);
1006	}
1007}
1008
1009
1010void
1011xpt_announce_periph(struct cam_periph *periph, char *announce_string)
1012{
1013	struct	cam_path *path = periph->path;
1014
1015	cam_periph_assert(periph, MA_OWNED);
1016	periph->flags |= CAM_PERIPH_ANNOUNCED;
1017
1018	printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1019	       periph->periph_name, periph->unit_number,
1020	       path->bus->sim->sim_name,
1021	       path->bus->sim->unit_number,
1022	       path->bus->sim->bus_id,
1023	       path->bus->path_id,
1024	       path->target->target_id,
1025	       (uintmax_t)path->device->lun_id);
1026	printf("%s%d: ", periph->periph_name, periph->unit_number);
1027	if (path->device->protocol == PROTO_SCSI)
1028		scsi_print_inquiry(&path->device->inq_data);
1029	else if (path->device->protocol == PROTO_ATA ||
1030	    path->device->protocol == PROTO_SATAPM)
1031		ata_print_ident(&path->device->ident_data);
1032	else if (path->device->protocol == PROTO_SEMB)
1033		semb_print_ident(
1034		    (struct sep_identify_data *)&path->device->ident_data);
1035	else
1036		printf("Unknown protocol device\n");
1037	if (path->device->serial_num_len > 0) {
1038		/* Don't wrap the screen  - print only the first 60 chars */
1039		printf("%s%d: Serial Number %.60s\n", periph->periph_name,
1040		       periph->unit_number, path->device->serial_num);
1041	}
1042	/* Announce transport details. */
1043	(*(path->bus->xport->announce))(periph);
1044	/* Announce command queueing. */
1045	if (path->device->inq_flags & SID_CmdQue
1046	 || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1047		printf("%s%d: Command Queueing enabled\n",
1048		       periph->periph_name, periph->unit_number);
1049	}
1050	/* Announce caller's details if they've passed in. */
1051	if (announce_string != NULL)
1052		printf("%s%d: %s\n", periph->periph_name,
1053		       periph->unit_number, announce_string);
1054}
1055
1056void
1057xpt_announce_quirks(struct cam_periph *periph, int quirks, char *bit_string)
1058{
1059	if (quirks != 0) {
1060		printf("%s%d: quirks=0x%b\n", periph->periph_name,
1061		    periph->unit_number, quirks, bit_string);
1062	}
1063}
1064
1065void
1066xpt_denounce_periph(struct cam_periph *periph)
1067{
1068	struct	cam_path *path = periph->path;
1069
1070	cam_periph_assert(periph, MA_OWNED);
1071	printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1072	       periph->periph_name, periph->unit_number,
1073	       path->bus->sim->sim_name,
1074	       path->bus->sim->unit_number,
1075	       path->bus->sim->bus_id,
1076	       path->bus->path_id,
1077	       path->target->target_id,
1078	       (uintmax_t)path->device->lun_id);
1079	printf("%s%d: ", periph->periph_name, periph->unit_number);
1080	if (path->device->protocol == PROTO_SCSI)
1081		scsi_print_inquiry_short(&path->device->inq_data);
1082	else if (path->device->protocol == PROTO_ATA ||
1083	    path->device->protocol == PROTO_SATAPM)
1084		ata_print_ident_short(&path->device->ident_data);
1085	else if (path->device->protocol == PROTO_SEMB)
1086		semb_print_ident_short(
1087		    (struct sep_identify_data *)&path->device->ident_data);
1088	else
1089		printf("Unknown protocol device");
1090	if (path->device->serial_num_len > 0)
1091		printf(" s/n %.60s", path->device->serial_num);
1092	printf(" detached\n");
1093}
1094
1095
1096int
1097xpt_getattr(char *buf, size_t len, const char *attr, struct cam_path *path)
1098{
1099	int ret = -1, l;
1100	struct ccb_dev_advinfo cdai;
1101	struct scsi_vpd_id_descriptor *idd;
1102
1103	xpt_path_assert(path, MA_OWNED);
1104
1105	memset(&cdai, 0, sizeof(cdai));
1106	xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
1107	cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
1108	cdai.bufsiz = len;
1109
1110	if (!strcmp(attr, "GEOM::ident"))
1111		cdai.buftype = CDAI_TYPE_SERIAL_NUM;
1112	else if (!strcmp(attr, "GEOM::physpath"))
1113		cdai.buftype = CDAI_TYPE_PHYS_PATH;
1114	else if (strcmp(attr, "GEOM::lunid") == 0 ||
1115		 strcmp(attr, "GEOM::lunname") == 0) {
1116		cdai.buftype = CDAI_TYPE_SCSI_DEVID;
1117		cdai.bufsiz = CAM_SCSI_DEVID_MAXLEN;
1118	} else
1119		goto out;
1120
1121	cdai.buf = malloc(cdai.bufsiz, M_CAMXPT, M_NOWAIT|M_ZERO);
1122	if (cdai.buf == NULL) {
1123		ret = ENOMEM;
1124		goto out;
1125	}
1126	xpt_action((union ccb *)&cdai); /* can only be synchronous */
1127	if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
1128		cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
1129	if (cdai.provsiz == 0)
1130		goto out;
1131	if (cdai.buftype == CDAI_TYPE_SCSI_DEVID) {
1132		if (strcmp(attr, "GEOM::lunid") == 0) {
1133			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1134			    cdai.provsiz, scsi_devid_is_lun_naa);
1135			if (idd == NULL)
1136				idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1137				    cdai.provsiz, scsi_devid_is_lun_eui64);
1138		} else
1139			idd = NULL;
1140		if (idd == NULL)
1141			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1142			    cdai.provsiz, scsi_devid_is_lun_t10);
1143		if (idd == NULL)
1144			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1145			    cdai.provsiz, scsi_devid_is_lun_name);
1146		if (idd == NULL)
1147			goto out;
1148		ret = 0;
1149		if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_ASCII) {
1150			if (idd->length < len) {
1151				for (l = 0; l < idd->length; l++)
1152					buf[l] = idd->identifier[l] ?
1153					    idd->identifier[l] : ' ';
1154				buf[l] = 0;
1155			} else
1156				ret = EFAULT;
1157		} else if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_UTF8) {
1158			l = strnlen(idd->identifier, idd->length);
1159			if (l < len) {
1160				bcopy(idd->identifier, buf, l);
1161				buf[l] = 0;
1162			} else
1163				ret = EFAULT;
1164		} else {
1165			if (idd->length * 2 < len) {
1166				for (l = 0; l < idd->length; l++)
1167					sprintf(buf + l * 2, "%02x",
1168					    idd->identifier[l]);
1169			} else
1170				ret = EFAULT;
1171		}
1172	} else {
1173		ret = 0;
1174		if (strlcpy(buf, cdai.buf, len) >= len)
1175			ret = EFAULT;
1176	}
1177
1178out:
1179	if (cdai.buf != NULL)
1180		free(cdai.buf, M_CAMXPT);
1181	return ret;
1182}
1183
1184static dev_match_ret
1185xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1186	    struct cam_eb *bus)
1187{
1188	dev_match_ret retval;
1189	int i;
1190
1191	retval = DM_RET_NONE;
1192
1193	/*
1194	 * If we aren't given something to match against, that's an error.
1195	 */
1196	if (bus == NULL)
1197		return(DM_RET_ERROR);
1198
1199	/*
1200	 * If there are no match entries, then this bus matches no
1201	 * matter what.
1202	 */
1203	if ((patterns == NULL) || (num_patterns == 0))
1204		return(DM_RET_DESCEND | DM_RET_COPY);
1205
1206	for (i = 0; i < num_patterns; i++) {
1207		struct bus_match_pattern *cur_pattern;
1208
1209		/*
1210		 * If the pattern in question isn't for a bus node, we
1211		 * aren't interested.  However, we do indicate to the
1212		 * calling routine that we should continue descending the
1213		 * tree, since the user wants to match against lower-level
1214		 * EDT elements.
1215		 */
1216		if (patterns[i].type != DEV_MATCH_BUS) {
1217			if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1218				retval |= DM_RET_DESCEND;
1219			continue;
1220		}
1221
1222		cur_pattern = &patterns[i].pattern.bus_pattern;
1223
1224		/*
1225		 * If they want to match any bus node, we give them any
1226		 * device node.
1227		 */
1228		if (cur_pattern->flags == BUS_MATCH_ANY) {
1229			/* set the copy flag */
1230			retval |= DM_RET_COPY;
1231
1232			/*
1233			 * If we've already decided on an action, go ahead
1234			 * and return.
1235			 */
1236			if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1237				return(retval);
1238		}
1239
1240		/*
1241		 * Not sure why someone would do this...
1242		 */
1243		if (cur_pattern->flags == BUS_MATCH_NONE)
1244			continue;
1245
1246		if (((cur_pattern->flags & BUS_MATCH_PATH) != 0)
1247		 && (cur_pattern->path_id != bus->path_id))
1248			continue;
1249
1250		if (((cur_pattern->flags & BUS_MATCH_BUS_ID) != 0)
1251		 && (cur_pattern->bus_id != bus->sim->bus_id))
1252			continue;
1253
1254		if (((cur_pattern->flags & BUS_MATCH_UNIT) != 0)
1255		 && (cur_pattern->unit_number != bus->sim->unit_number))
1256			continue;
1257
1258		if (((cur_pattern->flags & BUS_MATCH_NAME) != 0)
1259		 && (strncmp(cur_pattern->dev_name, bus->sim->sim_name,
1260			     DEV_IDLEN) != 0))
1261			continue;
1262
1263		/*
1264		 * If we get to this point, the user definitely wants
1265		 * information on this bus.  So tell the caller to copy the
1266		 * data out.
1267		 */
1268		retval |= DM_RET_COPY;
1269
1270		/*
1271		 * If the return action has been set to descend, then we
1272		 * know that we've already seen a non-bus matching
1273		 * expression, therefore we need to further descend the tree.
1274		 * This won't change by continuing around the loop, so we
1275		 * go ahead and return.  If we haven't seen a non-bus
1276		 * matching expression, we keep going around the loop until
1277		 * we exhaust the matching expressions.  We'll set the stop
1278		 * flag once we fall out of the loop.
1279		 */
1280		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1281			return(retval);
1282	}
1283
1284	/*
1285	 * If the return action hasn't been set to descend yet, that means
1286	 * we haven't seen anything other than bus matching patterns.  So
1287	 * tell the caller to stop descending the tree -- the user doesn't
1288	 * want to match against lower level tree elements.
1289	 */
1290	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1291		retval |= DM_RET_STOP;
1292
1293	return(retval);
1294}
1295
1296static dev_match_ret
1297xptdevicematch(struct dev_match_pattern *patterns, u_int num_patterns,
1298	       struct cam_ed *device)
1299{
1300	dev_match_ret retval;
1301	int i;
1302
1303	retval = DM_RET_NONE;
1304
1305	/*
1306	 * If we aren't given something to match against, that's an error.
1307	 */
1308	if (device == NULL)
1309		return(DM_RET_ERROR);
1310
1311	/*
1312	 * If there are no match entries, then this device matches no
1313	 * matter what.
1314	 */
1315	if ((patterns == NULL) || (num_patterns == 0))
1316		return(DM_RET_DESCEND | DM_RET_COPY);
1317
1318	for (i = 0; i < num_patterns; i++) {
1319		struct device_match_pattern *cur_pattern;
1320		struct scsi_vpd_device_id *device_id_page;
1321
1322		/*
1323		 * If the pattern in question isn't for a device node, we
1324		 * aren't interested.
1325		 */
1326		if (patterns[i].type != DEV_MATCH_DEVICE) {
1327			if ((patterns[i].type == DEV_MATCH_PERIPH)
1328			 && ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE))
1329				retval |= DM_RET_DESCEND;
1330			continue;
1331		}
1332
1333		cur_pattern = &patterns[i].pattern.device_pattern;
1334
1335		/* Error out if mutually exclusive options are specified. */
1336		if ((cur_pattern->flags & (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1337		 == (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1338			return(DM_RET_ERROR);
1339
1340		/*
1341		 * If they want to match any device node, we give them any
1342		 * device node.
1343		 */
1344		if (cur_pattern->flags == DEV_MATCH_ANY)
1345			goto copy_dev_node;
1346
1347		/*
1348		 * Not sure why someone would do this...
1349		 */
1350		if (cur_pattern->flags == DEV_MATCH_NONE)
1351			continue;
1352
1353		if (((cur_pattern->flags & DEV_MATCH_PATH) != 0)
1354		 && (cur_pattern->path_id != device->target->bus->path_id))
1355			continue;
1356
1357		if (((cur_pattern->flags & DEV_MATCH_TARGET) != 0)
1358		 && (cur_pattern->target_id != device->target->target_id))
1359			continue;
1360
1361		if (((cur_pattern->flags & DEV_MATCH_LUN) != 0)
1362		 && (cur_pattern->target_lun != device->lun_id))
1363			continue;
1364
1365		if (((cur_pattern->flags & DEV_MATCH_INQUIRY) != 0)
1366		 && (cam_quirkmatch((caddr_t)&device->inq_data,
1367				    (caddr_t)&cur_pattern->data.inq_pat,
1368				    1, sizeof(cur_pattern->data.inq_pat),
1369				    scsi_static_inquiry_match) == NULL))
1370			continue;
1371
1372		device_id_page = (struct scsi_vpd_device_id *)device->device_id;
1373		if (((cur_pattern->flags & DEV_MATCH_DEVID) != 0)
1374		 && (device->device_id_len < SVPD_DEVICE_ID_HDR_LEN
1375		  || scsi_devid_match((uint8_t *)device_id_page->desc_list,
1376				      device->device_id_len
1377				    - SVPD_DEVICE_ID_HDR_LEN,
1378				      cur_pattern->data.devid_pat.id,
1379				      cur_pattern->data.devid_pat.id_len) != 0))
1380			continue;
1381
1382copy_dev_node:
1383		/*
1384		 * If we get to this point, the user definitely wants
1385		 * information on this device.  So tell the caller to copy
1386		 * the data out.
1387		 */
1388		retval |= DM_RET_COPY;
1389
1390		/*
1391		 * If the return action has been set to descend, then we
1392		 * know that we've already seen a peripheral matching
1393		 * expression, therefore we need to further descend the tree.
1394		 * This won't change by continuing around the loop, so we
1395		 * go ahead and return.  If we haven't seen a peripheral
1396		 * matching expression, we keep going around the loop until
1397		 * we exhaust the matching expressions.  We'll set the stop
1398		 * flag once we fall out of the loop.
1399		 */
1400		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1401			return(retval);
1402	}
1403
1404	/*
1405	 * If the return action hasn't been set to descend yet, that means
1406	 * we haven't seen any peripheral matching patterns.  So tell the
1407	 * caller to stop descending the tree -- the user doesn't want to
1408	 * match against lower level tree elements.
1409	 */
1410	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1411		retval |= DM_RET_STOP;
1412
1413	return(retval);
1414}
1415
1416/*
1417 * Match a single peripheral against any number of match patterns.
1418 */
1419static dev_match_ret
1420xptperiphmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1421	       struct cam_periph *periph)
1422{
1423	dev_match_ret retval;
1424	int i;
1425
1426	/*
1427	 * If we aren't given something to match against, that's an error.
1428	 */
1429	if (periph == NULL)
1430		return(DM_RET_ERROR);
1431
1432	/*
1433	 * If there are no match entries, then this peripheral matches no
1434	 * matter what.
1435	 */
1436	if ((patterns == NULL) || (num_patterns == 0))
1437		return(DM_RET_STOP | DM_RET_COPY);
1438
1439	/*
1440	 * There aren't any nodes below a peripheral node, so there's no
1441	 * reason to descend the tree any further.
1442	 */
1443	retval = DM_RET_STOP;
1444
1445	for (i = 0; i < num_patterns; i++) {
1446		struct periph_match_pattern *cur_pattern;
1447
1448		/*
1449		 * If the pattern in question isn't for a peripheral, we
1450		 * aren't interested.
1451		 */
1452		if (patterns[i].type != DEV_MATCH_PERIPH)
1453			continue;
1454
1455		cur_pattern = &patterns[i].pattern.periph_pattern;
1456
1457		/*
1458		 * If they want to match on anything, then we will do so.
1459		 */
1460		if (cur_pattern->flags == PERIPH_MATCH_ANY) {
1461			/* set the copy flag */
1462			retval |= DM_RET_COPY;
1463
1464			/*
1465			 * We've already set the return action to stop,
1466			 * since there are no nodes below peripherals in
1467			 * the tree.
1468			 */
1469			return(retval);
1470		}
1471
1472		/*
1473		 * Not sure why someone would do this...
1474		 */
1475		if (cur_pattern->flags == PERIPH_MATCH_NONE)
1476			continue;
1477
1478		if (((cur_pattern->flags & PERIPH_MATCH_PATH) != 0)
1479		 && (cur_pattern->path_id != periph->path->bus->path_id))
1480			continue;
1481
1482		/*
1483		 * For the target and lun id's, we have to make sure the
1484		 * target and lun pointers aren't NULL.  The xpt peripheral
1485		 * has a wildcard target and device.
1486		 */
1487		if (((cur_pattern->flags & PERIPH_MATCH_TARGET) != 0)
1488		 && ((periph->path->target == NULL)
1489		 ||(cur_pattern->target_id != periph->path->target->target_id)))
1490			continue;
1491
1492		if (((cur_pattern->flags & PERIPH_MATCH_LUN) != 0)
1493		 && ((periph->path->device == NULL)
1494		 || (cur_pattern->target_lun != periph->path->device->lun_id)))
1495			continue;
1496
1497		if (((cur_pattern->flags & PERIPH_MATCH_UNIT) != 0)
1498		 && (cur_pattern->unit_number != periph->unit_number))
1499			continue;
1500
1501		if (((cur_pattern->flags & PERIPH_MATCH_NAME) != 0)
1502		 && (strncmp(cur_pattern->periph_name, periph->periph_name,
1503			     DEV_IDLEN) != 0))
1504			continue;
1505
1506		/*
1507		 * If we get to this point, the user definitely wants
1508		 * information on this peripheral.  So tell the caller to
1509		 * copy the data out.
1510		 */
1511		retval |= DM_RET_COPY;
1512
1513		/*
1514		 * The return action has already been set to stop, since
1515		 * peripherals don't have any nodes below them in the EDT.
1516		 */
1517		return(retval);
1518	}
1519
1520	/*
1521	 * If we get to this point, the peripheral that was passed in
1522	 * doesn't match any of the patterns.
1523	 */
1524	return(retval);
1525}
1526
1527static int
1528xptedtbusfunc(struct cam_eb *bus, void *arg)
1529{
1530	struct ccb_dev_match *cdm;
1531	struct cam_et *target;
1532	dev_match_ret retval;
1533
1534	cdm = (struct ccb_dev_match *)arg;
1535
1536	/*
1537	 * If our position is for something deeper in the tree, that means
1538	 * that we've already seen this node.  So, we keep going down.
1539	 */
1540	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1541	 && (cdm->pos.cookie.bus == bus)
1542	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1543	 && (cdm->pos.cookie.target != NULL))
1544		retval = DM_RET_DESCEND;
1545	else
1546		retval = xptbusmatch(cdm->patterns, cdm->num_patterns, bus);
1547
1548	/*
1549	 * If we got an error, bail out of the search.
1550	 */
1551	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1552		cdm->status = CAM_DEV_MATCH_ERROR;
1553		return(0);
1554	}
1555
1556	/*
1557	 * If the copy flag is set, copy this bus out.
1558	 */
1559	if (retval & DM_RET_COPY) {
1560		int spaceleft, j;
1561
1562		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1563			sizeof(struct dev_match_result));
1564
1565		/*
1566		 * If we don't have enough space to put in another
1567		 * match result, save our position and tell the
1568		 * user there are more devices to check.
1569		 */
1570		if (spaceleft < sizeof(struct dev_match_result)) {
1571			bzero(&cdm->pos, sizeof(cdm->pos));
1572			cdm->pos.position_type =
1573				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS;
1574
1575			cdm->pos.cookie.bus = bus;
1576			cdm->pos.generations[CAM_BUS_GENERATION]=
1577				xsoftc.bus_generation;
1578			cdm->status = CAM_DEV_MATCH_MORE;
1579			return(0);
1580		}
1581		j = cdm->num_matches;
1582		cdm->num_matches++;
1583		cdm->matches[j].type = DEV_MATCH_BUS;
1584		cdm->matches[j].result.bus_result.path_id = bus->path_id;
1585		cdm->matches[j].result.bus_result.bus_id = bus->sim->bus_id;
1586		cdm->matches[j].result.bus_result.unit_number =
1587			bus->sim->unit_number;
1588		strncpy(cdm->matches[j].result.bus_result.dev_name,
1589			bus->sim->sim_name, DEV_IDLEN);
1590	}
1591
1592	/*
1593	 * If the user is only interested in busses, there's no
1594	 * reason to descend to the next level in the tree.
1595	 */
1596	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1597		return(1);
1598
1599	/*
1600	 * If there is a target generation recorded, check it to
1601	 * make sure the target list hasn't changed.
1602	 */
1603	mtx_lock(&bus->eb_mtx);
1604	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1605	 && (cdm->pos.cookie.bus == bus)
1606	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1607	 && (cdm->pos.cookie.target != NULL)) {
1608		if ((cdm->pos.generations[CAM_TARGET_GENERATION] !=
1609		    bus->generation)) {
1610			mtx_unlock(&bus->eb_mtx);
1611			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1612			return (0);
1613		}
1614		target = (struct cam_et *)cdm->pos.cookie.target;
1615		target->refcount++;
1616	} else
1617		target = NULL;
1618	mtx_unlock(&bus->eb_mtx);
1619
1620	return (xpttargettraverse(bus, target, xptedttargetfunc, arg));
1621}
1622
1623static int
1624xptedttargetfunc(struct cam_et *target, void *arg)
1625{
1626	struct ccb_dev_match *cdm;
1627	struct cam_eb *bus;
1628	struct cam_ed *device;
1629
1630	cdm = (struct ccb_dev_match *)arg;
1631	bus = target->bus;
1632
1633	/*
1634	 * If there is a device list generation recorded, check it to
1635	 * make sure the device list hasn't changed.
1636	 */
1637	mtx_lock(&bus->eb_mtx);
1638	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1639	 && (cdm->pos.cookie.bus == bus)
1640	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1641	 && (cdm->pos.cookie.target == target)
1642	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1643	 && (cdm->pos.cookie.device != NULL)) {
1644		if (cdm->pos.generations[CAM_DEV_GENERATION] !=
1645		    target->generation) {
1646			mtx_unlock(&bus->eb_mtx);
1647			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1648			return(0);
1649		}
1650		device = (struct cam_ed *)cdm->pos.cookie.device;
1651		device->refcount++;
1652	} else
1653		device = NULL;
1654	mtx_unlock(&bus->eb_mtx);
1655
1656	return (xptdevicetraverse(target, device, xptedtdevicefunc, arg));
1657}
1658
1659static int
1660xptedtdevicefunc(struct cam_ed *device, void *arg)
1661{
1662	struct cam_eb *bus;
1663	struct cam_periph *periph;
1664	struct ccb_dev_match *cdm;
1665	dev_match_ret retval;
1666
1667	cdm = (struct ccb_dev_match *)arg;
1668	bus = device->target->bus;
1669
1670	/*
1671	 * If our position is for something deeper in the tree, that means
1672	 * that we've already seen this node.  So, we keep going down.
1673	 */
1674	if ((cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1675	 && (cdm->pos.cookie.device == device)
1676	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1677	 && (cdm->pos.cookie.periph != NULL))
1678		retval = DM_RET_DESCEND;
1679	else
1680		retval = xptdevicematch(cdm->patterns, cdm->num_patterns,
1681					device);
1682
1683	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1684		cdm->status = CAM_DEV_MATCH_ERROR;
1685		return(0);
1686	}
1687
1688	/*
1689	 * If the copy flag is set, copy this device out.
1690	 */
1691	if (retval & DM_RET_COPY) {
1692		int spaceleft, j;
1693
1694		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1695			sizeof(struct dev_match_result));
1696
1697		/*
1698		 * If we don't have enough space to put in another
1699		 * match result, save our position and tell the
1700		 * user there are more devices to check.
1701		 */
1702		if (spaceleft < sizeof(struct dev_match_result)) {
1703			bzero(&cdm->pos, sizeof(cdm->pos));
1704			cdm->pos.position_type =
1705				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1706				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE;
1707
1708			cdm->pos.cookie.bus = device->target->bus;
1709			cdm->pos.generations[CAM_BUS_GENERATION]=
1710				xsoftc.bus_generation;
1711			cdm->pos.cookie.target = device->target;
1712			cdm->pos.generations[CAM_TARGET_GENERATION] =
1713				device->target->bus->generation;
1714			cdm->pos.cookie.device = device;
1715			cdm->pos.generations[CAM_DEV_GENERATION] =
1716				device->target->generation;
1717			cdm->status = CAM_DEV_MATCH_MORE;
1718			return(0);
1719		}
1720		j = cdm->num_matches;
1721		cdm->num_matches++;
1722		cdm->matches[j].type = DEV_MATCH_DEVICE;
1723		cdm->matches[j].result.device_result.path_id =
1724			device->target->bus->path_id;
1725		cdm->matches[j].result.device_result.target_id =
1726			device->target->target_id;
1727		cdm->matches[j].result.device_result.target_lun =
1728			device->lun_id;
1729		cdm->matches[j].result.device_result.protocol =
1730			device->protocol;
1731		bcopy(&device->inq_data,
1732		      &cdm->matches[j].result.device_result.inq_data,
1733		      sizeof(struct scsi_inquiry_data));
1734		bcopy(&device->ident_data,
1735		      &cdm->matches[j].result.device_result.ident_data,
1736		      sizeof(struct ata_params));
1737
1738		/* Let the user know whether this device is unconfigured */
1739		if (device->flags & CAM_DEV_UNCONFIGURED)
1740			cdm->matches[j].result.device_result.flags =
1741				DEV_RESULT_UNCONFIGURED;
1742		else
1743			cdm->matches[j].result.device_result.flags =
1744				DEV_RESULT_NOFLAG;
1745	}
1746
1747	/*
1748	 * If the user isn't interested in peripherals, don't descend
1749	 * the tree any further.
1750	 */
1751	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1752		return(1);
1753
1754	/*
1755	 * If there is a peripheral list generation recorded, make sure
1756	 * it hasn't changed.
1757	 */
1758	xpt_lock_buses();
1759	mtx_lock(&bus->eb_mtx);
1760	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1761	 && (cdm->pos.cookie.bus == bus)
1762	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1763	 && (cdm->pos.cookie.target == device->target)
1764	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1765	 && (cdm->pos.cookie.device == device)
1766	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1767	 && (cdm->pos.cookie.periph != NULL)) {
1768		if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1769		    device->generation) {
1770			mtx_unlock(&bus->eb_mtx);
1771			xpt_unlock_buses();
1772			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1773			return(0);
1774		}
1775		periph = (struct cam_periph *)cdm->pos.cookie.periph;
1776		periph->refcount++;
1777	} else
1778		periph = NULL;
1779	mtx_unlock(&bus->eb_mtx);
1780	xpt_unlock_buses();
1781
1782	return (xptperiphtraverse(device, periph, xptedtperiphfunc, arg));
1783}
1784
1785static int
1786xptedtperiphfunc(struct cam_periph *periph, void *arg)
1787{
1788	struct ccb_dev_match *cdm;
1789	dev_match_ret retval;
1790
1791	cdm = (struct ccb_dev_match *)arg;
1792
1793	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1794
1795	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1796		cdm->status = CAM_DEV_MATCH_ERROR;
1797		return(0);
1798	}
1799
1800	/*
1801	 * If the copy flag is set, copy this peripheral out.
1802	 */
1803	if (retval & DM_RET_COPY) {
1804		int spaceleft, j;
1805
1806		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1807			sizeof(struct dev_match_result));
1808
1809		/*
1810		 * If we don't have enough space to put in another
1811		 * match result, save our position and tell the
1812		 * user there are more devices to check.
1813		 */
1814		if (spaceleft < sizeof(struct dev_match_result)) {
1815			bzero(&cdm->pos, sizeof(cdm->pos));
1816			cdm->pos.position_type =
1817				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1818				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE |
1819				CAM_DEV_POS_PERIPH;
1820
1821			cdm->pos.cookie.bus = periph->path->bus;
1822			cdm->pos.generations[CAM_BUS_GENERATION]=
1823				xsoftc.bus_generation;
1824			cdm->pos.cookie.target = periph->path->target;
1825			cdm->pos.generations[CAM_TARGET_GENERATION] =
1826				periph->path->bus->generation;
1827			cdm->pos.cookie.device = periph->path->device;
1828			cdm->pos.generations[CAM_DEV_GENERATION] =
1829				periph->path->target->generation;
1830			cdm->pos.cookie.periph = periph;
1831			cdm->pos.generations[CAM_PERIPH_GENERATION] =
1832				periph->path->device->generation;
1833			cdm->status = CAM_DEV_MATCH_MORE;
1834			return(0);
1835		}
1836
1837		j = cdm->num_matches;
1838		cdm->num_matches++;
1839		cdm->matches[j].type = DEV_MATCH_PERIPH;
1840		cdm->matches[j].result.periph_result.path_id =
1841			periph->path->bus->path_id;
1842		cdm->matches[j].result.periph_result.target_id =
1843			periph->path->target->target_id;
1844		cdm->matches[j].result.periph_result.target_lun =
1845			periph->path->device->lun_id;
1846		cdm->matches[j].result.periph_result.unit_number =
1847			periph->unit_number;
1848		strncpy(cdm->matches[j].result.periph_result.periph_name,
1849			periph->periph_name, DEV_IDLEN);
1850	}
1851
1852	return(1);
1853}
1854
1855static int
1856xptedtmatch(struct ccb_dev_match *cdm)
1857{
1858	struct cam_eb *bus;
1859	int ret;
1860
1861	cdm->num_matches = 0;
1862
1863	/*
1864	 * Check the bus list generation.  If it has changed, the user
1865	 * needs to reset everything and start over.
1866	 */
1867	xpt_lock_buses();
1868	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1869	 && (cdm->pos.cookie.bus != NULL)) {
1870		if (cdm->pos.generations[CAM_BUS_GENERATION] !=
1871		    xsoftc.bus_generation) {
1872			xpt_unlock_buses();
1873			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1874			return(0);
1875		}
1876		bus = (struct cam_eb *)cdm->pos.cookie.bus;
1877		bus->refcount++;
1878	} else
1879		bus = NULL;
1880	xpt_unlock_buses();
1881
1882	ret = xptbustraverse(bus, xptedtbusfunc, cdm);
1883
1884	/*
1885	 * If we get back 0, that means that we had to stop before fully
1886	 * traversing the EDT.  It also means that one of the subroutines
1887	 * has set the status field to the proper value.  If we get back 1,
1888	 * we've fully traversed the EDT and copied out any matching entries.
1889	 */
1890	if (ret == 1)
1891		cdm->status = CAM_DEV_MATCH_LAST;
1892
1893	return(ret);
1894}
1895
1896static int
1897xptplistpdrvfunc(struct periph_driver **pdrv, void *arg)
1898{
1899	struct cam_periph *periph;
1900	struct ccb_dev_match *cdm;
1901
1902	cdm = (struct ccb_dev_match *)arg;
1903
1904	xpt_lock_buses();
1905	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
1906	 && (cdm->pos.cookie.pdrv == pdrv)
1907	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1908	 && (cdm->pos.cookie.periph != NULL)) {
1909		if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1910		    (*pdrv)->generation) {
1911			xpt_unlock_buses();
1912			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1913			return(0);
1914		}
1915		periph = (struct cam_periph *)cdm->pos.cookie.periph;
1916		periph->refcount++;
1917	} else
1918		periph = NULL;
1919	xpt_unlock_buses();
1920
1921	return (xptpdperiphtraverse(pdrv, periph, xptplistperiphfunc, arg));
1922}
1923
1924static int
1925xptplistperiphfunc(struct cam_periph *periph, void *arg)
1926{
1927	struct ccb_dev_match *cdm;
1928	dev_match_ret retval;
1929
1930	cdm = (struct ccb_dev_match *)arg;
1931
1932	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1933
1934	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1935		cdm->status = CAM_DEV_MATCH_ERROR;
1936		return(0);
1937	}
1938
1939	/*
1940	 * If the copy flag is set, copy this peripheral out.
1941	 */
1942	if (retval & DM_RET_COPY) {
1943		int spaceleft, j;
1944
1945		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1946			sizeof(struct dev_match_result));
1947
1948		/*
1949		 * If we don't have enough space to put in another
1950		 * match result, save our position and tell the
1951		 * user there are more devices to check.
1952		 */
1953		if (spaceleft < sizeof(struct dev_match_result)) {
1954			struct periph_driver **pdrv;
1955
1956			pdrv = NULL;
1957			bzero(&cdm->pos, sizeof(cdm->pos));
1958			cdm->pos.position_type =
1959				CAM_DEV_POS_PDRV | CAM_DEV_POS_PDPTR |
1960				CAM_DEV_POS_PERIPH;
1961
1962			/*
1963			 * This may look a bit non-sensical, but it is
1964			 * actually quite logical.  There are very few
1965			 * peripheral drivers, and bloating every peripheral
1966			 * structure with a pointer back to its parent
1967			 * peripheral driver linker set entry would cost
1968			 * more in the long run than doing this quick lookup.
1969			 */
1970			for (pdrv = periph_drivers; *pdrv != NULL; pdrv++) {
1971				if (strcmp((*pdrv)->driver_name,
1972				    periph->periph_name) == 0)
1973					break;
1974			}
1975
1976			if (*pdrv == NULL) {
1977				cdm->status = CAM_DEV_MATCH_ERROR;
1978				return(0);
1979			}
1980
1981			cdm->pos.cookie.pdrv = pdrv;
1982			/*
1983			 * The periph generation slot does double duty, as
1984			 * does the periph pointer slot.  They are used for
1985			 * both edt and pdrv lookups and positioning.
1986			 */
1987			cdm->pos.cookie.periph = periph;
1988			cdm->pos.generations[CAM_PERIPH_GENERATION] =
1989				(*pdrv)->generation;
1990			cdm->status = CAM_DEV_MATCH_MORE;
1991			return(0);
1992		}
1993
1994		j = cdm->num_matches;
1995		cdm->num_matches++;
1996		cdm->matches[j].type = DEV_MATCH_PERIPH;
1997		cdm->matches[j].result.periph_result.path_id =
1998			periph->path->bus->path_id;
1999
2000		/*
2001		 * The transport layer peripheral doesn't have a target or
2002		 * lun.
2003		 */
2004		if (periph->path->target)
2005			cdm->matches[j].result.periph_result.target_id =
2006				periph->path->target->target_id;
2007		else
2008			cdm->matches[j].result.periph_result.target_id =
2009				CAM_TARGET_WILDCARD;
2010
2011		if (periph->path->device)
2012			cdm->matches[j].result.periph_result.target_lun =
2013				periph->path->device->lun_id;
2014		else
2015			cdm->matches[j].result.periph_result.target_lun =
2016				CAM_LUN_WILDCARD;
2017
2018		cdm->matches[j].result.periph_result.unit_number =
2019			periph->unit_number;
2020		strncpy(cdm->matches[j].result.periph_result.periph_name,
2021			periph->periph_name, DEV_IDLEN);
2022	}
2023
2024	return(1);
2025}
2026
2027static int
2028xptperiphlistmatch(struct ccb_dev_match *cdm)
2029{
2030	int ret;
2031
2032	cdm->num_matches = 0;
2033
2034	/*
2035	 * At this point in the edt traversal function, we check the bus
2036	 * list generation to make sure that no busses have been added or
2037	 * removed since the user last sent a XPT_DEV_MATCH ccb through.
2038	 * For the peripheral driver list traversal function, however, we
2039	 * don't have to worry about new peripheral driver types coming or
2040	 * going; they're in a linker set, and therefore can't change
2041	 * without a recompile.
2042	 */
2043
2044	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2045	 && (cdm->pos.cookie.pdrv != NULL))
2046		ret = xptpdrvtraverse(
2047				(struct periph_driver **)cdm->pos.cookie.pdrv,
2048				xptplistpdrvfunc, cdm);
2049	else
2050		ret = xptpdrvtraverse(NULL, xptplistpdrvfunc, cdm);
2051
2052	/*
2053	 * If we get back 0, that means that we had to stop before fully
2054	 * traversing the peripheral driver tree.  It also means that one of
2055	 * the subroutines has set the status field to the proper value.  If
2056	 * we get back 1, we've fully traversed the EDT and copied out any
2057	 * matching entries.
2058	 */
2059	if (ret == 1)
2060		cdm->status = CAM_DEV_MATCH_LAST;
2061
2062	return(ret);
2063}
2064
2065static int
2066xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
2067{
2068	struct cam_eb *bus, *next_bus;
2069	int retval;
2070
2071	retval = 1;
2072	if (start_bus)
2073		bus = start_bus;
2074	else {
2075		xpt_lock_buses();
2076		bus = TAILQ_FIRST(&xsoftc.xpt_busses);
2077		if (bus == NULL) {
2078			xpt_unlock_buses();
2079			return (retval);
2080		}
2081		bus->refcount++;
2082		xpt_unlock_buses();
2083	}
2084	for (; bus != NULL; bus = next_bus) {
2085		retval = tr_func(bus, arg);
2086		if (retval == 0) {
2087			xpt_release_bus(bus);
2088			break;
2089		}
2090		xpt_lock_buses();
2091		next_bus = TAILQ_NEXT(bus, links);
2092		if (next_bus)
2093			next_bus->refcount++;
2094		xpt_unlock_buses();
2095		xpt_release_bus(bus);
2096	}
2097	return(retval);
2098}
2099
2100static int
2101xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target,
2102		  xpt_targetfunc_t *tr_func, void *arg)
2103{
2104	struct cam_et *target, *next_target;
2105	int retval;
2106
2107	retval = 1;
2108	if (start_target)
2109		target = start_target;
2110	else {
2111		mtx_lock(&bus->eb_mtx);
2112		target = TAILQ_FIRST(&bus->et_entries);
2113		if (target == NULL) {
2114			mtx_unlock(&bus->eb_mtx);
2115			return (retval);
2116		}
2117		target->refcount++;
2118		mtx_unlock(&bus->eb_mtx);
2119	}
2120	for (; target != NULL; target = next_target) {
2121		retval = tr_func(target, arg);
2122		if (retval == 0) {
2123			xpt_release_target(target);
2124			break;
2125		}
2126		mtx_lock(&bus->eb_mtx);
2127		next_target = TAILQ_NEXT(target, links);
2128		if (next_target)
2129			next_target->refcount++;
2130		mtx_unlock(&bus->eb_mtx);
2131		xpt_release_target(target);
2132	}
2133	return(retval);
2134}
2135
2136static int
2137xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device,
2138		  xpt_devicefunc_t *tr_func, void *arg)
2139{
2140	struct cam_eb *bus;
2141	struct cam_ed *device, *next_device;
2142	int retval;
2143
2144	retval = 1;
2145	bus = target->bus;
2146	if (start_device)
2147		device = start_device;
2148	else {
2149		mtx_lock(&bus->eb_mtx);
2150		device = TAILQ_FIRST(&target->ed_entries);
2151		if (device == NULL) {
2152			mtx_unlock(&bus->eb_mtx);
2153			return (retval);
2154		}
2155		device->refcount++;
2156		mtx_unlock(&bus->eb_mtx);
2157	}
2158	for (; device != NULL; device = next_device) {
2159		mtx_lock(&device->device_mtx);
2160		retval = tr_func(device, arg);
2161		mtx_unlock(&device->device_mtx);
2162		if (retval == 0) {
2163			xpt_release_device(device);
2164			break;
2165		}
2166		mtx_lock(&bus->eb_mtx);
2167		next_device = TAILQ_NEXT(device, links);
2168		if (next_device)
2169			next_device->refcount++;
2170		mtx_unlock(&bus->eb_mtx);
2171		xpt_release_device(device);
2172	}
2173	return(retval);
2174}
2175
2176static int
2177xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph,
2178		  xpt_periphfunc_t *tr_func, void *arg)
2179{
2180	struct cam_eb *bus;
2181	struct cam_periph *periph, *next_periph;
2182	int retval;
2183
2184	retval = 1;
2185
2186	bus = device->target->bus;
2187	if (start_periph)
2188		periph = start_periph;
2189	else {
2190		xpt_lock_buses();
2191		mtx_lock(&bus->eb_mtx);
2192		periph = SLIST_FIRST(&device->periphs);
2193		while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2194			periph = SLIST_NEXT(periph, periph_links);
2195		if (periph == NULL) {
2196			mtx_unlock(&bus->eb_mtx);
2197			xpt_unlock_buses();
2198			return (retval);
2199		}
2200		periph->refcount++;
2201		mtx_unlock(&bus->eb_mtx);
2202		xpt_unlock_buses();
2203	}
2204	for (; periph != NULL; periph = next_periph) {
2205		retval = tr_func(periph, arg);
2206		if (retval == 0) {
2207			cam_periph_release_locked(periph);
2208			break;
2209		}
2210		xpt_lock_buses();
2211		mtx_lock(&bus->eb_mtx);
2212		next_periph = SLIST_NEXT(periph, periph_links);
2213		while (next_periph != NULL &&
2214		    (next_periph->flags & CAM_PERIPH_FREE) != 0)
2215			next_periph = SLIST_NEXT(next_periph, periph_links);
2216		if (next_periph)
2217			next_periph->refcount++;
2218		mtx_unlock(&bus->eb_mtx);
2219		xpt_unlock_buses();
2220		cam_periph_release_locked(periph);
2221	}
2222	return(retval);
2223}
2224
2225static int
2226xptpdrvtraverse(struct periph_driver **start_pdrv,
2227		xpt_pdrvfunc_t *tr_func, void *arg)
2228{
2229	struct periph_driver **pdrv;
2230	int retval;
2231
2232	retval = 1;
2233
2234	/*
2235	 * We don't traverse the peripheral driver list like we do the
2236	 * other lists, because it is a linker set, and therefore cannot be
2237	 * changed during runtime.  If the peripheral driver list is ever
2238	 * re-done to be something other than a linker set (i.e. it can
2239	 * change while the system is running), the list traversal should
2240	 * be modified to work like the other traversal functions.
2241	 */
2242	for (pdrv = (start_pdrv ? start_pdrv : periph_drivers);
2243	     *pdrv != NULL; pdrv++) {
2244		retval = tr_func(pdrv, arg);
2245
2246		if (retval == 0)
2247			return(retval);
2248	}
2249
2250	return(retval);
2251}
2252
2253static int
2254xptpdperiphtraverse(struct periph_driver **pdrv,
2255		    struct cam_periph *start_periph,
2256		    xpt_periphfunc_t *tr_func, void *arg)
2257{
2258	struct cam_periph *periph, *next_periph;
2259	int retval;
2260
2261	retval = 1;
2262
2263	if (start_periph)
2264		periph = start_periph;
2265	else {
2266		xpt_lock_buses();
2267		periph = TAILQ_FIRST(&(*pdrv)->units);
2268		while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2269			periph = TAILQ_NEXT(periph, unit_links);
2270		if (periph == NULL) {
2271			xpt_unlock_buses();
2272			return (retval);
2273		}
2274		periph->refcount++;
2275		xpt_unlock_buses();
2276	}
2277	for (; periph != NULL; periph = next_periph) {
2278		cam_periph_lock(periph);
2279		retval = tr_func(periph, arg);
2280		cam_periph_unlock(periph);
2281		if (retval == 0) {
2282			cam_periph_release(periph);
2283			break;
2284		}
2285		xpt_lock_buses();
2286		next_periph = TAILQ_NEXT(periph, unit_links);
2287		while (next_periph != NULL &&
2288		    (next_periph->flags & CAM_PERIPH_FREE) != 0)
2289			next_periph = TAILQ_NEXT(next_periph, unit_links);
2290		if (next_periph)
2291			next_periph->refcount++;
2292		xpt_unlock_buses();
2293		cam_periph_release(periph);
2294	}
2295	return(retval);
2296}
2297
2298static int
2299xptdefbusfunc(struct cam_eb *bus, void *arg)
2300{
2301	struct xpt_traverse_config *tr_config;
2302
2303	tr_config = (struct xpt_traverse_config *)arg;
2304
2305	if (tr_config->depth == XPT_DEPTH_BUS) {
2306		xpt_busfunc_t *tr_func;
2307
2308		tr_func = (xpt_busfunc_t *)tr_config->tr_func;
2309
2310		return(tr_func(bus, tr_config->tr_arg));
2311	} else
2312		return(xpttargettraverse(bus, NULL, xptdeftargetfunc, arg));
2313}
2314
2315static int
2316xptdeftargetfunc(struct cam_et *target, void *arg)
2317{
2318	struct xpt_traverse_config *tr_config;
2319
2320	tr_config = (struct xpt_traverse_config *)arg;
2321
2322	if (tr_config->depth == XPT_DEPTH_TARGET) {
2323		xpt_targetfunc_t *tr_func;
2324
2325		tr_func = (xpt_targetfunc_t *)tr_config->tr_func;
2326
2327		return(tr_func(target, tr_config->tr_arg));
2328	} else
2329		return(xptdevicetraverse(target, NULL, xptdefdevicefunc, arg));
2330}
2331
2332static int
2333xptdefdevicefunc(struct cam_ed *device, void *arg)
2334{
2335	struct xpt_traverse_config *tr_config;
2336
2337	tr_config = (struct xpt_traverse_config *)arg;
2338
2339	if (tr_config->depth == XPT_DEPTH_DEVICE) {
2340		xpt_devicefunc_t *tr_func;
2341
2342		tr_func = (xpt_devicefunc_t *)tr_config->tr_func;
2343
2344		return(tr_func(device, tr_config->tr_arg));
2345	} else
2346		return(xptperiphtraverse(device, NULL, xptdefperiphfunc, arg));
2347}
2348
2349static int
2350xptdefperiphfunc(struct cam_periph *periph, void *arg)
2351{
2352	struct xpt_traverse_config *tr_config;
2353	xpt_periphfunc_t *tr_func;
2354
2355	tr_config = (struct xpt_traverse_config *)arg;
2356
2357	tr_func = (xpt_periphfunc_t *)tr_config->tr_func;
2358
2359	/*
2360	 * Unlike the other default functions, we don't check for depth
2361	 * here.  The peripheral driver level is the last level in the EDT,
2362	 * so if we're here, we should execute the function in question.
2363	 */
2364	return(tr_func(periph, tr_config->tr_arg));
2365}
2366
2367/*
2368 * Execute the given function for every bus in the EDT.
2369 */
2370static int
2371xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
2372{
2373	struct xpt_traverse_config tr_config;
2374
2375	tr_config.depth = XPT_DEPTH_BUS;
2376	tr_config.tr_func = tr_func;
2377	tr_config.tr_arg = arg;
2378
2379	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2380}
2381
2382/*
2383 * Execute the given function for every device in the EDT.
2384 */
2385static int
2386xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
2387{
2388	struct xpt_traverse_config tr_config;
2389
2390	tr_config.depth = XPT_DEPTH_DEVICE;
2391	tr_config.tr_func = tr_func;
2392	tr_config.tr_arg = arg;
2393
2394	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2395}
2396
2397static int
2398xptsetasyncfunc(struct cam_ed *device, void *arg)
2399{
2400	struct cam_path path;
2401	struct ccb_getdev cgd;
2402	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2403
2404	/*
2405	 * Don't report unconfigured devices (Wildcard devs,
2406	 * devices only for target mode, device instances
2407	 * that have been invalidated but are waiting for
2408	 * their last reference count to be released).
2409	 */
2410	if ((device->flags & CAM_DEV_UNCONFIGURED) != 0)
2411		return (1);
2412
2413	xpt_compile_path(&path,
2414			 NULL,
2415			 device->target->bus->path_id,
2416			 device->target->target_id,
2417			 device->lun_id);
2418	xpt_setup_ccb(&cgd.ccb_h, &path, CAM_PRIORITY_NORMAL);
2419	cgd.ccb_h.func_code = XPT_GDEV_TYPE;
2420	xpt_action((union ccb *)&cgd);
2421	csa->callback(csa->callback_arg,
2422			    AC_FOUND_DEVICE,
2423			    &path, &cgd);
2424	xpt_release_path(&path);
2425
2426	return(1);
2427}
2428
2429static int
2430xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
2431{
2432	struct cam_path path;
2433	struct ccb_pathinq cpi;
2434	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2435
2436	xpt_compile_path(&path, /*periph*/NULL,
2437			 bus->path_id,
2438			 CAM_TARGET_WILDCARD,
2439			 CAM_LUN_WILDCARD);
2440	xpt_path_lock(&path);
2441	xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL);
2442	cpi.ccb_h.func_code = XPT_PATH_INQ;
2443	xpt_action((union ccb *)&cpi);
2444	csa->callback(csa->callback_arg,
2445			    AC_PATH_REGISTERED,
2446			    &path, &cpi);
2447	xpt_path_unlock(&path);
2448	xpt_release_path(&path);
2449
2450	return(1);
2451}
2452
2453void
2454xpt_action(union ccb *start_ccb)
2455{
2456
2457	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_action\n"));
2458
2459	start_ccb->ccb_h.status = CAM_REQ_INPROG;
2460	(*(start_ccb->ccb_h.path->bus->xport->action))(start_ccb);
2461}
2462
2463void
2464xpt_action_default(union ccb *start_ccb)
2465{
2466	struct cam_path *path;
2467	struct cam_sim *sim;
2468	int lock;
2469
2470	path = start_ccb->ccb_h.path;
2471	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_action_default\n"));
2472
2473	switch (start_ccb->ccb_h.func_code) {
2474	case XPT_SCSI_IO:
2475	{
2476		struct cam_ed *device;
2477
2478		/*
2479		 * For the sake of compatibility with SCSI-1
2480		 * devices that may not understand the identify
2481		 * message, we include lun information in the
2482		 * second byte of all commands.  SCSI-1 specifies
2483		 * that luns are a 3 bit value and reserves only 3
2484		 * bits for lun information in the CDB.  Later
2485		 * revisions of the SCSI spec allow for more than 8
2486		 * luns, but have deprecated lun information in the
2487		 * CDB.  So, if the lun won't fit, we must omit.
2488		 *
2489		 * Also be aware that during initial probing for devices,
2490		 * the inquiry information is unknown but initialized to 0.
2491		 * This means that this code will be exercised while probing
2492		 * devices with an ANSI revision greater than 2.
2493		 */
2494		device = path->device;
2495		if (device->protocol_version <= SCSI_REV_2
2496		 && start_ccb->ccb_h.target_lun < 8
2497		 && (start_ccb->ccb_h.flags & CAM_CDB_POINTER) == 0) {
2498
2499			start_ccb->csio.cdb_io.cdb_bytes[1] |=
2500			    start_ccb->ccb_h.target_lun << 5;
2501		}
2502		start_ccb->csio.scsi_status = SCSI_STATUS_OK;
2503	}
2504	/* FALLTHROUGH */
2505	case XPT_TARGET_IO:
2506	case XPT_CONT_TARGET_IO:
2507		start_ccb->csio.sense_resid = 0;
2508		start_ccb->csio.resid = 0;
2509		/* FALLTHROUGH */
2510	case XPT_ATA_IO:
2511		if (start_ccb->ccb_h.func_code == XPT_ATA_IO)
2512			start_ccb->ataio.resid = 0;
2513		/* FALLTHROUGH */
2514	case XPT_RESET_DEV:
2515	case XPT_ENG_EXEC:
2516	case XPT_SMP_IO:
2517	{
2518		struct cam_devq *devq;
2519
2520		devq = path->bus->sim->devq;
2521		mtx_lock(&devq->send_mtx);
2522		cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
2523		if (xpt_schedule_devq(devq, path->device) != 0)
2524			xpt_run_devq(devq);
2525		mtx_unlock(&devq->send_mtx);
2526		break;
2527	}
2528	case XPT_CALC_GEOMETRY:
2529		/* Filter out garbage */
2530		if (start_ccb->ccg.block_size == 0
2531		 || start_ccb->ccg.volume_size == 0) {
2532			start_ccb->ccg.cylinders = 0;
2533			start_ccb->ccg.heads = 0;
2534			start_ccb->ccg.secs_per_track = 0;
2535			start_ccb->ccb_h.status = CAM_REQ_CMP;
2536			break;
2537		}
2538#if defined(PC98) || defined(__sparc64__)
2539		/*
2540		 * In a PC-98 system, geometry translation depens on
2541		 * the "real" device geometry obtained from mode page 4.
2542		 * SCSI geometry translation is performed in the
2543		 * initialization routine of the SCSI BIOS and the result
2544		 * stored in host memory.  If the translation is available
2545		 * in host memory, use it.  If not, rely on the default
2546		 * translation the device driver performs.
2547		 * For sparc64, we may need adjust the geometry of large
2548		 * disks in order to fit the limitations of the 16-bit
2549		 * fields of the VTOC8 disk label.
2550		 */
2551		if (scsi_da_bios_params(&start_ccb->ccg) != 0) {
2552			start_ccb->ccb_h.status = CAM_REQ_CMP;
2553			break;
2554		}
2555#endif
2556		goto call_sim;
2557	case XPT_ABORT:
2558	{
2559		union ccb* abort_ccb;
2560
2561		abort_ccb = start_ccb->cab.abort_ccb;
2562		if (XPT_FC_IS_DEV_QUEUED(abort_ccb)) {
2563
2564			if (abort_ccb->ccb_h.pinfo.index >= 0) {
2565				struct cam_ccbq *ccbq;
2566				struct cam_ed *device;
2567
2568				device = abort_ccb->ccb_h.path->device;
2569				ccbq = &device->ccbq;
2570				cam_ccbq_remove_ccb(ccbq, abort_ccb);
2571				abort_ccb->ccb_h.status =
2572				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2573				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2574				xpt_done(abort_ccb);
2575				start_ccb->ccb_h.status = CAM_REQ_CMP;
2576				break;
2577			}
2578			if (abort_ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX
2579			 && (abort_ccb->ccb_h.status & CAM_SIM_QUEUED) == 0) {
2580				/*
2581				 * We've caught this ccb en route to
2582				 * the SIM.  Flag it for abort and the
2583				 * SIM will do so just before starting
2584				 * real work on the CCB.
2585				 */
2586				abort_ccb->ccb_h.status =
2587				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2588				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2589				start_ccb->ccb_h.status = CAM_REQ_CMP;
2590				break;
2591			}
2592		}
2593		if (XPT_FC_IS_QUEUED(abort_ccb)
2594		 && (abort_ccb->ccb_h.pinfo.index == CAM_DONEQ_INDEX)) {
2595			/*
2596			 * It's already completed but waiting
2597			 * for our SWI to get to it.
2598			 */
2599			start_ccb->ccb_h.status = CAM_UA_ABORT;
2600			break;
2601		}
2602		/*
2603		 * If we weren't able to take care of the abort request
2604		 * in the XPT, pass the request down to the SIM for processing.
2605		 */
2606	}
2607	/* FALLTHROUGH */
2608	case XPT_ACCEPT_TARGET_IO:
2609	case XPT_EN_LUN:
2610	case XPT_IMMED_NOTIFY:
2611	case XPT_NOTIFY_ACK:
2612	case XPT_RESET_BUS:
2613	case XPT_IMMEDIATE_NOTIFY:
2614	case XPT_NOTIFY_ACKNOWLEDGE:
2615	case XPT_GET_SIM_KNOB:
2616	case XPT_SET_SIM_KNOB:
2617	case XPT_GET_TRAN_SETTINGS:
2618	case XPT_SET_TRAN_SETTINGS:
2619	case XPT_PATH_INQ:
2620call_sim:
2621		sim = path->bus->sim;
2622		lock = (mtx_owned(sim->mtx) == 0);
2623		if (lock)
2624			CAM_SIM_LOCK(sim);
2625		(*(sim->sim_action))(sim, start_ccb);
2626		if (lock)
2627			CAM_SIM_UNLOCK(sim);
2628		break;
2629	case XPT_PATH_STATS:
2630		start_ccb->cpis.last_reset = path->bus->last_reset;
2631		start_ccb->ccb_h.status = CAM_REQ_CMP;
2632		break;
2633	case XPT_GDEV_TYPE:
2634	{
2635		struct cam_ed *dev;
2636
2637		dev = path->device;
2638		if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2639			start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2640		} else {
2641			struct ccb_getdev *cgd;
2642
2643			cgd = &start_ccb->cgd;
2644			cgd->protocol = dev->protocol;
2645			cgd->inq_data = dev->inq_data;
2646			cgd->ident_data = dev->ident_data;
2647			cgd->inq_flags = dev->inq_flags;
2648			cgd->ccb_h.status = CAM_REQ_CMP;
2649			cgd->serial_num_len = dev->serial_num_len;
2650			if ((dev->serial_num_len > 0)
2651			 && (dev->serial_num != NULL))
2652				bcopy(dev->serial_num, cgd->serial_num,
2653				      dev->serial_num_len);
2654		}
2655		break;
2656	}
2657	case XPT_GDEV_STATS:
2658	{
2659		struct cam_ed *dev;
2660
2661		dev = path->device;
2662		if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2663			start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2664		} else {
2665			struct ccb_getdevstats *cgds;
2666			struct cam_eb *bus;
2667			struct cam_et *tar;
2668			struct cam_devq *devq;
2669
2670			cgds = &start_ccb->cgds;
2671			bus = path->bus;
2672			tar = path->target;
2673			devq = bus->sim->devq;
2674			mtx_lock(&devq->send_mtx);
2675			cgds->dev_openings = dev->ccbq.dev_openings;
2676			cgds->dev_active = dev->ccbq.dev_active;
2677			cgds->allocated = dev->ccbq.allocated;
2678			cgds->queued = cam_ccbq_pending_ccb_count(&dev->ccbq);
2679			cgds->held = cgds->allocated - cgds->dev_active -
2680			    cgds->queued;
2681			cgds->last_reset = tar->last_reset;
2682			cgds->maxtags = dev->maxtags;
2683			cgds->mintags = dev->mintags;
2684			if (timevalcmp(&tar->last_reset, &bus->last_reset, <))
2685				cgds->last_reset = bus->last_reset;
2686			mtx_unlock(&devq->send_mtx);
2687			cgds->ccb_h.status = CAM_REQ_CMP;
2688		}
2689		break;
2690	}
2691	case XPT_GDEVLIST:
2692	{
2693		struct cam_periph	*nperiph;
2694		struct periph_list	*periph_head;
2695		struct ccb_getdevlist	*cgdl;
2696		u_int			i;
2697		struct cam_ed		*device;
2698		int			found;
2699
2700
2701		found = 0;
2702
2703		/*
2704		 * Don't want anyone mucking with our data.
2705		 */
2706		device = path->device;
2707		periph_head = &device->periphs;
2708		cgdl = &start_ccb->cgdl;
2709
2710		/*
2711		 * Check and see if the list has changed since the user
2712		 * last requested a list member.  If so, tell them that the
2713		 * list has changed, and therefore they need to start over
2714		 * from the beginning.
2715		 */
2716		if ((cgdl->index != 0) &&
2717		    (cgdl->generation != device->generation)) {
2718			cgdl->status = CAM_GDEVLIST_LIST_CHANGED;
2719			break;
2720		}
2721
2722		/*
2723		 * Traverse the list of peripherals and attempt to find
2724		 * the requested peripheral.
2725		 */
2726		for (nperiph = SLIST_FIRST(periph_head), i = 0;
2727		     (nperiph != NULL) && (i <= cgdl->index);
2728		     nperiph = SLIST_NEXT(nperiph, periph_links), i++) {
2729			if (i == cgdl->index) {
2730				strncpy(cgdl->periph_name,
2731					nperiph->periph_name,
2732					DEV_IDLEN);
2733				cgdl->unit_number = nperiph->unit_number;
2734				found = 1;
2735			}
2736		}
2737		if (found == 0) {
2738			cgdl->status = CAM_GDEVLIST_ERROR;
2739			break;
2740		}
2741
2742		if (nperiph == NULL)
2743			cgdl->status = CAM_GDEVLIST_LAST_DEVICE;
2744		else
2745			cgdl->status = CAM_GDEVLIST_MORE_DEVS;
2746
2747		cgdl->index++;
2748		cgdl->generation = device->generation;
2749
2750		cgdl->ccb_h.status = CAM_REQ_CMP;
2751		break;
2752	}
2753	case XPT_DEV_MATCH:
2754	{
2755		dev_pos_type position_type;
2756		struct ccb_dev_match *cdm;
2757
2758		cdm = &start_ccb->cdm;
2759
2760		/*
2761		 * There are two ways of getting at information in the EDT.
2762		 * The first way is via the primary EDT tree.  It starts
2763		 * with a list of busses, then a list of targets on a bus,
2764		 * then devices/luns on a target, and then peripherals on a
2765		 * device/lun.  The "other" way is by the peripheral driver
2766		 * lists.  The peripheral driver lists are organized by
2767		 * peripheral driver.  (obviously)  So it makes sense to
2768		 * use the peripheral driver list if the user is looking
2769		 * for something like "da1", or all "da" devices.  If the
2770		 * user is looking for something on a particular bus/target
2771		 * or lun, it's generally better to go through the EDT tree.
2772		 */
2773
2774		if (cdm->pos.position_type != CAM_DEV_POS_NONE)
2775			position_type = cdm->pos.position_type;
2776		else {
2777			u_int i;
2778
2779			position_type = CAM_DEV_POS_NONE;
2780
2781			for (i = 0; i < cdm->num_patterns; i++) {
2782				if ((cdm->patterns[i].type == DEV_MATCH_BUS)
2783				 ||(cdm->patterns[i].type == DEV_MATCH_DEVICE)){
2784					position_type = CAM_DEV_POS_EDT;
2785					break;
2786				}
2787			}
2788
2789			if (cdm->num_patterns == 0)
2790				position_type = CAM_DEV_POS_EDT;
2791			else if (position_type == CAM_DEV_POS_NONE)
2792				position_type = CAM_DEV_POS_PDRV;
2793		}
2794
2795		switch(position_type & CAM_DEV_POS_TYPEMASK) {
2796		case CAM_DEV_POS_EDT:
2797			xptedtmatch(cdm);
2798			break;
2799		case CAM_DEV_POS_PDRV:
2800			xptperiphlistmatch(cdm);
2801			break;
2802		default:
2803			cdm->status = CAM_DEV_MATCH_ERROR;
2804			break;
2805		}
2806
2807		if (cdm->status == CAM_DEV_MATCH_ERROR)
2808			start_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2809		else
2810			start_ccb->ccb_h.status = CAM_REQ_CMP;
2811
2812		break;
2813	}
2814	case XPT_SASYNC_CB:
2815	{
2816		struct ccb_setasync *csa;
2817		struct async_node *cur_entry;
2818		struct async_list *async_head;
2819		u_int32_t added;
2820
2821		csa = &start_ccb->csa;
2822		added = csa->event_enable;
2823		async_head = &path->device->asyncs;
2824
2825		/*
2826		 * If there is already an entry for us, simply
2827		 * update it.
2828		 */
2829		cur_entry = SLIST_FIRST(async_head);
2830		while (cur_entry != NULL) {
2831			if ((cur_entry->callback_arg == csa->callback_arg)
2832			 && (cur_entry->callback == csa->callback))
2833				break;
2834			cur_entry = SLIST_NEXT(cur_entry, links);
2835		}
2836
2837		if (cur_entry != NULL) {
2838		 	/*
2839			 * If the request has no flags set,
2840			 * remove the entry.
2841			 */
2842			added &= ~cur_entry->event_enable;
2843			if (csa->event_enable == 0) {
2844				SLIST_REMOVE(async_head, cur_entry,
2845					     async_node, links);
2846				xpt_release_device(path->device);
2847				free(cur_entry, M_CAMXPT);
2848			} else {
2849				cur_entry->event_enable = csa->event_enable;
2850			}
2851			csa->event_enable = added;
2852		} else {
2853			cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
2854					   M_NOWAIT);
2855			if (cur_entry == NULL) {
2856				csa->ccb_h.status = CAM_RESRC_UNAVAIL;
2857				break;
2858			}
2859			cur_entry->event_enable = csa->event_enable;
2860			cur_entry->event_lock =
2861			    mtx_owned(path->bus->sim->mtx) ? 1 : 0;
2862			cur_entry->callback_arg = csa->callback_arg;
2863			cur_entry->callback = csa->callback;
2864			SLIST_INSERT_HEAD(async_head, cur_entry, links);
2865			xpt_acquire_device(path->device);
2866		}
2867		start_ccb->ccb_h.status = CAM_REQ_CMP;
2868		break;
2869	}
2870	case XPT_REL_SIMQ:
2871	{
2872		struct ccb_relsim *crs;
2873		struct cam_ed *dev;
2874
2875		crs = &start_ccb->crs;
2876		dev = path->device;
2877		if (dev == NULL) {
2878
2879			crs->ccb_h.status = CAM_DEV_NOT_THERE;
2880			break;
2881		}
2882
2883		if ((crs->release_flags & RELSIM_ADJUST_OPENINGS) != 0) {
2884
2885			/* Don't ever go below one opening */
2886			if (crs->openings > 0) {
2887				xpt_dev_ccbq_resize(path, crs->openings);
2888				if (bootverbose) {
2889					xpt_print(path,
2890					    "number of openings is now %d\n",
2891					    crs->openings);
2892				}
2893			}
2894		}
2895
2896		mtx_lock(&dev->sim->devq->send_mtx);
2897		if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) {
2898
2899			if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
2900
2901				/*
2902				 * Just extend the old timeout and decrement
2903				 * the freeze count so that a single timeout
2904				 * is sufficient for releasing the queue.
2905				 */
2906				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2907				callout_stop(&dev->callout);
2908			} else {
2909
2910				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2911			}
2912
2913			callout_reset_sbt(&dev->callout,
2914			    SBT_1MS * crs->release_timeout, 0,
2915			    xpt_release_devq_timeout, dev, 0);
2916
2917			dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
2918
2919		}
2920
2921		if ((crs->release_flags & RELSIM_RELEASE_AFTER_CMDCMPLT) != 0) {
2922
2923			if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0) {
2924				/*
2925				 * Decrement the freeze count so that a single
2926				 * completion is still sufficient to unfreeze
2927				 * the queue.
2928				 */
2929				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2930			} else {
2931
2932				dev->flags |= CAM_DEV_REL_ON_COMPLETE;
2933				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2934			}
2935		}
2936
2937		if ((crs->release_flags & RELSIM_RELEASE_AFTER_QEMPTY) != 0) {
2938
2939			if ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
2940			 || (dev->ccbq.dev_active == 0)) {
2941
2942				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2943			} else {
2944
2945				dev->flags |= CAM_DEV_REL_ON_QUEUE_EMPTY;
2946				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2947			}
2948		}
2949		mtx_unlock(&dev->sim->devq->send_mtx);
2950
2951		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0)
2952			xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
2953		start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt;
2954		start_ccb->ccb_h.status = CAM_REQ_CMP;
2955		break;
2956	}
2957	case XPT_DEBUG: {
2958		struct cam_path *oldpath;
2959
2960		/* Check that all request bits are supported. */
2961		if (start_ccb->cdbg.flags & ~(CAM_DEBUG_COMPILE)) {
2962			start_ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2963			break;
2964		}
2965
2966		cam_dflags = CAM_DEBUG_NONE;
2967		if (cam_dpath != NULL) {
2968			oldpath = cam_dpath;
2969			cam_dpath = NULL;
2970			xpt_free_path(oldpath);
2971		}
2972		if (start_ccb->cdbg.flags != CAM_DEBUG_NONE) {
2973			if (xpt_create_path(&cam_dpath, NULL,
2974					    start_ccb->ccb_h.path_id,
2975					    start_ccb->ccb_h.target_id,
2976					    start_ccb->ccb_h.target_lun) !=
2977					    CAM_REQ_CMP) {
2978				start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2979			} else {
2980				cam_dflags = start_ccb->cdbg.flags;
2981				start_ccb->ccb_h.status = CAM_REQ_CMP;
2982				xpt_print(cam_dpath, "debugging flags now %x\n",
2983				    cam_dflags);
2984			}
2985		} else
2986			start_ccb->ccb_h.status = CAM_REQ_CMP;
2987		break;
2988	}
2989	case XPT_NOOP:
2990		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
2991			xpt_freeze_devq(path, 1);
2992		start_ccb->ccb_h.status = CAM_REQ_CMP;
2993		break;
2994	default:
2995	case XPT_SDEV_TYPE:
2996	case XPT_TERM_IO:
2997	case XPT_ENG_INQ:
2998		/* XXX Implement */
2999		printf("%s: CCB type %#x not supported\n", __func__,
3000		       start_ccb->ccb_h.func_code);
3001		start_ccb->ccb_h.status = CAM_PROVIDE_FAIL;
3002		if (start_ccb->ccb_h.func_code & XPT_FC_DEV_QUEUED) {
3003			xpt_done(start_ccb);
3004		}
3005		break;
3006	}
3007}
3008
3009void
3010xpt_polled_action(union ccb *start_ccb)
3011{
3012	u_int32_t timeout;
3013	struct	  cam_sim *sim;
3014	struct	  cam_devq *devq;
3015	struct	  cam_ed *dev;
3016
3017	timeout = start_ccb->ccb_h.timeout * 10;
3018	sim = start_ccb->ccb_h.path->bus->sim;
3019	devq = sim->devq;
3020	dev = start_ccb->ccb_h.path->device;
3021
3022	mtx_unlock(&dev->device_mtx);
3023
3024	/*
3025	 * Steal an opening so that no other queued requests
3026	 * can get it before us while we simulate interrupts.
3027	 */
3028	mtx_lock(&devq->send_mtx);
3029	dev->ccbq.dev_openings--;
3030	while((devq->send_openings <= 0 || dev->ccbq.dev_openings < 0) &&
3031	    (--timeout > 0)) {
3032		mtx_unlock(&devq->send_mtx);
3033		DELAY(100);
3034		CAM_SIM_LOCK(sim);
3035		(*(sim->sim_poll))(sim);
3036		CAM_SIM_UNLOCK(sim);
3037		camisr_runqueue();
3038		mtx_lock(&devq->send_mtx);
3039	}
3040	dev->ccbq.dev_openings++;
3041	mtx_unlock(&devq->send_mtx);
3042
3043	if (timeout != 0) {
3044		xpt_action(start_ccb);
3045		while(--timeout > 0) {
3046			CAM_SIM_LOCK(sim);
3047			(*(sim->sim_poll))(sim);
3048			CAM_SIM_UNLOCK(sim);
3049			camisr_runqueue();
3050			if ((start_ccb->ccb_h.status  & CAM_STATUS_MASK)
3051			    != CAM_REQ_INPROG)
3052				break;
3053			DELAY(100);
3054		}
3055		if (timeout == 0) {
3056			/*
3057			 * XXX Is it worth adding a sim_timeout entry
3058			 * point so we can attempt recovery?  If
3059			 * this is only used for dumps, I don't think
3060			 * it is.
3061			 */
3062			start_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
3063		}
3064	} else {
3065		start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3066	}
3067
3068	mtx_lock(&dev->device_mtx);
3069}
3070
3071/*
3072 * Schedule a peripheral driver to receive a ccb when its
3073 * target device has space for more transactions.
3074 */
3075void
3076xpt_schedule(struct cam_periph *periph, u_int32_t new_priority)
3077{
3078
3079	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
3080	cam_periph_assert(periph, MA_OWNED);
3081	if (new_priority < periph->scheduled_priority) {
3082		periph->scheduled_priority = new_priority;
3083		xpt_run_allocq(periph, 0);
3084	}
3085}
3086
3087
3088/*
3089 * Schedule a device to run on a given queue.
3090 * If the device was inserted as a new entry on the queue,
3091 * return 1 meaning the device queue should be run. If we
3092 * were already queued, implying someone else has already
3093 * started the queue, return 0 so the caller doesn't attempt
3094 * to run the queue.
3095 */
3096static int
3097xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
3098		 u_int32_t new_priority)
3099{
3100	int retval;
3101	u_int32_t old_priority;
3102
3103	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n"));
3104
3105	old_priority = pinfo->priority;
3106
3107	/*
3108	 * Are we already queued?
3109	 */
3110	if (pinfo->index != CAM_UNQUEUED_INDEX) {
3111		/* Simply reorder based on new priority */
3112		if (new_priority < old_priority) {
3113			camq_change_priority(queue, pinfo->index,
3114					     new_priority);
3115			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3116					("changed priority to %d\n",
3117					 new_priority));
3118			retval = 1;
3119		} else
3120			retval = 0;
3121	} else {
3122		/* New entry on the queue */
3123		if (new_priority < old_priority)
3124			pinfo->priority = new_priority;
3125
3126		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3127				("Inserting onto queue\n"));
3128		pinfo->generation = ++queue->generation;
3129		camq_insert(queue, pinfo);
3130		retval = 1;
3131	}
3132	return (retval);
3133}
3134
3135static void
3136xpt_run_allocq_task(void *context, int pending)
3137{
3138	struct cam_periph *periph = context;
3139
3140	cam_periph_lock(periph);
3141	periph->flags &= ~CAM_PERIPH_RUN_TASK;
3142	xpt_run_allocq(periph, 1);
3143	cam_periph_unlock(periph);
3144	cam_periph_release(periph);
3145}
3146
3147static void
3148xpt_run_allocq(struct cam_periph *periph, int sleep)
3149{
3150	struct cam_ed	*device;
3151	union ccb	*ccb;
3152	uint32_t	 prio;
3153
3154	cam_periph_assert(periph, MA_OWNED);
3155	if (periph->periph_allocating)
3156		return;
3157	periph->periph_allocating = 1;
3158	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_allocq(%p)\n", periph));
3159	device = periph->path->device;
3160	ccb = NULL;
3161restart:
3162	while ((prio = min(periph->scheduled_priority,
3163	    periph->immediate_priority)) != CAM_PRIORITY_NONE &&
3164	    (periph->periph_allocated - (ccb != NULL ? 1 : 0) <
3165	     device->ccbq.total_openings || prio <= CAM_PRIORITY_OOB)) {
3166
3167		if (ccb == NULL &&
3168		    (ccb = xpt_get_ccb_nowait(periph)) == NULL) {
3169			if (sleep) {
3170				ccb = xpt_get_ccb(periph);
3171				goto restart;
3172			}
3173			if (periph->flags & CAM_PERIPH_RUN_TASK)
3174				break;
3175			cam_periph_doacquire(periph);
3176			periph->flags |= CAM_PERIPH_RUN_TASK;
3177			taskqueue_enqueue(xsoftc.xpt_taskq,
3178			    &periph->periph_run_task);
3179			break;
3180		}
3181		xpt_setup_ccb(&ccb->ccb_h, periph->path, prio);
3182		if (prio == periph->immediate_priority) {
3183			periph->immediate_priority = CAM_PRIORITY_NONE;
3184			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3185					("waking cam_periph_getccb()\n"));
3186			SLIST_INSERT_HEAD(&periph->ccb_list, &ccb->ccb_h,
3187					  periph_links.sle);
3188			wakeup(&periph->ccb_list);
3189		} else {
3190			periph->scheduled_priority = CAM_PRIORITY_NONE;
3191			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3192					("calling periph_start()\n"));
3193			periph->periph_start(periph, ccb);
3194		}
3195		ccb = NULL;
3196	}
3197	if (ccb != NULL)
3198		xpt_release_ccb(ccb);
3199	periph->periph_allocating = 0;
3200}
3201
3202static void
3203xpt_run_devq(struct cam_devq *devq)
3204{
3205	char cdb_str[(SCSI_MAX_CDBLEN * 3) + 1];
3206	int lock;
3207
3208	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_devq\n"));
3209
3210	devq->send_queue.qfrozen_cnt++;
3211	while ((devq->send_queue.entries > 0)
3212	    && (devq->send_openings > 0)
3213	    && (devq->send_queue.qfrozen_cnt <= 1)) {
3214		struct	cam_ed *device;
3215		union ccb *work_ccb;
3216		struct	cam_sim *sim;
3217
3218		device = (struct cam_ed *)camq_remove(&devq->send_queue,
3219							   CAMQ_HEAD);
3220		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3221				("running device %p\n", device));
3222
3223		work_ccb = cam_ccbq_peek_ccb(&device->ccbq, CAMQ_HEAD);
3224		if (work_ccb == NULL) {
3225			printf("device on run queue with no ccbs???\n");
3226			continue;
3227		}
3228
3229		if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
3230
3231			mtx_lock(&xsoftc.xpt_highpower_lock);
3232		 	if (xsoftc.num_highpower <= 0) {
3233				/*
3234				 * We got a high power command, but we
3235				 * don't have any available slots.  Freeze
3236				 * the device queue until we have a slot
3237				 * available.
3238				 */
3239				xpt_freeze_devq_device(device, 1);
3240				STAILQ_INSERT_TAIL(&xsoftc.highpowerq, device,
3241						   highpowerq_entry);
3242
3243				mtx_unlock(&xsoftc.xpt_highpower_lock);
3244				continue;
3245			} else {
3246				/*
3247				 * Consume a high power slot while
3248				 * this ccb runs.
3249				 */
3250				xsoftc.num_highpower--;
3251			}
3252			mtx_unlock(&xsoftc.xpt_highpower_lock);
3253		}
3254		cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
3255		cam_ccbq_send_ccb(&device->ccbq, work_ccb);
3256		devq->send_openings--;
3257		devq->send_active++;
3258		xpt_schedule_devq(devq, device);
3259		mtx_unlock(&devq->send_mtx);
3260
3261		if ((work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0) {
3262			/*
3263			 * The client wants to freeze the queue
3264			 * after this CCB is sent.
3265			 */
3266			xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3267		}
3268
3269		/* In Target mode, the peripheral driver knows best... */
3270		if (work_ccb->ccb_h.func_code == XPT_SCSI_IO) {
3271			if ((device->inq_flags & SID_CmdQue) != 0
3272			 && work_ccb->csio.tag_action != CAM_TAG_ACTION_NONE)
3273				work_ccb->ccb_h.flags |= CAM_TAG_ACTION_VALID;
3274			else
3275				/*
3276				 * Clear this in case of a retried CCB that
3277				 * failed due to a rejected tag.
3278				 */
3279				work_ccb->ccb_h.flags &= ~CAM_TAG_ACTION_VALID;
3280		}
3281
3282		switch (work_ccb->ccb_h.func_code) {
3283		case XPT_SCSI_IO:
3284			CAM_DEBUG(work_ccb->ccb_h.path,
3285			    CAM_DEBUG_CDB,("%s. CDB: %s\n",
3286			     scsi_op_desc(work_ccb->csio.cdb_io.cdb_bytes[0],
3287					  &device->inq_data),
3288			     scsi_cdb_string(work_ccb->csio.cdb_io.cdb_bytes,
3289					     cdb_str, sizeof(cdb_str))));
3290			break;
3291		case XPT_ATA_IO:
3292			CAM_DEBUG(work_ccb->ccb_h.path,
3293			    CAM_DEBUG_CDB,("%s. ACB: %s\n",
3294			     ata_op_string(&work_ccb->ataio.cmd),
3295			     ata_cmd_string(&work_ccb->ataio.cmd,
3296					    cdb_str, sizeof(cdb_str))));
3297			break;
3298		default:
3299			break;
3300		}
3301
3302		/*
3303		 * Device queues can be shared among multiple SIM instances
3304		 * that reside on different busses.  Use the SIM from the
3305		 * queued device, rather than the one from the calling bus.
3306		 */
3307		sim = device->sim;
3308		lock = (mtx_owned(sim->mtx) == 0);
3309		if (lock)
3310			CAM_SIM_LOCK(sim);
3311		(*(sim->sim_action))(sim, work_ccb);
3312		if (lock)
3313			CAM_SIM_UNLOCK(sim);
3314		mtx_lock(&devq->send_mtx);
3315	}
3316	devq->send_queue.qfrozen_cnt--;
3317}
3318
3319/*
3320 * This function merges stuff from the slave ccb into the master ccb, while
3321 * keeping important fields in the master ccb constant.
3322 */
3323void
3324xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
3325{
3326
3327	/*
3328	 * Pull fields that are valid for peripheral drivers to set
3329	 * into the master CCB along with the CCB "payload".
3330	 */
3331	master_ccb->ccb_h.retry_count = slave_ccb->ccb_h.retry_count;
3332	master_ccb->ccb_h.func_code = slave_ccb->ccb_h.func_code;
3333	master_ccb->ccb_h.timeout = slave_ccb->ccb_h.timeout;
3334	master_ccb->ccb_h.flags = slave_ccb->ccb_h.flags;
3335	bcopy(&(&slave_ccb->ccb_h)[1], &(&master_ccb->ccb_h)[1],
3336	      sizeof(union ccb) - sizeof(struct ccb_hdr));
3337}
3338
3339void
3340xpt_setup_ccb_flags(struct ccb_hdr *ccb_h, struct cam_path *path,
3341		    u_int32_t priority, u_int32_t flags)
3342{
3343
3344	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
3345	ccb_h->pinfo.priority = priority;
3346	ccb_h->path = path;
3347	ccb_h->path_id = path->bus->path_id;
3348	if (path->target)
3349		ccb_h->target_id = path->target->target_id;
3350	else
3351		ccb_h->target_id = CAM_TARGET_WILDCARD;
3352	if (path->device) {
3353		ccb_h->target_lun = path->device->lun_id;
3354		ccb_h->pinfo.generation = ++path->device->ccbq.queue.generation;
3355	} else {
3356		ccb_h->target_lun = CAM_TARGET_WILDCARD;
3357	}
3358	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
3359	ccb_h->flags = flags;
3360	ccb_h->xflags = 0;
3361}
3362
3363void
3364xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
3365{
3366	xpt_setup_ccb_flags(ccb_h, path, priority, /*flags*/ 0);
3367}
3368
3369/* Path manipulation functions */
3370cam_status
3371xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
3372		path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3373{
3374	struct	   cam_path *path;
3375	cam_status status;
3376
3377	path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3378
3379	if (path == NULL) {
3380		status = CAM_RESRC_UNAVAIL;
3381		return(status);
3382	}
3383	status = xpt_compile_path(path, perph, path_id, target_id, lun_id);
3384	if (status != CAM_REQ_CMP) {
3385		free(path, M_CAMPATH);
3386		path = NULL;
3387	}
3388	*new_path_ptr = path;
3389	return (status);
3390}
3391
3392cam_status
3393xpt_create_path_unlocked(struct cam_path **new_path_ptr,
3394			 struct cam_periph *periph, path_id_t path_id,
3395			 target_id_t target_id, lun_id_t lun_id)
3396{
3397
3398	return (xpt_create_path(new_path_ptr, periph, path_id, target_id,
3399	    lun_id));
3400}
3401
3402cam_status
3403xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
3404		 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3405{
3406	struct	     cam_eb *bus;
3407	struct	     cam_et *target;
3408	struct	     cam_ed *device;
3409	cam_status   status;
3410
3411	status = CAM_REQ_CMP;	/* Completed without error */
3412	target = NULL;		/* Wildcarded */
3413	device = NULL;		/* Wildcarded */
3414
3415	/*
3416	 * We will potentially modify the EDT, so block interrupts
3417	 * that may attempt to create cam paths.
3418	 */
3419	bus = xpt_find_bus(path_id);
3420	if (bus == NULL) {
3421		status = CAM_PATH_INVALID;
3422	} else {
3423		xpt_lock_buses();
3424		mtx_lock(&bus->eb_mtx);
3425		target = xpt_find_target(bus, target_id);
3426		if (target == NULL) {
3427			/* Create one */
3428			struct cam_et *new_target;
3429
3430			new_target = xpt_alloc_target(bus, target_id);
3431			if (new_target == NULL) {
3432				status = CAM_RESRC_UNAVAIL;
3433			} else {
3434				target = new_target;
3435			}
3436		}
3437		xpt_unlock_buses();
3438		if (target != NULL) {
3439			device = xpt_find_device(target, lun_id);
3440			if (device == NULL) {
3441				/* Create one */
3442				struct cam_ed *new_device;
3443
3444				new_device =
3445				    (*(bus->xport->alloc_device))(bus,
3446								      target,
3447								      lun_id);
3448				if (new_device == NULL) {
3449					status = CAM_RESRC_UNAVAIL;
3450				} else {
3451					device = new_device;
3452				}
3453			}
3454		}
3455		mtx_unlock(&bus->eb_mtx);
3456	}
3457
3458	/*
3459	 * Only touch the user's data if we are successful.
3460	 */
3461	if (status == CAM_REQ_CMP) {
3462		new_path->periph = perph;
3463		new_path->bus = bus;
3464		new_path->target = target;
3465		new_path->device = device;
3466		CAM_DEBUG(new_path, CAM_DEBUG_TRACE, ("xpt_compile_path\n"));
3467	} else {
3468		if (device != NULL)
3469			xpt_release_device(device);
3470		if (target != NULL)
3471			xpt_release_target(target);
3472		if (bus != NULL)
3473			xpt_release_bus(bus);
3474	}
3475	return (status);
3476}
3477
3478cam_status
3479xpt_clone_path(struct cam_path **new_path_ptr, struct cam_path *path)
3480{
3481	struct	   cam_path *new_path;
3482
3483	new_path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3484	if (new_path == NULL)
3485		return(CAM_RESRC_UNAVAIL);
3486	xpt_copy_path(new_path, path);
3487	*new_path_ptr = new_path;
3488	return (CAM_REQ_CMP);
3489}
3490
3491void
3492xpt_copy_path(struct cam_path *new_path, struct cam_path *path)
3493{
3494
3495	*new_path = *path;
3496	if (path->bus != NULL)
3497		xpt_acquire_bus(path->bus);
3498	if (path->target != NULL)
3499		xpt_acquire_target(path->target);
3500	if (path->device != NULL)
3501		xpt_acquire_device(path->device);
3502}
3503
3504void
3505xpt_release_path(struct cam_path *path)
3506{
3507	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_path\n"));
3508	if (path->device != NULL) {
3509		xpt_release_device(path->device);
3510		path->device = NULL;
3511	}
3512	if (path->target != NULL) {
3513		xpt_release_target(path->target);
3514		path->target = NULL;
3515	}
3516	if (path->bus != NULL) {
3517		xpt_release_bus(path->bus);
3518		path->bus = NULL;
3519	}
3520}
3521
3522void
3523xpt_free_path(struct cam_path *path)
3524{
3525
3526	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
3527	xpt_release_path(path);
3528	free(path, M_CAMPATH);
3529}
3530
3531void
3532xpt_path_counts(struct cam_path *path, uint32_t *bus_ref,
3533    uint32_t *periph_ref, uint32_t *target_ref, uint32_t *device_ref)
3534{
3535
3536	xpt_lock_buses();
3537	if (bus_ref) {
3538		if (path->bus)
3539			*bus_ref = path->bus->refcount;
3540		else
3541			*bus_ref = 0;
3542	}
3543	if (periph_ref) {
3544		if (path->periph)
3545			*periph_ref = path->periph->refcount;
3546		else
3547			*periph_ref = 0;
3548	}
3549	xpt_unlock_buses();
3550	if (target_ref) {
3551		if (path->target)
3552			*target_ref = path->target->refcount;
3553		else
3554			*target_ref = 0;
3555	}
3556	if (device_ref) {
3557		if (path->device)
3558			*device_ref = path->device->refcount;
3559		else
3560			*device_ref = 0;
3561	}
3562}
3563
3564/*
3565 * Return -1 for failure, 0 for exact match, 1 for match with wildcards
3566 * in path1, 2 for match with wildcards in path2.
3567 */
3568int
3569xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
3570{
3571	int retval = 0;
3572
3573	if (path1->bus != path2->bus) {
3574		if (path1->bus->path_id == CAM_BUS_WILDCARD)
3575			retval = 1;
3576		else if (path2->bus->path_id == CAM_BUS_WILDCARD)
3577			retval = 2;
3578		else
3579			return (-1);
3580	}
3581	if (path1->target != path2->target) {
3582		if (path1->target->target_id == CAM_TARGET_WILDCARD) {
3583			if (retval == 0)
3584				retval = 1;
3585		} else if (path2->target->target_id == CAM_TARGET_WILDCARD)
3586			retval = 2;
3587		else
3588			return (-1);
3589	}
3590	if (path1->device != path2->device) {
3591		if (path1->device->lun_id == CAM_LUN_WILDCARD) {
3592			if (retval == 0)
3593				retval = 1;
3594		} else if (path2->device->lun_id == CAM_LUN_WILDCARD)
3595			retval = 2;
3596		else
3597			return (-1);
3598	}
3599	return (retval);
3600}
3601
3602int
3603xpt_path_comp_dev(struct cam_path *path, struct cam_ed *dev)
3604{
3605	int retval = 0;
3606
3607	if (path->bus != dev->target->bus) {
3608		if (path->bus->path_id == CAM_BUS_WILDCARD)
3609			retval = 1;
3610		else if (dev->target->bus->path_id == CAM_BUS_WILDCARD)
3611			retval = 2;
3612		else
3613			return (-1);
3614	}
3615	if (path->target != dev->target) {
3616		if (path->target->target_id == CAM_TARGET_WILDCARD) {
3617			if (retval == 0)
3618				retval = 1;
3619		} else if (dev->target->target_id == CAM_TARGET_WILDCARD)
3620			retval = 2;
3621		else
3622			return (-1);
3623	}
3624	if (path->device != dev) {
3625		if (path->device->lun_id == CAM_LUN_WILDCARD) {
3626			if (retval == 0)
3627				retval = 1;
3628		} else if (dev->lun_id == CAM_LUN_WILDCARD)
3629			retval = 2;
3630		else
3631			return (-1);
3632	}
3633	return (retval);
3634}
3635
3636void
3637xpt_print_path(struct cam_path *path)
3638{
3639
3640	if (path == NULL)
3641		printf("(nopath): ");
3642	else {
3643		if (path->periph != NULL)
3644			printf("(%s%d:", path->periph->periph_name,
3645			       path->periph->unit_number);
3646		else
3647			printf("(noperiph:");
3648
3649		if (path->bus != NULL)
3650			printf("%s%d:%d:", path->bus->sim->sim_name,
3651			       path->bus->sim->unit_number,
3652			       path->bus->sim->bus_id);
3653		else
3654			printf("nobus:");
3655
3656		if (path->target != NULL)
3657			printf("%d:", path->target->target_id);
3658		else
3659			printf("X:");
3660
3661		if (path->device != NULL)
3662			printf("%jx): ", (uintmax_t)path->device->lun_id);
3663		else
3664			printf("X): ");
3665	}
3666}
3667
3668void
3669xpt_print_device(struct cam_ed *device)
3670{
3671
3672	if (device == NULL)
3673		printf("(nopath): ");
3674	else {
3675		printf("(noperiph:%s%d:%d:%d:%jx): ", device->sim->sim_name,
3676		       device->sim->unit_number,
3677		       device->sim->bus_id,
3678		       device->target->target_id,
3679		       (uintmax_t)device->lun_id);
3680	}
3681}
3682
3683void
3684xpt_print(struct cam_path *path, const char *fmt, ...)
3685{
3686	va_list ap;
3687	xpt_print_path(path);
3688	va_start(ap, fmt);
3689	vprintf(fmt, ap);
3690	va_end(ap);
3691}
3692
3693int
3694xpt_path_string(struct cam_path *path, char *str, size_t str_len)
3695{
3696	struct sbuf sb;
3697
3698	sbuf_new(&sb, str, str_len, 0);
3699
3700	if (path == NULL)
3701		sbuf_printf(&sb, "(nopath): ");
3702	else {
3703		if (path->periph != NULL)
3704			sbuf_printf(&sb, "(%s%d:", path->periph->periph_name,
3705				    path->periph->unit_number);
3706		else
3707			sbuf_printf(&sb, "(noperiph:");
3708
3709		if (path->bus != NULL)
3710			sbuf_printf(&sb, "%s%d:%d:", path->bus->sim->sim_name,
3711				    path->bus->sim->unit_number,
3712				    path->bus->sim->bus_id);
3713		else
3714			sbuf_printf(&sb, "nobus:");
3715
3716		if (path->target != NULL)
3717			sbuf_printf(&sb, "%d:", path->target->target_id);
3718		else
3719			sbuf_printf(&sb, "X:");
3720
3721		if (path->device != NULL)
3722			sbuf_printf(&sb, "%jx): ",
3723			    (uintmax_t)path->device->lun_id);
3724		else
3725			sbuf_printf(&sb, "X): ");
3726	}
3727	sbuf_finish(&sb);
3728
3729	return(sbuf_len(&sb));
3730}
3731
3732path_id_t
3733xpt_path_path_id(struct cam_path *path)
3734{
3735	return(path->bus->path_id);
3736}
3737
3738target_id_t
3739xpt_path_target_id(struct cam_path *path)
3740{
3741	if (path->target != NULL)
3742		return (path->target->target_id);
3743	else
3744		return (CAM_TARGET_WILDCARD);
3745}
3746
3747lun_id_t
3748xpt_path_lun_id(struct cam_path *path)
3749{
3750	if (path->device != NULL)
3751		return (path->device->lun_id);
3752	else
3753		return (CAM_LUN_WILDCARD);
3754}
3755
3756struct cam_sim *
3757xpt_path_sim(struct cam_path *path)
3758{
3759
3760	return (path->bus->sim);
3761}
3762
3763struct cam_periph*
3764xpt_path_periph(struct cam_path *path)
3765{
3766
3767	return (path->periph);
3768}
3769
3770int
3771xpt_path_legacy_ata_id(struct cam_path *path)
3772{
3773	struct cam_eb *bus;
3774	int bus_id;
3775
3776	if ((strcmp(path->bus->sim->sim_name, "ata") != 0) &&
3777	    strcmp(path->bus->sim->sim_name, "ahcich") != 0 &&
3778	    strcmp(path->bus->sim->sim_name, "mvsch") != 0 &&
3779	    strcmp(path->bus->sim->sim_name, "siisch") != 0)
3780		return (-1);
3781
3782	if (strcmp(path->bus->sim->sim_name, "ata") == 0 &&
3783	    path->bus->sim->unit_number < 2) {
3784		bus_id = path->bus->sim->unit_number;
3785	} else {
3786		bus_id = 2;
3787		xpt_lock_buses();
3788		TAILQ_FOREACH(bus, &xsoftc.xpt_busses, links) {
3789			if (bus == path->bus)
3790				break;
3791			if ((strcmp(bus->sim->sim_name, "ata") == 0 &&
3792			     bus->sim->unit_number >= 2) ||
3793			    strcmp(bus->sim->sim_name, "ahcich") == 0 ||
3794			    strcmp(bus->sim->sim_name, "mvsch") == 0 ||
3795			    strcmp(bus->sim->sim_name, "siisch") == 0)
3796				bus_id++;
3797		}
3798		xpt_unlock_buses();
3799	}
3800	if (path->target != NULL) {
3801		if (path->target->target_id < 2)
3802			return (bus_id * 2 + path->target->target_id);
3803		else
3804			return (-1);
3805	} else
3806		return (bus_id * 2);
3807}
3808
3809/*
3810 * Release a CAM control block for the caller.  Remit the cost of the structure
3811 * to the device referenced by the path.  If the this device had no 'credits'
3812 * and peripheral drivers have registered async callbacks for this notification
3813 * call them now.
3814 */
3815void
3816xpt_release_ccb(union ccb *free_ccb)
3817{
3818	struct	 cam_ed *device;
3819	struct	 cam_periph *periph;
3820
3821	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
3822	xpt_path_assert(free_ccb->ccb_h.path, MA_OWNED);
3823	device = free_ccb->ccb_h.path->device;
3824	periph = free_ccb->ccb_h.path->periph;
3825
3826	xpt_free_ccb(free_ccb);
3827	periph->periph_allocated--;
3828	cam_ccbq_release_opening(&device->ccbq);
3829	xpt_run_allocq(periph, 0);
3830}
3831
3832/* Functions accessed by SIM drivers */
3833
3834static struct xpt_xport xport_default = {
3835	.alloc_device = xpt_alloc_device_default,
3836	.action = xpt_action_default,
3837	.async = xpt_dev_async_default,
3838};
3839
3840/*
3841 * A sim structure, listing the SIM entry points and instance
3842 * identification info is passed to xpt_bus_register to hook the SIM
3843 * into the CAM framework.  xpt_bus_register creates a cam_eb entry
3844 * for this new bus and places it in the array of busses and assigns
3845 * it a path_id.  The path_id may be influenced by "hard wiring"
3846 * information specified by the user.  Once interrupt services are
3847 * available, the bus will be probed.
3848 */
3849int32_t
3850xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
3851{
3852	struct cam_eb *new_bus;
3853	struct cam_eb *old_bus;
3854	struct ccb_pathinq cpi;
3855	struct cam_path *path;
3856	cam_status status;
3857
3858	mtx_assert(sim->mtx, MA_OWNED);
3859
3860	sim->bus_id = bus;
3861	new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
3862					  M_CAMXPT, M_NOWAIT|M_ZERO);
3863	if (new_bus == NULL) {
3864		/* Couldn't satisfy request */
3865		return (CAM_RESRC_UNAVAIL);
3866	}
3867
3868	mtx_init(&new_bus->eb_mtx, "CAM bus lock", NULL, MTX_DEF);
3869	TAILQ_INIT(&new_bus->et_entries);
3870	cam_sim_hold(sim);
3871	new_bus->sim = sim;
3872	timevalclear(&new_bus->last_reset);
3873	new_bus->flags = 0;
3874	new_bus->refcount = 1;	/* Held until a bus_deregister event */
3875	new_bus->generation = 0;
3876
3877	xpt_lock_buses();
3878	sim->path_id = new_bus->path_id =
3879	    xptpathid(sim->sim_name, sim->unit_number, sim->bus_id);
3880	old_bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3881	while (old_bus != NULL
3882	    && old_bus->path_id < new_bus->path_id)
3883		old_bus = TAILQ_NEXT(old_bus, links);
3884	if (old_bus != NULL)
3885		TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
3886	else
3887		TAILQ_INSERT_TAIL(&xsoftc.xpt_busses, new_bus, links);
3888	xsoftc.bus_generation++;
3889	xpt_unlock_buses();
3890
3891	/*
3892	 * Set a default transport so that a PATH_INQ can be issued to
3893	 * the SIM.  This will then allow for probing and attaching of
3894	 * a more appropriate transport.
3895	 */
3896	new_bus->xport = &xport_default;
3897
3898	status = xpt_create_path(&path, /*periph*/NULL, sim->path_id,
3899				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3900	if (status != CAM_REQ_CMP) {
3901		xpt_release_bus(new_bus);
3902		free(path, M_CAMXPT);
3903		return (CAM_RESRC_UNAVAIL);
3904	}
3905
3906	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
3907	cpi.ccb_h.func_code = XPT_PATH_INQ;
3908	xpt_action((union ccb *)&cpi);
3909
3910	if (cpi.ccb_h.status == CAM_REQ_CMP) {
3911		switch (cpi.transport) {
3912		case XPORT_SPI:
3913		case XPORT_SAS:
3914		case XPORT_FC:
3915		case XPORT_USB:
3916		case XPORT_ISCSI:
3917		case XPORT_SRP:
3918		case XPORT_PPB:
3919			new_bus->xport = scsi_get_xport();
3920			break;
3921		case XPORT_ATA:
3922		case XPORT_SATA:
3923			new_bus->xport = ata_get_xport();
3924			break;
3925		default:
3926			new_bus->xport = &xport_default;
3927			break;
3928		}
3929	}
3930
3931	/* Notify interested parties */
3932	if (sim->path_id != CAM_XPT_PATH_ID) {
3933
3934		xpt_async(AC_PATH_REGISTERED, path, &cpi);
3935		if ((cpi.hba_misc & PIM_NOSCAN) == 0) {
3936			union	ccb *scan_ccb;
3937
3938			/* Initiate bus rescan. */
3939			scan_ccb = xpt_alloc_ccb_nowait();
3940			if (scan_ccb != NULL) {
3941				scan_ccb->ccb_h.path = path;
3942				scan_ccb->ccb_h.func_code = XPT_SCAN_BUS;
3943				scan_ccb->crcn.flags = 0;
3944				xpt_rescan(scan_ccb);
3945			} else {
3946				xpt_print(path,
3947					  "Can't allocate CCB to scan bus\n");
3948				xpt_free_path(path);
3949			}
3950		} else
3951			xpt_free_path(path);
3952	} else
3953		xpt_free_path(path);
3954	return (CAM_SUCCESS);
3955}
3956
3957int32_t
3958xpt_bus_deregister(path_id_t pathid)
3959{
3960	struct cam_path bus_path;
3961	cam_status status;
3962
3963	status = xpt_compile_path(&bus_path, NULL, pathid,
3964				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3965	if (status != CAM_REQ_CMP)
3966		return (status);
3967
3968	xpt_async(AC_LOST_DEVICE, &bus_path, NULL);
3969	xpt_async(AC_PATH_DEREGISTERED, &bus_path, NULL);
3970
3971	/* Release the reference count held while registered. */
3972	xpt_release_bus(bus_path.bus);
3973	xpt_release_path(&bus_path);
3974
3975	return (CAM_REQ_CMP);
3976}
3977
3978static path_id_t
3979xptnextfreepathid(void)
3980{
3981	struct cam_eb *bus;
3982	path_id_t pathid;
3983	const char *strval;
3984
3985	mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
3986	pathid = 0;
3987	bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3988retry:
3989	/* Find an unoccupied pathid */
3990	while (bus != NULL && bus->path_id <= pathid) {
3991		if (bus->path_id == pathid)
3992			pathid++;
3993		bus = TAILQ_NEXT(bus, links);
3994	}
3995
3996	/*
3997	 * Ensure that this pathid is not reserved for
3998	 * a bus that may be registered in the future.
3999	 */
4000	if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
4001		++pathid;
4002		/* Start the search over */
4003		goto retry;
4004	}
4005	return (pathid);
4006}
4007
4008static path_id_t
4009xptpathid(const char *sim_name, int sim_unit, int sim_bus)
4010{
4011	path_id_t pathid;
4012	int i, dunit, val;
4013	char buf[32];
4014	const char *dname;
4015
4016	pathid = CAM_XPT_PATH_ID;
4017	snprintf(buf, sizeof(buf), "%s%d", sim_name, sim_unit);
4018	if (strcmp(buf, "xpt0") == 0 && sim_bus == 0)
4019		return (pathid);
4020	i = 0;
4021	while ((resource_find_match(&i, &dname, &dunit, "at", buf)) == 0) {
4022		if (strcmp(dname, "scbus")) {
4023			/* Avoid a bit of foot shooting. */
4024			continue;
4025		}
4026		if (dunit < 0)		/* unwired?! */
4027			continue;
4028		if (resource_int_value("scbus", dunit, "bus", &val) == 0) {
4029			if (sim_bus == val) {
4030				pathid = dunit;
4031				break;
4032			}
4033		} else if (sim_bus == 0) {
4034			/* Unspecified matches bus 0 */
4035			pathid = dunit;
4036			break;
4037		} else {
4038			printf("Ambiguous scbus configuration for %s%d "
4039			       "bus %d, cannot wire down.  The kernel "
4040			       "config entry for scbus%d should "
4041			       "specify a controller bus.\n"
4042			       "Scbus will be assigned dynamically.\n",
4043			       sim_name, sim_unit, sim_bus, dunit);
4044			break;
4045		}
4046	}
4047
4048	if (pathid == CAM_XPT_PATH_ID)
4049		pathid = xptnextfreepathid();
4050	return (pathid);
4051}
4052
4053static const char *
4054xpt_async_string(u_int32_t async_code)
4055{
4056
4057	switch (async_code) {
4058	case AC_BUS_RESET: return ("AC_BUS_RESET");
4059	case AC_UNSOL_RESEL: return ("AC_UNSOL_RESEL");
4060	case AC_SCSI_AEN: return ("AC_SCSI_AEN");
4061	case AC_SENT_BDR: return ("AC_SENT_BDR");
4062	case AC_PATH_REGISTERED: return ("AC_PATH_REGISTERED");
4063	case AC_PATH_DEREGISTERED: return ("AC_PATH_DEREGISTERED");
4064	case AC_FOUND_DEVICE: return ("AC_FOUND_DEVICE");
4065	case AC_LOST_DEVICE: return ("AC_LOST_DEVICE");
4066	case AC_TRANSFER_NEG: return ("AC_TRANSFER_NEG");
4067	case AC_INQ_CHANGED: return ("AC_INQ_CHANGED");
4068	case AC_GETDEV_CHANGED: return ("AC_GETDEV_CHANGED");
4069	case AC_CONTRACT: return ("AC_CONTRACT");
4070	case AC_ADVINFO_CHANGED: return ("AC_ADVINFO_CHANGED");
4071	case AC_UNIT_ATTENTION: return ("AC_UNIT_ATTENTION");
4072	}
4073	return ("AC_UNKNOWN");
4074}
4075
4076static int
4077xpt_async_size(u_int32_t async_code)
4078{
4079
4080	switch (async_code) {
4081	case AC_BUS_RESET: return (0);
4082	case AC_UNSOL_RESEL: return (0);
4083	case AC_SCSI_AEN: return (0);
4084	case AC_SENT_BDR: return (0);
4085	case AC_PATH_REGISTERED: return (sizeof(struct ccb_pathinq));
4086	case AC_PATH_DEREGISTERED: return (0);
4087	case AC_FOUND_DEVICE: return (sizeof(struct ccb_getdev));
4088	case AC_LOST_DEVICE: return (0);
4089	case AC_TRANSFER_NEG: return (sizeof(struct ccb_trans_settings));
4090	case AC_INQ_CHANGED: return (0);
4091	case AC_GETDEV_CHANGED: return (0);
4092	case AC_CONTRACT: return (sizeof(struct ac_contract));
4093	case AC_ADVINFO_CHANGED: return (-1);
4094	case AC_UNIT_ATTENTION: return (sizeof(struct ccb_scsiio));
4095	}
4096	return (0);
4097}
4098
4099static int
4100xpt_async_process_dev(struct cam_ed *device, void *arg)
4101{
4102	union ccb *ccb = arg;
4103	struct cam_path *path = ccb->ccb_h.path;
4104	void *async_arg = ccb->casync.async_arg_ptr;
4105	u_int32_t async_code = ccb->casync.async_code;
4106	int relock;
4107
4108	if (path->device != device
4109	 && path->device->lun_id != CAM_LUN_WILDCARD
4110	 && device->lun_id != CAM_LUN_WILDCARD)
4111		return (1);
4112
4113	/*
4114	 * The async callback could free the device.
4115	 * If it is a broadcast async, it doesn't hold
4116	 * device reference, so take our own reference.
4117	 */
4118	xpt_acquire_device(device);
4119
4120	/*
4121	 * If async for specific device is to be delivered to
4122	 * the wildcard client, take the specific device lock.
4123	 * XXX: We may need a way for client to specify it.
4124	 */
4125	if ((device->lun_id == CAM_LUN_WILDCARD &&
4126	     path->device->lun_id != CAM_LUN_WILDCARD) ||
4127	    (device->target->target_id == CAM_TARGET_WILDCARD &&
4128	     path->target->target_id != CAM_TARGET_WILDCARD) ||
4129	    (device->target->bus->path_id == CAM_BUS_WILDCARD &&
4130	     path->target->bus->path_id != CAM_BUS_WILDCARD)) {
4131		mtx_unlock(&device->device_mtx);
4132		xpt_path_lock(path);
4133		relock = 1;
4134	} else
4135		relock = 0;
4136
4137	(*(device->target->bus->xport->async))(async_code,
4138	    device->target->bus, device->target, device, async_arg);
4139	xpt_async_bcast(&device->asyncs, async_code, path, async_arg);
4140
4141	if (relock) {
4142		xpt_path_unlock(path);
4143		mtx_lock(&device->device_mtx);
4144	}
4145	xpt_release_device(device);
4146	return (1);
4147}
4148
4149static int
4150xpt_async_process_tgt(struct cam_et *target, void *arg)
4151{
4152	union ccb *ccb = arg;
4153	struct cam_path *path = ccb->ccb_h.path;
4154
4155	if (path->target != target
4156	 && path->target->target_id != CAM_TARGET_WILDCARD
4157	 && target->target_id != CAM_TARGET_WILDCARD)
4158		return (1);
4159
4160	if (ccb->casync.async_code == AC_SENT_BDR) {
4161		/* Update our notion of when the last reset occurred */
4162		microtime(&target->last_reset);
4163	}
4164
4165	return (xptdevicetraverse(target, NULL, xpt_async_process_dev, ccb));
4166}
4167
4168static void
4169xpt_async_process(struct cam_periph *periph, union ccb *ccb)
4170{
4171	struct cam_eb *bus;
4172	struct cam_path *path;
4173	void *async_arg;
4174	u_int32_t async_code;
4175
4176	path = ccb->ccb_h.path;
4177	async_code = ccb->casync.async_code;
4178	async_arg = ccb->casync.async_arg_ptr;
4179	CAM_DEBUG(path, CAM_DEBUG_TRACE | CAM_DEBUG_INFO,
4180	    ("xpt_async(%s)\n", xpt_async_string(async_code)));
4181	bus = path->bus;
4182
4183	if (async_code == AC_BUS_RESET) {
4184		/* Update our notion of when the last reset occurred */
4185		microtime(&bus->last_reset);
4186	}
4187
4188	xpttargettraverse(bus, NULL, xpt_async_process_tgt, ccb);
4189
4190	/*
4191	 * If this wasn't a fully wildcarded async, tell all
4192	 * clients that want all async events.
4193	 */
4194	if (bus != xpt_periph->path->bus) {
4195		xpt_path_lock(xpt_periph->path);
4196		xpt_async_process_dev(xpt_periph->path->device, ccb);
4197		xpt_path_unlock(xpt_periph->path);
4198	}
4199
4200	if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4201		xpt_release_devq(path, 1, TRUE);
4202	else
4203		xpt_release_simq(path->bus->sim, TRUE);
4204	if (ccb->casync.async_arg_size > 0)
4205		free(async_arg, M_CAMXPT);
4206	xpt_free_path(path);
4207	xpt_free_ccb(ccb);
4208}
4209
4210static void
4211xpt_async_bcast(struct async_list *async_head,
4212		u_int32_t async_code,
4213		struct cam_path *path, void *async_arg)
4214{
4215	struct async_node *cur_entry;
4216	int lock;
4217
4218	cur_entry = SLIST_FIRST(async_head);
4219	while (cur_entry != NULL) {
4220		struct async_node *next_entry;
4221		/*
4222		 * Grab the next list entry before we call the current
4223		 * entry's callback.  This is because the callback function
4224		 * can delete its async callback entry.
4225		 */
4226		next_entry = SLIST_NEXT(cur_entry, links);
4227		if ((cur_entry->event_enable & async_code) != 0) {
4228			lock = cur_entry->event_lock;
4229			if (lock)
4230				CAM_SIM_LOCK(path->device->sim);
4231			cur_entry->callback(cur_entry->callback_arg,
4232					    async_code, path,
4233					    async_arg);
4234			if (lock)
4235				CAM_SIM_UNLOCK(path->device->sim);
4236		}
4237		cur_entry = next_entry;
4238	}
4239}
4240
4241void
4242xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
4243{
4244	union ccb *ccb;
4245	int size;
4246
4247	ccb = xpt_alloc_ccb_nowait();
4248	if (ccb == NULL) {
4249		xpt_print(path, "Can't allocate CCB to send %s\n",
4250		    xpt_async_string(async_code));
4251		return;
4252	}
4253
4254	if (xpt_clone_path(&ccb->ccb_h.path, path) != CAM_REQ_CMP) {
4255		xpt_print(path, "Can't allocate path to send %s\n",
4256		    xpt_async_string(async_code));
4257		xpt_free_ccb(ccb);
4258		return;
4259	}
4260	ccb->ccb_h.path->periph = NULL;
4261	ccb->ccb_h.func_code = XPT_ASYNC;
4262	ccb->ccb_h.cbfcnp = xpt_async_process;
4263	ccb->ccb_h.flags |= CAM_UNLOCKED;
4264	ccb->casync.async_code = async_code;
4265	ccb->casync.async_arg_size = 0;
4266	size = xpt_async_size(async_code);
4267	if (size > 0 && async_arg != NULL) {
4268		ccb->casync.async_arg_ptr = malloc(size, M_CAMXPT, M_NOWAIT);
4269		if (ccb->casync.async_arg_ptr == NULL) {
4270			xpt_print(path, "Can't allocate argument to send %s\n",
4271			    xpt_async_string(async_code));
4272			xpt_free_path(ccb->ccb_h.path);
4273			xpt_free_ccb(ccb);
4274			return;
4275		}
4276		memcpy(ccb->casync.async_arg_ptr, async_arg, size);
4277		ccb->casync.async_arg_size = size;
4278	} else if (size < 0) {
4279		ccb->casync.async_arg_ptr = async_arg;
4280		ccb->casync.async_arg_size = size;
4281	}
4282	if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4283		xpt_freeze_devq(path, 1);
4284	else
4285		xpt_freeze_simq(path->bus->sim, 1);
4286	xpt_done(ccb);
4287}
4288
4289static void
4290xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus,
4291		      struct cam_et *target, struct cam_ed *device,
4292		      void *async_arg)
4293{
4294
4295	/*
4296	 * We only need to handle events for real devices.
4297	 */
4298	if (target->target_id == CAM_TARGET_WILDCARD
4299	 || device->lun_id == CAM_LUN_WILDCARD)
4300		return;
4301
4302	printf("%s called\n", __func__);
4303}
4304
4305static uint32_t
4306xpt_freeze_devq_device(struct cam_ed *dev, u_int count)
4307{
4308	struct cam_devq	*devq;
4309	uint32_t freeze;
4310
4311	devq = dev->sim->devq;
4312	mtx_assert(&devq->send_mtx, MA_OWNED);
4313	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4314	    ("xpt_freeze_devq_device(%d) %u->%u\n", count,
4315	    dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt + count));
4316	freeze = (dev->ccbq.queue.qfrozen_cnt += count);
4317	/* Remove frozen device from sendq. */
4318	if (device_is_queued(dev))
4319		camq_remove(&devq->send_queue, dev->devq_entry.index);
4320	return (freeze);
4321}
4322
4323u_int32_t
4324xpt_freeze_devq(struct cam_path *path, u_int count)
4325{
4326	struct cam_ed	*dev = path->device;
4327	struct cam_devq	*devq;
4328	uint32_t	 freeze;
4329
4330	devq = dev->sim->devq;
4331	mtx_lock(&devq->send_mtx);
4332	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_freeze_devq(%d)\n", count));
4333	freeze = xpt_freeze_devq_device(dev, count);
4334	mtx_unlock(&devq->send_mtx);
4335	return (freeze);
4336}
4337
4338u_int32_t
4339xpt_freeze_simq(struct cam_sim *sim, u_int count)
4340{
4341	struct cam_devq	*devq;
4342	uint32_t	 freeze;
4343
4344	devq = sim->devq;
4345	mtx_lock(&devq->send_mtx);
4346	freeze = (devq->send_queue.qfrozen_cnt += count);
4347	mtx_unlock(&devq->send_mtx);
4348	return (freeze);
4349}
4350
4351static void
4352xpt_release_devq_timeout(void *arg)
4353{
4354	struct cam_ed *dev;
4355	struct cam_devq *devq;
4356
4357	dev = (struct cam_ed *)arg;
4358	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, ("xpt_release_devq_timeout\n"));
4359	devq = dev->sim->devq;
4360	mtx_assert(&devq->send_mtx, MA_OWNED);
4361	if (xpt_release_devq_device(dev, /*count*/1, /*run_queue*/TRUE))
4362		xpt_run_devq(devq);
4363}
4364
4365void
4366xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
4367{
4368	struct cam_ed *dev;
4369	struct cam_devq *devq;
4370
4371	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_devq(%d, %d)\n",
4372	    count, run_queue));
4373	dev = path->device;
4374	devq = dev->sim->devq;
4375	mtx_lock(&devq->send_mtx);
4376	if (xpt_release_devq_device(dev, count, run_queue))
4377		xpt_run_devq(dev->sim->devq);
4378	mtx_unlock(&devq->send_mtx);
4379}
4380
4381static int
4382xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
4383{
4384
4385	mtx_assert(&dev->sim->devq->send_mtx, MA_OWNED);
4386	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4387	    ("xpt_release_devq_device(%d, %d) %u->%u\n", count, run_queue,
4388	    dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt - count));
4389	if (count > dev->ccbq.queue.qfrozen_cnt) {
4390#ifdef INVARIANTS
4391		printf("xpt_release_devq(): requested %u > present %u\n",
4392		    count, dev->ccbq.queue.qfrozen_cnt);
4393#endif
4394		count = dev->ccbq.queue.qfrozen_cnt;
4395	}
4396	dev->ccbq.queue.qfrozen_cnt -= count;
4397	if (dev->ccbq.queue.qfrozen_cnt == 0) {
4398		/*
4399		 * No longer need to wait for a successful
4400		 * command completion.
4401		 */
4402		dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
4403		/*
4404		 * Remove any timeouts that might be scheduled
4405		 * to release this queue.
4406		 */
4407		if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
4408			callout_stop(&dev->callout);
4409			dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
4410		}
4411		/*
4412		 * Now that we are unfrozen schedule the
4413		 * device so any pending transactions are
4414		 * run.
4415		 */
4416		xpt_schedule_devq(dev->sim->devq, dev);
4417	} else
4418		run_queue = 0;
4419	return (run_queue);
4420}
4421
4422void
4423xpt_release_simq(struct cam_sim *sim, int run_queue)
4424{
4425	struct cam_devq	*devq;
4426
4427	devq = sim->devq;
4428	mtx_lock(&devq->send_mtx);
4429	if (devq->send_queue.qfrozen_cnt <= 0) {
4430#ifdef INVARIANTS
4431		printf("xpt_release_simq: requested 1 > present %u\n",
4432		    devq->send_queue.qfrozen_cnt);
4433#endif
4434	} else
4435		devq->send_queue.qfrozen_cnt--;
4436	if (devq->send_queue.qfrozen_cnt == 0) {
4437		/*
4438		 * If there is a timeout scheduled to release this
4439		 * sim queue, remove it.  The queue frozen count is
4440		 * already at 0.
4441		 */
4442		if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
4443			callout_stop(&sim->callout);
4444			sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
4445		}
4446		if (run_queue) {
4447			/*
4448			 * Now that we are unfrozen run the send queue.
4449			 */
4450			xpt_run_devq(sim->devq);
4451		}
4452	}
4453	mtx_unlock(&devq->send_mtx);
4454}
4455
4456/*
4457 * XXX Appears to be unused.
4458 */
4459static void
4460xpt_release_simq_timeout(void *arg)
4461{
4462	struct cam_sim *sim;
4463
4464	sim = (struct cam_sim *)arg;
4465	xpt_release_simq(sim, /* run_queue */ TRUE);
4466}
4467
4468void
4469xpt_done(union ccb *done_ccb)
4470{
4471	struct cam_doneq *queue;
4472	int	run, hash;
4473
4474	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done\n"));
4475	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4476		return;
4477
4478	hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id +
4479	    done_ccb->ccb_h.target_lun) % cam_num_doneqs;
4480	queue = &cam_doneqs[hash];
4481	mtx_lock(&queue->cam_doneq_mtx);
4482	run = (queue->cam_doneq_sleep && STAILQ_EMPTY(&queue->cam_doneq));
4483	STAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.stqe);
4484	done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4485	mtx_unlock(&queue->cam_doneq_mtx);
4486	if (run)
4487		wakeup(&queue->cam_doneq);
4488}
4489
4490void
4491xpt_done_direct(union ccb *done_ccb)
4492{
4493
4494	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done_direct\n"));
4495	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4496		return;
4497
4498	xpt_done_process(&done_ccb->ccb_h);
4499}
4500
4501union ccb *
4502xpt_alloc_ccb()
4503{
4504	union ccb *new_ccb;
4505
4506	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4507	return (new_ccb);
4508}
4509
4510union ccb *
4511xpt_alloc_ccb_nowait()
4512{
4513	union ccb *new_ccb;
4514
4515	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4516	return (new_ccb);
4517}
4518
4519void
4520xpt_free_ccb(union ccb *free_ccb)
4521{
4522	free(free_ccb, M_CAMCCB);
4523}
4524
4525
4526
4527/* Private XPT functions */
4528
4529/*
4530 * Get a CAM control block for the caller. Charge the structure to the device
4531 * referenced by the path.  If we don't have sufficient resources to allocate
4532 * more ccbs, we return NULL.
4533 */
4534static union ccb *
4535xpt_get_ccb_nowait(struct cam_periph *periph)
4536{
4537	union ccb *new_ccb;
4538
4539	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4540	if (new_ccb == NULL)
4541		return (NULL);
4542	periph->periph_allocated++;
4543	cam_ccbq_take_opening(&periph->path->device->ccbq);
4544	return (new_ccb);
4545}
4546
4547static union ccb *
4548xpt_get_ccb(struct cam_periph *periph)
4549{
4550	union ccb *new_ccb;
4551
4552	cam_periph_unlock(periph);
4553	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4554	cam_periph_lock(periph);
4555	periph->periph_allocated++;
4556	cam_ccbq_take_opening(&periph->path->device->ccbq);
4557	return (new_ccb);
4558}
4559
4560union ccb *
4561cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
4562{
4563	struct ccb_hdr *ccb_h;
4564
4565	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("cam_periph_getccb\n"));
4566	cam_periph_assert(periph, MA_OWNED);
4567	while ((ccb_h = SLIST_FIRST(&periph->ccb_list)) == NULL ||
4568	    ccb_h->pinfo.priority != priority) {
4569		if (priority < periph->immediate_priority) {
4570			periph->immediate_priority = priority;
4571			xpt_run_allocq(periph, 0);
4572		} else
4573			cam_periph_sleep(periph, &periph->ccb_list, PRIBIO,
4574			    "cgticb", 0);
4575	}
4576	SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle);
4577	return ((union ccb *)ccb_h);
4578}
4579
4580static void
4581xpt_acquire_bus(struct cam_eb *bus)
4582{
4583
4584	xpt_lock_buses();
4585	bus->refcount++;
4586	xpt_unlock_buses();
4587}
4588
4589static void
4590xpt_release_bus(struct cam_eb *bus)
4591{
4592
4593	xpt_lock_buses();
4594	KASSERT(bus->refcount >= 1, ("bus->refcount >= 1"));
4595	if (--bus->refcount > 0) {
4596		xpt_unlock_buses();
4597		return;
4598	}
4599	TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
4600	xsoftc.bus_generation++;
4601	xpt_unlock_buses();
4602	KASSERT(TAILQ_EMPTY(&bus->et_entries),
4603	    ("destroying bus, but target list is not empty"));
4604	cam_sim_release(bus->sim);
4605	mtx_destroy(&bus->eb_mtx);
4606	free(bus, M_CAMXPT);
4607}
4608
4609static struct cam_et *
4610xpt_alloc_target(struct cam_eb *bus, target_id_t target_id)
4611{
4612	struct cam_et *cur_target, *target;
4613
4614	mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4615	mtx_assert(&bus->eb_mtx, MA_OWNED);
4616	target = (struct cam_et *)malloc(sizeof(*target), M_CAMXPT,
4617					 M_NOWAIT|M_ZERO);
4618	if (target == NULL)
4619		return (NULL);
4620
4621	TAILQ_INIT(&target->ed_entries);
4622	target->bus = bus;
4623	target->target_id = target_id;
4624	target->refcount = 1;
4625	target->generation = 0;
4626	target->luns = NULL;
4627	mtx_init(&target->luns_mtx, "CAM LUNs lock", NULL, MTX_DEF);
4628	timevalclear(&target->last_reset);
4629	/*
4630	 * Hold a reference to our parent bus so it
4631	 * will not go away before we do.
4632	 */
4633	bus->refcount++;
4634
4635	/* Insertion sort into our bus's target list */
4636	cur_target = TAILQ_FIRST(&bus->et_entries);
4637	while (cur_target != NULL && cur_target->target_id < target_id)
4638		cur_target = TAILQ_NEXT(cur_target, links);
4639	if (cur_target != NULL) {
4640		TAILQ_INSERT_BEFORE(cur_target, target, links);
4641	} else {
4642		TAILQ_INSERT_TAIL(&bus->et_entries, target, links);
4643	}
4644	bus->generation++;
4645	return (target);
4646}
4647
4648static void
4649xpt_acquire_target(struct cam_et *target)
4650{
4651	struct cam_eb *bus = target->bus;
4652
4653	mtx_lock(&bus->eb_mtx);
4654	target->refcount++;
4655	mtx_unlock(&bus->eb_mtx);
4656}
4657
4658static void
4659xpt_release_target(struct cam_et *target)
4660{
4661	struct cam_eb *bus = target->bus;
4662
4663	mtx_lock(&bus->eb_mtx);
4664	if (--target->refcount > 0) {
4665		mtx_unlock(&bus->eb_mtx);
4666		return;
4667	}
4668	TAILQ_REMOVE(&bus->et_entries, target, links);
4669	bus->generation++;
4670	mtx_unlock(&bus->eb_mtx);
4671	KASSERT(TAILQ_EMPTY(&target->ed_entries),
4672	    ("destroying target, but device list is not empty"));
4673	xpt_release_bus(bus);
4674	mtx_destroy(&target->luns_mtx);
4675	if (target->luns)
4676		free(target->luns, M_CAMXPT);
4677	free(target, M_CAMXPT);
4678}
4679
4680static struct cam_ed *
4681xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target,
4682			 lun_id_t lun_id)
4683{
4684	struct cam_ed *device;
4685
4686	device = xpt_alloc_device(bus, target, lun_id);
4687	if (device == NULL)
4688		return (NULL);
4689
4690	device->mintags = 1;
4691	device->maxtags = 1;
4692	return (device);
4693}
4694
4695static void
4696xpt_destroy_device(void *context, int pending)
4697{
4698	struct cam_ed	*device = context;
4699
4700	mtx_lock(&device->device_mtx);
4701	mtx_destroy(&device->device_mtx);
4702	free(device, M_CAMDEV);
4703}
4704
4705struct cam_ed *
4706xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
4707{
4708	struct cam_ed	*cur_device, *device;
4709	struct cam_devq	*devq;
4710	cam_status status;
4711
4712	mtx_assert(&bus->eb_mtx, MA_OWNED);
4713	/* Make space for us in the device queue on our bus */
4714	devq = bus->sim->devq;
4715	mtx_lock(&devq->send_mtx);
4716	status = cam_devq_resize(devq, devq->send_queue.array_size + 1);
4717	mtx_unlock(&devq->send_mtx);
4718	if (status != CAM_REQ_CMP)
4719		return (NULL);
4720
4721	device = (struct cam_ed *)malloc(sizeof(*device),
4722					 M_CAMDEV, M_NOWAIT|M_ZERO);
4723	if (device == NULL)
4724		return (NULL);
4725
4726	cam_init_pinfo(&device->devq_entry);
4727	device->target = target;
4728	device->lun_id = lun_id;
4729	device->sim = bus->sim;
4730	if (cam_ccbq_init(&device->ccbq,
4731			  bus->sim->max_dev_openings) != 0) {
4732		free(device, M_CAMDEV);
4733		return (NULL);
4734	}
4735	SLIST_INIT(&device->asyncs);
4736	SLIST_INIT(&device->periphs);
4737	device->generation = 0;
4738	device->flags = CAM_DEV_UNCONFIGURED;
4739	device->tag_delay_count = 0;
4740	device->tag_saved_openings = 0;
4741	device->refcount = 1;
4742	mtx_init(&device->device_mtx, "CAM device lock", NULL, MTX_DEF);
4743	callout_init_mtx(&device->callout, &devq->send_mtx, 0);
4744	TASK_INIT(&device->device_destroy_task, 0, xpt_destroy_device, device);
4745	/*
4746	 * Hold a reference to our parent bus so it
4747	 * will not go away before we do.
4748	 */
4749	target->refcount++;
4750
4751	cur_device = TAILQ_FIRST(&target->ed_entries);
4752	while (cur_device != NULL && cur_device->lun_id < lun_id)
4753		cur_device = TAILQ_NEXT(cur_device, links);
4754	if (cur_device != NULL)
4755		TAILQ_INSERT_BEFORE(cur_device, device, links);
4756	else
4757		TAILQ_INSERT_TAIL(&target->ed_entries, device, links);
4758	target->generation++;
4759	return (device);
4760}
4761
4762void
4763xpt_acquire_device(struct cam_ed *device)
4764{
4765	struct cam_eb *bus = device->target->bus;
4766
4767	mtx_lock(&bus->eb_mtx);
4768	device->refcount++;
4769	mtx_unlock(&bus->eb_mtx);
4770}
4771
4772void
4773xpt_release_device(struct cam_ed *device)
4774{
4775	struct cam_eb *bus = device->target->bus;
4776	struct cam_devq *devq;
4777
4778	mtx_lock(&bus->eb_mtx);
4779	if (--device->refcount > 0) {
4780		mtx_unlock(&bus->eb_mtx);
4781		return;
4782	}
4783
4784	TAILQ_REMOVE(&device->target->ed_entries, device,links);
4785	device->target->generation++;
4786	mtx_unlock(&bus->eb_mtx);
4787
4788	/* Release our slot in the devq */
4789	devq = bus->sim->devq;
4790	mtx_lock(&devq->send_mtx);
4791	cam_devq_resize(devq, devq->send_queue.array_size - 1);
4792	mtx_unlock(&devq->send_mtx);
4793
4794	KASSERT(SLIST_EMPTY(&device->periphs),
4795	    ("destroying device, but periphs list is not empty"));
4796	KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX,
4797	    ("destroying device while still queued for ccbs"));
4798
4799	if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
4800		callout_stop(&device->callout);
4801
4802	xpt_release_target(device->target);
4803
4804	cam_ccbq_fini(&device->ccbq);
4805	/*
4806	 * Free allocated memory.  free(9) does nothing if the
4807	 * supplied pointer is NULL, so it is safe to call without
4808	 * checking.
4809	 */
4810	free(device->supported_vpds, M_CAMXPT);
4811	free(device->device_id, M_CAMXPT);
4812	free(device->ext_inq, M_CAMXPT);
4813	free(device->physpath, M_CAMXPT);
4814	free(device->rcap_buf, M_CAMXPT);
4815	free(device->serial_num, M_CAMXPT);
4816	taskqueue_enqueue(xsoftc.xpt_taskq, &device->device_destroy_task);
4817}
4818
4819u_int32_t
4820xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
4821{
4822	int	result;
4823	struct	cam_ed *dev;
4824
4825	dev = path->device;
4826	mtx_lock(&dev->sim->devq->send_mtx);
4827	result = cam_ccbq_resize(&dev->ccbq, newopenings);
4828	mtx_unlock(&dev->sim->devq->send_mtx);
4829	if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
4830	 || (dev->inq_flags & SID_CmdQue) != 0)
4831		dev->tag_saved_openings = newopenings;
4832	return (result);
4833}
4834
4835static struct cam_eb *
4836xpt_find_bus(path_id_t path_id)
4837{
4838	struct cam_eb *bus;
4839
4840	xpt_lock_buses();
4841	for (bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4842	     bus != NULL;
4843	     bus = TAILQ_NEXT(bus, links)) {
4844		if (bus->path_id == path_id) {
4845			bus->refcount++;
4846			break;
4847		}
4848	}
4849	xpt_unlock_buses();
4850	return (bus);
4851}
4852
4853static struct cam_et *
4854xpt_find_target(struct cam_eb *bus, target_id_t	target_id)
4855{
4856	struct cam_et *target;
4857
4858	mtx_assert(&bus->eb_mtx, MA_OWNED);
4859	for (target = TAILQ_FIRST(&bus->et_entries);
4860	     target != NULL;
4861	     target = TAILQ_NEXT(target, links)) {
4862		if (target->target_id == target_id) {
4863			target->refcount++;
4864			break;
4865		}
4866	}
4867	return (target);
4868}
4869
4870static struct cam_ed *
4871xpt_find_device(struct cam_et *target, lun_id_t lun_id)
4872{
4873	struct cam_ed *device;
4874
4875	mtx_assert(&target->bus->eb_mtx, MA_OWNED);
4876	for (device = TAILQ_FIRST(&target->ed_entries);
4877	     device != NULL;
4878	     device = TAILQ_NEXT(device, links)) {
4879		if (device->lun_id == lun_id) {
4880			device->refcount++;
4881			break;
4882		}
4883	}
4884	return (device);
4885}
4886
4887void
4888xpt_start_tags(struct cam_path *path)
4889{
4890	struct ccb_relsim crs;
4891	struct cam_ed *device;
4892	struct cam_sim *sim;
4893	int    newopenings;
4894
4895	device = path->device;
4896	sim = path->bus->sim;
4897	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4898	xpt_freeze_devq(path, /*count*/1);
4899	device->inq_flags |= SID_CmdQue;
4900	if (device->tag_saved_openings != 0)
4901		newopenings = device->tag_saved_openings;
4902	else
4903		newopenings = min(device->maxtags,
4904				  sim->max_tagged_dev_openings);
4905	xpt_dev_ccbq_resize(path, newopenings);
4906	xpt_async(AC_GETDEV_CHANGED, path, NULL);
4907	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4908	crs.ccb_h.func_code = XPT_REL_SIMQ;
4909	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4910	crs.openings
4911	    = crs.release_timeout
4912	    = crs.qfrozen_cnt
4913	    = 0;
4914	xpt_action((union ccb *)&crs);
4915}
4916
4917void
4918xpt_stop_tags(struct cam_path *path)
4919{
4920	struct ccb_relsim crs;
4921	struct cam_ed *device;
4922	struct cam_sim *sim;
4923
4924	device = path->device;
4925	sim = path->bus->sim;
4926	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4927	device->tag_delay_count = 0;
4928	xpt_freeze_devq(path, /*count*/1);
4929	device->inq_flags &= ~SID_CmdQue;
4930	xpt_dev_ccbq_resize(path, sim->max_dev_openings);
4931	xpt_async(AC_GETDEV_CHANGED, path, NULL);
4932	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4933	crs.ccb_h.func_code = XPT_REL_SIMQ;
4934	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4935	crs.openings
4936	    = crs.release_timeout
4937	    = crs.qfrozen_cnt
4938	    = 0;
4939	xpt_action((union ccb *)&crs);
4940}
4941
4942static void
4943xpt_boot_delay(void *arg)
4944{
4945
4946	xpt_release_boot();
4947}
4948
4949static void
4950xpt_config(void *arg)
4951{
4952	/*
4953	 * Now that interrupts are enabled, go find our devices
4954	 */
4955	if (taskqueue_start_threads(&xsoftc.xpt_taskq, 1, PRIBIO, "CAM taskq"))
4956		printf("xpt_config: failed to create taskqueue thread.\n");
4957
4958	/* Setup debugging path */
4959	if (cam_dflags != CAM_DEBUG_NONE) {
4960		if (xpt_create_path(&cam_dpath, NULL,
4961				    CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
4962				    CAM_DEBUG_LUN) != CAM_REQ_CMP) {
4963			printf("xpt_config: xpt_create_path() failed for debug"
4964			       " target %d:%d:%d, debugging disabled\n",
4965			       CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN);
4966			cam_dflags = CAM_DEBUG_NONE;
4967		}
4968	} else
4969		cam_dpath = NULL;
4970
4971	periphdriver_init(1);
4972	xpt_hold_boot();
4973	callout_init(&xsoftc.boot_callout, 1);
4974	callout_reset_sbt(&xsoftc.boot_callout, SBT_1MS * xsoftc.boot_delay, 0,
4975	    xpt_boot_delay, NULL, 0);
4976	/* Fire up rescan thread. */
4977	if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0,
4978	    "cam", "scanner")) {
4979		printf("xpt_config: failed to create rescan thread.\n");
4980	}
4981}
4982
4983void
4984xpt_hold_boot(void)
4985{
4986	xpt_lock_buses();
4987	xsoftc.buses_to_config++;
4988	xpt_unlock_buses();
4989}
4990
4991void
4992xpt_release_boot(void)
4993{
4994	xpt_lock_buses();
4995	xsoftc.buses_to_config--;
4996	if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
4997		struct	xpt_task *task;
4998
4999		xsoftc.buses_config_done = 1;
5000		xpt_unlock_buses();
5001		/* Call manually because we don't have any busses */
5002		task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
5003		if (task != NULL) {
5004			TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
5005			taskqueue_enqueue(taskqueue_thread, &task->task);
5006		}
5007	} else
5008		xpt_unlock_buses();
5009}
5010
5011/*
5012 * If the given device only has one peripheral attached to it, and if that
5013 * peripheral is the passthrough driver, announce it.  This insures that the
5014 * user sees some sort of announcement for every peripheral in their system.
5015 */
5016static int
5017xptpassannouncefunc(struct cam_ed *device, void *arg)
5018{
5019	struct cam_periph *periph;
5020	int i;
5021
5022	for (periph = SLIST_FIRST(&device->periphs), i = 0; periph != NULL;
5023	     periph = SLIST_NEXT(periph, periph_links), i++);
5024
5025	periph = SLIST_FIRST(&device->periphs);
5026	if ((i == 1)
5027	 && (strncmp(periph->periph_name, "pass", 4) == 0))
5028		xpt_announce_periph(periph, NULL);
5029
5030	return(1);
5031}
5032
5033static void
5034xpt_finishconfig_task(void *context, int pending)
5035{
5036
5037	periphdriver_init(2);
5038	/*
5039	 * Check for devices with no "standard" peripheral driver
5040	 * attached.  For any devices like that, announce the
5041	 * passthrough driver so the user will see something.
5042	 */
5043	if (!bootverbose)
5044		xpt_for_all_devices(xptpassannouncefunc, NULL);
5045
5046	/* Release our hook so that the boot can continue. */
5047	config_intrhook_disestablish(xsoftc.xpt_config_hook);
5048	free(xsoftc.xpt_config_hook, M_CAMXPT);
5049	xsoftc.xpt_config_hook = NULL;
5050
5051	free(context, M_CAMXPT);
5052}
5053
5054cam_status
5055xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
5056		   struct cam_path *path)
5057{
5058	struct ccb_setasync csa;
5059	cam_status status;
5060	int xptpath = 0;
5061
5062	if (path == NULL) {
5063		status = xpt_create_path(&path, /*periph*/NULL, CAM_XPT_PATH_ID,
5064					 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
5065		if (status != CAM_REQ_CMP)
5066			return (status);
5067		xpt_path_lock(path);
5068		xptpath = 1;
5069	}
5070
5071	xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL);
5072	csa.ccb_h.func_code = XPT_SASYNC_CB;
5073	csa.event_enable = event;
5074	csa.callback = cbfunc;
5075	csa.callback_arg = cbarg;
5076	xpt_action((union ccb *)&csa);
5077	status = csa.ccb_h.status;
5078
5079	if (xptpath) {
5080		xpt_path_unlock(path);
5081		xpt_free_path(path);
5082	}
5083
5084	if ((status == CAM_REQ_CMP) &&
5085	    (csa.event_enable & AC_FOUND_DEVICE)) {
5086		/*
5087		 * Get this peripheral up to date with all
5088		 * the currently existing devices.
5089		 */
5090		xpt_for_all_devices(xptsetasyncfunc, &csa);
5091	}
5092	if ((status == CAM_REQ_CMP) &&
5093	    (csa.event_enable & AC_PATH_REGISTERED)) {
5094		/*
5095		 * Get this peripheral up to date with all
5096		 * the currently existing busses.
5097		 */
5098		xpt_for_all_busses(xptsetasyncbusfunc, &csa);
5099	}
5100
5101	return (status);
5102}
5103
5104static void
5105xptaction(struct cam_sim *sim, union ccb *work_ccb)
5106{
5107	CAM_DEBUG(work_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xptaction\n"));
5108
5109	switch (work_ccb->ccb_h.func_code) {
5110	/* Common cases first */
5111	case XPT_PATH_INQ:		/* Path routing inquiry */
5112	{
5113		struct ccb_pathinq *cpi;
5114
5115		cpi = &work_ccb->cpi;
5116		cpi->version_num = 1; /* XXX??? */
5117		cpi->hba_inquiry = 0;
5118		cpi->target_sprt = 0;
5119		cpi->hba_misc = 0;
5120		cpi->hba_eng_cnt = 0;
5121		cpi->max_target = 0;
5122		cpi->max_lun = 0;
5123		cpi->initiator_id = 0;
5124		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
5125		strncpy(cpi->hba_vid, "", HBA_IDLEN);
5126		strncpy(cpi->dev_name, sim->sim_name, DEV_IDLEN);
5127		cpi->unit_number = sim->unit_number;
5128		cpi->bus_id = sim->bus_id;
5129		cpi->base_transfer_speed = 0;
5130		cpi->protocol = PROTO_UNSPECIFIED;
5131		cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
5132		cpi->transport = XPORT_UNSPECIFIED;
5133		cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
5134		cpi->ccb_h.status = CAM_REQ_CMP;
5135		xpt_done(work_ccb);
5136		break;
5137	}
5138	default:
5139		work_ccb->ccb_h.status = CAM_REQ_INVALID;
5140		xpt_done(work_ccb);
5141		break;
5142	}
5143}
5144
5145/*
5146 * The xpt as a "controller" has no interrupt sources, so polling
5147 * is a no-op.
5148 */
5149static void
5150xptpoll(struct cam_sim *sim)
5151{
5152}
5153
5154void
5155xpt_lock_buses(void)
5156{
5157	mtx_lock(&xsoftc.xpt_topo_lock);
5158}
5159
5160void
5161xpt_unlock_buses(void)
5162{
5163	mtx_unlock(&xsoftc.xpt_topo_lock);
5164}
5165
5166struct mtx *
5167xpt_path_mtx(struct cam_path *path)
5168{
5169
5170	return (&path->device->device_mtx);
5171}
5172
5173static void
5174xpt_done_process(struct ccb_hdr *ccb_h)
5175{
5176	struct cam_sim *sim;
5177	struct cam_devq *devq;
5178	struct mtx *mtx = NULL;
5179
5180	if (ccb_h->flags & CAM_HIGH_POWER) {
5181		struct highpowerlist	*hphead;
5182		struct cam_ed		*device;
5183
5184		mtx_lock(&xsoftc.xpt_highpower_lock);
5185		hphead = &xsoftc.highpowerq;
5186
5187		device = STAILQ_FIRST(hphead);
5188
5189		/*
5190		 * Increment the count since this command is done.
5191		 */
5192		xsoftc.num_highpower++;
5193
5194		/*
5195		 * Any high powered commands queued up?
5196		 */
5197		if (device != NULL) {
5198
5199			STAILQ_REMOVE_HEAD(hphead, highpowerq_entry);
5200			mtx_unlock(&xsoftc.xpt_highpower_lock);
5201
5202			mtx_lock(&device->sim->devq->send_mtx);
5203			xpt_release_devq_device(device,
5204					 /*count*/1, /*runqueue*/TRUE);
5205			mtx_unlock(&device->sim->devq->send_mtx);
5206		} else
5207			mtx_unlock(&xsoftc.xpt_highpower_lock);
5208	}
5209
5210	sim = ccb_h->path->bus->sim;
5211
5212	if (ccb_h->status & CAM_RELEASE_SIMQ) {
5213		xpt_release_simq(sim, /*run_queue*/FALSE);
5214		ccb_h->status &= ~CAM_RELEASE_SIMQ;
5215	}
5216
5217	if ((ccb_h->flags & CAM_DEV_QFRZDIS)
5218	 && (ccb_h->status & CAM_DEV_QFRZN)) {
5219		xpt_release_devq(ccb_h->path, /*count*/1, /*run_queue*/TRUE);
5220		ccb_h->status &= ~CAM_DEV_QFRZN;
5221	}
5222
5223	devq = sim->devq;
5224	if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
5225		struct cam_ed *dev = ccb_h->path->device;
5226
5227		mtx_lock(&devq->send_mtx);
5228		devq->send_active--;
5229		devq->send_openings++;
5230		cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
5231
5232		if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
5233		  && (dev->ccbq.dev_active == 0))) {
5234			dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY;
5235			xpt_release_devq_device(dev, /*count*/1,
5236					 /*run_queue*/FALSE);
5237		}
5238
5239		if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
5240		  && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) {
5241			dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
5242			xpt_release_devq_device(dev, /*count*/1,
5243					 /*run_queue*/FALSE);
5244		}
5245
5246		if (!device_is_queued(dev))
5247			(void)xpt_schedule_devq(devq, dev);
5248		xpt_run_devq(devq);
5249		mtx_unlock(&devq->send_mtx);
5250
5251		if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0) {
5252			mtx = xpt_path_mtx(ccb_h->path);
5253			mtx_lock(mtx);
5254
5255			if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5256			 && (--dev->tag_delay_count == 0))
5257				xpt_start_tags(ccb_h->path);
5258		}
5259	}
5260
5261	if ((ccb_h->flags & CAM_UNLOCKED) == 0) {
5262		if (mtx == NULL) {
5263			mtx = xpt_path_mtx(ccb_h->path);
5264			mtx_lock(mtx);
5265		}
5266	} else {
5267		if (mtx != NULL) {
5268			mtx_unlock(mtx);
5269			mtx = NULL;
5270		}
5271	}
5272
5273	/* Call the peripheral driver's callback */
5274	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
5275	(*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
5276	if (mtx != NULL)
5277		mtx_unlock(mtx);
5278}
5279
5280void
5281xpt_done_td(void *arg)
5282{
5283	struct cam_doneq *queue = arg;
5284	struct ccb_hdr *ccb_h;
5285	STAILQ_HEAD(, ccb_hdr)	doneq;
5286
5287	STAILQ_INIT(&doneq);
5288	mtx_lock(&queue->cam_doneq_mtx);
5289	while (1) {
5290		while (STAILQ_EMPTY(&queue->cam_doneq)) {
5291			queue->cam_doneq_sleep = 1;
5292			msleep(&queue->cam_doneq, &queue->cam_doneq_mtx,
5293			    PRIBIO, "-", 0);
5294			queue->cam_doneq_sleep = 0;
5295		}
5296		STAILQ_CONCAT(&doneq, &queue->cam_doneq);
5297		mtx_unlock(&queue->cam_doneq_mtx);
5298
5299		THREAD_NO_SLEEPING();
5300		while ((ccb_h = STAILQ_FIRST(&doneq)) != NULL) {
5301			STAILQ_REMOVE_HEAD(&doneq, sim_links.stqe);
5302			xpt_done_process(ccb_h);
5303		}
5304		THREAD_SLEEPING_OK();
5305
5306		mtx_lock(&queue->cam_doneq_mtx);
5307	}
5308}
5309
5310static void
5311camisr_runqueue(void)
5312{
5313	struct	ccb_hdr *ccb_h;
5314	struct cam_doneq *queue;
5315	int i;
5316
5317	/* Process global queues. */
5318	for (i = 0; i < cam_num_doneqs; i++) {
5319		queue = &cam_doneqs[i];
5320		mtx_lock(&queue->cam_doneq_mtx);
5321		while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) {
5322			STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe);
5323			mtx_unlock(&queue->cam_doneq_mtx);
5324			xpt_done_process(ccb_h);
5325			mtx_lock(&queue->cam_doneq_mtx);
5326		}
5327		mtx_unlock(&queue->cam_doneq_mtx);
5328	}
5329}
5330