cam_xpt.c revision 298810 
1/*-
2 * Implementation of the Common Access Method Transport (XPT) layer.
3 *
4 * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs.
5 * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions, and the following disclaimer,
13 *    without modification, immediately at the beginning of the file.
14 * 2. The name of the author may not be used to endorse or promote products
15 *    derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
21 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/cam/cam_xpt.c 298810 2016-04-29 21:05:48Z pfg $");
32
33#include <sys/param.h>
34#include <sys/bus.h>
35#include <sys/systm.h>
36#include <sys/types.h>
37#include <sys/malloc.h>
38#include <sys/kernel.h>
39#include <sys/time.h>
40#include <sys/conf.h>
41#include <sys/fcntl.h>
42#include <sys/interrupt.h>
43#include <sys/proc.h>
44#include <sys/sbuf.h>
45#include <sys/smp.h>
46#include <sys/taskqueue.h>
47
48#include <sys/lock.h>
49#include <sys/mutex.h>
50#include <sys/sysctl.h>
51#include <sys/kthread.h>
52
53#include <cam/cam.h>
54#include <cam/cam_ccb.h>
55#include <cam/cam_periph.h>
56#include <cam/cam_queue.h>
57#include <cam/cam_sim.h>
58#include <cam/cam_xpt.h>
59#include <cam/cam_xpt_sim.h>
60#include <cam/cam_xpt_periph.h>
61#include <cam/cam_xpt_internal.h>
62#include <cam/cam_debug.h>
63#include <cam/cam_compat.h>
64
65#include <cam/scsi/scsi_all.h>
66#include <cam/scsi/scsi_message.h>
67#include <cam/scsi/scsi_pass.h>
68
69#include <machine/md_var.h>	/* geometry translation */
70#include <machine/stdarg.h>	/* for xpt_print below */
71
72#include "opt_cam.h"
73
74/*
75 * This is the maximum number of high powered commands (e.g. start unit)
76 * that can be outstanding at a particular time.
77 */
78#ifndef CAM_MAX_HIGHPOWER
79#define CAM_MAX_HIGHPOWER  4
80#endif
81
82/* Datastructures internal to the xpt layer */
83MALLOC_DEFINE(M_CAMXPT, "CAM XPT", "CAM XPT buffers");
84MALLOC_DEFINE(M_CAMDEV, "CAM DEV", "CAM devices");
85MALLOC_DEFINE(M_CAMCCB, "CAM CCB", "CAM CCBs");
86MALLOC_DEFINE(M_CAMPATH, "CAM path", "CAM paths");
87
88/* Object for defering XPT actions to a taskqueue */
89struct xpt_task {
90	struct task	task;
91	void		*data1;
92	uintptr_t	data2;
93};
94
95struct xpt_softc {
96	uint32_t		xpt_generation;
97
98	/* number of high powered commands that can go through right now */
99	struct mtx		xpt_highpower_lock;
100	STAILQ_HEAD(highpowerlist, cam_ed)	highpowerq;
101	int			num_highpower;
102
103	/* queue for handling async rescan requests. */
104	TAILQ_HEAD(, ccb_hdr) ccb_scanq;
105	int buses_to_config;
106	int buses_config_done;
107
108	/* Registered busses */
109	TAILQ_HEAD(,cam_eb)	xpt_busses;
110	u_int			bus_generation;
111
112	struct intr_config_hook	*xpt_config_hook;
113
114	int			boot_delay;
115	struct callout 		boot_callout;
116
117	struct mtx		xpt_topo_lock;
118	struct mtx		xpt_lock;
119	struct taskqueue	*xpt_taskq;
120};
121
122typedef enum {
123	DM_RET_COPY		= 0x01,
124	DM_RET_FLAG_MASK	= 0x0f,
125	DM_RET_NONE		= 0x00,
126	DM_RET_STOP		= 0x10,
127	DM_RET_DESCEND		= 0x20,
128	DM_RET_ERROR		= 0x30,
129	DM_RET_ACTION_MASK	= 0xf0
130} dev_match_ret;
131
132typedef enum {
133	XPT_DEPTH_BUS,
134	XPT_DEPTH_TARGET,
135	XPT_DEPTH_DEVICE,
136	XPT_DEPTH_PERIPH
137} xpt_traverse_depth;
138
139struct xpt_traverse_config {
140	xpt_traverse_depth	depth;
141	void			*tr_func;
142	void			*tr_arg;
143};
144
145typedef	int	xpt_busfunc_t (struct cam_eb *bus, void *arg);
146typedef	int	xpt_targetfunc_t (struct cam_et *target, void *arg);
147typedef	int	xpt_devicefunc_t (struct cam_ed *device, void *arg);
148typedef	int	xpt_periphfunc_t (struct cam_periph *periph, void *arg);
149typedef int	xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
150
151/* Transport layer configuration information */
152static struct xpt_softc xsoftc;
153
154MTX_SYSINIT(xpt_topo_init, &xsoftc.xpt_topo_lock, "XPT topology lock", MTX_DEF);
155
156SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
157           &xsoftc.boot_delay, 0, "Bus registration wait time");
158SYSCTL_UINT(_kern_cam, OID_AUTO, xpt_generation, CTLFLAG_RD,
159	    &xsoftc.xpt_generation, 0, "CAM peripheral generation count");
160
161struct cam_doneq {
162	struct mtx_padalign	cam_doneq_mtx;
163	STAILQ_HEAD(, ccb_hdr)	cam_doneq;
164	int			cam_doneq_sleep;
165};
166
167static struct cam_doneq cam_doneqs[MAXCPU];
168static int cam_num_doneqs;
169static struct proc *cam_proc;
170
171SYSCTL_INT(_kern_cam, OID_AUTO, num_doneqs, CTLFLAG_RDTUN,
172           &cam_num_doneqs, 0, "Number of completion queues/threads");
173
174struct cam_periph *xpt_periph;
175
176static periph_init_t xpt_periph_init;
177
178static struct periph_driver xpt_driver =
179{
180	xpt_periph_init, "xpt",
181	TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
182	CAM_PERIPH_DRV_EARLY
183};
184
185PERIPHDRIVER_DECLARE(xpt, xpt_driver);
186
187static d_open_t xptopen;
188static d_close_t xptclose;
189static d_ioctl_t xptioctl;
190static d_ioctl_t xptdoioctl;
191
192static struct cdevsw xpt_cdevsw = {
193	.d_version =	D_VERSION,
194	.d_flags =	0,
195	.d_open =	xptopen,
196	.d_close =	xptclose,
197	.d_ioctl =	xptioctl,
198	.d_name =	"xpt",
199};
200
201/* Storage for debugging datastructures */
202struct cam_path *cam_dpath;
203u_int32_t cam_dflags = CAM_DEBUG_FLAGS;
204SYSCTL_UINT(_kern_cam, OID_AUTO, dflags, CTLFLAG_RWTUN,
205	&cam_dflags, 0, "Enabled debug flags");
206u_int32_t cam_debug_delay = CAM_DEBUG_DELAY;
207SYSCTL_UINT(_kern_cam, OID_AUTO, debug_delay, CTLFLAG_RWTUN,
208	&cam_debug_delay, 0, "Delay in us after each debug message");
209
210/* Our boot-time initialization hook */
211static int cam_module_event_handler(module_t, int /*modeventtype_t*/, void *);
212
213static moduledata_t cam_moduledata = {
214	"cam",
215	cam_module_event_handler,
216	NULL
217};
218
219static int	xpt_init(void *);
220
221DECLARE_MODULE(cam, cam_moduledata, SI_SUB_CONFIGURE, SI_ORDER_SECOND);
222MODULE_VERSION(cam, 1);
223
224
225static void		xpt_async_bcast(struct async_list *async_head,
226					u_int32_t async_code,
227					struct cam_path *path,
228					void *async_arg);
229static path_id_t xptnextfreepathid(void);
230static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
231static union ccb *xpt_get_ccb(struct cam_periph *periph);
232static union ccb *xpt_get_ccb_nowait(struct cam_periph *periph);
233static void	 xpt_run_allocq(struct cam_periph *periph, int sleep);
234static void	 xpt_run_allocq_task(void *context, int pending);
235static void	 xpt_run_devq(struct cam_devq *devq);
236static timeout_t xpt_release_devq_timeout;
237static void	 xpt_release_simq_timeout(void *arg) __unused;
238static void	 xpt_acquire_bus(struct cam_eb *bus);
239static void	 xpt_release_bus(struct cam_eb *bus);
240static uint32_t	 xpt_freeze_devq_device(struct cam_ed *dev, u_int count);
241static int	 xpt_release_devq_device(struct cam_ed *dev, u_int count,
242		    int run_queue);
243static struct cam_et*
244		 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
245static void	 xpt_acquire_target(struct cam_et *target);
246static void	 xpt_release_target(struct cam_et *target);
247static struct cam_eb*
248		 xpt_find_bus(path_id_t path_id);
249static struct cam_et*
250		 xpt_find_target(struct cam_eb *bus, target_id_t target_id);
251static struct cam_ed*
252		 xpt_find_device(struct cam_et *target, lun_id_t lun_id);
253static void	 xpt_config(void *arg);
254static int	 xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
255				 u_int32_t new_priority);
256static xpt_devicefunc_t xptpassannouncefunc;
257static void	 xptaction(struct cam_sim *sim, union ccb *work_ccb);
258static void	 xptpoll(struct cam_sim *sim);
259static void	 camisr_runqueue(void);
260static void	 xpt_done_process(struct ccb_hdr *ccb_h);
261static void	 xpt_done_td(void *);
262static dev_match_ret	xptbusmatch(struct dev_match_pattern *patterns,
263				    u_int num_patterns, struct cam_eb *bus);
264static dev_match_ret	xptdevicematch(struct dev_match_pattern *patterns,
265				       u_int num_patterns,
266				       struct cam_ed *device);
267static dev_match_ret	xptperiphmatch(struct dev_match_pattern *patterns,
268				       u_int num_patterns,
269				       struct cam_periph *periph);
270static xpt_busfunc_t	xptedtbusfunc;
271static xpt_targetfunc_t	xptedttargetfunc;
272static xpt_devicefunc_t	xptedtdevicefunc;
273static xpt_periphfunc_t	xptedtperiphfunc;
274static xpt_pdrvfunc_t	xptplistpdrvfunc;
275static xpt_periphfunc_t	xptplistperiphfunc;
276static int		xptedtmatch(struct ccb_dev_match *cdm);
277static int		xptperiphlistmatch(struct ccb_dev_match *cdm);
278static int		xptbustraverse(struct cam_eb *start_bus,
279				       xpt_busfunc_t *tr_func, void *arg);
280static int		xpttargettraverse(struct cam_eb *bus,
281					  struct cam_et *start_target,
282					  xpt_targetfunc_t *tr_func, void *arg);
283static int		xptdevicetraverse(struct cam_et *target,
284					  struct cam_ed *start_device,
285					  xpt_devicefunc_t *tr_func, void *arg);
286static int		xptperiphtraverse(struct cam_ed *device,
287					  struct cam_periph *start_periph,
288					  xpt_periphfunc_t *tr_func, void *arg);
289static int		xptpdrvtraverse(struct periph_driver **start_pdrv,
290					xpt_pdrvfunc_t *tr_func, void *arg);
291static int		xptpdperiphtraverse(struct periph_driver **pdrv,
292					    struct cam_periph *start_periph,
293					    xpt_periphfunc_t *tr_func,
294					    void *arg);
295static xpt_busfunc_t	xptdefbusfunc;
296static xpt_targetfunc_t	xptdeftargetfunc;
297static xpt_devicefunc_t	xptdefdevicefunc;
298static xpt_periphfunc_t	xptdefperiphfunc;
299static void		xpt_finishconfig_task(void *context, int pending);
300static void		xpt_dev_async_default(u_int32_t async_code,
301					      struct cam_eb *bus,
302					      struct cam_et *target,
303					      struct cam_ed *device,
304					      void *async_arg);
305static struct cam_ed *	xpt_alloc_device_default(struct cam_eb *bus,
306						 struct cam_et *target,
307						 lun_id_t lun_id);
308static xpt_devicefunc_t	xptsetasyncfunc;
309static xpt_busfunc_t	xptsetasyncbusfunc;
310static cam_status	xptregister(struct cam_periph *periph,
311				    void *arg);
312static __inline int device_is_queued(struct cam_ed *device);
313
314static __inline int
315xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev)
316{
317	int	retval;
318
319	mtx_assert(&devq->send_mtx, MA_OWNED);
320	if ((dev->ccbq.queue.entries > 0) &&
321	    (dev->ccbq.dev_openings > 0) &&
322	    (dev->ccbq.queue.qfrozen_cnt == 0)) {
323		/*
324		 * The priority of a device waiting for controller
325		 * resources is that of the highest priority CCB
326		 * enqueued.
327		 */
328		retval =
329		    xpt_schedule_dev(&devq->send_queue,
330				     &dev->devq_entry,
331				     CAMQ_GET_PRIO(&dev->ccbq.queue));
332	} else {
333		retval = 0;
334	}
335	return (retval);
336}
337
338static __inline int
339device_is_queued(struct cam_ed *device)
340{
341	return (device->devq_entry.index != CAM_UNQUEUED_INDEX);
342}
343
344static void
345xpt_periph_init()
346{
347	make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0");
348}
349
350static int
351xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
352{
353
354	/*
355	 * Only allow read-write access.
356	 */
357	if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0))
358		return(EPERM);
359
360	/*
361	 * We don't allow nonblocking access.
362	 */
363	if ((flags & O_NONBLOCK) != 0) {
364		printf("%s: can't do nonblocking access\n", devtoname(dev));
365		return(ENODEV);
366	}
367
368	return(0);
369}
370
371static int
372xptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
373{
374
375	return(0);
376}
377
378/*
379 * Don't automatically grab the xpt softc lock here even though this is going
380 * through the xpt device.  The xpt device is really just a back door for
381 * accessing other devices and SIMs, so the right thing to do is to grab
382 * the appropriate SIM lock once the bus/SIM is located.
383 */
384static int
385xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
386{
387	int error;
388
389	if ((error = xptdoioctl(dev, cmd, addr, flag, td)) == ENOTTY) {
390		error = cam_compat_ioctl(dev, cmd, addr, flag, td, xptdoioctl);
391	}
392	return (error);
393}
394
395static int
396xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
397{
398	int error;
399
400	error = 0;
401
402	switch(cmd) {
403	/*
404	 * For the transport layer CAMIOCOMMAND ioctl, we really only want
405	 * to accept CCB types that don't quite make sense to send through a
406	 * passthrough driver. XPT_PATH_INQ is an exception to this, as stated
407	 * in the CAM spec.
408	 */
409	case CAMIOCOMMAND: {
410		union ccb *ccb;
411		union ccb *inccb;
412		struct cam_eb *bus;
413
414		inccb = (union ccb *)addr;
415
416		bus = xpt_find_bus(inccb->ccb_h.path_id);
417		if (bus == NULL)
418			return (EINVAL);
419
420		switch (inccb->ccb_h.func_code) {
421		case XPT_SCAN_BUS:
422		case XPT_RESET_BUS:
423			if (inccb->ccb_h.target_id != CAM_TARGET_WILDCARD ||
424			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
425				xpt_release_bus(bus);
426				return (EINVAL);
427			}
428			break;
429		case XPT_SCAN_TGT:
430			if (inccb->ccb_h.target_id == CAM_TARGET_WILDCARD ||
431			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
432				xpt_release_bus(bus);
433				return (EINVAL);
434			}
435			break;
436		default:
437			break;
438		}
439
440		switch(inccb->ccb_h.func_code) {
441		case XPT_SCAN_BUS:
442		case XPT_RESET_BUS:
443		case XPT_PATH_INQ:
444		case XPT_ENG_INQ:
445		case XPT_SCAN_LUN:
446		case XPT_SCAN_TGT:
447
448			ccb = xpt_alloc_ccb();
449
450			/*
451			 * Create a path using the bus, target, and lun the
452			 * user passed in.
453			 */
454			if (xpt_create_path(&ccb->ccb_h.path, NULL,
455					    inccb->ccb_h.path_id,
456					    inccb->ccb_h.target_id,
457					    inccb->ccb_h.target_lun) !=
458					    CAM_REQ_CMP){
459				error = EINVAL;
460				xpt_free_ccb(ccb);
461				break;
462			}
463			/* Ensure all of our fields are correct */
464			xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path,
465				      inccb->ccb_h.pinfo.priority);
466			xpt_merge_ccb(ccb, inccb);
467			xpt_path_lock(ccb->ccb_h.path);
468			cam_periph_runccb(ccb, NULL, 0, 0, NULL);
469			xpt_path_unlock(ccb->ccb_h.path);
470			bcopy(ccb, inccb, sizeof(union ccb));
471			xpt_free_path(ccb->ccb_h.path);
472			xpt_free_ccb(ccb);
473			break;
474
475		case XPT_DEBUG: {
476			union ccb ccb;
477
478			/*
479			 * This is an immediate CCB, so it's okay to
480			 * allocate it on the stack.
481			 */
482
483			/*
484			 * Create a path using the bus, target, and lun the
485			 * user passed in.
486			 */
487			if (xpt_create_path(&ccb.ccb_h.path, NULL,
488					    inccb->ccb_h.path_id,
489					    inccb->ccb_h.target_id,
490					    inccb->ccb_h.target_lun) !=
491					    CAM_REQ_CMP){
492				error = EINVAL;
493				break;
494			}
495			/* Ensure all of our fields are correct */
496			xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path,
497				      inccb->ccb_h.pinfo.priority);
498			xpt_merge_ccb(&ccb, inccb);
499			xpt_action(&ccb);
500			bcopy(&ccb, inccb, sizeof(union ccb));
501			xpt_free_path(ccb.ccb_h.path);
502			break;
503
504		}
505		case XPT_DEV_MATCH: {
506			struct cam_periph_map_info mapinfo;
507			struct cam_path *old_path;
508
509			/*
510			 * We can't deal with physical addresses for this
511			 * type of transaction.
512			 */
513			if ((inccb->ccb_h.flags & CAM_DATA_MASK) !=
514			    CAM_DATA_VADDR) {
515				error = EINVAL;
516				break;
517			}
518
519			/*
520			 * Save this in case the caller had it set to
521			 * something in particular.
522			 */
523			old_path = inccb->ccb_h.path;
524
525			/*
526			 * We really don't need a path for the matching
527			 * code.  The path is needed because of the
528			 * debugging statements in xpt_action().  They
529			 * assume that the CCB has a valid path.
530			 */
531			inccb->ccb_h.path = xpt_periph->path;
532
533			bzero(&mapinfo, sizeof(mapinfo));
534
535			/*
536			 * Map the pattern and match buffers into kernel
537			 * virtual address space.
538			 */
539			error = cam_periph_mapmem(inccb, &mapinfo, MAXPHYS);
540
541			if (error) {
542				inccb->ccb_h.path = old_path;
543				break;
544			}
545
546			/*
547			 * This is an immediate CCB, we can send it on directly.
548			 */
549			xpt_action(inccb);
550
551			/*
552			 * Map the buffers back into user space.
553			 */
554			cam_periph_unmapmem(inccb, &mapinfo);
555
556			inccb->ccb_h.path = old_path;
557
558			error = 0;
559			break;
560		}
561		default:
562			error = ENOTSUP;
563			break;
564		}
565		xpt_release_bus(bus);
566		break;
567	}
568	/*
569	 * This is the getpassthru ioctl. It takes a XPT_GDEVLIST ccb as input,
570	 * with the periphal driver name and unit name filled in.  The other
571	 * fields don't really matter as input.  The passthrough driver name
572	 * ("pass"), and unit number are passed back in the ccb.  The current
573	 * device generation number, and the index into the device peripheral
574	 * driver list, and the status are also passed back.  Note that
575	 * since we do everything in one pass, unlike the XPT_GDEVLIST ccb,
576	 * we never return a status of CAM_GDEVLIST_LIST_CHANGED.  It is
577	 * (or rather should be) impossible for the device peripheral driver
578	 * list to change since we look at the whole thing in one pass, and
579	 * we do it with lock protection.
580	 *
581	 */
582	case CAMGETPASSTHRU: {
583		union ccb *ccb;
584		struct cam_periph *periph;
585		struct periph_driver **p_drv;
586		char   *name;
587		u_int unit;
588		int base_periph_found;
589
590		ccb = (union ccb *)addr;
591		unit = ccb->cgdl.unit_number;
592		name = ccb->cgdl.periph_name;
593		base_periph_found = 0;
594
595		/*
596		 * Sanity check -- make sure we don't get a null peripheral
597		 * driver name.
598		 */
599		if (*ccb->cgdl.periph_name == '\0') {
600			error = EINVAL;
601			break;
602		}
603
604		/* Keep the list from changing while we traverse it */
605		xpt_lock_buses();
606
607		/* first find our driver in the list of drivers */
608		for (p_drv = periph_drivers; *p_drv != NULL; p_drv++)
609			if (strcmp((*p_drv)->driver_name, name) == 0)
610				break;
611
612		if (*p_drv == NULL) {
613			xpt_unlock_buses();
614			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
615			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
616			*ccb->cgdl.periph_name = '\0';
617			ccb->cgdl.unit_number = 0;
618			error = ENOENT;
619			break;
620		}
621
622		/*
623		 * Run through every peripheral instance of this driver
624		 * and check to see whether it matches the unit passed
625		 * in by the user.  If it does, get out of the loops and
626		 * find the passthrough driver associated with that
627		 * peripheral driver.
628		 */
629		for (periph = TAILQ_FIRST(&(*p_drv)->units); periph != NULL;
630		     periph = TAILQ_NEXT(periph, unit_links)) {
631
632			if (periph->unit_number == unit)
633				break;
634		}
635		/*
636		 * If we found the peripheral driver that the user passed
637		 * in, go through all of the peripheral drivers for that
638		 * particular device and look for a passthrough driver.
639		 */
640		if (periph != NULL) {
641			struct cam_ed *device;
642			int i;
643
644			base_periph_found = 1;
645			device = periph->path->device;
646			for (i = 0, periph = SLIST_FIRST(&device->periphs);
647			     periph != NULL;
648			     periph = SLIST_NEXT(periph, periph_links), i++) {
649				/*
650				 * Check to see whether we have a
651				 * passthrough device or not.
652				 */
653				if (strcmp(periph->periph_name, "pass") == 0) {
654					/*
655					 * Fill in the getdevlist fields.
656					 */
657					strcpy(ccb->cgdl.periph_name,
658					       periph->periph_name);
659					ccb->cgdl.unit_number =
660						periph->unit_number;
661					if (SLIST_NEXT(periph, periph_links))
662						ccb->cgdl.status =
663							CAM_GDEVLIST_MORE_DEVS;
664					else
665						ccb->cgdl.status =
666						       CAM_GDEVLIST_LAST_DEVICE;
667					ccb->cgdl.generation =
668						device->generation;
669					ccb->cgdl.index = i;
670					/*
671					 * Fill in some CCB header fields
672					 * that the user may want.
673					 */
674					ccb->ccb_h.path_id =
675						periph->path->bus->path_id;
676					ccb->ccb_h.target_id =
677						periph->path->target->target_id;
678					ccb->ccb_h.target_lun =
679						periph->path->device->lun_id;
680					ccb->ccb_h.status = CAM_REQ_CMP;
681					break;
682				}
683			}
684		}
685
686		/*
687		 * If the periph is null here, one of two things has
688		 * happened.  The first possibility is that we couldn't
689		 * find the unit number of the particular peripheral driver
690		 * that the user is asking about.  e.g. the user asks for
691		 * the passthrough driver for "da11".  We find the list of
692		 * "da" peripherals all right, but there is no unit 11.
693		 * The other possibility is that we went through the list
694		 * of peripheral drivers attached to the device structure,
695		 * but didn't find one with the name "pass".  Either way,
696		 * we return ENOENT, since we couldn't find something.
697		 */
698		if (periph == NULL) {
699			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
700			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
701			*ccb->cgdl.periph_name = '\0';
702			ccb->cgdl.unit_number = 0;
703			error = ENOENT;
704			/*
705			 * It is unfortunate that this is even necessary,
706			 * but there are many, many clueless users out there.
707			 * If this is true, the user is looking for the
708			 * passthrough driver, but doesn't have one in his
709			 * kernel.
710			 */
711			if (base_periph_found == 1) {
712				printf("xptioctl: pass driver is not in the "
713				       "kernel\n");
714				printf("xptioctl: put \"device pass\" in "
715				       "your kernel config file\n");
716			}
717		}
718		xpt_unlock_buses();
719		break;
720		}
721	default:
722		error = ENOTTY;
723		break;
724	}
725
726	return(error);
727}
728
729static int
730cam_module_event_handler(module_t mod, int what, void *arg)
731{
732	int error;
733
734	switch (what) {
735	case MOD_LOAD:
736		if ((error = xpt_init(NULL)) != 0)
737			return (error);
738		break;
739	case MOD_UNLOAD:
740		return EBUSY;
741	default:
742		return EOPNOTSUPP;
743	}
744
745	return 0;
746}
747
748static void
749xpt_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
750{
751
752	if (done_ccb->ccb_h.ppriv_ptr1 == NULL) {
753		xpt_free_path(done_ccb->ccb_h.path);
754		xpt_free_ccb(done_ccb);
755	} else {
756		done_ccb->ccb_h.cbfcnp = done_ccb->ccb_h.ppriv_ptr1;
757		(*done_ccb->ccb_h.cbfcnp)(periph, done_ccb);
758	}
759	xpt_release_boot();
760}
761
762/* thread to handle bus rescans */
763static void
764xpt_scanner_thread(void *dummy)
765{
766	union ccb	*ccb;
767	struct cam_path	 path;
768
769	xpt_lock_buses();
770	for (;;) {
771		if (TAILQ_EMPTY(&xsoftc.ccb_scanq))
772			msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
773			       "-", 0);
774		if ((ccb = (union ccb *)TAILQ_FIRST(&xsoftc.ccb_scanq)) != NULL) {
775			TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
776			xpt_unlock_buses();
777
778			/*
779			 * Since lock can be dropped inside and path freed
780			 * by completion callback even before return here,
781			 * take our own path copy for reference.
782			 */
783			xpt_copy_path(&path, ccb->ccb_h.path);
784			xpt_path_lock(&path);
785			xpt_action(ccb);
786			xpt_path_unlock(&path);
787			xpt_release_path(&path);
788
789			xpt_lock_buses();
790		}
791	}
792}
793
794void
795xpt_rescan(union ccb *ccb)
796{
797	struct ccb_hdr *hdr;
798
799	/* Prepare request */
800	if (ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD &&
801	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
802		ccb->ccb_h.func_code = XPT_SCAN_BUS;
803	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
804	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
805		ccb->ccb_h.func_code = XPT_SCAN_TGT;
806	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
807	    ccb->ccb_h.path->device->lun_id != CAM_LUN_WILDCARD)
808		ccb->ccb_h.func_code = XPT_SCAN_LUN;
809	else {
810		xpt_print(ccb->ccb_h.path, "illegal scan path\n");
811		xpt_free_path(ccb->ccb_h.path);
812		xpt_free_ccb(ccb);
813		return;
814	}
815	ccb->ccb_h.ppriv_ptr1 = ccb->ccb_h.cbfcnp;
816	ccb->ccb_h.cbfcnp = xpt_rescan_done;
817	xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_XPT);
818	/* Don't make duplicate entries for the same paths. */
819	xpt_lock_buses();
820	if (ccb->ccb_h.ppriv_ptr1 == NULL) {
821		TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
822			if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
823				wakeup(&xsoftc.ccb_scanq);
824				xpt_unlock_buses();
825				xpt_print(ccb->ccb_h.path, "rescan already queued\n");
826				xpt_free_path(ccb->ccb_h.path);
827				xpt_free_ccb(ccb);
828				return;
829			}
830		}
831	}
832	TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
833	xsoftc.buses_to_config++;
834	wakeup(&xsoftc.ccb_scanq);
835	xpt_unlock_buses();
836}
837
838/* Functions accessed by the peripheral drivers */
839static int
840xpt_init(void *dummy)
841{
842	struct cam_sim *xpt_sim;
843	struct cam_path *path;
844	struct cam_devq *devq;
845	cam_status status;
846	int error, i;
847
848	TAILQ_INIT(&xsoftc.xpt_busses);
849	TAILQ_INIT(&xsoftc.ccb_scanq);
850	STAILQ_INIT(&xsoftc.highpowerq);
851	xsoftc.num_highpower = CAM_MAX_HIGHPOWER;
852
853	mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF);
854	mtx_init(&xsoftc.xpt_highpower_lock, "XPT highpower lock", NULL, MTX_DEF);
855	xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK,
856	    taskqueue_thread_enqueue, /*context*/&xsoftc.xpt_taskq);
857
858#ifdef CAM_BOOT_DELAY
859	/*
860	 * Override this value at compile time to assist our users
861	 * who don't use loader to boot a kernel.
862	 */
863	xsoftc.boot_delay = CAM_BOOT_DELAY;
864#endif
865	/*
866	 * The xpt layer is, itself, the equivalent of a SIM.
867	 * Allow 16 ccbs in the ccb pool for it.  This should
868	 * give decent parallelism when we probe busses and
869	 * perform other XPT functions.
870	 */
871	devq = cam_simq_alloc(16);
872	xpt_sim = cam_sim_alloc(xptaction,
873				xptpoll,
874				"xpt",
875				/*softc*/NULL,
876				/*unit*/0,
877				/*mtx*/&xsoftc.xpt_lock,
878				/*max_dev_transactions*/0,
879				/*max_tagged_dev_transactions*/0,
880				devq);
881	if (xpt_sim == NULL)
882		return (ENOMEM);
883
884	mtx_lock(&xsoftc.xpt_lock);
885	if ((status = xpt_bus_register(xpt_sim, NULL, 0)) != CAM_SUCCESS) {
886		mtx_unlock(&xsoftc.xpt_lock);
887		printf("xpt_init: xpt_bus_register failed with status %#x,"
888		       " failing attach\n", status);
889		return (EINVAL);
890	}
891	mtx_unlock(&xsoftc.xpt_lock);
892
893	/*
894	 * Looking at the XPT from the SIM layer, the XPT is
895	 * the equivalent of a peripheral driver.  Allocate
896	 * a peripheral driver entry for us.
897	 */
898	if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
899				      CAM_TARGET_WILDCARD,
900				      CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
901		printf("xpt_init: xpt_create_path failed with status %#x,"
902		       " failing attach\n", status);
903		return (EINVAL);
904	}
905	xpt_path_lock(path);
906	cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
907			 path, NULL, 0, xpt_sim);
908	xpt_path_unlock(path);
909	xpt_free_path(path);
910
911	if (cam_num_doneqs < 1)
912		cam_num_doneqs = 1 + mp_ncpus / 6;
913	else if (cam_num_doneqs > MAXCPU)
914		cam_num_doneqs = MAXCPU;
915	for (i = 0; i < cam_num_doneqs; i++) {
916		mtx_init(&cam_doneqs[i].cam_doneq_mtx, "CAM doneq", NULL,
917		    MTX_DEF);
918		STAILQ_INIT(&cam_doneqs[i].cam_doneq);
919		error = kproc_kthread_add(xpt_done_td, &cam_doneqs[i],
920		    &cam_proc, NULL, 0, 0, "cam", "doneq%d", i);
921		if (error != 0) {
922			cam_num_doneqs = i;
923			break;
924		}
925	}
926	if (cam_num_doneqs < 1) {
927		printf("xpt_init: Cannot init completion queues "
928		       "- failing attach\n");
929		return (ENOMEM);
930	}
931	/*
932	 * Register a callback for when interrupts are enabled.
933	 */
934	xsoftc.xpt_config_hook =
935	    (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
936					      M_CAMXPT, M_NOWAIT | M_ZERO);
937	if (xsoftc.xpt_config_hook == NULL) {
938		printf("xpt_init: Cannot malloc config hook "
939		       "- failing attach\n");
940		return (ENOMEM);
941	}
942	xsoftc.xpt_config_hook->ich_func = xpt_config;
943	if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
944		free (xsoftc.xpt_config_hook, M_CAMXPT);
945		printf("xpt_init: config_intrhook_establish failed "
946		       "- failing attach\n");
947	}
948
949	return (0);
950}
951
952static cam_status
953xptregister(struct cam_periph *periph, void *arg)
954{
955	struct cam_sim *xpt_sim;
956
957	if (periph == NULL) {
958		printf("xptregister: periph was NULL!!\n");
959		return(CAM_REQ_CMP_ERR);
960	}
961
962	xpt_sim = (struct cam_sim *)arg;
963	xpt_sim->softc = periph;
964	xpt_periph = periph;
965	periph->softc = NULL;
966
967	return(CAM_REQ_CMP);
968}
969
970int32_t
971xpt_add_periph(struct cam_periph *periph)
972{
973	struct cam_ed *device;
974	int32_t	 status;
975
976	TASK_INIT(&periph->periph_run_task, 0, xpt_run_allocq_task, periph);
977	device = periph->path->device;
978	status = CAM_REQ_CMP;
979	if (device != NULL) {
980		mtx_lock(&device->target->bus->eb_mtx);
981		device->generation++;
982		SLIST_INSERT_HEAD(&device->periphs, periph, periph_links);
983		mtx_unlock(&device->target->bus->eb_mtx);
984		atomic_add_32(&xsoftc.xpt_generation, 1);
985	}
986
987	return (status);
988}
989
990void
991xpt_remove_periph(struct cam_periph *periph)
992{
993	struct cam_ed *device;
994
995	device = periph->path->device;
996	if (device != NULL) {
997		mtx_lock(&device->target->bus->eb_mtx);
998		device->generation++;
999		SLIST_REMOVE(&device->periphs, periph, cam_periph, periph_links);
1000		mtx_unlock(&device->target->bus->eb_mtx);
1001		atomic_add_32(&xsoftc.xpt_generation, 1);
1002	}
1003}
1004
1005
1006void
1007xpt_announce_periph(struct cam_periph *periph, char *announce_string)
1008{
1009	struct	cam_path *path = periph->path;
1010
1011	cam_periph_assert(periph, MA_OWNED);
1012	periph->flags |= CAM_PERIPH_ANNOUNCED;
1013
1014	printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1015	       periph->periph_name, periph->unit_number,
1016	       path->bus->sim->sim_name,
1017	       path->bus->sim->unit_number,
1018	       path->bus->sim->bus_id,
1019	       path->bus->path_id,
1020	       path->target->target_id,
1021	       (uintmax_t)path->device->lun_id);
1022	printf("%s%d: ", periph->periph_name, periph->unit_number);
1023	if (path->device->protocol == PROTO_SCSI)
1024		scsi_print_inquiry(&path->device->inq_data);
1025	else if (path->device->protocol == PROTO_ATA ||
1026	    path->device->protocol == PROTO_SATAPM)
1027		ata_print_ident(&path->device->ident_data);
1028	else if (path->device->protocol == PROTO_SEMB)
1029		semb_print_ident(
1030		    (struct sep_identify_data *)&path->device->ident_data);
1031	else
1032		printf("Unknown protocol device\n");
1033	if (path->device->serial_num_len > 0) {
1034		/* Don't wrap the screen  - print only the first 60 chars */
1035		printf("%s%d: Serial Number %.60s\n", periph->periph_name,
1036		       periph->unit_number, path->device->serial_num);
1037	}
1038	/* Announce transport details. */
1039	(*(path->bus->xport->announce))(periph);
1040	/* Announce command queueing. */
1041	if (path->device->inq_flags & SID_CmdQue
1042	 || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1043		printf("%s%d: Command Queueing enabled\n",
1044		       periph->periph_name, periph->unit_number);
1045	}
1046	/* Announce caller's details if they've passed in. */
1047	if (announce_string != NULL)
1048		printf("%s%d: %s\n", periph->periph_name,
1049		       periph->unit_number, announce_string);
1050}
1051
1052void
1053xpt_announce_quirks(struct cam_periph *periph, int quirks, char *bit_string)
1054{
1055	if (quirks != 0) {
1056		printf("%s%d: quirks=0x%b\n", periph->periph_name,
1057		    periph->unit_number, quirks, bit_string);
1058	}
1059}
1060
1061void
1062xpt_denounce_periph(struct cam_periph *periph)
1063{
1064	struct	cam_path *path = periph->path;
1065
1066	cam_periph_assert(periph, MA_OWNED);
1067	printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1068	       periph->periph_name, periph->unit_number,
1069	       path->bus->sim->sim_name,
1070	       path->bus->sim->unit_number,
1071	       path->bus->sim->bus_id,
1072	       path->bus->path_id,
1073	       path->target->target_id,
1074	       (uintmax_t)path->device->lun_id);
1075	printf("%s%d: ", periph->periph_name, periph->unit_number);
1076	if (path->device->protocol == PROTO_SCSI)
1077		scsi_print_inquiry_short(&path->device->inq_data);
1078	else if (path->device->protocol == PROTO_ATA ||
1079	    path->device->protocol == PROTO_SATAPM)
1080		ata_print_ident_short(&path->device->ident_data);
1081	else if (path->device->protocol == PROTO_SEMB)
1082		semb_print_ident_short(
1083		    (struct sep_identify_data *)&path->device->ident_data);
1084	else
1085		printf("Unknown protocol device");
1086	if (path->device->serial_num_len > 0)
1087		printf(" s/n %.60s", path->device->serial_num);
1088	printf(" detached\n");
1089}
1090
1091
1092int
1093xpt_getattr(char *buf, size_t len, const char *attr, struct cam_path *path)
1094{
1095	int ret = -1, l;
1096	struct ccb_dev_advinfo cdai;
1097	struct scsi_vpd_id_descriptor *idd;
1098
1099	xpt_path_assert(path, MA_OWNED);
1100
1101	memset(&cdai, 0, sizeof(cdai));
1102	xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
1103	cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
1104	cdai.bufsiz = len;
1105
1106	if (!strcmp(attr, "GEOM::ident"))
1107		cdai.buftype = CDAI_TYPE_SERIAL_NUM;
1108	else if (!strcmp(attr, "GEOM::physpath"))
1109		cdai.buftype = CDAI_TYPE_PHYS_PATH;
1110	else if (strcmp(attr, "GEOM::lunid") == 0 ||
1111		 strcmp(attr, "GEOM::lunname") == 0) {
1112		cdai.buftype = CDAI_TYPE_SCSI_DEVID;
1113		cdai.bufsiz = CAM_SCSI_DEVID_MAXLEN;
1114	} else
1115		goto out;
1116
1117	cdai.buf = malloc(cdai.bufsiz, M_CAMXPT, M_NOWAIT|M_ZERO);
1118	if (cdai.buf == NULL) {
1119		ret = ENOMEM;
1120		goto out;
1121	}
1122	xpt_action((union ccb *)&cdai); /* can only be synchronous */
1123	if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
1124		cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
1125	if (cdai.provsiz == 0)
1126		goto out;
1127	if (cdai.buftype == CDAI_TYPE_SCSI_DEVID) {
1128		if (strcmp(attr, "GEOM::lunid") == 0) {
1129			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1130			    cdai.provsiz, scsi_devid_is_lun_naa);
1131			if (idd == NULL)
1132				idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1133				    cdai.provsiz, scsi_devid_is_lun_eui64);
1134		} else
1135			idd = NULL;
1136		if (idd == NULL)
1137			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1138			    cdai.provsiz, scsi_devid_is_lun_t10);
1139		if (idd == NULL)
1140			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1141			    cdai.provsiz, scsi_devid_is_lun_name);
1142		if (idd == NULL)
1143			goto out;
1144		ret = 0;
1145		if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_ASCII) {
1146			if (idd->length < len) {
1147				for (l = 0; l < idd->length; l++)
1148					buf[l] = idd->identifier[l] ?
1149					    idd->identifier[l] : ' ';
1150				buf[l] = 0;
1151			} else
1152				ret = EFAULT;
1153		} else if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_UTF8) {
1154			l = strnlen(idd->identifier, idd->length);
1155			if (l < len) {
1156				bcopy(idd->identifier, buf, l);
1157				buf[l] = 0;
1158			} else
1159				ret = EFAULT;
1160		} else {
1161			if (idd->length * 2 < len) {
1162				for (l = 0; l < idd->length; l++)
1163					sprintf(buf + l * 2, "%02x",
1164					    idd->identifier[l]);
1165			} else
1166				ret = EFAULT;
1167		}
1168	} else {
1169		ret = 0;
1170		if (strlcpy(buf, cdai.buf, len) >= len)
1171			ret = EFAULT;
1172	}
1173
1174out:
1175	if (cdai.buf != NULL)
1176		free(cdai.buf, M_CAMXPT);
1177	return ret;
1178}
1179
1180static dev_match_ret
1181xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1182	    struct cam_eb *bus)
1183{
1184	dev_match_ret retval;
1185	u_int i;
1186
1187	retval = DM_RET_NONE;
1188
1189	/*
1190	 * If we aren't given something to match against, that's an error.
1191	 */
1192	if (bus == NULL)
1193		return(DM_RET_ERROR);
1194
1195	/*
1196	 * If there are no match entries, then this bus matches no
1197	 * matter what.
1198	 */
1199	if ((patterns == NULL) || (num_patterns == 0))
1200		return(DM_RET_DESCEND | DM_RET_COPY);
1201
1202	for (i = 0; i < num_patterns; i++) {
1203		struct bus_match_pattern *cur_pattern;
1204
1205		/*
1206		 * If the pattern in question isn't for a bus node, we
1207		 * aren't interested.  However, we do indicate to the
1208		 * calling routine that we should continue descending the
1209		 * tree, since the user wants to match against lower-level
1210		 * EDT elements.
1211		 */
1212		if (patterns[i].type != DEV_MATCH_BUS) {
1213			if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1214				retval |= DM_RET_DESCEND;
1215			continue;
1216		}
1217
1218		cur_pattern = &patterns[i].pattern.bus_pattern;
1219
1220		/*
1221		 * If they want to match any bus node, we give them any
1222		 * device node.
1223		 */
1224		if (cur_pattern->flags == BUS_MATCH_ANY) {
1225			/* set the copy flag */
1226			retval |= DM_RET_COPY;
1227
1228			/*
1229			 * If we've already decided on an action, go ahead
1230			 * and return.
1231			 */
1232			if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1233				return(retval);
1234		}
1235
1236		/*
1237		 * Not sure why someone would do this...
1238		 */
1239		if (cur_pattern->flags == BUS_MATCH_NONE)
1240			continue;
1241
1242		if (((cur_pattern->flags & BUS_MATCH_PATH) != 0)
1243		 && (cur_pattern->path_id != bus->path_id))
1244			continue;
1245
1246		if (((cur_pattern->flags & BUS_MATCH_BUS_ID) != 0)
1247		 && (cur_pattern->bus_id != bus->sim->bus_id))
1248			continue;
1249
1250		if (((cur_pattern->flags & BUS_MATCH_UNIT) != 0)
1251		 && (cur_pattern->unit_number != bus->sim->unit_number))
1252			continue;
1253
1254		if (((cur_pattern->flags & BUS_MATCH_NAME) != 0)
1255		 && (strncmp(cur_pattern->dev_name, bus->sim->sim_name,
1256			     DEV_IDLEN) != 0))
1257			continue;
1258
1259		/*
1260		 * If we get to this point, the user definitely wants
1261		 * information on this bus.  So tell the caller to copy the
1262		 * data out.
1263		 */
1264		retval |= DM_RET_COPY;
1265
1266		/*
1267		 * If the return action has been set to descend, then we
1268		 * know that we've already seen a non-bus matching
1269		 * expression, therefore we need to further descend the tree.
1270		 * This won't change by continuing around the loop, so we
1271		 * go ahead and return.  If we haven't seen a non-bus
1272		 * matching expression, we keep going around the loop until
1273		 * we exhaust the matching expressions.  We'll set the stop
1274		 * flag once we fall out of the loop.
1275		 */
1276		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1277			return(retval);
1278	}
1279
1280	/*
1281	 * If the return action hasn't been set to descend yet, that means
1282	 * we haven't seen anything other than bus matching patterns.  So
1283	 * tell the caller to stop descending the tree -- the user doesn't
1284	 * want to match against lower level tree elements.
1285	 */
1286	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1287		retval |= DM_RET_STOP;
1288
1289	return(retval);
1290}
1291
1292static dev_match_ret
1293xptdevicematch(struct dev_match_pattern *patterns, u_int num_patterns,
1294	       struct cam_ed *device)
1295{
1296	dev_match_ret retval;
1297	u_int i;
1298
1299	retval = DM_RET_NONE;
1300
1301	/*
1302	 * If we aren't given something to match against, that's an error.
1303	 */
1304	if (device == NULL)
1305		return(DM_RET_ERROR);
1306
1307	/*
1308	 * If there are no match entries, then this device matches no
1309	 * matter what.
1310	 */
1311	if ((patterns == NULL) || (num_patterns == 0))
1312		return(DM_RET_DESCEND | DM_RET_COPY);
1313
1314	for (i = 0; i < num_patterns; i++) {
1315		struct device_match_pattern *cur_pattern;
1316		struct scsi_vpd_device_id *device_id_page;
1317
1318		/*
1319		 * If the pattern in question isn't for a device node, we
1320		 * aren't interested.
1321		 */
1322		if (patterns[i].type != DEV_MATCH_DEVICE) {
1323			if ((patterns[i].type == DEV_MATCH_PERIPH)
1324			 && ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE))
1325				retval |= DM_RET_DESCEND;
1326			continue;
1327		}
1328
1329		cur_pattern = &patterns[i].pattern.device_pattern;
1330
1331		/* Error out if mutually exclusive options are specified. */
1332		if ((cur_pattern->flags & (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1333		 == (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1334			return(DM_RET_ERROR);
1335
1336		/*
1337		 * If they want to match any device node, we give them any
1338		 * device node.
1339		 */
1340		if (cur_pattern->flags == DEV_MATCH_ANY)
1341			goto copy_dev_node;
1342
1343		/*
1344		 * Not sure why someone would do this...
1345		 */
1346		if (cur_pattern->flags == DEV_MATCH_NONE)
1347			continue;
1348
1349		if (((cur_pattern->flags & DEV_MATCH_PATH) != 0)
1350		 && (cur_pattern->path_id != device->target->bus->path_id))
1351			continue;
1352
1353		if (((cur_pattern->flags & DEV_MATCH_TARGET) != 0)
1354		 && (cur_pattern->target_id != device->target->target_id))
1355			continue;
1356
1357		if (((cur_pattern->flags & DEV_MATCH_LUN) != 0)
1358		 && (cur_pattern->target_lun != device->lun_id))
1359			continue;
1360
1361		if (((cur_pattern->flags & DEV_MATCH_INQUIRY) != 0)
1362		 && (cam_quirkmatch((caddr_t)&device->inq_data,
1363				    (caddr_t)&cur_pattern->data.inq_pat,
1364				    1, sizeof(cur_pattern->data.inq_pat),
1365				    scsi_static_inquiry_match) == NULL))
1366			continue;
1367
1368		device_id_page = (struct scsi_vpd_device_id *)device->device_id;
1369		if (((cur_pattern->flags & DEV_MATCH_DEVID) != 0)
1370		 && (device->device_id_len < SVPD_DEVICE_ID_HDR_LEN
1371		  || scsi_devid_match((uint8_t *)device_id_page->desc_list,
1372				      device->device_id_len
1373				    - SVPD_DEVICE_ID_HDR_LEN,
1374				      cur_pattern->data.devid_pat.id,
1375				      cur_pattern->data.devid_pat.id_len) != 0))
1376			continue;
1377
1378copy_dev_node:
1379		/*
1380		 * If we get to this point, the user definitely wants
1381		 * information on this device.  So tell the caller to copy
1382		 * the data out.
1383		 */
1384		retval |= DM_RET_COPY;
1385
1386		/*
1387		 * If the return action has been set to descend, then we
1388		 * know that we've already seen a peripheral matching
1389		 * expression, therefore we need to further descend the tree.
1390		 * This won't change by continuing around the loop, so we
1391		 * go ahead and return.  If we haven't seen a peripheral
1392		 * matching expression, we keep going around the loop until
1393		 * we exhaust the matching expressions.  We'll set the stop
1394		 * flag once we fall out of the loop.
1395		 */
1396		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1397			return(retval);
1398	}
1399
1400	/*
1401	 * If the return action hasn't been set to descend yet, that means
1402	 * we haven't seen any peripheral matching patterns.  So tell the
1403	 * caller to stop descending the tree -- the user doesn't want to
1404	 * match against lower level tree elements.
1405	 */
1406	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1407		retval |= DM_RET_STOP;
1408
1409	return(retval);
1410}
1411
1412/*
1413 * Match a single peripheral against any number of match patterns.
1414 */
1415static dev_match_ret
1416xptperiphmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1417	       struct cam_periph *periph)
1418{
1419	dev_match_ret retval;
1420	u_int i;
1421
1422	/*
1423	 * If we aren't given something to match against, that's an error.
1424	 */
1425	if (periph == NULL)
1426		return(DM_RET_ERROR);
1427
1428	/*
1429	 * If there are no match entries, then this peripheral matches no
1430	 * matter what.
1431	 */
1432	if ((patterns == NULL) || (num_patterns == 0))
1433		return(DM_RET_STOP | DM_RET_COPY);
1434
1435	/*
1436	 * There aren't any nodes below a peripheral node, so there's no
1437	 * reason to descend the tree any further.
1438	 */
1439	retval = DM_RET_STOP;
1440
1441	for (i = 0; i < num_patterns; i++) {
1442		struct periph_match_pattern *cur_pattern;
1443
1444		/*
1445		 * If the pattern in question isn't for a peripheral, we
1446		 * aren't interested.
1447		 */
1448		if (patterns[i].type != DEV_MATCH_PERIPH)
1449			continue;
1450
1451		cur_pattern = &patterns[i].pattern.periph_pattern;
1452
1453		/*
1454		 * If they want to match on anything, then we will do so.
1455		 */
1456		if (cur_pattern->flags == PERIPH_MATCH_ANY) {
1457			/* set the copy flag */
1458			retval |= DM_RET_COPY;
1459
1460			/*
1461			 * We've already set the return action to stop,
1462			 * since there are no nodes below peripherals in
1463			 * the tree.
1464			 */
1465			return(retval);
1466		}
1467
1468		/*
1469		 * Not sure why someone would do this...
1470		 */
1471		if (cur_pattern->flags == PERIPH_MATCH_NONE)
1472			continue;
1473
1474		if (((cur_pattern->flags & PERIPH_MATCH_PATH) != 0)
1475		 && (cur_pattern->path_id != periph->path->bus->path_id))
1476			continue;
1477
1478		/*
1479		 * For the target and lun id's, we have to make sure the
1480		 * target and lun pointers aren't NULL.  The xpt peripheral
1481		 * has a wildcard target and device.
1482		 */
1483		if (((cur_pattern->flags & PERIPH_MATCH_TARGET) != 0)
1484		 && ((periph->path->target == NULL)
1485		 ||(cur_pattern->target_id != periph->path->target->target_id)))
1486			continue;
1487
1488		if (((cur_pattern->flags & PERIPH_MATCH_LUN) != 0)
1489		 && ((periph->path->device == NULL)
1490		 || (cur_pattern->target_lun != periph->path->device->lun_id)))
1491			continue;
1492
1493		if (((cur_pattern->flags & PERIPH_MATCH_UNIT) != 0)
1494		 && (cur_pattern->unit_number != periph->unit_number))
1495			continue;
1496
1497		if (((cur_pattern->flags & PERIPH_MATCH_NAME) != 0)
1498		 && (strncmp(cur_pattern->periph_name, periph->periph_name,
1499			     DEV_IDLEN) != 0))
1500			continue;
1501
1502		/*
1503		 * If we get to this point, the user definitely wants
1504		 * information on this peripheral.  So tell the caller to
1505		 * copy the data out.
1506		 */
1507		retval |= DM_RET_COPY;
1508
1509		/*
1510		 * The return action has already been set to stop, since
1511		 * peripherals don't have any nodes below them in the EDT.
1512		 */
1513		return(retval);
1514	}
1515
1516	/*
1517	 * If we get to this point, the peripheral that was passed in
1518	 * doesn't match any of the patterns.
1519	 */
1520	return(retval);
1521}
1522
1523static int
1524xptedtbusfunc(struct cam_eb *bus, void *arg)
1525{
1526	struct ccb_dev_match *cdm;
1527	struct cam_et *target;
1528	dev_match_ret retval;
1529
1530	cdm = (struct ccb_dev_match *)arg;
1531
1532	/*
1533	 * If our position is for something deeper in the tree, that means
1534	 * that we've already seen this node.  So, we keep going down.
1535	 */
1536	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1537	 && (cdm->pos.cookie.bus == bus)
1538	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1539	 && (cdm->pos.cookie.target != NULL))
1540		retval = DM_RET_DESCEND;
1541	else
1542		retval = xptbusmatch(cdm->patterns, cdm->num_patterns, bus);
1543
1544	/*
1545	 * If we got an error, bail out of the search.
1546	 */
1547	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1548		cdm->status = CAM_DEV_MATCH_ERROR;
1549		return(0);
1550	}
1551
1552	/*
1553	 * If the copy flag is set, copy this bus out.
1554	 */
1555	if (retval & DM_RET_COPY) {
1556		int spaceleft, j;
1557
1558		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1559			sizeof(struct dev_match_result));
1560
1561		/*
1562		 * If we don't have enough space to put in another
1563		 * match result, save our position and tell the
1564		 * user there are more devices to check.
1565		 */
1566		if (spaceleft < sizeof(struct dev_match_result)) {
1567			bzero(&cdm->pos, sizeof(cdm->pos));
1568			cdm->pos.position_type =
1569				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS;
1570
1571			cdm->pos.cookie.bus = bus;
1572			cdm->pos.generations[CAM_BUS_GENERATION]=
1573				xsoftc.bus_generation;
1574			cdm->status = CAM_DEV_MATCH_MORE;
1575			return(0);
1576		}
1577		j = cdm->num_matches;
1578		cdm->num_matches++;
1579		cdm->matches[j].type = DEV_MATCH_BUS;
1580		cdm->matches[j].result.bus_result.path_id = bus->path_id;
1581		cdm->matches[j].result.bus_result.bus_id = bus->sim->bus_id;
1582		cdm->matches[j].result.bus_result.unit_number =
1583			bus->sim->unit_number;
1584		strncpy(cdm->matches[j].result.bus_result.dev_name,
1585			bus->sim->sim_name, DEV_IDLEN);
1586	}
1587
1588	/*
1589	 * If the user is only interested in busses, there's no
1590	 * reason to descend to the next level in the tree.
1591	 */
1592	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1593		return(1);
1594
1595	/*
1596	 * If there is a target generation recorded, check it to
1597	 * make sure the target list hasn't changed.
1598	 */
1599	mtx_lock(&bus->eb_mtx);
1600	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1601	 && (cdm->pos.cookie.bus == bus)
1602	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1603	 && (cdm->pos.cookie.target != NULL)) {
1604		if ((cdm->pos.generations[CAM_TARGET_GENERATION] !=
1605		    bus->generation)) {
1606			mtx_unlock(&bus->eb_mtx);
1607			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1608			return (0);
1609		}
1610		target = (struct cam_et *)cdm->pos.cookie.target;
1611		target->refcount++;
1612	} else
1613		target = NULL;
1614	mtx_unlock(&bus->eb_mtx);
1615
1616	return (xpttargettraverse(bus, target, xptedttargetfunc, arg));
1617}
1618
1619static int
1620xptedttargetfunc(struct cam_et *target, void *arg)
1621{
1622	struct ccb_dev_match *cdm;
1623	struct cam_eb *bus;
1624	struct cam_ed *device;
1625
1626	cdm = (struct ccb_dev_match *)arg;
1627	bus = target->bus;
1628
1629	/*
1630	 * If there is a device list generation recorded, check it to
1631	 * make sure the device list hasn't changed.
1632	 */
1633	mtx_lock(&bus->eb_mtx);
1634	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1635	 && (cdm->pos.cookie.bus == bus)
1636	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1637	 && (cdm->pos.cookie.target == target)
1638	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1639	 && (cdm->pos.cookie.device != NULL)) {
1640		if (cdm->pos.generations[CAM_DEV_GENERATION] !=
1641		    target->generation) {
1642			mtx_unlock(&bus->eb_mtx);
1643			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1644			return(0);
1645		}
1646		device = (struct cam_ed *)cdm->pos.cookie.device;
1647		device->refcount++;
1648	} else
1649		device = NULL;
1650	mtx_unlock(&bus->eb_mtx);
1651
1652	return (xptdevicetraverse(target, device, xptedtdevicefunc, arg));
1653}
1654
1655static int
1656xptedtdevicefunc(struct cam_ed *device, void *arg)
1657{
1658	struct cam_eb *bus;
1659	struct cam_periph *periph;
1660	struct ccb_dev_match *cdm;
1661	dev_match_ret retval;
1662
1663	cdm = (struct ccb_dev_match *)arg;
1664	bus = device->target->bus;
1665
1666	/*
1667	 * If our position is for something deeper in the tree, that means
1668	 * that we've already seen this node.  So, we keep going down.
1669	 */
1670	if ((cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1671	 && (cdm->pos.cookie.device == device)
1672	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1673	 && (cdm->pos.cookie.periph != NULL))
1674		retval = DM_RET_DESCEND;
1675	else
1676		retval = xptdevicematch(cdm->patterns, cdm->num_patterns,
1677					device);
1678
1679	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1680		cdm->status = CAM_DEV_MATCH_ERROR;
1681		return(0);
1682	}
1683
1684	/*
1685	 * If the copy flag is set, copy this device out.
1686	 */
1687	if (retval & DM_RET_COPY) {
1688		int spaceleft, j;
1689
1690		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1691			sizeof(struct dev_match_result));
1692
1693		/*
1694		 * If we don't have enough space to put in another
1695		 * match result, save our position and tell the
1696		 * user there are more devices to check.
1697		 */
1698		if (spaceleft < sizeof(struct dev_match_result)) {
1699			bzero(&cdm->pos, sizeof(cdm->pos));
1700			cdm->pos.position_type =
1701				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1702				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE;
1703
1704			cdm->pos.cookie.bus = device->target->bus;
1705			cdm->pos.generations[CAM_BUS_GENERATION]=
1706				xsoftc.bus_generation;
1707			cdm->pos.cookie.target = device->target;
1708			cdm->pos.generations[CAM_TARGET_GENERATION] =
1709				device->target->bus->generation;
1710			cdm->pos.cookie.device = device;
1711			cdm->pos.generations[CAM_DEV_GENERATION] =
1712				device->target->generation;
1713			cdm->status = CAM_DEV_MATCH_MORE;
1714			return(0);
1715		}
1716		j = cdm->num_matches;
1717		cdm->num_matches++;
1718		cdm->matches[j].type = DEV_MATCH_DEVICE;
1719		cdm->matches[j].result.device_result.path_id =
1720			device->target->bus->path_id;
1721		cdm->matches[j].result.device_result.target_id =
1722			device->target->target_id;
1723		cdm->matches[j].result.device_result.target_lun =
1724			device->lun_id;
1725		cdm->matches[j].result.device_result.protocol =
1726			device->protocol;
1727		bcopy(&device->inq_data,
1728		      &cdm->matches[j].result.device_result.inq_data,
1729		      sizeof(struct scsi_inquiry_data));
1730		bcopy(&device->ident_data,
1731		      &cdm->matches[j].result.device_result.ident_data,
1732		      sizeof(struct ata_params));
1733
1734		/* Let the user know whether this device is unconfigured */
1735		if (device->flags & CAM_DEV_UNCONFIGURED)
1736			cdm->matches[j].result.device_result.flags =
1737				DEV_RESULT_UNCONFIGURED;
1738		else
1739			cdm->matches[j].result.device_result.flags =
1740				DEV_RESULT_NOFLAG;
1741	}
1742
1743	/*
1744	 * If the user isn't interested in peripherals, don't descend
1745	 * the tree any further.
1746	 */
1747	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1748		return(1);
1749
1750	/*
1751	 * If there is a peripheral list generation recorded, make sure
1752	 * it hasn't changed.
1753	 */
1754	xpt_lock_buses();
1755	mtx_lock(&bus->eb_mtx);
1756	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1757	 && (cdm->pos.cookie.bus == bus)
1758	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1759	 && (cdm->pos.cookie.target == device->target)
1760	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1761	 && (cdm->pos.cookie.device == device)
1762	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1763	 && (cdm->pos.cookie.periph != NULL)) {
1764		if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1765		    device->generation) {
1766			mtx_unlock(&bus->eb_mtx);
1767			xpt_unlock_buses();
1768			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1769			return(0);
1770		}
1771		periph = (struct cam_periph *)cdm->pos.cookie.periph;
1772		periph->refcount++;
1773	} else
1774		periph = NULL;
1775	mtx_unlock(&bus->eb_mtx);
1776	xpt_unlock_buses();
1777
1778	return (xptperiphtraverse(device, periph, xptedtperiphfunc, arg));
1779}
1780
1781static int
1782xptedtperiphfunc(struct cam_periph *periph, void *arg)
1783{
1784	struct ccb_dev_match *cdm;
1785	dev_match_ret retval;
1786
1787	cdm = (struct ccb_dev_match *)arg;
1788
1789	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1790
1791	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1792		cdm->status = CAM_DEV_MATCH_ERROR;
1793		return(0);
1794	}
1795
1796	/*
1797	 * If the copy flag is set, copy this peripheral out.
1798	 */
1799	if (retval & DM_RET_COPY) {
1800		int spaceleft, j;
1801
1802		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1803			sizeof(struct dev_match_result));
1804
1805		/*
1806		 * If we don't have enough space to put in another
1807		 * match result, save our position and tell the
1808		 * user there are more devices to check.
1809		 */
1810		if (spaceleft < sizeof(struct dev_match_result)) {
1811			bzero(&cdm->pos, sizeof(cdm->pos));
1812			cdm->pos.position_type =
1813				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1814				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE |
1815				CAM_DEV_POS_PERIPH;
1816
1817			cdm->pos.cookie.bus = periph->path->bus;
1818			cdm->pos.generations[CAM_BUS_GENERATION]=
1819				xsoftc.bus_generation;
1820			cdm->pos.cookie.target = periph->path->target;
1821			cdm->pos.generations[CAM_TARGET_GENERATION] =
1822				periph->path->bus->generation;
1823			cdm->pos.cookie.device = periph->path->device;
1824			cdm->pos.generations[CAM_DEV_GENERATION] =
1825				periph->path->target->generation;
1826			cdm->pos.cookie.periph = periph;
1827			cdm->pos.generations[CAM_PERIPH_GENERATION] =
1828				periph->path->device->generation;
1829			cdm->status = CAM_DEV_MATCH_MORE;
1830			return(0);
1831		}
1832
1833		j = cdm->num_matches;
1834		cdm->num_matches++;
1835		cdm->matches[j].type = DEV_MATCH_PERIPH;
1836		cdm->matches[j].result.periph_result.path_id =
1837			periph->path->bus->path_id;
1838		cdm->matches[j].result.periph_result.target_id =
1839			periph->path->target->target_id;
1840		cdm->matches[j].result.periph_result.target_lun =
1841			periph->path->device->lun_id;
1842		cdm->matches[j].result.periph_result.unit_number =
1843			periph->unit_number;
1844		strncpy(cdm->matches[j].result.periph_result.periph_name,
1845			periph->periph_name, DEV_IDLEN);
1846	}
1847
1848	return(1);
1849}
1850
1851static int
1852xptedtmatch(struct ccb_dev_match *cdm)
1853{
1854	struct cam_eb *bus;
1855	int ret;
1856
1857	cdm->num_matches = 0;
1858
1859	/*
1860	 * Check the bus list generation.  If it has changed, the user
1861	 * needs to reset everything and start over.
1862	 */
1863	xpt_lock_buses();
1864	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1865	 && (cdm->pos.cookie.bus != NULL)) {
1866		if (cdm->pos.generations[CAM_BUS_GENERATION] !=
1867		    xsoftc.bus_generation) {
1868			xpt_unlock_buses();
1869			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1870			return(0);
1871		}
1872		bus = (struct cam_eb *)cdm->pos.cookie.bus;
1873		bus->refcount++;
1874	} else
1875		bus = NULL;
1876	xpt_unlock_buses();
1877
1878	ret = xptbustraverse(bus, xptedtbusfunc, cdm);
1879
1880	/*
1881	 * If we get back 0, that means that we had to stop before fully
1882	 * traversing the EDT.  It also means that one of the subroutines
1883	 * has set the status field to the proper value.  If we get back 1,
1884	 * we've fully traversed the EDT and copied out any matching entries.
1885	 */
1886	if (ret == 1)
1887		cdm->status = CAM_DEV_MATCH_LAST;
1888
1889	return(ret);
1890}
1891
1892static int
1893xptplistpdrvfunc(struct periph_driver **pdrv, void *arg)
1894{
1895	struct cam_periph *periph;
1896	struct ccb_dev_match *cdm;
1897
1898	cdm = (struct ccb_dev_match *)arg;
1899
1900	xpt_lock_buses();
1901	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
1902	 && (cdm->pos.cookie.pdrv == pdrv)
1903	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1904	 && (cdm->pos.cookie.periph != NULL)) {
1905		if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1906		    (*pdrv)->generation) {
1907			xpt_unlock_buses();
1908			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1909			return(0);
1910		}
1911		periph = (struct cam_periph *)cdm->pos.cookie.periph;
1912		periph->refcount++;
1913	} else
1914		periph = NULL;
1915	xpt_unlock_buses();
1916
1917	return (xptpdperiphtraverse(pdrv, periph, xptplistperiphfunc, arg));
1918}
1919
1920static int
1921xptplistperiphfunc(struct cam_periph *periph, void *arg)
1922{
1923	struct ccb_dev_match *cdm;
1924	dev_match_ret retval;
1925
1926	cdm = (struct ccb_dev_match *)arg;
1927
1928	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1929
1930	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1931		cdm->status = CAM_DEV_MATCH_ERROR;
1932		return(0);
1933	}
1934
1935	/*
1936	 * If the copy flag is set, copy this peripheral out.
1937	 */
1938	if (retval & DM_RET_COPY) {
1939		int spaceleft, j;
1940
1941		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1942			sizeof(struct dev_match_result));
1943
1944		/*
1945		 * If we don't have enough space to put in another
1946		 * match result, save our position and tell the
1947		 * user there are more devices to check.
1948		 */
1949		if (spaceleft < sizeof(struct dev_match_result)) {
1950			struct periph_driver **pdrv;
1951
1952			pdrv = NULL;
1953			bzero(&cdm->pos, sizeof(cdm->pos));
1954			cdm->pos.position_type =
1955				CAM_DEV_POS_PDRV | CAM_DEV_POS_PDPTR |
1956				CAM_DEV_POS_PERIPH;
1957
1958			/*
1959			 * This may look a bit non-sensical, but it is
1960			 * actually quite logical.  There are very few
1961			 * peripheral drivers, and bloating every peripheral
1962			 * structure with a pointer back to its parent
1963			 * peripheral driver linker set entry would cost
1964			 * more in the long run than doing this quick lookup.
1965			 */
1966			for (pdrv = periph_drivers; *pdrv != NULL; pdrv++) {
1967				if (strcmp((*pdrv)->driver_name,
1968				    periph->periph_name) == 0)
1969					break;
1970			}
1971
1972			if (*pdrv == NULL) {
1973				cdm->status = CAM_DEV_MATCH_ERROR;
1974				return(0);
1975			}
1976
1977			cdm->pos.cookie.pdrv = pdrv;
1978			/*
1979			 * The periph generation slot does double duty, as
1980			 * does the periph pointer slot.  They are used for
1981			 * both edt and pdrv lookups and positioning.
1982			 */
1983			cdm->pos.cookie.periph = periph;
1984			cdm->pos.generations[CAM_PERIPH_GENERATION] =
1985				(*pdrv)->generation;
1986			cdm->status = CAM_DEV_MATCH_MORE;
1987			return(0);
1988		}
1989
1990		j = cdm->num_matches;
1991		cdm->num_matches++;
1992		cdm->matches[j].type = DEV_MATCH_PERIPH;
1993		cdm->matches[j].result.periph_result.path_id =
1994			periph->path->bus->path_id;
1995
1996		/*
1997		 * The transport layer peripheral doesn't have a target or
1998		 * lun.
1999		 */
2000		if (periph->path->target)
2001			cdm->matches[j].result.periph_result.target_id =
2002				periph->path->target->target_id;
2003		else
2004			cdm->matches[j].result.periph_result.target_id =
2005				CAM_TARGET_WILDCARD;
2006
2007		if (periph->path->device)
2008			cdm->matches[j].result.periph_result.target_lun =
2009				periph->path->device->lun_id;
2010		else
2011			cdm->matches[j].result.periph_result.target_lun =
2012				CAM_LUN_WILDCARD;
2013
2014		cdm->matches[j].result.periph_result.unit_number =
2015			periph->unit_number;
2016		strncpy(cdm->matches[j].result.periph_result.periph_name,
2017			periph->periph_name, DEV_IDLEN);
2018	}
2019
2020	return(1);
2021}
2022
2023static int
2024xptperiphlistmatch(struct ccb_dev_match *cdm)
2025{
2026	int ret;
2027
2028	cdm->num_matches = 0;
2029
2030	/*
2031	 * At this point in the edt traversal function, we check the bus
2032	 * list generation to make sure that no busses have been added or
2033	 * removed since the user last sent a XPT_DEV_MATCH ccb through.
2034	 * For the peripheral driver list traversal function, however, we
2035	 * don't have to worry about new peripheral driver types coming or
2036	 * going; they're in a linker set, and therefore can't change
2037	 * without a recompile.
2038	 */
2039
2040	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2041	 && (cdm->pos.cookie.pdrv != NULL))
2042		ret = xptpdrvtraverse(
2043				(struct periph_driver **)cdm->pos.cookie.pdrv,
2044				xptplistpdrvfunc, cdm);
2045	else
2046		ret = xptpdrvtraverse(NULL, xptplistpdrvfunc, cdm);
2047
2048	/*
2049	 * If we get back 0, that means that we had to stop before fully
2050	 * traversing the peripheral driver tree.  It also means that one of
2051	 * the subroutines has set the status field to the proper value.  If
2052	 * we get back 1, we've fully traversed the EDT and copied out any
2053	 * matching entries.
2054	 */
2055	if (ret == 1)
2056		cdm->status = CAM_DEV_MATCH_LAST;
2057
2058	return(ret);
2059}
2060
2061static int
2062xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
2063{
2064	struct cam_eb *bus, *next_bus;
2065	int retval;
2066
2067	retval = 1;
2068	if (start_bus)
2069		bus = start_bus;
2070	else {
2071		xpt_lock_buses();
2072		bus = TAILQ_FIRST(&xsoftc.xpt_busses);
2073		if (bus == NULL) {
2074			xpt_unlock_buses();
2075			return (retval);
2076		}
2077		bus->refcount++;
2078		xpt_unlock_buses();
2079	}
2080	for (; bus != NULL; bus = next_bus) {
2081		retval = tr_func(bus, arg);
2082		if (retval == 0) {
2083			xpt_release_bus(bus);
2084			break;
2085		}
2086		xpt_lock_buses();
2087		next_bus = TAILQ_NEXT(bus, links);
2088		if (next_bus)
2089			next_bus->refcount++;
2090		xpt_unlock_buses();
2091		xpt_release_bus(bus);
2092	}
2093	return(retval);
2094}
2095
2096static int
2097xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target,
2098		  xpt_targetfunc_t *tr_func, void *arg)
2099{
2100	struct cam_et *target, *next_target;
2101	int retval;
2102
2103	retval = 1;
2104	if (start_target)
2105		target = start_target;
2106	else {
2107		mtx_lock(&bus->eb_mtx);
2108		target = TAILQ_FIRST(&bus->et_entries);
2109		if (target == NULL) {
2110			mtx_unlock(&bus->eb_mtx);
2111			return (retval);
2112		}
2113		target->refcount++;
2114		mtx_unlock(&bus->eb_mtx);
2115	}
2116	for (; target != NULL; target = next_target) {
2117		retval = tr_func(target, arg);
2118		if (retval == 0) {
2119			xpt_release_target(target);
2120			break;
2121		}
2122		mtx_lock(&bus->eb_mtx);
2123		next_target = TAILQ_NEXT(target, links);
2124		if (next_target)
2125			next_target->refcount++;
2126		mtx_unlock(&bus->eb_mtx);
2127		xpt_release_target(target);
2128	}
2129	return(retval);
2130}
2131
2132static int
2133xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device,
2134		  xpt_devicefunc_t *tr_func, void *arg)
2135{
2136	struct cam_eb *bus;
2137	struct cam_ed *device, *next_device;
2138	int retval;
2139
2140	retval = 1;
2141	bus = target->bus;
2142	if (start_device)
2143		device = start_device;
2144	else {
2145		mtx_lock(&bus->eb_mtx);
2146		device = TAILQ_FIRST(&target->ed_entries);
2147		if (device == NULL) {
2148			mtx_unlock(&bus->eb_mtx);
2149			return (retval);
2150		}
2151		device->refcount++;
2152		mtx_unlock(&bus->eb_mtx);
2153	}
2154	for (; device != NULL; device = next_device) {
2155		mtx_lock(&device->device_mtx);
2156		retval = tr_func(device, arg);
2157		mtx_unlock(&device->device_mtx);
2158		if (retval == 0) {
2159			xpt_release_device(device);
2160			break;
2161		}
2162		mtx_lock(&bus->eb_mtx);
2163		next_device = TAILQ_NEXT(device, links);
2164		if (next_device)
2165			next_device->refcount++;
2166		mtx_unlock(&bus->eb_mtx);
2167		xpt_release_device(device);
2168	}
2169	return(retval);
2170}
2171
2172static int
2173xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph,
2174		  xpt_periphfunc_t *tr_func, void *arg)
2175{
2176	struct cam_eb *bus;
2177	struct cam_periph *periph, *next_periph;
2178	int retval;
2179
2180	retval = 1;
2181
2182	bus = device->target->bus;
2183	if (start_periph)
2184		periph = start_periph;
2185	else {
2186		xpt_lock_buses();
2187		mtx_lock(&bus->eb_mtx);
2188		periph = SLIST_FIRST(&device->periphs);
2189		while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2190			periph = SLIST_NEXT(periph, periph_links);
2191		if (periph == NULL) {
2192			mtx_unlock(&bus->eb_mtx);
2193			xpt_unlock_buses();
2194			return (retval);
2195		}
2196		periph->refcount++;
2197		mtx_unlock(&bus->eb_mtx);
2198		xpt_unlock_buses();
2199	}
2200	for (; periph != NULL; periph = next_periph) {
2201		retval = tr_func(periph, arg);
2202		if (retval == 0) {
2203			cam_periph_release_locked(periph);
2204			break;
2205		}
2206		xpt_lock_buses();
2207		mtx_lock(&bus->eb_mtx);
2208		next_periph = SLIST_NEXT(periph, periph_links);
2209		while (next_periph != NULL &&
2210		    (next_periph->flags & CAM_PERIPH_FREE) != 0)
2211			next_periph = SLIST_NEXT(next_periph, periph_links);
2212		if (next_periph)
2213			next_periph->refcount++;
2214		mtx_unlock(&bus->eb_mtx);
2215		xpt_unlock_buses();
2216		cam_periph_release_locked(periph);
2217	}
2218	return(retval);
2219}
2220
2221static int
2222xptpdrvtraverse(struct periph_driver **start_pdrv,
2223		xpt_pdrvfunc_t *tr_func, void *arg)
2224{
2225	struct periph_driver **pdrv;
2226	int retval;
2227
2228	retval = 1;
2229
2230	/*
2231	 * We don't traverse the peripheral driver list like we do the
2232	 * other lists, because it is a linker set, and therefore cannot be
2233	 * changed during runtime.  If the peripheral driver list is ever
2234	 * re-done to be something other than a linker set (i.e. it can
2235	 * change while the system is running), the list traversal should
2236	 * be modified to work like the other traversal functions.
2237	 */
2238	for (pdrv = (start_pdrv ? start_pdrv : periph_drivers);
2239	     *pdrv != NULL; pdrv++) {
2240		retval = tr_func(pdrv, arg);
2241
2242		if (retval == 0)
2243			return(retval);
2244	}
2245
2246	return(retval);
2247}
2248
2249static int
2250xptpdperiphtraverse(struct periph_driver **pdrv,
2251		    struct cam_periph *start_periph,
2252		    xpt_periphfunc_t *tr_func, void *arg)
2253{
2254	struct cam_periph *periph, *next_periph;
2255	int retval;
2256
2257	retval = 1;
2258
2259	if (start_periph)
2260		periph = start_periph;
2261	else {
2262		xpt_lock_buses();
2263		periph = TAILQ_FIRST(&(*pdrv)->units);
2264		while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2265			periph = TAILQ_NEXT(periph, unit_links);
2266		if (periph == NULL) {
2267			xpt_unlock_buses();
2268			return (retval);
2269		}
2270		periph->refcount++;
2271		xpt_unlock_buses();
2272	}
2273	for (; periph != NULL; periph = next_periph) {
2274		cam_periph_lock(periph);
2275		retval = tr_func(periph, arg);
2276		cam_periph_unlock(periph);
2277		if (retval == 0) {
2278			cam_periph_release(periph);
2279			break;
2280		}
2281		xpt_lock_buses();
2282		next_periph = TAILQ_NEXT(periph, unit_links);
2283		while (next_periph != NULL &&
2284		    (next_periph->flags & CAM_PERIPH_FREE) != 0)
2285			next_periph = TAILQ_NEXT(next_periph, unit_links);
2286		if (next_periph)
2287			next_periph->refcount++;
2288		xpt_unlock_buses();
2289		cam_periph_release(periph);
2290	}
2291	return(retval);
2292}
2293
2294static int
2295xptdefbusfunc(struct cam_eb *bus, void *arg)
2296{
2297	struct xpt_traverse_config *tr_config;
2298
2299	tr_config = (struct xpt_traverse_config *)arg;
2300
2301	if (tr_config->depth == XPT_DEPTH_BUS) {
2302		xpt_busfunc_t *tr_func;
2303
2304		tr_func = (xpt_busfunc_t *)tr_config->tr_func;
2305
2306		return(tr_func(bus, tr_config->tr_arg));
2307	} else
2308		return(xpttargettraverse(bus, NULL, xptdeftargetfunc, arg));
2309}
2310
2311static int
2312xptdeftargetfunc(struct cam_et *target, void *arg)
2313{
2314	struct xpt_traverse_config *tr_config;
2315
2316	tr_config = (struct xpt_traverse_config *)arg;
2317
2318	if (tr_config->depth == XPT_DEPTH_TARGET) {
2319		xpt_targetfunc_t *tr_func;
2320
2321		tr_func = (xpt_targetfunc_t *)tr_config->tr_func;
2322
2323		return(tr_func(target, tr_config->tr_arg));
2324	} else
2325		return(xptdevicetraverse(target, NULL, xptdefdevicefunc, arg));
2326}
2327
2328static int
2329xptdefdevicefunc(struct cam_ed *device, void *arg)
2330{
2331	struct xpt_traverse_config *tr_config;
2332
2333	tr_config = (struct xpt_traverse_config *)arg;
2334
2335	if (tr_config->depth == XPT_DEPTH_DEVICE) {
2336		xpt_devicefunc_t *tr_func;
2337
2338		tr_func = (xpt_devicefunc_t *)tr_config->tr_func;
2339
2340		return(tr_func(device, tr_config->tr_arg));
2341	} else
2342		return(xptperiphtraverse(device, NULL, xptdefperiphfunc, arg));
2343}
2344
2345static int
2346xptdefperiphfunc(struct cam_periph *periph, void *arg)
2347{
2348	struct xpt_traverse_config *tr_config;
2349	xpt_periphfunc_t *tr_func;
2350
2351	tr_config = (struct xpt_traverse_config *)arg;
2352
2353	tr_func = (xpt_periphfunc_t *)tr_config->tr_func;
2354
2355	/*
2356	 * Unlike the other default functions, we don't check for depth
2357	 * here.  The peripheral driver level is the last level in the EDT,
2358	 * so if we're here, we should execute the function in question.
2359	 */
2360	return(tr_func(periph, tr_config->tr_arg));
2361}
2362
2363/*
2364 * Execute the given function for every bus in the EDT.
2365 */
2366static int
2367xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
2368{
2369	struct xpt_traverse_config tr_config;
2370
2371	tr_config.depth = XPT_DEPTH_BUS;
2372	tr_config.tr_func = tr_func;
2373	tr_config.tr_arg = arg;
2374
2375	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2376}
2377
2378/*
2379 * Execute the given function for every device in the EDT.
2380 */
2381static int
2382xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
2383{
2384	struct xpt_traverse_config tr_config;
2385
2386	tr_config.depth = XPT_DEPTH_DEVICE;
2387	tr_config.tr_func = tr_func;
2388	tr_config.tr_arg = arg;
2389
2390	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2391}
2392
2393static int
2394xptsetasyncfunc(struct cam_ed *device, void *arg)
2395{
2396	struct cam_path path;
2397	struct ccb_getdev cgd;
2398	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2399
2400	/*
2401	 * Don't report unconfigured devices (Wildcard devs,
2402	 * devices only for target mode, device instances
2403	 * that have been invalidated but are waiting for
2404	 * their last reference count to be released).
2405	 */
2406	if ((device->flags & CAM_DEV_UNCONFIGURED) != 0)
2407		return (1);
2408
2409	xpt_compile_path(&path,
2410			 NULL,
2411			 device->target->bus->path_id,
2412			 device->target->target_id,
2413			 device->lun_id);
2414	xpt_setup_ccb(&cgd.ccb_h, &path, CAM_PRIORITY_NORMAL);
2415	cgd.ccb_h.func_code = XPT_GDEV_TYPE;
2416	xpt_action((union ccb *)&cgd);
2417	csa->callback(csa->callback_arg,
2418			    AC_FOUND_DEVICE,
2419			    &path, &cgd);
2420	xpt_release_path(&path);
2421
2422	return(1);
2423}
2424
2425static int
2426xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
2427{
2428	struct cam_path path;
2429	struct ccb_pathinq cpi;
2430	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2431
2432	xpt_compile_path(&path, /*periph*/NULL,
2433			 bus->path_id,
2434			 CAM_TARGET_WILDCARD,
2435			 CAM_LUN_WILDCARD);
2436	xpt_path_lock(&path);
2437	xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL);
2438	cpi.ccb_h.func_code = XPT_PATH_INQ;
2439	xpt_action((union ccb *)&cpi);
2440	csa->callback(csa->callback_arg,
2441			    AC_PATH_REGISTERED,
2442			    &path, &cpi);
2443	xpt_path_unlock(&path);
2444	xpt_release_path(&path);
2445
2446	return(1);
2447}
2448
2449void
2450xpt_action(union ccb *start_ccb)
2451{
2452
2453	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE,
2454	    ("xpt_action: func=%#x\n", start_ccb->ccb_h.func_code));
2455
2456	start_ccb->ccb_h.status = CAM_REQ_INPROG;
2457	(*(start_ccb->ccb_h.path->bus->xport->action))(start_ccb);
2458}
2459
2460void
2461xpt_action_default(union ccb *start_ccb)
2462{
2463	struct cam_path *path;
2464	struct cam_sim *sim;
2465	int lock;
2466
2467	path = start_ccb->ccb_h.path;
2468	CAM_DEBUG(path, CAM_DEBUG_TRACE,
2469	    ("xpt_action_default: func=%#x\n", start_ccb->ccb_h.func_code));
2470
2471	switch (start_ccb->ccb_h.func_code) {
2472	case XPT_SCSI_IO:
2473	{
2474		struct cam_ed *device;
2475
2476		/*
2477		 * For the sake of compatibility with SCSI-1
2478		 * devices that may not understand the identify
2479		 * message, we include lun information in the
2480		 * second byte of all commands.  SCSI-1 specifies
2481		 * that luns are a 3 bit value and reserves only 3
2482		 * bits for lun information in the CDB.  Later
2483		 * revisions of the SCSI spec allow for more than 8
2484		 * luns, but have deprecated lun information in the
2485		 * CDB.  So, if the lun won't fit, we must omit.
2486		 *
2487		 * Also be aware that during initial probing for devices,
2488		 * the inquiry information is unknown but initialized to 0.
2489		 * This means that this code will be exercised while probing
2490		 * devices with an ANSI revision greater than 2.
2491		 */
2492		device = path->device;
2493		if (device->protocol_version <= SCSI_REV_2
2494		 && start_ccb->ccb_h.target_lun < 8
2495		 && (start_ccb->ccb_h.flags & CAM_CDB_POINTER) == 0) {
2496
2497			start_ccb->csio.cdb_io.cdb_bytes[1] |=
2498			    start_ccb->ccb_h.target_lun << 5;
2499		}
2500		start_ccb->csio.scsi_status = SCSI_STATUS_OK;
2501	}
2502	/* FALLTHROUGH */
2503	case XPT_TARGET_IO:
2504	case XPT_CONT_TARGET_IO:
2505		start_ccb->csio.sense_resid = 0;
2506		start_ccb->csio.resid = 0;
2507		/* FALLTHROUGH */
2508	case XPT_ATA_IO:
2509		if (start_ccb->ccb_h.func_code == XPT_ATA_IO)
2510			start_ccb->ataio.resid = 0;
2511		/* FALLTHROUGH */
2512	case XPT_RESET_DEV:
2513	case XPT_ENG_EXEC:
2514	case XPT_SMP_IO:
2515	{
2516		struct cam_devq *devq;
2517
2518		devq = path->bus->sim->devq;
2519		mtx_lock(&devq->send_mtx);
2520		cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
2521		if (xpt_schedule_devq(devq, path->device) != 0)
2522			xpt_run_devq(devq);
2523		mtx_unlock(&devq->send_mtx);
2524		break;
2525	}
2526	case XPT_CALC_GEOMETRY:
2527		/* Filter out garbage */
2528		if (start_ccb->ccg.block_size == 0
2529		 || start_ccb->ccg.volume_size == 0) {
2530			start_ccb->ccg.cylinders = 0;
2531			start_ccb->ccg.heads = 0;
2532			start_ccb->ccg.secs_per_track = 0;
2533			start_ccb->ccb_h.status = CAM_REQ_CMP;
2534			break;
2535		}
2536#if defined(PC98) || defined(__sparc64__)
2537		/*
2538		 * In a PC-98 system, geometry translation depens on
2539		 * the "real" device geometry obtained from mode page 4.
2540		 * SCSI geometry translation is performed in the
2541		 * initialization routine of the SCSI BIOS and the result
2542		 * stored in host memory.  If the translation is available
2543		 * in host memory, use it.  If not, rely on the default
2544		 * translation the device driver performs.
2545		 * For sparc64, we may need adjust the geometry of large
2546		 * disks in order to fit the limitations of the 16-bit
2547		 * fields of the VTOC8 disk label.
2548		 */
2549		if (scsi_da_bios_params(&start_ccb->ccg) != 0) {
2550			start_ccb->ccb_h.status = CAM_REQ_CMP;
2551			break;
2552		}
2553#endif
2554		goto call_sim;
2555	case XPT_ABORT:
2556	{
2557		union ccb* abort_ccb;
2558
2559		abort_ccb = start_ccb->cab.abort_ccb;
2560		if (XPT_FC_IS_DEV_QUEUED(abort_ccb)) {
2561
2562			if (abort_ccb->ccb_h.pinfo.index >= 0) {
2563				struct cam_ccbq *ccbq;
2564				struct cam_ed *device;
2565
2566				device = abort_ccb->ccb_h.path->device;
2567				ccbq = &device->ccbq;
2568				cam_ccbq_remove_ccb(ccbq, abort_ccb);
2569				abort_ccb->ccb_h.status =
2570				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2571				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2572				xpt_done(abort_ccb);
2573				start_ccb->ccb_h.status = CAM_REQ_CMP;
2574				break;
2575			}
2576			if (abort_ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX
2577			 && (abort_ccb->ccb_h.status & CAM_SIM_QUEUED) == 0) {
2578				/*
2579				 * We've caught this ccb en route to
2580				 * the SIM.  Flag it for abort and the
2581				 * SIM will do so just before starting
2582				 * real work on the CCB.
2583				 */
2584				abort_ccb->ccb_h.status =
2585				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2586				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2587				start_ccb->ccb_h.status = CAM_REQ_CMP;
2588				break;
2589			}
2590		}
2591		if (XPT_FC_IS_QUEUED(abort_ccb)
2592		 && (abort_ccb->ccb_h.pinfo.index == CAM_DONEQ_INDEX)) {
2593			/*
2594			 * It's already completed but waiting
2595			 * for our SWI to get to it.
2596			 */
2597			start_ccb->ccb_h.status = CAM_UA_ABORT;
2598			break;
2599		}
2600		/*
2601		 * If we weren't able to take care of the abort request
2602		 * in the XPT, pass the request down to the SIM for processing.
2603		 */
2604	}
2605	/* FALLTHROUGH */
2606	case XPT_ACCEPT_TARGET_IO:
2607	case XPT_EN_LUN:
2608	case XPT_IMMED_NOTIFY:
2609	case XPT_NOTIFY_ACK:
2610	case XPT_RESET_BUS:
2611	case XPT_IMMEDIATE_NOTIFY:
2612	case XPT_NOTIFY_ACKNOWLEDGE:
2613	case XPT_GET_SIM_KNOB_OLD:
2614	case XPT_GET_SIM_KNOB:
2615	case XPT_SET_SIM_KNOB:
2616	case XPT_GET_TRAN_SETTINGS:
2617	case XPT_SET_TRAN_SETTINGS:
2618	case XPT_PATH_INQ:
2619call_sim:
2620		sim = path->bus->sim;
2621		lock = (mtx_owned(sim->mtx) == 0);
2622		if (lock)
2623			CAM_SIM_LOCK(sim);
2624		CAM_DEBUG(path, CAM_DEBUG_TRACE,
2625		    ("sim->sim_action: func=%#x\n", start_ccb->ccb_h.func_code));
2626		(*(sim->sim_action))(sim, start_ccb);
2627		CAM_DEBUG(path, CAM_DEBUG_TRACE,
2628		    ("sim->sim_action: status=%#x\n", start_ccb->ccb_h.status));
2629		if (lock)
2630			CAM_SIM_UNLOCK(sim);
2631		break;
2632	case XPT_PATH_STATS:
2633		start_ccb->cpis.last_reset = path->bus->last_reset;
2634		start_ccb->ccb_h.status = CAM_REQ_CMP;
2635		break;
2636	case XPT_GDEV_TYPE:
2637	{
2638		struct cam_ed *dev;
2639
2640		dev = path->device;
2641		if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2642			start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2643		} else {
2644			struct ccb_getdev *cgd;
2645
2646			cgd = &start_ccb->cgd;
2647			cgd->protocol = dev->protocol;
2648			cgd->inq_data = dev->inq_data;
2649			cgd->ident_data = dev->ident_data;
2650			cgd->inq_flags = dev->inq_flags;
2651			cgd->ccb_h.status = CAM_REQ_CMP;
2652			cgd->serial_num_len = dev->serial_num_len;
2653			if ((dev->serial_num_len > 0)
2654			 && (dev->serial_num != NULL))
2655				bcopy(dev->serial_num, cgd->serial_num,
2656				      dev->serial_num_len);
2657		}
2658		break;
2659	}
2660	case XPT_GDEV_STATS:
2661	{
2662		struct cam_ed *dev;
2663
2664		dev = path->device;
2665		if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2666			start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2667		} else {
2668			struct ccb_getdevstats *cgds;
2669			struct cam_eb *bus;
2670			struct cam_et *tar;
2671			struct cam_devq *devq;
2672
2673			cgds = &start_ccb->cgds;
2674			bus = path->bus;
2675			tar = path->target;
2676			devq = bus->sim->devq;
2677			mtx_lock(&devq->send_mtx);
2678			cgds->dev_openings = dev->ccbq.dev_openings;
2679			cgds->dev_active = dev->ccbq.dev_active;
2680			cgds->allocated = dev->ccbq.allocated;
2681			cgds->queued = cam_ccbq_pending_ccb_count(&dev->ccbq);
2682			cgds->held = cgds->allocated - cgds->dev_active -
2683			    cgds->queued;
2684			cgds->last_reset = tar->last_reset;
2685			cgds->maxtags = dev->maxtags;
2686			cgds->mintags = dev->mintags;
2687			if (timevalcmp(&tar->last_reset, &bus->last_reset, <))
2688				cgds->last_reset = bus->last_reset;
2689			mtx_unlock(&devq->send_mtx);
2690			cgds->ccb_h.status = CAM_REQ_CMP;
2691		}
2692		break;
2693	}
2694	case XPT_GDEVLIST:
2695	{
2696		struct cam_periph	*nperiph;
2697		struct periph_list	*periph_head;
2698		struct ccb_getdevlist	*cgdl;
2699		u_int			i;
2700		struct cam_ed		*device;
2701		int			found;
2702
2703
2704		found = 0;
2705
2706		/*
2707		 * Don't want anyone mucking with our data.
2708		 */
2709		device = path->device;
2710		periph_head = &device->periphs;
2711		cgdl = &start_ccb->cgdl;
2712
2713		/*
2714		 * Check and see if the list has changed since the user
2715		 * last requested a list member.  If so, tell them that the
2716		 * list has changed, and therefore they need to start over
2717		 * from the beginning.
2718		 */
2719		if ((cgdl->index != 0) &&
2720		    (cgdl->generation != device->generation)) {
2721			cgdl->status = CAM_GDEVLIST_LIST_CHANGED;
2722			break;
2723		}
2724
2725		/*
2726		 * Traverse the list of peripherals and attempt to find
2727		 * the requested peripheral.
2728		 */
2729		for (nperiph = SLIST_FIRST(periph_head), i = 0;
2730		     (nperiph != NULL) && (i <= cgdl->index);
2731		     nperiph = SLIST_NEXT(nperiph, periph_links), i++) {
2732			if (i == cgdl->index) {
2733				strncpy(cgdl->periph_name,
2734					nperiph->periph_name,
2735					DEV_IDLEN);
2736				cgdl->unit_number = nperiph->unit_number;
2737				found = 1;
2738			}
2739		}
2740		if (found == 0) {
2741			cgdl->status = CAM_GDEVLIST_ERROR;
2742			break;
2743		}
2744
2745		if (nperiph == NULL)
2746			cgdl->status = CAM_GDEVLIST_LAST_DEVICE;
2747		else
2748			cgdl->status = CAM_GDEVLIST_MORE_DEVS;
2749
2750		cgdl->index++;
2751		cgdl->generation = device->generation;
2752
2753		cgdl->ccb_h.status = CAM_REQ_CMP;
2754		break;
2755	}
2756	case XPT_DEV_MATCH:
2757	{
2758		dev_pos_type position_type;
2759		struct ccb_dev_match *cdm;
2760
2761		cdm = &start_ccb->cdm;
2762
2763		/*
2764		 * There are two ways of getting at information in the EDT.
2765		 * The first way is via the primary EDT tree.  It starts
2766		 * with a list of busses, then a list of targets on a bus,
2767		 * then devices/luns on a target, and then peripherals on a
2768		 * device/lun.  The "other" way is by the peripheral driver
2769		 * lists.  The peripheral driver lists are organized by
2770		 * peripheral driver.  (obviously)  So it makes sense to
2771		 * use the peripheral driver list if the user is looking
2772		 * for something like "da1", or all "da" devices.  If the
2773		 * user is looking for something on a particular bus/target
2774		 * or lun, it's generally better to go through the EDT tree.
2775		 */
2776
2777		if (cdm->pos.position_type != CAM_DEV_POS_NONE)
2778			position_type = cdm->pos.position_type;
2779		else {
2780			u_int i;
2781
2782			position_type = CAM_DEV_POS_NONE;
2783
2784			for (i = 0; i < cdm->num_patterns; i++) {
2785				if ((cdm->patterns[i].type == DEV_MATCH_BUS)
2786				 ||(cdm->patterns[i].type == DEV_MATCH_DEVICE)){
2787					position_type = CAM_DEV_POS_EDT;
2788					break;
2789				}
2790			}
2791
2792			if (cdm->num_patterns == 0)
2793				position_type = CAM_DEV_POS_EDT;
2794			else if (position_type == CAM_DEV_POS_NONE)
2795				position_type = CAM_DEV_POS_PDRV;
2796		}
2797
2798		switch(position_type & CAM_DEV_POS_TYPEMASK) {
2799		case CAM_DEV_POS_EDT:
2800			xptedtmatch(cdm);
2801			break;
2802		case CAM_DEV_POS_PDRV:
2803			xptperiphlistmatch(cdm);
2804			break;
2805		default:
2806			cdm->status = CAM_DEV_MATCH_ERROR;
2807			break;
2808		}
2809
2810		if (cdm->status == CAM_DEV_MATCH_ERROR)
2811			start_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2812		else
2813			start_ccb->ccb_h.status = CAM_REQ_CMP;
2814
2815		break;
2816	}
2817	case XPT_SASYNC_CB:
2818	{
2819		struct ccb_setasync *csa;
2820		struct async_node *cur_entry;
2821		struct async_list *async_head;
2822		u_int32_t added;
2823
2824		csa = &start_ccb->csa;
2825		added = csa->event_enable;
2826		async_head = &path->device->asyncs;
2827
2828		/*
2829		 * If there is already an entry for us, simply
2830		 * update it.
2831		 */
2832		cur_entry = SLIST_FIRST(async_head);
2833		while (cur_entry != NULL) {
2834			if ((cur_entry->callback_arg == csa->callback_arg)
2835			 && (cur_entry->callback == csa->callback))
2836				break;
2837			cur_entry = SLIST_NEXT(cur_entry, links);
2838		}
2839
2840		if (cur_entry != NULL) {
2841		 	/*
2842			 * If the request has no flags set,
2843			 * remove the entry.
2844			 */
2845			added &= ~cur_entry->event_enable;
2846			if (csa->event_enable == 0) {
2847				SLIST_REMOVE(async_head, cur_entry,
2848					     async_node, links);
2849				xpt_release_device(path->device);
2850				free(cur_entry, M_CAMXPT);
2851			} else {
2852				cur_entry->event_enable = csa->event_enable;
2853			}
2854			csa->event_enable = added;
2855		} else {
2856			cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
2857					   M_NOWAIT);
2858			if (cur_entry == NULL) {
2859				csa->ccb_h.status = CAM_RESRC_UNAVAIL;
2860				break;
2861			}
2862			cur_entry->event_enable = csa->event_enable;
2863			cur_entry->event_lock =
2864			    mtx_owned(path->bus->sim->mtx) ? 1 : 0;
2865			cur_entry->callback_arg = csa->callback_arg;
2866			cur_entry->callback = csa->callback;
2867			SLIST_INSERT_HEAD(async_head, cur_entry, links);
2868			xpt_acquire_device(path->device);
2869		}
2870		start_ccb->ccb_h.status = CAM_REQ_CMP;
2871		break;
2872	}
2873	case XPT_REL_SIMQ:
2874	{
2875		struct ccb_relsim *crs;
2876		struct cam_ed *dev;
2877
2878		crs = &start_ccb->crs;
2879		dev = path->device;
2880		if (dev == NULL) {
2881
2882			crs->ccb_h.status = CAM_DEV_NOT_THERE;
2883			break;
2884		}
2885
2886		if ((crs->release_flags & RELSIM_ADJUST_OPENINGS) != 0) {
2887
2888			/* Don't ever go below one opening */
2889			if (crs->openings > 0) {
2890				xpt_dev_ccbq_resize(path, crs->openings);
2891				if (bootverbose) {
2892					xpt_print(path,
2893					    "number of openings is now %d\n",
2894					    crs->openings);
2895				}
2896			}
2897		}
2898
2899		mtx_lock(&dev->sim->devq->send_mtx);
2900		if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) {
2901
2902			if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
2903
2904				/*
2905				 * Just extend the old timeout and decrement
2906				 * the freeze count so that a single timeout
2907				 * is sufficient for releasing the queue.
2908				 */
2909				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2910				callout_stop(&dev->callout);
2911			} else {
2912
2913				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2914			}
2915
2916			callout_reset_sbt(&dev->callout,
2917			    SBT_1MS * crs->release_timeout, 0,
2918			    xpt_release_devq_timeout, dev, 0);
2919
2920			dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
2921
2922		}
2923
2924		if ((crs->release_flags & RELSIM_RELEASE_AFTER_CMDCMPLT) != 0) {
2925
2926			if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0) {
2927				/*
2928				 * Decrement the freeze count so that a single
2929				 * completion is still sufficient to unfreeze
2930				 * the queue.
2931				 */
2932				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2933			} else {
2934
2935				dev->flags |= CAM_DEV_REL_ON_COMPLETE;
2936				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2937			}
2938		}
2939
2940		if ((crs->release_flags & RELSIM_RELEASE_AFTER_QEMPTY) != 0) {
2941
2942			if ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
2943			 || (dev->ccbq.dev_active == 0)) {
2944
2945				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2946			} else {
2947
2948				dev->flags |= CAM_DEV_REL_ON_QUEUE_EMPTY;
2949				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2950			}
2951		}
2952		mtx_unlock(&dev->sim->devq->send_mtx);
2953
2954		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0)
2955			xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
2956		start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt;
2957		start_ccb->ccb_h.status = CAM_REQ_CMP;
2958		break;
2959	}
2960	case XPT_DEBUG: {
2961		struct cam_path *oldpath;
2962
2963		/* Check that all request bits are supported. */
2964		if (start_ccb->cdbg.flags & ~(CAM_DEBUG_COMPILE)) {
2965			start_ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2966			break;
2967		}
2968
2969		cam_dflags = CAM_DEBUG_NONE;
2970		if (cam_dpath != NULL) {
2971			oldpath = cam_dpath;
2972			cam_dpath = NULL;
2973			xpt_free_path(oldpath);
2974		}
2975		if (start_ccb->cdbg.flags != CAM_DEBUG_NONE) {
2976			if (xpt_create_path(&cam_dpath, NULL,
2977					    start_ccb->ccb_h.path_id,
2978					    start_ccb->ccb_h.target_id,
2979					    start_ccb->ccb_h.target_lun) !=
2980					    CAM_REQ_CMP) {
2981				start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2982			} else {
2983				cam_dflags = start_ccb->cdbg.flags;
2984				start_ccb->ccb_h.status = CAM_REQ_CMP;
2985				xpt_print(cam_dpath, "debugging flags now %x\n",
2986				    cam_dflags);
2987			}
2988		} else
2989			start_ccb->ccb_h.status = CAM_REQ_CMP;
2990		break;
2991	}
2992	case XPT_NOOP:
2993		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
2994			xpt_freeze_devq(path, 1);
2995		start_ccb->ccb_h.status = CAM_REQ_CMP;
2996		break;
2997	default:
2998	case XPT_SDEV_TYPE:
2999	case XPT_TERM_IO:
3000	case XPT_ENG_INQ:
3001		/* XXX Implement */
3002		printf("%s: CCB type %#x not supported\n", __func__,
3003		       start_ccb->ccb_h.func_code);
3004		start_ccb->ccb_h.status = CAM_PROVIDE_FAIL;
3005		if (start_ccb->ccb_h.func_code & XPT_FC_DEV_QUEUED) {
3006			xpt_done(start_ccb);
3007		}
3008		break;
3009	}
3010}
3011
3012void
3013xpt_polled_action(union ccb *start_ccb)
3014{
3015	u_int32_t timeout;
3016	struct	  cam_sim *sim;
3017	struct	  cam_devq *devq;
3018	struct	  cam_ed *dev;
3019
3020	timeout = start_ccb->ccb_h.timeout * 10;
3021	sim = start_ccb->ccb_h.path->bus->sim;
3022	devq = sim->devq;
3023	dev = start_ccb->ccb_h.path->device;
3024
3025	mtx_unlock(&dev->device_mtx);
3026
3027	/*
3028	 * Steal an opening so that no other queued requests
3029	 * can get it before us while we simulate interrupts.
3030	 */
3031	mtx_lock(&devq->send_mtx);
3032	dev->ccbq.dev_openings--;
3033	while((devq->send_openings <= 0 || dev->ccbq.dev_openings < 0) &&
3034	    (--timeout > 0)) {
3035		mtx_unlock(&devq->send_mtx);
3036		DELAY(100);
3037		CAM_SIM_LOCK(sim);
3038		(*(sim->sim_poll))(sim);
3039		CAM_SIM_UNLOCK(sim);
3040		camisr_runqueue();
3041		mtx_lock(&devq->send_mtx);
3042	}
3043	dev->ccbq.dev_openings++;
3044	mtx_unlock(&devq->send_mtx);
3045
3046	if (timeout != 0) {
3047		xpt_action(start_ccb);
3048		while(--timeout > 0) {
3049			CAM_SIM_LOCK(sim);
3050			(*(sim->sim_poll))(sim);
3051			CAM_SIM_UNLOCK(sim);
3052			camisr_runqueue();
3053			if ((start_ccb->ccb_h.status  & CAM_STATUS_MASK)
3054			    != CAM_REQ_INPROG)
3055				break;
3056			DELAY(100);
3057		}
3058		if (timeout == 0) {
3059			/*
3060			 * XXX Is it worth adding a sim_timeout entry
3061			 * point so we can attempt recovery?  If
3062			 * this is only used for dumps, I don't think
3063			 * it is.
3064			 */
3065			start_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
3066		}
3067	} else {
3068		start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3069	}
3070
3071	mtx_lock(&dev->device_mtx);
3072}
3073
3074/*
3075 * Schedule a peripheral driver to receive a ccb when its
3076 * target device has space for more transactions.
3077 */
3078void
3079xpt_schedule(struct cam_periph *periph, u_int32_t new_priority)
3080{
3081
3082	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
3083	cam_periph_assert(periph, MA_OWNED);
3084	if (new_priority < periph->scheduled_priority) {
3085		periph->scheduled_priority = new_priority;
3086		xpt_run_allocq(periph, 0);
3087	}
3088}
3089
3090
3091/*
3092 * Schedule a device to run on a given queue.
3093 * If the device was inserted as a new entry on the queue,
3094 * return 1 meaning the device queue should be run. If we
3095 * were already queued, implying someone else has already
3096 * started the queue, return 0 so the caller doesn't attempt
3097 * to run the queue.
3098 */
3099static int
3100xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
3101		 u_int32_t new_priority)
3102{
3103	int retval;
3104	u_int32_t old_priority;
3105
3106	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n"));
3107
3108	old_priority = pinfo->priority;
3109
3110	/*
3111	 * Are we already queued?
3112	 */
3113	if (pinfo->index != CAM_UNQUEUED_INDEX) {
3114		/* Simply reorder based on new priority */
3115		if (new_priority < old_priority) {
3116			camq_change_priority(queue, pinfo->index,
3117					     new_priority);
3118			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3119					("changed priority to %d\n",
3120					 new_priority));
3121			retval = 1;
3122		} else
3123			retval = 0;
3124	} else {
3125		/* New entry on the queue */
3126		if (new_priority < old_priority)
3127			pinfo->priority = new_priority;
3128
3129		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3130				("Inserting onto queue\n"));
3131		pinfo->generation = ++queue->generation;
3132		camq_insert(queue, pinfo);
3133		retval = 1;
3134	}
3135	return (retval);
3136}
3137
3138static void
3139xpt_run_allocq_task(void *context, int pending)
3140{
3141	struct cam_periph *periph = context;
3142
3143	cam_periph_lock(periph);
3144	periph->flags &= ~CAM_PERIPH_RUN_TASK;
3145	xpt_run_allocq(periph, 1);
3146	cam_periph_unlock(periph);
3147	cam_periph_release(periph);
3148}
3149
3150static void
3151xpt_run_allocq(struct cam_periph *periph, int sleep)
3152{
3153	struct cam_ed	*device;
3154	union ccb	*ccb;
3155	uint32_t	 prio;
3156
3157	cam_periph_assert(periph, MA_OWNED);
3158	if (periph->periph_allocating)
3159		return;
3160	periph->periph_allocating = 1;
3161	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_allocq(%p)\n", periph));
3162	device = periph->path->device;
3163	ccb = NULL;
3164restart:
3165	while ((prio = min(periph->scheduled_priority,
3166	    periph->immediate_priority)) != CAM_PRIORITY_NONE &&
3167	    (periph->periph_allocated - (ccb != NULL ? 1 : 0) <
3168	     device->ccbq.total_openings || prio <= CAM_PRIORITY_OOB)) {
3169
3170		if (ccb == NULL &&
3171		    (ccb = xpt_get_ccb_nowait(periph)) == NULL) {
3172			if (sleep) {
3173				ccb = xpt_get_ccb(periph);
3174				goto restart;
3175			}
3176			if (periph->flags & CAM_PERIPH_RUN_TASK)
3177				break;
3178			cam_periph_doacquire(periph);
3179			periph->flags |= CAM_PERIPH_RUN_TASK;
3180			taskqueue_enqueue(xsoftc.xpt_taskq,
3181			    &periph->periph_run_task);
3182			break;
3183		}
3184		xpt_setup_ccb(&ccb->ccb_h, periph->path, prio);
3185		if (prio == periph->immediate_priority) {
3186			periph->immediate_priority = CAM_PRIORITY_NONE;
3187			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3188					("waking cam_periph_getccb()\n"));
3189			SLIST_INSERT_HEAD(&periph->ccb_list, &ccb->ccb_h,
3190					  periph_links.sle);
3191			wakeup(&periph->ccb_list);
3192		} else {
3193			periph->scheduled_priority = CAM_PRIORITY_NONE;
3194			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3195					("calling periph_start()\n"));
3196			periph->periph_start(periph, ccb);
3197		}
3198		ccb = NULL;
3199	}
3200	if (ccb != NULL)
3201		xpt_release_ccb(ccb);
3202	periph->periph_allocating = 0;
3203}
3204
3205static void
3206xpt_run_devq(struct cam_devq *devq)
3207{
3208	char cdb_str[(SCSI_MAX_CDBLEN * 3) + 1];
3209	int lock;
3210
3211	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_devq\n"));
3212
3213	devq->send_queue.qfrozen_cnt++;
3214	while ((devq->send_queue.entries > 0)
3215	    && (devq->send_openings > 0)
3216	    && (devq->send_queue.qfrozen_cnt <= 1)) {
3217		struct	cam_ed *device;
3218		union ccb *work_ccb;
3219		struct	cam_sim *sim;
3220
3221		device = (struct cam_ed *)camq_remove(&devq->send_queue,
3222							   CAMQ_HEAD);
3223		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3224				("running device %p\n", device));
3225
3226		work_ccb = cam_ccbq_peek_ccb(&device->ccbq, CAMQ_HEAD);
3227		if (work_ccb == NULL) {
3228			printf("device on run queue with no ccbs???\n");
3229			continue;
3230		}
3231
3232		if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
3233
3234			mtx_lock(&xsoftc.xpt_highpower_lock);
3235		 	if (xsoftc.num_highpower <= 0) {
3236				/*
3237				 * We got a high power command, but we
3238				 * don't have any available slots.  Freeze
3239				 * the device queue until we have a slot
3240				 * available.
3241				 */
3242				xpt_freeze_devq_device(device, 1);
3243				STAILQ_INSERT_TAIL(&xsoftc.highpowerq, device,
3244						   highpowerq_entry);
3245
3246				mtx_unlock(&xsoftc.xpt_highpower_lock);
3247				continue;
3248			} else {
3249				/*
3250				 * Consume a high power slot while
3251				 * this ccb runs.
3252				 */
3253				xsoftc.num_highpower--;
3254			}
3255			mtx_unlock(&xsoftc.xpt_highpower_lock);
3256		}
3257		cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
3258		cam_ccbq_send_ccb(&device->ccbq, work_ccb);
3259		devq->send_openings--;
3260		devq->send_active++;
3261		xpt_schedule_devq(devq, device);
3262		mtx_unlock(&devq->send_mtx);
3263
3264		if ((work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0) {
3265			/*
3266			 * The client wants to freeze the queue
3267			 * after this CCB is sent.
3268			 */
3269			xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3270		}
3271
3272		/* In Target mode, the peripheral driver knows best... */
3273		if (work_ccb->ccb_h.func_code == XPT_SCSI_IO) {
3274			if ((device->inq_flags & SID_CmdQue) != 0
3275			 && work_ccb->csio.tag_action != CAM_TAG_ACTION_NONE)
3276				work_ccb->ccb_h.flags |= CAM_TAG_ACTION_VALID;
3277			else
3278				/*
3279				 * Clear this in case of a retried CCB that
3280				 * failed due to a rejected tag.
3281				 */
3282				work_ccb->ccb_h.flags &= ~CAM_TAG_ACTION_VALID;
3283		}
3284
3285		switch (work_ccb->ccb_h.func_code) {
3286		case XPT_SCSI_IO:
3287			CAM_DEBUG(work_ccb->ccb_h.path,
3288			    CAM_DEBUG_CDB,("%s. CDB: %s\n",
3289			     scsi_op_desc(work_ccb->csio.cdb_io.cdb_bytes[0],
3290					  &device->inq_data),
3291			     scsi_cdb_string(work_ccb->csio.cdb_io.cdb_bytes,
3292					     cdb_str, sizeof(cdb_str))));
3293			break;
3294		case XPT_ATA_IO:
3295			CAM_DEBUG(work_ccb->ccb_h.path,
3296			    CAM_DEBUG_CDB,("%s. ACB: %s\n",
3297			     ata_op_string(&work_ccb->ataio.cmd),
3298			     ata_cmd_string(&work_ccb->ataio.cmd,
3299					    cdb_str, sizeof(cdb_str))));
3300			break;
3301		default:
3302			break;
3303		}
3304
3305		/*
3306		 * Device queues can be shared among multiple SIM instances
3307		 * that reside on different busses.  Use the SIM from the
3308		 * queued device, rather than the one from the calling bus.
3309		 */
3310		sim = device->sim;
3311		lock = (mtx_owned(sim->mtx) == 0);
3312		if (lock)
3313			CAM_SIM_LOCK(sim);
3314		work_ccb->ccb_h.qos.sim_data = sbinuptime(); // xxx uintprt_t too small 32bit platforms
3315		(*(sim->sim_action))(sim, work_ccb);
3316		if (lock)
3317			CAM_SIM_UNLOCK(sim);
3318		mtx_lock(&devq->send_mtx);
3319	}
3320	devq->send_queue.qfrozen_cnt--;
3321}
3322
3323/*
3324 * This function merges stuff from the slave ccb into the master ccb, while
3325 * keeping important fields in the master ccb constant.
3326 */
3327void
3328xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
3329{
3330
3331	/*
3332	 * Pull fields that are valid for peripheral drivers to set
3333	 * into the master CCB along with the CCB "payload".
3334	 */
3335	master_ccb->ccb_h.retry_count = slave_ccb->ccb_h.retry_count;
3336	master_ccb->ccb_h.func_code = slave_ccb->ccb_h.func_code;
3337	master_ccb->ccb_h.timeout = slave_ccb->ccb_h.timeout;
3338	master_ccb->ccb_h.flags = slave_ccb->ccb_h.flags;
3339	bcopy(&(&slave_ccb->ccb_h)[1], &(&master_ccb->ccb_h)[1],
3340	      sizeof(union ccb) - sizeof(struct ccb_hdr));
3341}
3342
3343void
3344xpt_setup_ccb_flags(struct ccb_hdr *ccb_h, struct cam_path *path,
3345		    u_int32_t priority, u_int32_t flags)
3346{
3347
3348	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
3349	ccb_h->pinfo.priority = priority;
3350	ccb_h->path = path;
3351	ccb_h->path_id = path->bus->path_id;
3352	if (path->target)
3353		ccb_h->target_id = path->target->target_id;
3354	else
3355		ccb_h->target_id = CAM_TARGET_WILDCARD;
3356	if (path->device) {
3357		ccb_h->target_lun = path->device->lun_id;
3358		ccb_h->pinfo.generation = ++path->device->ccbq.queue.generation;
3359	} else {
3360		ccb_h->target_lun = CAM_TARGET_WILDCARD;
3361	}
3362	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
3363	ccb_h->flags = flags;
3364	ccb_h->xflags = 0;
3365}
3366
3367void
3368xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
3369{
3370	xpt_setup_ccb_flags(ccb_h, path, priority, /*flags*/ 0);
3371}
3372
3373/* Path manipulation functions */
3374cam_status
3375xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
3376		path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3377{
3378	struct	   cam_path *path;
3379	cam_status status;
3380
3381	path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3382
3383	if (path == NULL) {
3384		status = CAM_RESRC_UNAVAIL;
3385		return(status);
3386	}
3387	status = xpt_compile_path(path, perph, path_id, target_id, lun_id);
3388	if (status != CAM_REQ_CMP) {
3389		free(path, M_CAMPATH);
3390		path = NULL;
3391	}
3392	*new_path_ptr = path;
3393	return (status);
3394}
3395
3396cam_status
3397xpt_create_path_unlocked(struct cam_path **new_path_ptr,
3398			 struct cam_periph *periph, path_id_t path_id,
3399			 target_id_t target_id, lun_id_t lun_id)
3400{
3401
3402	return (xpt_create_path(new_path_ptr, periph, path_id, target_id,
3403	    lun_id));
3404}
3405
3406cam_status
3407xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
3408		 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3409{
3410	struct	     cam_eb *bus;
3411	struct	     cam_et *target;
3412	struct	     cam_ed *device;
3413	cam_status   status;
3414
3415	status = CAM_REQ_CMP;	/* Completed without error */
3416	target = NULL;		/* Wildcarded */
3417	device = NULL;		/* Wildcarded */
3418
3419	/*
3420	 * We will potentially modify the EDT, so block interrupts
3421	 * that may attempt to create cam paths.
3422	 */
3423	bus = xpt_find_bus(path_id);
3424	if (bus == NULL) {
3425		status = CAM_PATH_INVALID;
3426	} else {
3427		xpt_lock_buses();
3428		mtx_lock(&bus->eb_mtx);
3429		target = xpt_find_target(bus, target_id);
3430		if (target == NULL) {
3431			/* Create one */
3432			struct cam_et *new_target;
3433
3434			new_target = xpt_alloc_target(bus, target_id);
3435			if (new_target == NULL) {
3436				status = CAM_RESRC_UNAVAIL;
3437			} else {
3438				target = new_target;
3439			}
3440		}
3441		xpt_unlock_buses();
3442		if (target != NULL) {
3443			device = xpt_find_device(target, lun_id);
3444			if (device == NULL) {
3445				/* Create one */
3446				struct cam_ed *new_device;
3447
3448				new_device =
3449				    (*(bus->xport->alloc_device))(bus,
3450								      target,
3451								      lun_id);
3452				if (new_device == NULL) {
3453					status = CAM_RESRC_UNAVAIL;
3454				} else {
3455					device = new_device;
3456				}
3457			}
3458		}
3459		mtx_unlock(&bus->eb_mtx);
3460	}
3461
3462	/*
3463	 * Only touch the user's data if we are successful.
3464	 */
3465	if (status == CAM_REQ_CMP) {
3466		new_path->periph = perph;
3467		new_path->bus = bus;
3468		new_path->target = target;
3469		new_path->device = device;
3470		CAM_DEBUG(new_path, CAM_DEBUG_TRACE, ("xpt_compile_path\n"));
3471	} else {
3472		if (device != NULL)
3473			xpt_release_device(device);
3474		if (target != NULL)
3475			xpt_release_target(target);
3476		if (bus != NULL)
3477			xpt_release_bus(bus);
3478	}
3479	return (status);
3480}
3481
3482cam_status
3483xpt_clone_path(struct cam_path **new_path_ptr, struct cam_path *path)
3484{
3485	struct	   cam_path *new_path;
3486
3487	new_path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3488	if (new_path == NULL)
3489		return(CAM_RESRC_UNAVAIL);
3490	xpt_copy_path(new_path, path);
3491	*new_path_ptr = new_path;
3492	return (CAM_REQ_CMP);
3493}
3494
3495void
3496xpt_copy_path(struct cam_path *new_path, struct cam_path *path)
3497{
3498
3499	*new_path = *path;
3500	if (path->bus != NULL)
3501		xpt_acquire_bus(path->bus);
3502	if (path->target != NULL)
3503		xpt_acquire_target(path->target);
3504	if (path->device != NULL)
3505		xpt_acquire_device(path->device);
3506}
3507
3508void
3509xpt_release_path(struct cam_path *path)
3510{
3511	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_path\n"));
3512	if (path->device != NULL) {
3513		xpt_release_device(path->device);
3514		path->device = NULL;
3515	}
3516	if (path->target != NULL) {
3517		xpt_release_target(path->target);
3518		path->target = NULL;
3519	}
3520	if (path->bus != NULL) {
3521		xpt_release_bus(path->bus);
3522		path->bus = NULL;
3523	}
3524}
3525
3526void
3527xpt_free_path(struct cam_path *path)
3528{
3529
3530	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
3531	xpt_release_path(path);
3532	free(path, M_CAMPATH);
3533}
3534
3535void
3536xpt_path_counts(struct cam_path *path, uint32_t *bus_ref,
3537    uint32_t *periph_ref, uint32_t *target_ref, uint32_t *device_ref)
3538{
3539
3540	xpt_lock_buses();
3541	if (bus_ref) {
3542		if (path->bus)
3543			*bus_ref = path->bus->refcount;
3544		else
3545			*bus_ref = 0;
3546	}
3547	if (periph_ref) {
3548		if (path->periph)
3549			*periph_ref = path->periph->refcount;
3550		else
3551			*periph_ref = 0;
3552	}
3553	xpt_unlock_buses();
3554	if (target_ref) {
3555		if (path->target)
3556			*target_ref = path->target->refcount;
3557		else
3558			*target_ref = 0;
3559	}
3560	if (device_ref) {
3561		if (path->device)
3562			*device_ref = path->device->refcount;
3563		else
3564			*device_ref = 0;
3565	}
3566}
3567
3568/*
3569 * Return -1 for failure, 0 for exact match, 1 for match with wildcards
3570 * in path1, 2 for match with wildcards in path2.
3571 */
3572int
3573xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
3574{
3575	int retval = 0;
3576
3577	if (path1->bus != path2->bus) {
3578		if (path1->bus->path_id == CAM_BUS_WILDCARD)
3579			retval = 1;
3580		else if (path2->bus->path_id == CAM_BUS_WILDCARD)
3581			retval = 2;
3582		else
3583			return (-1);
3584	}
3585	if (path1->target != path2->target) {
3586		if (path1->target->target_id == CAM_TARGET_WILDCARD) {
3587			if (retval == 0)
3588				retval = 1;
3589		} else if (path2->target->target_id == CAM_TARGET_WILDCARD)
3590			retval = 2;
3591		else
3592			return (-1);
3593	}
3594	if (path1->device != path2->device) {
3595		if (path1->device->lun_id == CAM_LUN_WILDCARD) {
3596			if (retval == 0)
3597				retval = 1;
3598		} else if (path2->device->lun_id == CAM_LUN_WILDCARD)
3599			retval = 2;
3600		else
3601			return (-1);
3602	}
3603	return (retval);
3604}
3605
3606int
3607xpt_path_comp_dev(struct cam_path *path, struct cam_ed *dev)
3608{
3609	int retval = 0;
3610
3611	if (path->bus != dev->target->bus) {
3612		if (path->bus->path_id == CAM_BUS_WILDCARD)
3613			retval = 1;
3614		else if (dev->target->bus->path_id == CAM_BUS_WILDCARD)
3615			retval = 2;
3616		else
3617			return (-1);
3618	}
3619	if (path->target != dev->target) {
3620		if (path->target->target_id == CAM_TARGET_WILDCARD) {
3621			if (retval == 0)
3622				retval = 1;
3623		} else if (dev->target->target_id == CAM_TARGET_WILDCARD)
3624			retval = 2;
3625		else
3626			return (-1);
3627	}
3628	if (path->device != dev) {
3629		if (path->device->lun_id == CAM_LUN_WILDCARD) {
3630			if (retval == 0)
3631				retval = 1;
3632		} else if (dev->lun_id == CAM_LUN_WILDCARD)
3633			retval = 2;
3634		else
3635			return (-1);
3636	}
3637	return (retval);
3638}
3639
3640void
3641xpt_print_path(struct cam_path *path)
3642{
3643
3644	if (path == NULL)
3645		printf("(nopath): ");
3646	else {
3647		if (path->periph != NULL)
3648			printf("(%s%d:", path->periph->periph_name,
3649			       path->periph->unit_number);
3650		else
3651			printf("(noperiph:");
3652
3653		if (path->bus != NULL)
3654			printf("%s%d:%d:", path->bus->sim->sim_name,
3655			       path->bus->sim->unit_number,
3656			       path->bus->sim->bus_id);
3657		else
3658			printf("nobus:");
3659
3660		if (path->target != NULL)
3661			printf("%d:", path->target->target_id);
3662		else
3663			printf("X:");
3664
3665		if (path->device != NULL)
3666			printf("%jx): ", (uintmax_t)path->device->lun_id);
3667		else
3668			printf("X): ");
3669	}
3670}
3671
3672void
3673xpt_print_device(struct cam_ed *device)
3674{
3675
3676	if (device == NULL)
3677		printf("(nopath): ");
3678	else {
3679		printf("(noperiph:%s%d:%d:%d:%jx): ", device->sim->sim_name,
3680		       device->sim->unit_number,
3681		       device->sim->bus_id,
3682		       device->target->target_id,
3683		       (uintmax_t)device->lun_id);
3684	}
3685}
3686
3687void
3688xpt_print(struct cam_path *path, const char *fmt, ...)
3689{
3690	va_list ap;
3691	xpt_print_path(path);
3692	va_start(ap, fmt);
3693	vprintf(fmt, ap);
3694	va_end(ap);
3695}
3696
3697int
3698xpt_path_string(struct cam_path *path, char *str, size_t str_len)
3699{
3700	struct sbuf sb;
3701
3702	sbuf_new(&sb, str, str_len, 0);
3703
3704	if (path == NULL)
3705		sbuf_printf(&sb, "(nopath): ");
3706	else {
3707		if (path->periph != NULL)
3708			sbuf_printf(&sb, "(%s%d:", path->periph->periph_name,
3709				    path->periph->unit_number);
3710		else
3711			sbuf_printf(&sb, "(noperiph:");
3712
3713		if (path->bus != NULL)
3714			sbuf_printf(&sb, "%s%d:%d:", path->bus->sim->sim_name,
3715				    path->bus->sim->unit_number,
3716				    path->bus->sim->bus_id);
3717		else
3718			sbuf_printf(&sb, "nobus:");
3719
3720		if (path->target != NULL)
3721			sbuf_printf(&sb, "%d:", path->target->target_id);
3722		else
3723			sbuf_printf(&sb, "X:");
3724
3725		if (path->device != NULL)
3726			sbuf_printf(&sb, "%jx): ",
3727			    (uintmax_t)path->device->lun_id);
3728		else
3729			sbuf_printf(&sb, "X): ");
3730	}
3731	sbuf_finish(&sb);
3732
3733	return(sbuf_len(&sb));
3734}
3735
3736path_id_t
3737xpt_path_path_id(struct cam_path *path)
3738{
3739	return(path->bus->path_id);
3740}
3741
3742target_id_t
3743xpt_path_target_id(struct cam_path *path)
3744{
3745	if (path->target != NULL)
3746		return (path->target->target_id);
3747	else
3748		return (CAM_TARGET_WILDCARD);
3749}
3750
3751lun_id_t
3752xpt_path_lun_id(struct cam_path *path)
3753{
3754	if (path->device != NULL)
3755		return (path->device->lun_id);
3756	else
3757		return (CAM_LUN_WILDCARD);
3758}
3759
3760struct cam_sim *
3761xpt_path_sim(struct cam_path *path)
3762{
3763
3764	return (path->bus->sim);
3765}
3766
3767struct cam_periph*
3768xpt_path_periph(struct cam_path *path)
3769{
3770
3771	return (path->periph);
3772}
3773
3774/*
3775 * Release a CAM control block for the caller.  Remit the cost of the structure
3776 * to the device referenced by the path.  If the this device had no 'credits'
3777 * and peripheral drivers have registered async callbacks for this notification
3778 * call them now.
3779 */
3780void
3781xpt_release_ccb(union ccb *free_ccb)
3782{
3783	struct	 cam_ed *device;
3784	struct	 cam_periph *periph;
3785
3786	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
3787	xpt_path_assert(free_ccb->ccb_h.path, MA_OWNED);
3788	device = free_ccb->ccb_h.path->device;
3789	periph = free_ccb->ccb_h.path->periph;
3790
3791	xpt_free_ccb(free_ccb);
3792	periph->periph_allocated--;
3793	cam_ccbq_release_opening(&device->ccbq);
3794	xpt_run_allocq(periph, 0);
3795}
3796
3797/* Functions accessed by SIM drivers */
3798
3799static struct xpt_xport xport_default = {
3800	.alloc_device = xpt_alloc_device_default,
3801	.action = xpt_action_default,
3802	.async = xpt_dev_async_default,
3803};
3804
3805/*
3806 * A sim structure, listing the SIM entry points and instance
3807 * identification info is passed to xpt_bus_register to hook the SIM
3808 * into the CAM framework.  xpt_bus_register creates a cam_eb entry
3809 * for this new bus and places it in the array of busses and assigns
3810 * it a path_id.  The path_id may be influenced by "hard wiring"
3811 * information specified by the user.  Once interrupt services are
3812 * available, the bus will be probed.
3813 */
3814int32_t
3815xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
3816{
3817	struct cam_eb *new_bus;
3818	struct cam_eb *old_bus;
3819	struct ccb_pathinq cpi;
3820	struct cam_path *path;
3821	cam_status status;
3822
3823	mtx_assert(sim->mtx, MA_OWNED);
3824
3825	sim->bus_id = bus;
3826	new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
3827					  M_CAMXPT, M_NOWAIT|M_ZERO);
3828	if (new_bus == NULL) {
3829		/* Couldn't satisfy request */
3830		return (CAM_RESRC_UNAVAIL);
3831	}
3832
3833	mtx_init(&new_bus->eb_mtx, "CAM bus lock", NULL, MTX_DEF);
3834	TAILQ_INIT(&new_bus->et_entries);
3835	cam_sim_hold(sim);
3836	new_bus->sim = sim;
3837	timevalclear(&new_bus->last_reset);
3838	new_bus->flags = 0;
3839	new_bus->refcount = 1;	/* Held until a bus_deregister event */
3840	new_bus->generation = 0;
3841
3842	xpt_lock_buses();
3843	sim->path_id = new_bus->path_id =
3844	    xptpathid(sim->sim_name, sim->unit_number, sim->bus_id);
3845	old_bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3846	while (old_bus != NULL
3847	    && old_bus->path_id < new_bus->path_id)
3848		old_bus = TAILQ_NEXT(old_bus, links);
3849	if (old_bus != NULL)
3850		TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
3851	else
3852		TAILQ_INSERT_TAIL(&xsoftc.xpt_busses, new_bus, links);
3853	xsoftc.bus_generation++;
3854	xpt_unlock_buses();
3855
3856	/*
3857	 * Set a default transport so that a PATH_INQ can be issued to
3858	 * the SIM.  This will then allow for probing and attaching of
3859	 * a more appropriate transport.
3860	 */
3861	new_bus->xport = &xport_default;
3862
3863	status = xpt_create_path(&path, /*periph*/NULL, sim->path_id,
3864				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3865	if (status != CAM_REQ_CMP) {
3866		xpt_release_bus(new_bus);
3867		free(path, M_CAMXPT);
3868		return (CAM_RESRC_UNAVAIL);
3869	}
3870
3871	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
3872	cpi.ccb_h.func_code = XPT_PATH_INQ;
3873	xpt_action((union ccb *)&cpi);
3874
3875	if (cpi.ccb_h.status == CAM_REQ_CMP) {
3876		switch (cpi.transport) {
3877		case XPORT_SPI:
3878		case XPORT_SAS:
3879		case XPORT_FC:
3880		case XPORT_USB:
3881		case XPORT_ISCSI:
3882		case XPORT_SRP:
3883		case XPORT_PPB:
3884			new_bus->xport = scsi_get_xport();
3885			break;
3886		case XPORT_ATA:
3887		case XPORT_SATA:
3888			new_bus->xport = ata_get_xport();
3889			break;
3890		default:
3891			new_bus->xport = &xport_default;
3892			break;
3893		}
3894	}
3895
3896	/* Notify interested parties */
3897	if (sim->path_id != CAM_XPT_PATH_ID) {
3898
3899		xpt_async(AC_PATH_REGISTERED, path, &cpi);
3900		if ((cpi.hba_misc & PIM_NOSCAN) == 0) {
3901			union	ccb *scan_ccb;
3902
3903			/* Initiate bus rescan. */
3904			scan_ccb = xpt_alloc_ccb_nowait();
3905			if (scan_ccb != NULL) {
3906				scan_ccb->ccb_h.path = path;
3907				scan_ccb->ccb_h.func_code = XPT_SCAN_BUS;
3908				scan_ccb->crcn.flags = 0;
3909				xpt_rescan(scan_ccb);
3910			} else {
3911				xpt_print(path,
3912					  "Can't allocate CCB to scan bus\n");
3913				xpt_free_path(path);
3914			}
3915		} else
3916			xpt_free_path(path);
3917	} else
3918		xpt_free_path(path);
3919	return (CAM_SUCCESS);
3920}
3921
3922int32_t
3923xpt_bus_deregister(path_id_t pathid)
3924{
3925	struct cam_path bus_path;
3926	cam_status status;
3927
3928	status = xpt_compile_path(&bus_path, NULL, pathid,
3929				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3930	if (status != CAM_REQ_CMP)
3931		return (status);
3932
3933	xpt_async(AC_LOST_DEVICE, &bus_path, NULL);
3934	xpt_async(AC_PATH_DEREGISTERED, &bus_path, NULL);
3935
3936	/* Release the reference count held while registered. */
3937	xpt_release_bus(bus_path.bus);
3938	xpt_release_path(&bus_path);
3939
3940	return (CAM_REQ_CMP);
3941}
3942
3943static path_id_t
3944xptnextfreepathid(void)
3945{
3946	struct cam_eb *bus;
3947	path_id_t pathid;
3948	const char *strval;
3949
3950	mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
3951	pathid = 0;
3952	bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3953retry:
3954	/* Find an unoccupied pathid */
3955	while (bus != NULL && bus->path_id <= pathid) {
3956		if (bus->path_id == pathid)
3957			pathid++;
3958		bus = TAILQ_NEXT(bus, links);
3959	}
3960
3961	/*
3962	 * Ensure that this pathid is not reserved for
3963	 * a bus that may be registered in the future.
3964	 */
3965	if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
3966		++pathid;
3967		/* Start the search over */
3968		goto retry;
3969	}
3970	return (pathid);
3971}
3972
3973static path_id_t
3974xptpathid(const char *sim_name, int sim_unit, int sim_bus)
3975{
3976	path_id_t pathid;
3977	int i, dunit, val;
3978	char buf[32];
3979	const char *dname;
3980
3981	pathid = CAM_XPT_PATH_ID;
3982	snprintf(buf, sizeof(buf), "%s%d", sim_name, sim_unit);
3983	if (strcmp(buf, "xpt0") == 0 && sim_bus == 0)
3984		return (pathid);
3985	i = 0;
3986	while ((resource_find_match(&i, &dname, &dunit, "at", buf)) == 0) {
3987		if (strcmp(dname, "scbus")) {
3988			/* Avoid a bit of foot shooting. */
3989			continue;
3990		}
3991		if (dunit < 0)		/* unwired?! */
3992			continue;
3993		if (resource_int_value("scbus", dunit, "bus", &val) == 0) {
3994			if (sim_bus == val) {
3995				pathid = dunit;
3996				break;
3997			}
3998		} else if (sim_bus == 0) {
3999			/* Unspecified matches bus 0 */
4000			pathid = dunit;
4001			break;
4002		} else {
4003			printf("Ambiguous scbus configuration for %s%d "
4004			       "bus %d, cannot wire down.  The kernel "
4005			       "config entry for scbus%d should "
4006			       "specify a controller bus.\n"
4007			       "Scbus will be assigned dynamically.\n",
4008			       sim_name, sim_unit, sim_bus, dunit);
4009			break;
4010		}
4011	}
4012
4013	if (pathid == CAM_XPT_PATH_ID)
4014		pathid = xptnextfreepathid();
4015	return (pathid);
4016}
4017
4018static const char *
4019xpt_async_string(u_int32_t async_code)
4020{
4021
4022	switch (async_code) {
4023	case AC_BUS_RESET: return ("AC_BUS_RESET");
4024	case AC_UNSOL_RESEL: return ("AC_UNSOL_RESEL");
4025	case AC_SCSI_AEN: return ("AC_SCSI_AEN");
4026	case AC_SENT_BDR: return ("AC_SENT_BDR");
4027	case AC_PATH_REGISTERED: return ("AC_PATH_REGISTERED");
4028	case AC_PATH_DEREGISTERED: return ("AC_PATH_DEREGISTERED");
4029	case AC_FOUND_DEVICE: return ("AC_FOUND_DEVICE");
4030	case AC_LOST_DEVICE: return ("AC_LOST_DEVICE");
4031	case AC_TRANSFER_NEG: return ("AC_TRANSFER_NEG");
4032	case AC_INQ_CHANGED: return ("AC_INQ_CHANGED");
4033	case AC_GETDEV_CHANGED: return ("AC_GETDEV_CHANGED");
4034	case AC_CONTRACT: return ("AC_CONTRACT");
4035	case AC_ADVINFO_CHANGED: return ("AC_ADVINFO_CHANGED");
4036	case AC_UNIT_ATTENTION: return ("AC_UNIT_ATTENTION");
4037	}
4038	return ("AC_UNKNOWN");
4039}
4040
4041static int
4042xpt_async_size(u_int32_t async_code)
4043{
4044
4045	switch (async_code) {
4046	case AC_BUS_RESET: return (0);
4047	case AC_UNSOL_RESEL: return (0);
4048	case AC_SCSI_AEN: return (0);
4049	case AC_SENT_BDR: return (0);
4050	case AC_PATH_REGISTERED: return (sizeof(struct ccb_pathinq));
4051	case AC_PATH_DEREGISTERED: return (0);
4052	case AC_FOUND_DEVICE: return (sizeof(struct ccb_getdev));
4053	case AC_LOST_DEVICE: return (0);
4054	case AC_TRANSFER_NEG: return (sizeof(struct ccb_trans_settings));
4055	case AC_INQ_CHANGED: return (0);
4056	case AC_GETDEV_CHANGED: return (0);
4057	case AC_CONTRACT: return (sizeof(struct ac_contract));
4058	case AC_ADVINFO_CHANGED: return (-1);
4059	case AC_UNIT_ATTENTION: return (sizeof(struct ccb_scsiio));
4060	}
4061	return (0);
4062}
4063
4064static int
4065xpt_async_process_dev(struct cam_ed *device, void *arg)
4066{
4067	union ccb *ccb = arg;
4068	struct cam_path *path = ccb->ccb_h.path;
4069	void *async_arg = ccb->casync.async_arg_ptr;
4070	u_int32_t async_code = ccb->casync.async_code;
4071	int relock;
4072
4073	if (path->device != device
4074	 && path->device->lun_id != CAM_LUN_WILDCARD
4075	 && device->lun_id != CAM_LUN_WILDCARD)
4076		return (1);
4077
4078	/*
4079	 * The async callback could free the device.
4080	 * If it is a broadcast async, it doesn't hold
4081	 * device reference, so take our own reference.
4082	 */
4083	xpt_acquire_device(device);
4084
4085	/*
4086	 * If async for specific device is to be delivered to
4087	 * the wildcard client, take the specific device lock.
4088	 * XXX: We may need a way for client to specify it.
4089	 */
4090	if ((device->lun_id == CAM_LUN_WILDCARD &&
4091	     path->device->lun_id != CAM_LUN_WILDCARD) ||
4092	    (device->target->target_id == CAM_TARGET_WILDCARD &&
4093	     path->target->target_id != CAM_TARGET_WILDCARD) ||
4094	    (device->target->bus->path_id == CAM_BUS_WILDCARD &&
4095	     path->target->bus->path_id != CAM_BUS_WILDCARD)) {
4096		mtx_unlock(&device->device_mtx);
4097		xpt_path_lock(path);
4098		relock = 1;
4099	} else
4100		relock = 0;
4101
4102	(*(device->target->bus->xport->async))(async_code,
4103	    device->target->bus, device->target, device, async_arg);
4104	xpt_async_bcast(&device->asyncs, async_code, path, async_arg);
4105
4106	if (relock) {
4107		xpt_path_unlock(path);
4108		mtx_lock(&device->device_mtx);
4109	}
4110	xpt_release_device(device);
4111	return (1);
4112}
4113
4114static int
4115xpt_async_process_tgt(struct cam_et *target, void *arg)
4116{
4117	union ccb *ccb = arg;
4118	struct cam_path *path = ccb->ccb_h.path;
4119
4120	if (path->target != target
4121	 && path->target->target_id != CAM_TARGET_WILDCARD
4122	 && target->target_id != CAM_TARGET_WILDCARD)
4123		return (1);
4124
4125	if (ccb->casync.async_code == AC_SENT_BDR) {
4126		/* Update our notion of when the last reset occurred */
4127		microtime(&target->last_reset);
4128	}
4129
4130	return (xptdevicetraverse(target, NULL, xpt_async_process_dev, ccb));
4131}
4132
4133static void
4134xpt_async_process(struct cam_periph *periph, union ccb *ccb)
4135{
4136	struct cam_eb *bus;
4137	struct cam_path *path;
4138	void *async_arg;
4139	u_int32_t async_code;
4140
4141	path = ccb->ccb_h.path;
4142	async_code = ccb->casync.async_code;
4143	async_arg = ccb->casync.async_arg_ptr;
4144	CAM_DEBUG(path, CAM_DEBUG_TRACE | CAM_DEBUG_INFO,
4145	    ("xpt_async(%s)\n", xpt_async_string(async_code)));
4146	bus = path->bus;
4147
4148	if (async_code == AC_BUS_RESET) {
4149		/* Update our notion of when the last reset occurred */
4150		microtime(&bus->last_reset);
4151	}
4152
4153	xpttargettraverse(bus, NULL, xpt_async_process_tgt, ccb);
4154
4155	/*
4156	 * If this wasn't a fully wildcarded async, tell all
4157	 * clients that want all async events.
4158	 */
4159	if (bus != xpt_periph->path->bus) {
4160		xpt_path_lock(xpt_periph->path);
4161		xpt_async_process_dev(xpt_periph->path->device, ccb);
4162		xpt_path_unlock(xpt_periph->path);
4163	}
4164
4165	if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4166		xpt_release_devq(path, 1, TRUE);
4167	else
4168		xpt_release_simq(path->bus->sim, TRUE);
4169	if (ccb->casync.async_arg_size > 0)
4170		free(async_arg, M_CAMXPT);
4171	xpt_free_path(path);
4172	xpt_free_ccb(ccb);
4173}
4174
4175static void
4176xpt_async_bcast(struct async_list *async_head,
4177		u_int32_t async_code,
4178		struct cam_path *path, void *async_arg)
4179{
4180	struct async_node *cur_entry;
4181	int lock;
4182
4183	cur_entry = SLIST_FIRST(async_head);
4184	while (cur_entry != NULL) {
4185		struct async_node *next_entry;
4186		/*
4187		 * Grab the next list entry before we call the current
4188		 * entry's callback.  This is because the callback function
4189		 * can delete its async callback entry.
4190		 */
4191		next_entry = SLIST_NEXT(cur_entry, links);
4192		if ((cur_entry->event_enable & async_code) != 0) {
4193			lock = cur_entry->event_lock;
4194			if (lock)
4195				CAM_SIM_LOCK(path->device->sim);
4196			cur_entry->callback(cur_entry->callback_arg,
4197					    async_code, path,
4198					    async_arg);
4199			if (lock)
4200				CAM_SIM_UNLOCK(path->device->sim);
4201		}
4202		cur_entry = next_entry;
4203	}
4204}
4205
4206void
4207xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
4208{
4209	union ccb *ccb;
4210	int size;
4211
4212	ccb = xpt_alloc_ccb_nowait();
4213	if (ccb == NULL) {
4214		xpt_print(path, "Can't allocate CCB to send %s\n",
4215		    xpt_async_string(async_code));
4216		return;
4217	}
4218
4219	if (xpt_clone_path(&ccb->ccb_h.path, path) != CAM_REQ_CMP) {
4220		xpt_print(path, "Can't allocate path to send %s\n",
4221		    xpt_async_string(async_code));
4222		xpt_free_ccb(ccb);
4223		return;
4224	}
4225	ccb->ccb_h.path->periph = NULL;
4226	ccb->ccb_h.func_code = XPT_ASYNC;
4227	ccb->ccb_h.cbfcnp = xpt_async_process;
4228	ccb->ccb_h.flags |= CAM_UNLOCKED;
4229	ccb->casync.async_code = async_code;
4230	ccb->casync.async_arg_size = 0;
4231	size = xpt_async_size(async_code);
4232	if (size > 0 && async_arg != NULL) {
4233		ccb->casync.async_arg_ptr = malloc(size, M_CAMXPT, M_NOWAIT);
4234		if (ccb->casync.async_arg_ptr == NULL) {
4235			xpt_print(path, "Can't allocate argument to send %s\n",
4236			    xpt_async_string(async_code));
4237			xpt_free_path(ccb->ccb_h.path);
4238			xpt_free_ccb(ccb);
4239			return;
4240		}
4241		memcpy(ccb->casync.async_arg_ptr, async_arg, size);
4242		ccb->casync.async_arg_size = size;
4243	} else if (size < 0) {
4244		ccb->casync.async_arg_ptr = async_arg;
4245		ccb->casync.async_arg_size = size;
4246	}
4247	if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4248		xpt_freeze_devq(path, 1);
4249	else
4250		xpt_freeze_simq(path->bus->sim, 1);
4251	xpt_done(ccb);
4252}
4253
4254static void
4255xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus,
4256		      struct cam_et *target, struct cam_ed *device,
4257		      void *async_arg)
4258{
4259
4260	/*
4261	 * We only need to handle events for real devices.
4262	 */
4263	if (target->target_id == CAM_TARGET_WILDCARD
4264	 || device->lun_id == CAM_LUN_WILDCARD)
4265		return;
4266
4267	printf("%s called\n", __func__);
4268}
4269
4270static uint32_t
4271xpt_freeze_devq_device(struct cam_ed *dev, u_int count)
4272{
4273	struct cam_devq	*devq;
4274	uint32_t freeze;
4275
4276	devq = dev->sim->devq;
4277	mtx_assert(&devq->send_mtx, MA_OWNED);
4278	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4279	    ("xpt_freeze_devq_device(%d) %u->%u\n", count,
4280	    dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt + count));
4281	freeze = (dev->ccbq.queue.qfrozen_cnt += count);
4282	/* Remove frozen device from sendq. */
4283	if (device_is_queued(dev))
4284		camq_remove(&devq->send_queue, dev->devq_entry.index);
4285	return (freeze);
4286}
4287
4288u_int32_t
4289xpt_freeze_devq(struct cam_path *path, u_int count)
4290{
4291	struct cam_ed	*dev = path->device;
4292	struct cam_devq	*devq;
4293	uint32_t	 freeze;
4294
4295	devq = dev->sim->devq;
4296	mtx_lock(&devq->send_mtx);
4297	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_freeze_devq(%d)\n", count));
4298	freeze = xpt_freeze_devq_device(dev, count);
4299	mtx_unlock(&devq->send_mtx);
4300	return (freeze);
4301}
4302
4303u_int32_t
4304xpt_freeze_simq(struct cam_sim *sim, u_int count)
4305{
4306	struct cam_devq	*devq;
4307	uint32_t	 freeze;
4308
4309	devq = sim->devq;
4310	mtx_lock(&devq->send_mtx);
4311	freeze = (devq->send_queue.qfrozen_cnt += count);
4312	mtx_unlock(&devq->send_mtx);
4313	return (freeze);
4314}
4315
4316static void
4317xpt_release_devq_timeout(void *arg)
4318{
4319	struct cam_ed *dev;
4320	struct cam_devq *devq;
4321
4322	dev = (struct cam_ed *)arg;
4323	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, ("xpt_release_devq_timeout\n"));
4324	devq = dev->sim->devq;
4325	mtx_assert(&devq->send_mtx, MA_OWNED);
4326	if (xpt_release_devq_device(dev, /*count*/1, /*run_queue*/TRUE))
4327		xpt_run_devq(devq);
4328}
4329
4330void
4331xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
4332{
4333	struct cam_ed *dev;
4334	struct cam_devq *devq;
4335
4336	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_devq(%d, %d)\n",
4337	    count, run_queue));
4338	dev = path->device;
4339	devq = dev->sim->devq;
4340	mtx_lock(&devq->send_mtx);
4341	if (xpt_release_devq_device(dev, count, run_queue))
4342		xpt_run_devq(dev->sim->devq);
4343	mtx_unlock(&devq->send_mtx);
4344}
4345
4346static int
4347xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
4348{
4349
4350	mtx_assert(&dev->sim->devq->send_mtx, MA_OWNED);
4351	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4352	    ("xpt_release_devq_device(%d, %d) %u->%u\n", count, run_queue,
4353	    dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt - count));
4354	if (count > dev->ccbq.queue.qfrozen_cnt) {
4355#ifdef INVARIANTS
4356		printf("xpt_release_devq(): requested %u > present %u\n",
4357		    count, dev->ccbq.queue.qfrozen_cnt);
4358#endif
4359		count = dev->ccbq.queue.qfrozen_cnt;
4360	}
4361	dev->ccbq.queue.qfrozen_cnt -= count;
4362	if (dev->ccbq.queue.qfrozen_cnt == 0) {
4363		/*
4364		 * No longer need to wait for a successful
4365		 * command completion.
4366		 */
4367		dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
4368		/*
4369		 * Remove any timeouts that might be scheduled
4370		 * to release this queue.
4371		 */
4372		if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
4373			callout_stop(&dev->callout);
4374			dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
4375		}
4376		/*
4377		 * Now that we are unfrozen schedule the
4378		 * device so any pending transactions are
4379		 * run.
4380		 */
4381		xpt_schedule_devq(dev->sim->devq, dev);
4382	} else
4383		run_queue = 0;
4384	return (run_queue);
4385}
4386
4387void
4388xpt_release_simq(struct cam_sim *sim, int run_queue)
4389{
4390	struct cam_devq	*devq;
4391
4392	devq = sim->devq;
4393	mtx_lock(&devq->send_mtx);
4394	if (devq->send_queue.qfrozen_cnt <= 0) {
4395#ifdef INVARIANTS
4396		printf("xpt_release_simq: requested 1 > present %u\n",
4397		    devq->send_queue.qfrozen_cnt);
4398#endif
4399	} else
4400		devq->send_queue.qfrozen_cnt--;
4401	if (devq->send_queue.qfrozen_cnt == 0) {
4402		/*
4403		 * If there is a timeout scheduled to release this
4404		 * sim queue, remove it.  The queue frozen count is
4405		 * already at 0.
4406		 */
4407		if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
4408			callout_stop(&sim->callout);
4409			sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
4410		}
4411		if (run_queue) {
4412			/*
4413			 * Now that we are unfrozen run the send queue.
4414			 */
4415			xpt_run_devq(sim->devq);
4416		}
4417	}
4418	mtx_unlock(&devq->send_mtx);
4419}
4420
4421/*
4422 * XXX Appears to be unused.
4423 */
4424static void
4425xpt_release_simq_timeout(void *arg)
4426{
4427	struct cam_sim *sim;
4428
4429	sim = (struct cam_sim *)arg;
4430	xpt_release_simq(sim, /* run_queue */ TRUE);
4431}
4432
4433void
4434xpt_done(union ccb *done_ccb)
4435{
4436	struct cam_doneq *queue;
4437	int	run, hash;
4438
4439	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done\n"));
4440	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4441		return;
4442
4443	/* Store the time the ccb was in the sim */
4444	done_ccb->ccb_h.qos.sim_data = sbinuptime() - done_ccb->ccb_h.qos.sim_data;
4445	hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id +
4446	    done_ccb->ccb_h.target_lun) % cam_num_doneqs;
4447	queue = &cam_doneqs[hash];
4448	mtx_lock(&queue->cam_doneq_mtx);
4449	run = (queue->cam_doneq_sleep && STAILQ_EMPTY(&queue->cam_doneq));
4450	STAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.stqe);
4451	done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4452	mtx_unlock(&queue->cam_doneq_mtx);
4453	if (run)
4454		wakeup(&queue->cam_doneq);
4455}
4456
4457void
4458xpt_done_direct(union ccb *done_ccb)
4459{
4460
4461	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done_direct\n"));
4462	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4463		return;
4464
4465	/* Store the time the ccb was in the sim */
4466	done_ccb->ccb_h.qos.sim_data = sbinuptime() - done_ccb->ccb_h.qos.sim_data;
4467	xpt_done_process(&done_ccb->ccb_h);
4468}
4469
4470union ccb *
4471xpt_alloc_ccb()
4472{
4473	union ccb *new_ccb;
4474
4475	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4476	return (new_ccb);
4477}
4478
4479union ccb *
4480xpt_alloc_ccb_nowait()
4481{
4482	union ccb *new_ccb;
4483
4484	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4485	return (new_ccb);
4486}
4487
4488void
4489xpt_free_ccb(union ccb *free_ccb)
4490{
4491	free(free_ccb, M_CAMCCB);
4492}
4493
4494
4495
4496/* Private XPT functions */
4497
4498/*
4499 * Get a CAM control block for the caller. Charge the structure to the device
4500 * referenced by the path.  If we don't have sufficient resources to allocate
4501 * more ccbs, we return NULL.
4502 */
4503static union ccb *
4504xpt_get_ccb_nowait(struct cam_periph *periph)
4505{
4506	union ccb *new_ccb;
4507
4508	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4509	if (new_ccb == NULL)
4510		return (NULL);
4511	periph->periph_allocated++;
4512	cam_ccbq_take_opening(&periph->path->device->ccbq);
4513	return (new_ccb);
4514}
4515
4516static union ccb *
4517xpt_get_ccb(struct cam_periph *periph)
4518{
4519	union ccb *new_ccb;
4520
4521	cam_periph_unlock(periph);
4522	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4523	cam_periph_lock(periph);
4524	periph->periph_allocated++;
4525	cam_ccbq_take_opening(&periph->path->device->ccbq);
4526	return (new_ccb);
4527}
4528
4529union ccb *
4530cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
4531{
4532	struct ccb_hdr *ccb_h;
4533
4534	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("cam_periph_getccb\n"));
4535	cam_periph_assert(periph, MA_OWNED);
4536	while ((ccb_h = SLIST_FIRST(&periph->ccb_list)) == NULL ||
4537	    ccb_h->pinfo.priority != priority) {
4538		if (priority < periph->immediate_priority) {
4539			periph->immediate_priority = priority;
4540			xpt_run_allocq(periph, 0);
4541		} else
4542			cam_periph_sleep(periph, &periph->ccb_list, PRIBIO,
4543			    "cgticb", 0);
4544	}
4545	SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle);
4546	return ((union ccb *)ccb_h);
4547}
4548
4549static void
4550xpt_acquire_bus(struct cam_eb *bus)
4551{
4552
4553	xpt_lock_buses();
4554	bus->refcount++;
4555	xpt_unlock_buses();
4556}
4557
4558static void
4559xpt_release_bus(struct cam_eb *bus)
4560{
4561
4562	xpt_lock_buses();
4563	KASSERT(bus->refcount >= 1, ("bus->refcount >= 1"));
4564	if (--bus->refcount > 0) {
4565		xpt_unlock_buses();
4566		return;
4567	}
4568	TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
4569	xsoftc.bus_generation++;
4570	xpt_unlock_buses();
4571	KASSERT(TAILQ_EMPTY(&bus->et_entries),
4572	    ("destroying bus, but target list is not empty"));
4573	cam_sim_release(bus->sim);
4574	mtx_destroy(&bus->eb_mtx);
4575	free(bus, M_CAMXPT);
4576}
4577
4578static struct cam_et *
4579xpt_alloc_target(struct cam_eb *bus, target_id_t target_id)
4580{
4581	struct cam_et *cur_target, *target;
4582
4583	mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4584	mtx_assert(&bus->eb_mtx, MA_OWNED);
4585	target = (struct cam_et *)malloc(sizeof(*target), M_CAMXPT,
4586					 M_NOWAIT|M_ZERO);
4587	if (target == NULL)
4588		return (NULL);
4589
4590	TAILQ_INIT(&target->ed_entries);
4591	target->bus = bus;
4592	target->target_id = target_id;
4593	target->refcount = 1;
4594	target->generation = 0;
4595	target->luns = NULL;
4596	mtx_init(&target->luns_mtx, "CAM LUNs lock", NULL, MTX_DEF);
4597	timevalclear(&target->last_reset);
4598	/*
4599	 * Hold a reference to our parent bus so it
4600	 * will not go away before we do.
4601	 */
4602	bus->refcount++;
4603
4604	/* Insertion sort into our bus's target list */
4605	cur_target = TAILQ_FIRST(&bus->et_entries);
4606	while (cur_target != NULL && cur_target->target_id < target_id)
4607		cur_target = TAILQ_NEXT(cur_target, links);
4608	if (cur_target != NULL) {
4609		TAILQ_INSERT_BEFORE(cur_target, target, links);
4610	} else {
4611		TAILQ_INSERT_TAIL(&bus->et_entries, target, links);
4612	}
4613	bus->generation++;
4614	return (target);
4615}
4616
4617static void
4618xpt_acquire_target(struct cam_et *target)
4619{
4620	struct cam_eb *bus = target->bus;
4621
4622	mtx_lock(&bus->eb_mtx);
4623	target->refcount++;
4624	mtx_unlock(&bus->eb_mtx);
4625}
4626
4627static void
4628xpt_release_target(struct cam_et *target)
4629{
4630	struct cam_eb *bus = target->bus;
4631
4632	mtx_lock(&bus->eb_mtx);
4633	if (--target->refcount > 0) {
4634		mtx_unlock(&bus->eb_mtx);
4635		return;
4636	}
4637	TAILQ_REMOVE(&bus->et_entries, target, links);
4638	bus->generation++;
4639	mtx_unlock(&bus->eb_mtx);
4640	KASSERT(TAILQ_EMPTY(&target->ed_entries),
4641	    ("destroying target, but device list is not empty"));
4642	xpt_release_bus(bus);
4643	mtx_destroy(&target->luns_mtx);
4644	if (target->luns)
4645		free(target->luns, M_CAMXPT);
4646	free(target, M_CAMXPT);
4647}
4648
4649static struct cam_ed *
4650xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target,
4651			 lun_id_t lun_id)
4652{
4653	struct cam_ed *device;
4654
4655	device = xpt_alloc_device(bus, target, lun_id);
4656	if (device == NULL)
4657		return (NULL);
4658
4659	device->mintags = 1;
4660	device->maxtags = 1;
4661	return (device);
4662}
4663
4664static void
4665xpt_destroy_device(void *context, int pending)
4666{
4667	struct cam_ed	*device = context;
4668
4669	mtx_lock(&device->device_mtx);
4670	mtx_destroy(&device->device_mtx);
4671	free(device, M_CAMDEV);
4672}
4673
4674struct cam_ed *
4675xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
4676{
4677	struct cam_ed	*cur_device, *device;
4678	struct cam_devq	*devq;
4679	cam_status status;
4680
4681	mtx_assert(&bus->eb_mtx, MA_OWNED);
4682	/* Make space for us in the device queue on our bus */
4683	devq = bus->sim->devq;
4684	mtx_lock(&devq->send_mtx);
4685	status = cam_devq_resize(devq, devq->send_queue.array_size + 1);
4686	mtx_unlock(&devq->send_mtx);
4687	if (status != CAM_REQ_CMP)
4688		return (NULL);
4689
4690	device = (struct cam_ed *)malloc(sizeof(*device),
4691					 M_CAMDEV, M_NOWAIT|M_ZERO);
4692	if (device == NULL)
4693		return (NULL);
4694
4695	cam_init_pinfo(&device->devq_entry);
4696	device->target = target;
4697	device->lun_id = lun_id;
4698	device->sim = bus->sim;
4699	if (cam_ccbq_init(&device->ccbq,
4700			  bus->sim->max_dev_openings) != 0) {
4701		free(device, M_CAMDEV);
4702		return (NULL);
4703	}
4704	SLIST_INIT(&device->asyncs);
4705	SLIST_INIT(&device->periphs);
4706	device->generation = 0;
4707	device->flags = CAM_DEV_UNCONFIGURED;
4708	device->tag_delay_count = 0;
4709	device->tag_saved_openings = 0;
4710	device->refcount = 1;
4711	mtx_init(&device->device_mtx, "CAM device lock", NULL, MTX_DEF);
4712	callout_init_mtx(&device->callout, &devq->send_mtx, 0);
4713	TASK_INIT(&device->device_destroy_task, 0, xpt_destroy_device, device);
4714	/*
4715	 * Hold a reference to our parent bus so it
4716	 * will not go away before we do.
4717	 */
4718	target->refcount++;
4719
4720	cur_device = TAILQ_FIRST(&target->ed_entries);
4721	while (cur_device != NULL && cur_device->lun_id < lun_id)
4722		cur_device = TAILQ_NEXT(cur_device, links);
4723	if (cur_device != NULL)
4724		TAILQ_INSERT_BEFORE(cur_device, device, links);
4725	else
4726		TAILQ_INSERT_TAIL(&target->ed_entries, device, links);
4727	target->generation++;
4728	return (device);
4729}
4730
4731void
4732xpt_acquire_device(struct cam_ed *device)
4733{
4734	struct cam_eb *bus = device->target->bus;
4735
4736	mtx_lock(&bus->eb_mtx);
4737	device->refcount++;
4738	mtx_unlock(&bus->eb_mtx);
4739}
4740
4741void
4742xpt_release_device(struct cam_ed *device)
4743{
4744	struct cam_eb *bus = device->target->bus;
4745	struct cam_devq *devq;
4746
4747	mtx_lock(&bus->eb_mtx);
4748	if (--device->refcount > 0) {
4749		mtx_unlock(&bus->eb_mtx);
4750		return;
4751	}
4752
4753	TAILQ_REMOVE(&device->target->ed_entries, device,links);
4754	device->target->generation++;
4755	mtx_unlock(&bus->eb_mtx);
4756
4757	/* Release our slot in the devq */
4758	devq = bus->sim->devq;
4759	mtx_lock(&devq->send_mtx);
4760	cam_devq_resize(devq, devq->send_queue.array_size - 1);
4761	mtx_unlock(&devq->send_mtx);
4762
4763	KASSERT(SLIST_EMPTY(&device->periphs),
4764	    ("destroying device, but periphs list is not empty"));
4765	KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX,
4766	    ("destroying device while still queued for ccbs"));
4767
4768	if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
4769		callout_stop(&device->callout);
4770
4771	xpt_release_target(device->target);
4772
4773	cam_ccbq_fini(&device->ccbq);
4774	/*
4775	 * Free allocated memory.  free(9) does nothing if the
4776	 * supplied pointer is NULL, so it is safe to call without
4777	 * checking.
4778	 */
4779	free(device->supported_vpds, M_CAMXPT);
4780	free(device->device_id, M_CAMXPT);
4781	free(device->ext_inq, M_CAMXPT);
4782	free(device->physpath, M_CAMXPT);
4783	free(device->rcap_buf, M_CAMXPT);
4784	free(device->serial_num, M_CAMXPT);
4785	taskqueue_enqueue(xsoftc.xpt_taskq, &device->device_destroy_task);
4786}
4787
4788u_int32_t
4789xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
4790{
4791	int	result;
4792	struct	cam_ed *dev;
4793
4794	dev = path->device;
4795	mtx_lock(&dev->sim->devq->send_mtx);
4796	result = cam_ccbq_resize(&dev->ccbq, newopenings);
4797	mtx_unlock(&dev->sim->devq->send_mtx);
4798	if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
4799	 || (dev->inq_flags & SID_CmdQue) != 0)
4800		dev->tag_saved_openings = newopenings;
4801	return (result);
4802}
4803
4804static struct cam_eb *
4805xpt_find_bus(path_id_t path_id)
4806{
4807	struct cam_eb *bus;
4808
4809	xpt_lock_buses();
4810	for (bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4811	     bus != NULL;
4812	     bus = TAILQ_NEXT(bus, links)) {
4813		if (bus->path_id == path_id) {
4814			bus->refcount++;
4815			break;
4816		}
4817	}
4818	xpt_unlock_buses();
4819	return (bus);
4820}
4821
4822static struct cam_et *
4823xpt_find_target(struct cam_eb *bus, target_id_t	target_id)
4824{
4825	struct cam_et *target;
4826
4827	mtx_assert(&bus->eb_mtx, MA_OWNED);
4828	for (target = TAILQ_FIRST(&bus->et_entries);
4829	     target != NULL;
4830	     target = TAILQ_NEXT(target, links)) {
4831		if (target->target_id == target_id) {
4832			target->refcount++;
4833			break;
4834		}
4835	}
4836	return (target);
4837}
4838
4839static struct cam_ed *
4840xpt_find_device(struct cam_et *target, lun_id_t lun_id)
4841{
4842	struct cam_ed *device;
4843
4844	mtx_assert(&target->bus->eb_mtx, MA_OWNED);
4845	for (device = TAILQ_FIRST(&target->ed_entries);
4846	     device != NULL;
4847	     device = TAILQ_NEXT(device, links)) {
4848		if (device->lun_id == lun_id) {
4849			device->refcount++;
4850			break;
4851		}
4852	}
4853	return (device);
4854}
4855
4856void
4857xpt_start_tags(struct cam_path *path)
4858{
4859	struct ccb_relsim crs;
4860	struct cam_ed *device;
4861	struct cam_sim *sim;
4862	int    newopenings;
4863
4864	device = path->device;
4865	sim = path->bus->sim;
4866	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4867	xpt_freeze_devq(path, /*count*/1);
4868	device->inq_flags |= SID_CmdQue;
4869	if (device->tag_saved_openings != 0)
4870		newopenings = device->tag_saved_openings;
4871	else
4872		newopenings = min(device->maxtags,
4873				  sim->max_tagged_dev_openings);
4874	xpt_dev_ccbq_resize(path, newopenings);
4875	xpt_async(AC_GETDEV_CHANGED, path, NULL);
4876	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4877	crs.ccb_h.func_code = XPT_REL_SIMQ;
4878	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4879	crs.openings
4880	    = crs.release_timeout
4881	    = crs.qfrozen_cnt
4882	    = 0;
4883	xpt_action((union ccb *)&crs);
4884}
4885
4886void
4887xpt_stop_tags(struct cam_path *path)
4888{
4889	struct ccb_relsim crs;
4890	struct cam_ed *device;
4891	struct cam_sim *sim;
4892
4893	device = path->device;
4894	sim = path->bus->sim;
4895	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4896	device->tag_delay_count = 0;
4897	xpt_freeze_devq(path, /*count*/1);
4898	device->inq_flags &= ~SID_CmdQue;
4899	xpt_dev_ccbq_resize(path, sim->max_dev_openings);
4900	xpt_async(AC_GETDEV_CHANGED, path, NULL);
4901	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4902	crs.ccb_h.func_code = XPT_REL_SIMQ;
4903	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4904	crs.openings
4905	    = crs.release_timeout
4906	    = crs.qfrozen_cnt
4907	    = 0;
4908	xpt_action((union ccb *)&crs);
4909}
4910
4911static void
4912xpt_boot_delay(void *arg)
4913{
4914
4915	xpt_release_boot();
4916}
4917
4918static void
4919xpt_config(void *arg)
4920{
4921	/*
4922	 * Now that interrupts are enabled, go find our devices
4923	 */
4924	if (taskqueue_start_threads(&xsoftc.xpt_taskq, 1, PRIBIO, "CAM taskq"))
4925		printf("xpt_config: failed to create taskqueue thread.\n");
4926
4927	/* Setup debugging path */
4928	if (cam_dflags != CAM_DEBUG_NONE) {
4929		if (xpt_create_path(&cam_dpath, NULL,
4930				    CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
4931				    CAM_DEBUG_LUN) != CAM_REQ_CMP) {
4932			printf("xpt_config: xpt_create_path() failed for debug"
4933			       " target %d:%d:%d, debugging disabled\n",
4934			       CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN);
4935			cam_dflags = CAM_DEBUG_NONE;
4936		}
4937	} else
4938		cam_dpath = NULL;
4939
4940	periphdriver_init(1);
4941	xpt_hold_boot();
4942	callout_init(&xsoftc.boot_callout, 1);
4943	callout_reset_sbt(&xsoftc.boot_callout, SBT_1MS * xsoftc.boot_delay, 0,
4944	    xpt_boot_delay, NULL, 0);
4945	/* Fire up rescan thread. */
4946	if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0,
4947	    "cam", "scanner")) {
4948		printf("xpt_config: failed to create rescan thread.\n");
4949	}
4950}
4951
4952void
4953xpt_hold_boot(void)
4954{
4955	xpt_lock_buses();
4956	xsoftc.buses_to_config++;
4957	xpt_unlock_buses();
4958}
4959
4960void
4961xpt_release_boot(void)
4962{
4963	xpt_lock_buses();
4964	xsoftc.buses_to_config--;
4965	if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
4966		struct	xpt_task *task;
4967
4968		xsoftc.buses_config_done = 1;
4969		xpt_unlock_buses();
4970		/* Call manually because we don't have any busses */
4971		task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
4972		if (task != NULL) {
4973			TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
4974			taskqueue_enqueue(taskqueue_thread, &task->task);
4975		}
4976	} else
4977		xpt_unlock_buses();
4978}
4979
4980/*
4981 * If the given device only has one peripheral attached to it, and if that
4982 * peripheral is the passthrough driver, announce it.  This insures that the
4983 * user sees some sort of announcement for every peripheral in their system.
4984 */
4985static int
4986xptpassannouncefunc(struct cam_ed *device, void *arg)
4987{
4988	struct cam_periph *periph;
4989	int i;
4990
4991	for (periph = SLIST_FIRST(&device->periphs), i = 0; periph != NULL;
4992	     periph = SLIST_NEXT(periph, periph_links), i++);
4993
4994	periph = SLIST_FIRST(&device->periphs);
4995	if ((i == 1)
4996	 && (strncmp(periph->periph_name, "pass", 4) == 0))
4997		xpt_announce_periph(periph, NULL);
4998
4999	return(1);
5000}
5001
5002static void
5003xpt_finishconfig_task(void *context, int pending)
5004{
5005
5006	periphdriver_init(2);
5007	/*
5008	 * Check for devices with no "standard" peripheral driver
5009	 * attached.  For any devices like that, announce the
5010	 * passthrough driver so the user will see something.
5011	 */
5012	if (!bootverbose)
5013		xpt_for_all_devices(xptpassannouncefunc, NULL);
5014
5015	/* Release our hook so that the boot can continue. */
5016	config_intrhook_disestablish(xsoftc.xpt_config_hook);
5017	free(xsoftc.xpt_config_hook, M_CAMXPT);
5018	xsoftc.xpt_config_hook = NULL;
5019
5020	free(context, M_CAMXPT);
5021}
5022
5023cam_status
5024xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
5025		   struct cam_path *path)
5026{
5027	struct ccb_setasync csa;
5028	cam_status status;
5029	int xptpath = 0;
5030
5031	if (path == NULL) {
5032		status = xpt_create_path(&path, /*periph*/NULL, CAM_XPT_PATH_ID,
5033					 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
5034		if (status != CAM_REQ_CMP)
5035			return (status);
5036		xpt_path_lock(path);
5037		xptpath = 1;
5038	}
5039
5040	xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL);
5041	csa.ccb_h.func_code = XPT_SASYNC_CB;
5042	csa.event_enable = event;
5043	csa.callback = cbfunc;
5044	csa.callback_arg = cbarg;
5045	xpt_action((union ccb *)&csa);
5046	status = csa.ccb_h.status;
5047
5048	if (xptpath) {
5049		xpt_path_unlock(path);
5050		xpt_free_path(path);
5051	}
5052
5053	if ((status == CAM_REQ_CMP) &&
5054	    (csa.event_enable & AC_FOUND_DEVICE)) {
5055		/*
5056		 * Get this peripheral up to date with all
5057		 * the currently existing devices.
5058		 */
5059		xpt_for_all_devices(xptsetasyncfunc, &csa);
5060	}
5061	if ((status == CAM_REQ_CMP) &&
5062	    (csa.event_enable & AC_PATH_REGISTERED)) {
5063		/*
5064		 * Get this peripheral up to date with all
5065		 * the currently existing busses.
5066		 */
5067		xpt_for_all_busses(xptsetasyncbusfunc, &csa);
5068	}
5069
5070	return (status);
5071}
5072
5073static void
5074xptaction(struct cam_sim *sim, union ccb *work_ccb)
5075{
5076	CAM_DEBUG(work_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xptaction\n"));
5077
5078	switch (work_ccb->ccb_h.func_code) {
5079	/* Common cases first */
5080	case XPT_PATH_INQ:		/* Path routing inquiry */
5081	{
5082		struct ccb_pathinq *cpi;
5083
5084		cpi = &work_ccb->cpi;
5085		cpi->version_num = 1; /* XXX??? */
5086		cpi->hba_inquiry = 0;
5087		cpi->target_sprt = 0;
5088		cpi->hba_misc = 0;
5089		cpi->hba_eng_cnt = 0;
5090		cpi->max_target = 0;
5091		cpi->max_lun = 0;
5092		cpi->initiator_id = 0;
5093		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
5094		strncpy(cpi->hba_vid, "", HBA_IDLEN);
5095		strncpy(cpi->dev_name, sim->sim_name, DEV_IDLEN);
5096		cpi->unit_number = sim->unit_number;
5097		cpi->bus_id = sim->bus_id;
5098		cpi->base_transfer_speed = 0;
5099		cpi->protocol = PROTO_UNSPECIFIED;
5100		cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
5101		cpi->transport = XPORT_UNSPECIFIED;
5102		cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
5103		cpi->ccb_h.status = CAM_REQ_CMP;
5104		xpt_done(work_ccb);
5105		break;
5106	}
5107	default:
5108		work_ccb->ccb_h.status = CAM_REQ_INVALID;
5109		xpt_done(work_ccb);
5110		break;
5111	}
5112}
5113
5114/*
5115 * The xpt as a "controller" has no interrupt sources, so polling
5116 * is a no-op.
5117 */
5118static void
5119xptpoll(struct cam_sim *sim)
5120{
5121}
5122
5123void
5124xpt_lock_buses(void)
5125{
5126	mtx_lock(&xsoftc.xpt_topo_lock);
5127}
5128
5129void
5130xpt_unlock_buses(void)
5131{
5132	mtx_unlock(&xsoftc.xpt_topo_lock);
5133}
5134
5135struct mtx *
5136xpt_path_mtx(struct cam_path *path)
5137{
5138
5139	return (&path->device->device_mtx);
5140}
5141
5142static void
5143xpt_done_process(struct ccb_hdr *ccb_h)
5144{
5145	struct cam_sim *sim;
5146	struct cam_devq *devq;
5147	struct mtx *mtx = NULL;
5148
5149	if (ccb_h->flags & CAM_HIGH_POWER) {
5150		struct highpowerlist	*hphead;
5151		struct cam_ed		*device;
5152
5153		mtx_lock(&xsoftc.xpt_highpower_lock);
5154		hphead = &xsoftc.highpowerq;
5155
5156		device = STAILQ_FIRST(hphead);
5157
5158		/*
5159		 * Increment the count since this command is done.
5160		 */
5161		xsoftc.num_highpower++;
5162
5163		/*
5164		 * Any high powered commands queued up?
5165		 */
5166		if (device != NULL) {
5167
5168			STAILQ_REMOVE_HEAD(hphead, highpowerq_entry);
5169			mtx_unlock(&xsoftc.xpt_highpower_lock);
5170
5171			mtx_lock(&device->sim->devq->send_mtx);
5172			xpt_release_devq_device(device,
5173					 /*count*/1, /*runqueue*/TRUE);
5174			mtx_unlock(&device->sim->devq->send_mtx);
5175		} else
5176			mtx_unlock(&xsoftc.xpt_highpower_lock);
5177	}
5178
5179	sim = ccb_h->path->bus->sim;
5180
5181	if (ccb_h->status & CAM_RELEASE_SIMQ) {
5182		xpt_release_simq(sim, /*run_queue*/FALSE);
5183		ccb_h->status &= ~CAM_RELEASE_SIMQ;
5184	}
5185
5186	if ((ccb_h->flags & CAM_DEV_QFRZDIS)
5187	 && (ccb_h->status & CAM_DEV_QFRZN)) {
5188		xpt_release_devq(ccb_h->path, /*count*/1, /*run_queue*/TRUE);
5189		ccb_h->status &= ~CAM_DEV_QFRZN;
5190	}
5191
5192	devq = sim->devq;
5193	if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
5194		struct cam_ed *dev = ccb_h->path->device;
5195
5196		mtx_lock(&devq->send_mtx);
5197		devq->send_active--;
5198		devq->send_openings++;
5199		cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
5200
5201		if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
5202		  && (dev->ccbq.dev_active == 0))) {
5203			dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY;
5204			xpt_release_devq_device(dev, /*count*/1,
5205					 /*run_queue*/FALSE);
5206		}
5207
5208		if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
5209		  && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) {
5210			dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
5211			xpt_release_devq_device(dev, /*count*/1,
5212					 /*run_queue*/FALSE);
5213		}
5214
5215		if (!device_is_queued(dev))
5216			(void)xpt_schedule_devq(devq, dev);
5217		xpt_run_devq(devq);
5218		mtx_unlock(&devq->send_mtx);
5219
5220		if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0) {
5221			mtx = xpt_path_mtx(ccb_h->path);
5222			mtx_lock(mtx);
5223
5224			if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5225			 && (--dev->tag_delay_count == 0))
5226				xpt_start_tags(ccb_h->path);
5227		}
5228	}
5229
5230	if ((ccb_h->flags & CAM_UNLOCKED) == 0) {
5231		if (mtx == NULL) {
5232			mtx = xpt_path_mtx(ccb_h->path);
5233			mtx_lock(mtx);
5234		}
5235	} else {
5236		if (mtx != NULL) {
5237			mtx_unlock(mtx);
5238			mtx = NULL;
5239		}
5240	}
5241
5242	/* Call the peripheral driver's callback */
5243	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
5244	(*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
5245	if (mtx != NULL)
5246		mtx_unlock(mtx);
5247}
5248
5249void
5250xpt_done_td(void *arg)
5251{
5252	struct cam_doneq *queue = arg;
5253	struct ccb_hdr *ccb_h;
5254	STAILQ_HEAD(, ccb_hdr)	doneq;
5255
5256	STAILQ_INIT(&doneq);
5257	mtx_lock(&queue->cam_doneq_mtx);
5258	while (1) {
5259		while (STAILQ_EMPTY(&queue->cam_doneq)) {
5260			queue->cam_doneq_sleep = 1;
5261			msleep(&queue->cam_doneq, &queue->cam_doneq_mtx,
5262			    PRIBIO, "-", 0);
5263			queue->cam_doneq_sleep = 0;
5264		}
5265		STAILQ_CONCAT(&doneq, &queue->cam_doneq);
5266		mtx_unlock(&queue->cam_doneq_mtx);
5267
5268		THREAD_NO_SLEEPING();
5269		while ((ccb_h = STAILQ_FIRST(&doneq)) != NULL) {
5270			STAILQ_REMOVE_HEAD(&doneq, sim_links.stqe);
5271			xpt_done_process(ccb_h);
5272		}
5273		THREAD_SLEEPING_OK();
5274
5275		mtx_lock(&queue->cam_doneq_mtx);
5276	}
5277}
5278
5279static void
5280camisr_runqueue(void)
5281{
5282	struct	ccb_hdr *ccb_h;
5283	struct cam_doneq *queue;
5284	int i;
5285
5286	/* Process global queues. */
5287	for (i = 0; i < cam_num_doneqs; i++) {
5288		queue = &cam_doneqs[i];
5289		mtx_lock(&queue->cam_doneq_mtx);
5290		while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) {
5291			STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe);
5292			mtx_unlock(&queue->cam_doneq_mtx);
5293			xpt_done_process(ccb_h);
5294			mtx_lock(&queue->cam_doneq_mtx);
5295		}
5296		mtx_unlock(&queue->cam_doneq_mtx);
5297	}
5298}
5299