ahci_pci.c revision 206755
1/*-
2 * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer,
10 *    without modification, immediately at the beginning of the file.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/dev/ahci/ahci.c 206755 2010-04-17 11:40:39Z rpaulo $");
29
30#include <sys/param.h>
31#include <sys/module.h>
32#include <sys/systm.h>
33#include <sys/kernel.h>
34#include <sys/ata.h>
35#include <sys/bus.h>
36#include <sys/endian.h>
37#include <sys/malloc.h>
38#include <sys/lock.h>
39#include <sys/mutex.h>
40#include <sys/sema.h>
41#include <sys/taskqueue.h>
42#include <vm/uma.h>
43#include <machine/stdarg.h>
44#include <machine/resource.h>
45#include <machine/bus.h>
46#include <sys/rman.h>
47#include <dev/pci/pcivar.h>
48#include <dev/pci/pcireg.h>
49#include "ahci.h"
50
51#include <cam/cam.h>
52#include <cam/cam_ccb.h>
53#include <cam/cam_sim.h>
54#include <cam/cam_xpt_sim.h>
55#include <cam/cam_debug.h>
56
57/* local prototypes */
58static int ahci_setup_interrupt(device_t dev);
59static void ahci_intr(void *data);
60static void ahci_intr_one(void *data);
61static int ahci_suspend(device_t dev);
62static int ahci_resume(device_t dev);
63static int ahci_ch_suspend(device_t dev);
64static int ahci_ch_resume(device_t dev);
65static void ahci_ch_pm(void *arg);
66static void ahci_ch_intr_locked(void *data);
67static void ahci_ch_intr(void *data);
68static int ahci_ctlr_reset(device_t dev);
69static int ahci_ctlr_setup(device_t dev);
70static void ahci_begin_transaction(device_t dev, union ccb *ccb);
71static void ahci_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error);
72static void ahci_execute_transaction(struct ahci_slot *slot);
73static void ahci_timeout(struct ahci_slot *slot);
74static void ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et);
75static int ahci_setup_fis(device_t dev, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag);
76static void ahci_dmainit(device_t dev);
77static void ahci_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error);
78static void ahci_dmafini(device_t dev);
79static void ahci_slotsalloc(device_t dev);
80static void ahci_slotsfree(device_t dev);
81static void ahci_reset(device_t dev);
82static void ahci_start(device_t dev, int fbs);
83static void ahci_stop(device_t dev);
84static void ahci_clo(device_t dev);
85static void ahci_start_fr(device_t dev);
86static void ahci_stop_fr(device_t dev);
87
88static int ahci_sata_connect(struct ahci_channel *ch);
89static int ahci_sata_phy_reset(device_t dev);
90static int ahci_wait_ready(device_t dev, int t);
91
92static void ahci_issue_read_log(device_t dev);
93static void ahci_process_read_log(device_t dev, union ccb *ccb);
94
95static void ahciaction(struct cam_sim *sim, union ccb *ccb);
96static void ahcipoll(struct cam_sim *sim);
97
98MALLOC_DEFINE(M_AHCI, "AHCI driver", "AHCI driver data buffers");
99
100static struct {
101	uint32_t	id;
102	uint8_t		rev;
103	const char	*name;
104	int		quirks;
105#define AHCI_Q_NOFORCE	1
106#define AHCI_Q_NOPMP	2
107#define AHCI_Q_NONCQ	4
108#define AHCI_Q_1CH	8
109#define AHCI_Q_2CH	16
110#define AHCI_Q_4CH	32
111#define AHCI_Q_EDGEIS	64
112#define AHCI_Q_SATA2	128
113#define AHCI_Q_NOBSYRES	256
114} ahci_ids[] = {
115	{0x43801002, 0x00, "ATI IXP600",	0},
116	{0x43901002, 0x00, "ATI IXP700",	0},
117	{0x43911002, 0x00, "ATI IXP700",	0},
118	{0x43921002, 0x00, "ATI IXP700",	0},
119	{0x43931002, 0x00, "ATI IXP700",	0},
120	{0x43941002, 0x00, "ATI IXP800",	0},
121	{0x43951002, 0x00, "ATI IXP800",	0},
122	{0x26528086, 0x00, "Intel ICH6",	AHCI_Q_NOFORCE},
123	{0x26538086, 0x00, "Intel ICH6M",	AHCI_Q_NOFORCE},
124	{0x26818086, 0x00, "Intel ESB2",	0},
125	{0x26828086, 0x00, "Intel ESB2",	0},
126	{0x26838086, 0x00, "Intel ESB2",	0},
127	{0x27c18086, 0x00, "Intel ICH7",	0},
128	{0x27c38086, 0x00, "Intel ICH7",	0},
129	{0x27c48086, 0x00, "Intel ICH7M",	0},
130	{0x27c58086, 0x00, "Intel ICH7M",	0},
131	{0x27c68086, 0x00, "Intel ICH7M",	0},
132	{0x28218086, 0x00, "Intel ICH8",	0},
133	{0x28228086, 0x00, "Intel ICH8",	0},
134	{0x28248086, 0x00, "Intel ICH8",	0},
135	{0x28298086, 0x00, "Intel ICH8M",	0},
136	{0x282a8086, 0x00, "Intel ICH8M",	0},
137	{0x29228086, 0x00, "Intel ICH9",	0},
138	{0x29238086, 0x00, "Intel ICH9",	0},
139	{0x29248086, 0x00, "Intel ICH9",	0},
140	{0x29258086, 0x00, "Intel ICH9",	0},
141	{0x29278086, 0x00, "Intel ICH9",	0},
142	{0x29298086, 0x00, "Intel ICH9M",	0},
143	{0x292a8086, 0x00, "Intel ICH9M",	0},
144	{0x292b8086, 0x00, "Intel ICH9M",	0},
145	{0x292c8086, 0x00, "Intel ICH9M",	0},
146	{0x292f8086, 0x00, "Intel ICH9M",	0},
147	{0x294d8086, 0x00, "Intel ICH9",	0},
148	{0x294e8086, 0x00, "Intel ICH9M",	0},
149	{0x3a058086, 0x00, "Intel ICH10",	0},
150	{0x3a228086, 0x00, "Intel ICH10",	0},
151	{0x3a258086, 0x00, "Intel ICH10",	0},
152	{0x3b228086, 0x00, "Intel PCH",		0},
153	{0x3b238086, 0x00, "Intel PCH",		0},
154	{0x3b248086, 0x00, "Intel PCH",		0},
155	{0x3b258086, 0x00, "Intel PCH",		0},
156	{0x3b298086, 0x00, "Intel PCH",		0},
157	{0x3b2b8086, 0x00, "Intel PCH",		0},
158	{0x3b2c8086, 0x00, "Intel PCH",		0},
159	{0x3b2f8086, 0x00, "Intel PCH",		0},
160	{0x2361197b, 0x00, "JMicron JMB361",	AHCI_Q_NOFORCE},
161	{0x2363197b, 0x00, "JMicron JMB363",	AHCI_Q_NOFORCE},
162	{0x2365197b, 0x00, "JMicron JMB365",	AHCI_Q_NOFORCE},
163	{0x2366197b, 0x00, "JMicron JMB366",	AHCI_Q_NOFORCE},
164	{0x2368197b, 0x00, "JMicron JMB368",	AHCI_Q_NOFORCE},
165	{0x611111ab, 0x00, "Marvell 88SX6111",	AHCI_Q_NOFORCE|AHCI_Q_1CH|AHCI_Q_EDGEIS},
166	{0x612111ab, 0x00, "Marvell 88SX6121",	AHCI_Q_NOFORCE|AHCI_Q_2CH|AHCI_Q_EDGEIS},
167	{0x614111ab, 0x00, "Marvell 88SX6141",	AHCI_Q_NOFORCE|AHCI_Q_4CH|AHCI_Q_EDGEIS},
168	{0x614511ab, 0x00, "Marvell 88SX6145",	AHCI_Q_NOFORCE|AHCI_Q_4CH|AHCI_Q_EDGEIS},
169	{0x91231b4b, 0x11, "Marvell 88SE912x",	AHCI_Q_NOBSYRES},
170	{0x91231b4b, 0x00, "Marvell 88SE912x",	AHCI_Q_EDGEIS|AHCI_Q_SATA2|AHCI_Q_NOBSYRES},
171	{0x044c10de, 0x00, "NVIDIA MCP65",	0},
172	{0x044d10de, 0x00, "NVIDIA MCP65",	0},
173	{0x044e10de, 0x00, "NVIDIA MCP65",	0},
174	{0x044f10de, 0x00, "NVIDIA MCP65",	0},
175	{0x045c10de, 0x00, "NVIDIA MCP65",	0},
176	{0x045d10de, 0x00, "NVIDIA MCP65",	0},
177	{0x045e10de, 0x00, "NVIDIA MCP65",	0},
178	{0x045f10de, 0x00, "NVIDIA MCP65",	0},
179	{0x055010de, 0x00, "NVIDIA MCP67",	0},
180	{0x055110de, 0x00, "NVIDIA MCP67",	0},
181	{0x055210de, 0x00, "NVIDIA MCP67",	0},
182	{0x055310de, 0x00, "NVIDIA MCP67",	0},
183	{0x055410de, 0x00, "NVIDIA MCP67",	0},
184	{0x055510de, 0x00, "NVIDIA MCP67",	0},
185	{0x055610de, 0x00, "NVIDIA MCP67",	0},
186	{0x055710de, 0x00, "NVIDIA MCP67",	0},
187	{0x055810de, 0x00, "NVIDIA MCP67",	0},
188	{0x055910de, 0x00, "NVIDIA MCP67",	0},
189	{0x055A10de, 0x00, "NVIDIA MCP67",	0},
190	{0x055B10de, 0x00, "NVIDIA MCP67",	0},
191	{0x058410de, 0x00, "NVIDIA MCP67",	0},
192	{0x07f010de, 0x00, "NVIDIA MCP73",	0},
193	{0x07f110de, 0x00, "NVIDIA MCP73",	0},
194	{0x07f210de, 0x00, "NVIDIA MCP73",	0},
195	{0x07f310de, 0x00, "NVIDIA MCP73",	0},
196	{0x07f410de, 0x00, "NVIDIA MCP73",	0},
197	{0x07f510de, 0x00, "NVIDIA MCP73",	0},
198	{0x07f610de, 0x00, "NVIDIA MCP73",	0},
199	{0x07f710de, 0x00, "NVIDIA MCP73",	0},
200	{0x07f810de, 0x00, "NVIDIA MCP73",	0},
201	{0x07f910de, 0x00, "NVIDIA MCP73",	0},
202	{0x07fa10de, 0x00, "NVIDIA MCP73",	0},
203	{0x07fb10de, 0x00, "NVIDIA MCP73",	0},
204	{0x0ad010de, 0x00, "NVIDIA MCP77",	0},
205	{0x0ad110de, 0x00, "NVIDIA MCP77",	0},
206	{0x0ad210de, 0x00, "NVIDIA MCP77",	0},
207	{0x0ad310de, 0x00, "NVIDIA MCP77",	0},
208	{0x0ad410de, 0x00, "NVIDIA MCP77",	0},
209	{0x0ad510de, 0x00, "NVIDIA MCP77",	0},
210	{0x0ad610de, 0x00, "NVIDIA MCP77",	0},
211	{0x0ad710de, 0x00, "NVIDIA MCP77",	0},
212	{0x0ad810de, 0x00, "NVIDIA MCP77",	0},
213	{0x0ad910de, 0x00, "NVIDIA MCP77",	0},
214	{0x0ada10de, 0x00, "NVIDIA MCP77",	0},
215	{0x0adb10de, 0x00, "NVIDIA MCP77",	0},
216	{0x0ab410de, 0x00, "NVIDIA MCP79",	0},
217	{0x0ab510de, 0x00, "NVIDIA MCP79",	0},
218	{0x0ab610de, 0x00, "NVIDIA MCP79",	0},
219	{0x0ab710de, 0x00, "NVIDIA MCP79",	0},
220	{0x0ab810de, 0x00, "NVIDIA MCP79",	0},
221	{0x0ab910de, 0x00, "NVIDIA MCP79",	0},
222	{0x0aba10de, 0x00, "NVIDIA MCP79",	0},
223	{0x0abb10de, 0x00, "NVIDIA MCP79",	0},
224	{0x0abc10de, 0x00, "NVIDIA MCP79",	0},
225	{0x0abd10de, 0x00, "NVIDIA MCP79",	0},
226	{0x0abe10de, 0x00, "NVIDIA MCP79",	0},
227	{0x0abf10de, 0x00, "NVIDIA MCP79",	0},
228	{0x0d8410de, 0x00, "NVIDIA MCP89",	0},
229	{0x0d8510de, 0x00, "NVIDIA MCP89",	0},
230	{0x0d8610de, 0x00, "NVIDIA MCP89",	0},
231	{0x0d8710de, 0x00, "NVIDIA MCP89",	0},
232	{0x0d8810de, 0x00, "NVIDIA MCP89",	0},
233	{0x0d8910de, 0x00, "NVIDIA MCP89",	0},
234	{0x0d8a10de, 0x00, "NVIDIA MCP89",	0},
235	{0x0d8b10de, 0x00, "NVIDIA MCP89",	0},
236	{0x0d8c10de, 0x00, "NVIDIA MCP89",	0},
237	{0x0d8d10de, 0x00, "NVIDIA MCP89",	0},
238	{0x0d8e10de, 0x00, "NVIDIA MCP89",	0},
239	{0x0d8f10de, 0x00, "NVIDIA MCP89",	0},
240	{0x33491106, 0x00, "VIA VT8251",	0},
241	{0x62871106, 0x00, "VIA VT8251",	0},
242	{0x11841039, 0x00, "SiS 966",		0},
243	{0x11851039, 0x00, "SiS 968",		0},
244	{0x01861039, 0x00, "SiS 968",		0},
245	{0x00000000, 0x00, NULL,		0}
246};
247
248static int
249ahci_probe(device_t dev)
250{
251	char buf[64];
252	int i, valid = 0;
253	uint32_t devid = pci_get_devid(dev);
254	uint8_t revid = pci_get_revid(dev);
255
256	/* Is this a possible AHCI candidate? */
257	if (pci_get_class(dev) == PCIC_STORAGE &&
258	    pci_get_subclass(dev) == PCIS_STORAGE_SATA &&
259	    pci_get_progif(dev) == PCIP_STORAGE_SATA_AHCI_1_0)
260		valid = 1;
261	/* Is this a known AHCI chip? */
262	for (i = 0; ahci_ids[i].id != 0; i++) {
263		if (ahci_ids[i].id == devid &&
264		    ahci_ids[i].rev <= revid &&
265		    (valid || !(ahci_ids[i].quirks & AHCI_Q_NOFORCE))) {
266			/* Do not attach JMicrons with single PCI function. */
267			if (pci_get_vendor(dev) == 0x197b &&
268			    (pci_read_config(dev, 0xdf, 1) & 0x40) == 0)
269				return (ENXIO);
270			snprintf(buf, sizeof(buf), "%s AHCI SATA controller",
271			    ahci_ids[i].name);
272			device_set_desc_copy(dev, buf);
273			return (BUS_PROBE_VENDOR);
274		}
275	}
276	if (!valid)
277		return (ENXIO);
278	device_set_desc_copy(dev, "AHCI SATA controller");
279	return (BUS_PROBE_VENDOR);
280}
281
282static int
283ahci_ata_probe(device_t dev)
284{
285	char buf[64];
286	int i;
287	uint32_t devid = pci_get_devid(dev);
288	uint8_t revid = pci_get_revid(dev);
289
290	if ((intptr_t)device_get_ivars(dev) >= 0)
291		return (ENXIO);
292	/* Is this a known AHCI chip? */
293	for (i = 0; ahci_ids[i].id != 0; i++) {
294		if (ahci_ids[i].id == devid &&
295		    ahci_ids[i].rev <= revid) {
296			snprintf(buf, sizeof(buf), "%s AHCI SATA controller",
297			    ahci_ids[i].name);
298			device_set_desc_copy(dev, buf);
299			return (BUS_PROBE_VENDOR);
300		}
301	}
302	device_set_desc_copy(dev, "AHCI SATA controller");
303	return (BUS_PROBE_VENDOR);
304}
305
306static int
307ahci_attach(device_t dev)
308{
309	struct ahci_controller *ctlr = device_get_softc(dev);
310	device_t child;
311	int	error, unit, speed, i;
312	uint32_t devid = pci_get_devid(dev);
313	uint8_t revid = pci_get_revid(dev);
314	u_int32_t version;
315
316	ctlr->dev = dev;
317	i = 0;
318	while (ahci_ids[i].id != 0 &&
319	    (ahci_ids[i].id != devid ||
320	     ahci_ids[i].rev > revid))
321		i++;
322	ctlr->quirks = ahci_ids[i].quirks;
323	resource_int_value(device_get_name(dev),
324	    device_get_unit(dev), "ccc", &ctlr->ccc);
325	/* if we have a memory BAR(5) we are likely on an AHCI part */
326	ctlr->r_rid = PCIR_BAR(5);
327	if (!(ctlr->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
328	    &ctlr->r_rid, RF_ACTIVE)))
329		return ENXIO;
330	/* Setup our own memory management for channels. */
331	ctlr->sc_iomem.rm_type = RMAN_ARRAY;
332	ctlr->sc_iomem.rm_descr = "I/O memory addresses";
333	if ((error = rman_init(&ctlr->sc_iomem)) != 0) {
334		bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem);
335		return (error);
336	}
337	if ((error = rman_manage_region(&ctlr->sc_iomem,
338	    rman_get_start(ctlr->r_mem), rman_get_end(ctlr->r_mem))) != 0) {
339		bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem);
340		rman_fini(&ctlr->sc_iomem);
341		return (error);
342	}
343	/* Reset controller */
344	if ((error = ahci_ctlr_reset(dev)) != 0) {
345		bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem);
346		rman_fini(&ctlr->sc_iomem);
347		return (error);
348	};
349	/* Get the HW capabilities */
350	version = ATA_INL(ctlr->r_mem, AHCI_VS);
351	ctlr->caps = ATA_INL(ctlr->r_mem, AHCI_CAP);
352	if (version >= 0x00010020)
353		ctlr->caps2 = ATA_INL(ctlr->r_mem, AHCI_CAP2);
354	if (ctlr->caps & AHCI_CAP_EMS)
355		ctlr->capsem = ATA_INL(ctlr->r_mem, AHCI_EM_CTL);
356	ctlr->ichannels = ATA_INL(ctlr->r_mem, AHCI_PI);
357	if (ctlr->quirks & AHCI_Q_1CH) {
358		ctlr->caps &= ~AHCI_CAP_NPMASK;
359		ctlr->ichannels &= 0x01;
360	}
361	if (ctlr->quirks & AHCI_Q_2CH) {
362		ctlr->caps &= ~AHCI_CAP_NPMASK;
363		ctlr->caps |= 1;
364		ctlr->ichannels &= 0x03;
365	}
366	if (ctlr->quirks & AHCI_Q_4CH) {
367		ctlr->caps &= ~AHCI_CAP_NPMASK;
368		ctlr->caps |= 3;
369		ctlr->ichannels &= 0x0f;
370	}
371	ctlr->channels = MAX(flsl(ctlr->ichannels),
372	    (ctlr->caps & AHCI_CAP_NPMASK) + 1);
373	if (ctlr->quirks & AHCI_Q_NOPMP)
374		ctlr->caps &= ~AHCI_CAP_SPM;
375	if (ctlr->quirks & AHCI_Q_NONCQ)
376		ctlr->caps &= ~AHCI_CAP_SNCQ;
377	if ((ctlr->caps & AHCI_CAP_CCCS) == 0)
378		ctlr->ccc = 0;
379	ahci_ctlr_setup(dev);
380	/* Setup interrupts. */
381	if (ahci_setup_interrupt(dev)) {
382		bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem);
383		rman_fini(&ctlr->sc_iomem);
384		return ENXIO;
385	}
386	/* Announce HW capabilities. */
387	speed = (ctlr->caps & AHCI_CAP_ISS) >> AHCI_CAP_ISS_SHIFT;
388	device_printf(dev,
389		    "AHCI v%x.%02x with %d %sGbps ports, Port Multiplier %s%s\n",
390		    ((version >> 20) & 0xf0) + ((version >> 16) & 0x0f),
391		    ((version >> 4) & 0xf0) + (version & 0x0f),
392		    (ctlr->caps & AHCI_CAP_NPMASK) + 1,
393		    ((speed == 1) ? "1.5":((speed == 2) ? "3":
394		    ((speed == 3) ? "6":"?"))),
395		    (ctlr->caps & AHCI_CAP_SPM) ?
396		    "supported" : "not supported",
397		    (ctlr->caps & AHCI_CAP_FBSS) ?
398		    " with FBS" : "");
399	if (bootverbose) {
400		device_printf(dev, "Caps:%s%s%s%s%s%s%s%s %sGbps",
401		    (ctlr->caps & AHCI_CAP_64BIT) ? " 64bit":"",
402		    (ctlr->caps & AHCI_CAP_SNCQ) ? " NCQ":"",
403		    (ctlr->caps & AHCI_CAP_SSNTF) ? " SNTF":"",
404		    (ctlr->caps & AHCI_CAP_SMPS) ? " MPS":"",
405		    (ctlr->caps & AHCI_CAP_SSS) ? " SS":"",
406		    (ctlr->caps & AHCI_CAP_SALP) ? " ALP":"",
407		    (ctlr->caps & AHCI_CAP_SAL) ? " AL":"",
408		    (ctlr->caps & AHCI_CAP_SCLO) ? " CLO":"",
409		    ((speed == 1) ? "1.5":((speed == 2) ? "3":
410		    ((speed == 3) ? "6":"?"))));
411		printf("%s%s%s%s%s%s %dcmd%s%s%s %dports\n",
412		    (ctlr->caps & AHCI_CAP_SAM) ? " AM":"",
413		    (ctlr->caps & AHCI_CAP_SPM) ? " PM":"",
414		    (ctlr->caps & AHCI_CAP_FBSS) ? " FBS":"",
415		    (ctlr->caps & AHCI_CAP_PMD) ? " PMD":"",
416		    (ctlr->caps & AHCI_CAP_SSC) ? " SSC":"",
417		    (ctlr->caps & AHCI_CAP_PSC) ? " PSC":"",
418		    ((ctlr->caps & AHCI_CAP_NCS) >> AHCI_CAP_NCS_SHIFT) + 1,
419		    (ctlr->caps & AHCI_CAP_CCCS) ? " CCC":"",
420		    (ctlr->caps & AHCI_CAP_EMS) ? " EM":"",
421		    (ctlr->caps & AHCI_CAP_SXS) ? " eSATA":"",
422		    (ctlr->caps & AHCI_CAP_NPMASK) + 1);
423	}
424	if (bootverbose && version >= 0x00010020) {
425		device_printf(dev, "Caps2:%s%s%s\n",
426		    (ctlr->caps2 & AHCI_CAP2_APST) ? " APST":"",
427		    (ctlr->caps2 & AHCI_CAP2_NVMP) ? " NVMP":"",
428		    (ctlr->caps2 & AHCI_CAP2_BOH) ? " BOH":"");
429	}
430	if (bootverbose && (ctlr->caps & AHCI_CAP_EMS)) {
431		device_printf(dev, "EM Caps:%s%s%s%s%s%s%s%s\n",
432		    (ctlr->capsem & AHCI_EM_PM) ? " PM":"",
433		    (ctlr->capsem & AHCI_EM_ALHD) ? " ALHD":"",
434		    (ctlr->capsem & AHCI_EM_XMT) ? " XMT":"",
435		    (ctlr->capsem & AHCI_EM_SMB) ? " SMB":"",
436		    (ctlr->capsem & AHCI_EM_SGPIO) ? " SGPIO":"",
437		    (ctlr->capsem & AHCI_EM_SES2) ? " SES-2":"",
438		    (ctlr->capsem & AHCI_EM_SAFTE) ? " SAF-TE":"",
439		    (ctlr->capsem & AHCI_EM_LED) ? " LED":"");
440	}
441	/* Attach all channels on this controller */
442	for (unit = 0; unit < ctlr->channels; unit++) {
443		if ((ctlr->ichannels & (1 << unit)) == 0)
444			continue;
445		child = device_add_child(dev, "ahcich", -1);
446		if (child == NULL)
447			device_printf(dev, "failed to add channel device\n");
448		else
449			device_set_ivars(child, (void *)(intptr_t)unit);
450	}
451	bus_generic_attach(dev);
452	return 0;
453}
454
455static int
456ahci_detach(device_t dev)
457{
458	struct ahci_controller *ctlr = device_get_softc(dev);
459	device_t *children;
460	int nchildren, i;
461
462	/* Detach & delete all children */
463	if (!device_get_children(dev, &children, &nchildren)) {
464		for (i = 0; i < nchildren; i++)
465			device_delete_child(dev, children[i]);
466		free(children, M_TEMP);
467	}
468	/* Free interrupts. */
469	for (i = 0; i < ctlr->numirqs; i++) {
470		if (ctlr->irqs[i].r_irq) {
471			bus_teardown_intr(dev, ctlr->irqs[i].r_irq,
472			    ctlr->irqs[i].handle);
473			bus_release_resource(dev, SYS_RES_IRQ,
474			    ctlr->irqs[i].r_irq_rid, ctlr->irqs[i].r_irq);
475		}
476	}
477	pci_release_msi(dev);
478	/* Free memory. */
479	rman_fini(&ctlr->sc_iomem);
480	if (ctlr->r_mem)
481		bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem);
482	return (0);
483}
484
485static int
486ahci_ctlr_reset(device_t dev)
487{
488	struct ahci_controller *ctlr = device_get_softc(dev);
489	int timeout;
490
491	if (pci_read_config(dev, 0x00, 4) == 0x28298086 &&
492	    (pci_read_config(dev, 0x92, 1) & 0xfe) == 0x04)
493		pci_write_config(dev, 0x92, 0x01, 1);
494	/* Enable AHCI mode */
495	ATA_OUTL(ctlr->r_mem, AHCI_GHC, AHCI_GHC_AE);
496	/* Reset AHCI controller */
497	ATA_OUTL(ctlr->r_mem, AHCI_GHC, AHCI_GHC_AE|AHCI_GHC_HR);
498	for (timeout = 1000; timeout > 0; timeout--) {
499		DELAY(1000);
500		if ((ATA_INL(ctlr->r_mem, AHCI_GHC) & AHCI_GHC_HR) == 0)
501			break;
502	}
503	if (timeout == 0) {
504		device_printf(dev, "AHCI controller reset failure\n");
505		return ENXIO;
506	}
507	/* Reenable AHCI mode */
508	ATA_OUTL(ctlr->r_mem, AHCI_GHC, AHCI_GHC_AE);
509	return (0);
510}
511
512static int
513ahci_ctlr_setup(device_t dev)
514{
515	struct ahci_controller *ctlr = device_get_softc(dev);
516	/* Clear interrupts */
517	ATA_OUTL(ctlr->r_mem, AHCI_IS, ATA_INL(ctlr->r_mem, AHCI_IS));
518	/* Configure CCC */
519	if (ctlr->ccc) {
520		ATA_OUTL(ctlr->r_mem, AHCI_CCCP, ATA_INL(ctlr->r_mem, AHCI_PI));
521		ATA_OUTL(ctlr->r_mem, AHCI_CCCC,
522		    (ctlr->ccc << AHCI_CCCC_TV_SHIFT) |
523		    (4 << AHCI_CCCC_CC_SHIFT) |
524		    AHCI_CCCC_EN);
525		ctlr->cccv = (ATA_INL(ctlr->r_mem, AHCI_CCCC) &
526		    AHCI_CCCC_INT_MASK) >> AHCI_CCCC_INT_SHIFT;
527		if (bootverbose) {
528			device_printf(dev,
529			    "CCC with %dms/4cmd enabled on vector %d\n",
530			    ctlr->ccc, ctlr->cccv);
531		}
532	}
533	/* Enable AHCI interrupts */
534	ATA_OUTL(ctlr->r_mem, AHCI_GHC,
535	    ATA_INL(ctlr->r_mem, AHCI_GHC) | AHCI_GHC_IE);
536	return (0);
537}
538
539static int
540ahci_suspend(device_t dev)
541{
542	struct ahci_controller *ctlr = device_get_softc(dev);
543
544	bus_generic_suspend(dev);
545	/* Disable interupts, so the state change(s) doesn't trigger */
546	ATA_OUTL(ctlr->r_mem, AHCI_GHC,
547	     ATA_INL(ctlr->r_mem, AHCI_GHC) & (~AHCI_GHC_IE));
548	return 0;
549}
550
551static int
552ahci_resume(device_t dev)
553{
554	int res;
555
556	if ((res = ahci_ctlr_reset(dev)) != 0)
557		return (res);
558	ahci_ctlr_setup(dev);
559	return (bus_generic_resume(dev));
560}
561
562static int
563ahci_setup_interrupt(device_t dev)
564{
565	struct ahci_controller *ctlr = device_get_softc(dev);
566	int i, msi = 1;
567
568	/* Process hints. */
569	resource_int_value(device_get_name(dev),
570	    device_get_unit(dev), "msi", &msi);
571	if (msi < 0)
572		msi = 0;
573	else if (msi == 1)
574		msi = min(1, pci_msi_count(dev));
575	else if (msi > 1)
576		msi = pci_msi_count(dev);
577	/* Allocate MSI if needed/present. */
578	if (msi && pci_alloc_msi(dev, &msi) == 0) {
579		ctlr->numirqs = msi;
580	} else {
581		msi = 0;
582		ctlr->numirqs = 1;
583	}
584	/* Check for single MSI vector fallback. */
585	if (ctlr->numirqs > 1 &&
586	    (ATA_INL(ctlr->r_mem, AHCI_GHC) & AHCI_GHC_MRSM) != 0) {
587		device_printf(dev, "Falling back to one MSI\n");
588		ctlr->numirqs = 1;
589	}
590	/* Allocate all IRQs. */
591	for (i = 0; i < ctlr->numirqs; i++) {
592		ctlr->irqs[i].ctlr = ctlr;
593		ctlr->irqs[i].r_irq_rid = i + (msi ? 1 : 0);
594		if (ctlr->numirqs == 1 || i >= ctlr->channels ||
595		    (ctlr->ccc && i == ctlr->cccv))
596			ctlr->irqs[i].mode = AHCI_IRQ_MODE_ALL;
597		else if (i == ctlr->numirqs - 1)
598			ctlr->irqs[i].mode = AHCI_IRQ_MODE_AFTER;
599		else
600			ctlr->irqs[i].mode = AHCI_IRQ_MODE_ONE;
601		if (!(ctlr->irqs[i].r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
602		    &ctlr->irqs[i].r_irq_rid, RF_SHAREABLE | RF_ACTIVE))) {
603			device_printf(dev, "unable to map interrupt\n");
604			return ENXIO;
605		}
606		if ((bus_setup_intr(dev, ctlr->irqs[i].r_irq, ATA_INTR_FLAGS, NULL,
607		    (ctlr->irqs[i].mode == AHCI_IRQ_MODE_ONE) ? ahci_intr_one : ahci_intr,
608		    &ctlr->irqs[i], &ctlr->irqs[i].handle))) {
609			/* SOS XXX release r_irq */
610			device_printf(dev, "unable to setup interrupt\n");
611			return ENXIO;
612		}
613		if (ctlr->numirqs > 1) {
614			bus_describe_intr(dev, ctlr->irqs[i].r_irq,
615			    ctlr->irqs[i].handle,
616			    ctlr->irqs[i].mode == AHCI_IRQ_MODE_ONE ?
617			    "ch%d" : "%d", i);
618		}
619	}
620	return (0);
621}
622
623/*
624 * Common case interrupt handler.
625 */
626static void
627ahci_intr(void *data)
628{
629	struct ahci_controller_irq *irq = data;
630	struct ahci_controller *ctlr = irq->ctlr;
631	u_int32_t is, ise = 0;
632	void *arg;
633	int unit;
634
635	if (irq->mode == AHCI_IRQ_MODE_ALL) {
636		unit = 0;
637		if (ctlr->ccc)
638			is = ctlr->ichannels;
639		else
640			is = ATA_INL(ctlr->r_mem, AHCI_IS);
641	} else {	/* AHCI_IRQ_MODE_AFTER */
642		unit = irq->r_irq_rid - 1;
643		is = ATA_INL(ctlr->r_mem, AHCI_IS);
644	}
645	/* CCC interrupt is edge triggered. */
646	if (ctlr->ccc)
647		ise = 1 << ctlr->cccv;
648	/* Some controllers have edge triggered IS. */
649	if (ctlr->quirks & AHCI_Q_EDGEIS)
650		ise |= is;
651	if (ise != 0)
652		ATA_OUTL(ctlr->r_mem, AHCI_IS, ise);
653	for (; unit < ctlr->channels; unit++) {
654		if ((is & (1 << unit)) != 0 &&
655		    (arg = ctlr->interrupt[unit].argument)) {
656				ctlr->interrupt[unit].function(arg);
657		}
658	}
659	/* AHCI declares level triggered IS. */
660	if (!(ctlr->quirks & AHCI_Q_EDGEIS))
661		ATA_OUTL(ctlr->r_mem, AHCI_IS, is);
662}
663
664/*
665 * Simplified interrupt handler for multivector MSI mode.
666 */
667static void
668ahci_intr_one(void *data)
669{
670	struct ahci_controller_irq *irq = data;
671	struct ahci_controller *ctlr = irq->ctlr;
672	void *arg;
673	int unit;
674
675	unit = irq->r_irq_rid - 1;
676	/* Some controllers have edge triggered IS. */
677	if (ctlr->quirks & AHCI_Q_EDGEIS)
678		ATA_OUTL(ctlr->r_mem, AHCI_IS, 1 << unit);
679	if ((arg = ctlr->interrupt[unit].argument))
680	    ctlr->interrupt[unit].function(arg);
681	/* AHCI declares level triggered IS. */
682	if (!(ctlr->quirks & AHCI_Q_EDGEIS))
683		ATA_OUTL(ctlr->r_mem, AHCI_IS, 1 << unit);
684}
685
686static struct resource *
687ahci_alloc_resource(device_t dev, device_t child, int type, int *rid,
688		       u_long start, u_long end, u_long count, u_int flags)
689{
690	struct ahci_controller *ctlr = device_get_softc(dev);
691	int unit = ((struct ahci_channel *)device_get_softc(child))->unit;
692	struct resource *res = NULL;
693	int offset = AHCI_OFFSET + (unit << 7);
694	long st;
695
696	switch (type) {
697	case SYS_RES_MEMORY:
698		st = rman_get_start(ctlr->r_mem);
699		res = rman_reserve_resource(&ctlr->sc_iomem, st + offset,
700		    st + offset + 127, 128, RF_ACTIVE, child);
701		if (res) {
702			bus_space_handle_t bsh;
703			bus_space_tag_t bst;
704			bsh = rman_get_bushandle(ctlr->r_mem);
705			bst = rman_get_bustag(ctlr->r_mem);
706			bus_space_subregion(bst, bsh, offset, 128, &bsh);
707			rman_set_bushandle(res, bsh);
708			rman_set_bustag(res, bst);
709		}
710		break;
711	case SYS_RES_IRQ:
712		if (*rid == ATA_IRQ_RID)
713			res = ctlr->irqs[0].r_irq;
714		break;
715	}
716	return (res);
717}
718
719static int
720ahci_release_resource(device_t dev, device_t child, int type, int rid,
721			 struct resource *r)
722{
723
724	switch (type) {
725	case SYS_RES_MEMORY:
726		rman_release_resource(r);
727		return (0);
728	case SYS_RES_IRQ:
729		if (rid != ATA_IRQ_RID)
730			return ENOENT;
731		return (0);
732	}
733	return (EINVAL);
734}
735
736static int
737ahci_setup_intr(device_t dev, device_t child, struct resource *irq,
738		   int flags, driver_filter_t *filter, driver_intr_t *function,
739		   void *argument, void **cookiep)
740{
741	struct ahci_controller *ctlr = device_get_softc(dev);
742	int unit = (intptr_t)device_get_ivars(child);
743
744	if (filter != NULL) {
745		printf("ahci.c: we cannot use a filter here\n");
746		return (EINVAL);
747	}
748	ctlr->interrupt[unit].function = function;
749	ctlr->interrupt[unit].argument = argument;
750	return (0);
751}
752
753static int
754ahci_teardown_intr(device_t dev, device_t child, struct resource *irq,
755		      void *cookie)
756{
757	struct ahci_controller *ctlr = device_get_softc(dev);
758	int unit = (intptr_t)device_get_ivars(child);
759
760	ctlr->interrupt[unit].function = NULL;
761	ctlr->interrupt[unit].argument = NULL;
762	return (0);
763}
764
765static int
766ahci_print_child(device_t dev, device_t child)
767{
768	int retval;
769
770	retval = bus_print_child_header(dev, child);
771	retval += printf(" at channel %d",
772	    (int)(intptr_t)device_get_ivars(child));
773	retval += bus_print_child_footer(dev, child);
774
775	return (retval);
776}
777
778devclass_t ahci_devclass;
779static device_method_t ahci_methods[] = {
780	DEVMETHOD(device_probe,     ahci_probe),
781	DEVMETHOD(device_attach,    ahci_attach),
782	DEVMETHOD(device_detach,    ahci_detach),
783	DEVMETHOD(device_suspend,   ahci_suspend),
784	DEVMETHOD(device_resume,    ahci_resume),
785	DEVMETHOD(bus_print_child,  ahci_print_child),
786	DEVMETHOD(bus_alloc_resource,       ahci_alloc_resource),
787	DEVMETHOD(bus_release_resource,     ahci_release_resource),
788	DEVMETHOD(bus_setup_intr,   ahci_setup_intr),
789	DEVMETHOD(bus_teardown_intr,ahci_teardown_intr),
790	{ 0, 0 }
791};
792static driver_t ahci_driver = {
793        "ahci",
794        ahci_methods,
795        sizeof(struct ahci_controller)
796};
797DRIVER_MODULE(ahci, pci, ahci_driver, ahci_devclass, 0, 0);
798static device_method_t ahci_ata_methods[] = {
799	DEVMETHOD(device_probe,     ahci_ata_probe),
800	DEVMETHOD(device_attach,    ahci_attach),
801	DEVMETHOD(device_detach,    ahci_detach),
802	DEVMETHOD(device_suspend,   ahci_suspend),
803	DEVMETHOD(device_resume,    ahci_resume),
804	DEVMETHOD(bus_print_child,  ahci_print_child),
805	DEVMETHOD(bus_alloc_resource,       ahci_alloc_resource),
806	DEVMETHOD(bus_release_resource,     ahci_release_resource),
807	DEVMETHOD(bus_setup_intr,   ahci_setup_intr),
808	DEVMETHOD(bus_teardown_intr,ahci_teardown_intr),
809	{ 0, 0 }
810};
811static driver_t ahci_ata_driver = {
812        "ahci",
813        ahci_ata_methods,
814        sizeof(struct ahci_controller)
815};
816DRIVER_MODULE(ahci, atapci, ahci_ata_driver, ahci_devclass, 0, 0);
817MODULE_VERSION(ahci, 1);
818MODULE_DEPEND(ahci, cam, 1, 1, 1);
819
820static int
821ahci_ch_probe(device_t dev)
822{
823
824	device_set_desc_copy(dev, "AHCI channel");
825	return (0);
826}
827
828static int
829ahci_ch_attach(device_t dev)
830{
831	struct ahci_controller *ctlr = device_get_softc(device_get_parent(dev));
832	struct ahci_channel *ch = device_get_softc(dev);
833	struct cam_devq *devq;
834	int rid, error, i, sata_rev = 0;
835	u_int32_t version;
836
837	ch->dev = dev;
838	ch->unit = (intptr_t)device_get_ivars(dev);
839	ch->caps = ctlr->caps;
840	ch->caps2 = ctlr->caps2;
841	ch->quirks = ctlr->quirks;
842	ch->numslots = ((ch->caps & AHCI_CAP_NCS) >> AHCI_CAP_NCS_SHIFT) + 1,
843	mtx_init(&ch->mtx, "AHCI channel lock", NULL, MTX_DEF);
844	resource_int_value(device_get_name(dev),
845	    device_get_unit(dev), "pm_level", &ch->pm_level);
846	if (ch->pm_level > 3)
847		callout_init_mtx(&ch->pm_timer, &ch->mtx, 0);
848	/* Limit speed for my onboard JMicron external port.
849	 * It is not eSATA really. */
850	if (pci_get_devid(ctlr->dev) == 0x2363197b &&
851	    pci_get_subvendor(ctlr->dev) == 0x1043 &&
852	    pci_get_subdevice(ctlr->dev) == 0x81e4 &&
853	    ch->unit == 0)
854		sata_rev = 1;
855	if (ch->quirks & AHCI_Q_SATA2)
856		sata_rev = 2;
857	resource_int_value(device_get_name(dev),
858	    device_get_unit(dev), "sata_rev", &sata_rev);
859	for (i = 0; i < 16; i++) {
860		ch->user[i].revision = sata_rev;
861		ch->user[i].mode = 0;
862		ch->user[i].bytecount = 8192;
863		ch->user[i].tags = ch->numslots;
864		ch->curr[i] = ch->user[i];
865	}
866	rid = ch->unit;
867	if (!(ch->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
868	    &rid, RF_ACTIVE)))
869		return (ENXIO);
870	ahci_dmainit(dev);
871	ahci_slotsalloc(dev);
872	ahci_ch_resume(dev);
873	mtx_lock(&ch->mtx);
874	rid = ATA_IRQ_RID;
875	if (!(ch->r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
876	    &rid, RF_SHAREABLE | RF_ACTIVE))) {
877		bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
878		device_printf(dev, "Unable to map interrupt\n");
879		return (ENXIO);
880	}
881	if ((bus_setup_intr(dev, ch->r_irq, ATA_INTR_FLAGS, NULL,
882	    ahci_ch_intr_locked, dev, &ch->ih))) {
883		device_printf(dev, "Unable to setup interrupt\n");
884		error = ENXIO;
885		goto err1;
886	}
887	ch->chcaps = ATA_INL(ch->r_mem, AHCI_P_CMD);
888	version = ATA_INL(ctlr->r_mem, AHCI_VS);
889	if (version < 0x00010020 && (ctlr->caps & AHCI_CAP_FBSS))
890		ch->chcaps |= AHCI_P_CMD_FBSCP;
891	if (bootverbose) {
892		device_printf(dev, "Caps:%s%s%s%s%s\n",
893		    (ch->chcaps & AHCI_P_CMD_HPCP) ? " HPCP":"",
894		    (ch->chcaps & AHCI_P_CMD_MPSP) ? " MPSP":"",
895		    (ch->chcaps & AHCI_P_CMD_CPD) ? " CPD":"",
896		    (ch->chcaps & AHCI_P_CMD_ESP) ? " ESP":"",
897		    (ch->chcaps & AHCI_P_CMD_FBSCP) ? " FBSCP":"");
898	}
899	/* Create the device queue for our SIM. */
900	devq = cam_simq_alloc(ch->numslots);
901	if (devq == NULL) {
902		device_printf(dev, "Unable to allocate simq\n");
903		error = ENOMEM;
904		goto err1;
905	}
906	/* Construct SIM entry */
907	ch->sim = cam_sim_alloc(ahciaction, ahcipoll, "ahcich", ch,
908	    device_get_unit(dev), &ch->mtx,
909	    min(2, ch->numslots),
910	    (ch->caps & AHCI_CAP_SNCQ) ? ch->numslots : 0,
911	    devq);
912	if (ch->sim == NULL) {
913		device_printf(dev, "unable to allocate sim\n");
914		error = ENOMEM;
915		goto err2;
916	}
917	if (xpt_bus_register(ch->sim, dev, 0) != CAM_SUCCESS) {
918		device_printf(dev, "unable to register xpt bus\n");
919		error = ENXIO;
920		goto err2;
921	}
922	if (xpt_create_path(&ch->path, /*periph*/NULL, cam_sim_path(ch->sim),
923	    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
924		device_printf(dev, "unable to create path\n");
925		error = ENXIO;
926		goto err3;
927	}
928	if (ch->pm_level > 3) {
929		callout_reset(&ch->pm_timer,
930		    (ch->pm_level == 4) ? hz / 1000 : hz / 8,
931		    ahci_ch_pm, dev);
932	}
933	mtx_unlock(&ch->mtx);
934	return (0);
935
936err3:
937	xpt_bus_deregister(cam_sim_path(ch->sim));
938err2:
939	cam_sim_free(ch->sim, /*free_devq*/TRUE);
940err1:
941	bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
942	bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
943	mtx_unlock(&ch->mtx);
944	return (error);
945}
946
947static int
948ahci_ch_detach(device_t dev)
949{
950	struct ahci_channel *ch = device_get_softc(dev);
951
952	mtx_lock(&ch->mtx);
953	xpt_async(AC_LOST_DEVICE, ch->path, NULL);
954	xpt_free_path(ch->path);
955	xpt_bus_deregister(cam_sim_path(ch->sim));
956	cam_sim_free(ch->sim, /*free_devq*/TRUE);
957	mtx_unlock(&ch->mtx);
958
959	if (ch->pm_level > 3)
960		callout_drain(&ch->pm_timer);
961	bus_teardown_intr(dev, ch->r_irq, ch->ih);
962	bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
963
964	ahci_ch_suspend(dev);
965	ahci_slotsfree(dev);
966	ahci_dmafini(dev);
967
968	bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
969	mtx_destroy(&ch->mtx);
970	return (0);
971}
972
973static int
974ahci_ch_suspend(device_t dev)
975{
976	struct ahci_channel *ch = device_get_softc(dev);
977
978	/* Disable port interrupts. */
979	ATA_OUTL(ch->r_mem, AHCI_P_IE, 0);
980	/* Reset command register. */
981	ahci_stop(dev);
982	ahci_stop_fr(dev);
983	ATA_OUTL(ch->r_mem, AHCI_P_CMD, 0);
984	/* Allow everything, including partial and slumber modes. */
985	ATA_OUTL(ch->r_mem, AHCI_P_SCTL, 0);
986	/* Request slumber mode transition and give some time to get there. */
987	ATA_OUTL(ch->r_mem, AHCI_P_CMD, AHCI_P_CMD_SLUMBER);
988	DELAY(100);
989	/* Disable PHY. */
990	ATA_OUTL(ch->r_mem, AHCI_P_SCTL, ATA_SC_DET_DISABLE);
991	return (0);
992}
993
994static int
995ahci_ch_resume(device_t dev)
996{
997	struct ahci_channel *ch = device_get_softc(dev);
998	uint64_t work;
999
1000	/* Disable port interrupts */
1001	ATA_OUTL(ch->r_mem, AHCI_P_IE, 0);
1002	/* Setup work areas */
1003	work = ch->dma.work_bus + AHCI_CL_OFFSET;
1004	ATA_OUTL(ch->r_mem, AHCI_P_CLB, work & 0xffffffff);
1005	ATA_OUTL(ch->r_mem, AHCI_P_CLBU, work >> 32);
1006	work = ch->dma.rfis_bus;
1007	ATA_OUTL(ch->r_mem, AHCI_P_FB, work & 0xffffffff);
1008	ATA_OUTL(ch->r_mem, AHCI_P_FBU, work >> 32);
1009	/* Activate the channel and power/spin up device */
1010	ATA_OUTL(ch->r_mem, AHCI_P_CMD,
1011	     (AHCI_P_CMD_ACTIVE | AHCI_P_CMD_POD | AHCI_P_CMD_SUD |
1012	     ((ch->pm_level == 2 || ch->pm_level == 3) ? AHCI_P_CMD_ALPE : 0) |
1013	     ((ch->pm_level > 2) ? AHCI_P_CMD_ASP : 0 )));
1014	ahci_start_fr(dev);
1015	ahci_start(dev, 1);
1016	return (0);
1017}
1018
1019devclass_t ahcich_devclass;
1020static device_method_t ahcich_methods[] = {
1021	DEVMETHOD(device_probe,     ahci_ch_probe),
1022	DEVMETHOD(device_attach,    ahci_ch_attach),
1023	DEVMETHOD(device_detach,    ahci_ch_detach),
1024	DEVMETHOD(device_suspend,   ahci_ch_suspend),
1025	DEVMETHOD(device_resume,    ahci_ch_resume),
1026	{ 0, 0 }
1027};
1028static driver_t ahcich_driver = {
1029        "ahcich",
1030        ahcich_methods,
1031        sizeof(struct ahci_channel)
1032};
1033DRIVER_MODULE(ahcich, ahci, ahcich_driver, ahcich_devclass, 0, 0);
1034
1035struct ahci_dc_cb_args {
1036	bus_addr_t maddr;
1037	int error;
1038};
1039
1040static void
1041ahci_dmainit(device_t dev)
1042{
1043	struct ahci_channel *ch = device_get_softc(dev);
1044	struct ahci_dc_cb_args dcba;
1045	size_t rfsize;
1046
1047	if (ch->caps & AHCI_CAP_64BIT)
1048		ch->dma.max_address = BUS_SPACE_MAXADDR;
1049	else
1050		ch->dma.max_address = BUS_SPACE_MAXADDR_32BIT;
1051	/* Command area. */
1052	if (bus_dma_tag_create(bus_get_dma_tag(dev), 1024, 0,
1053	    ch->dma.max_address, BUS_SPACE_MAXADDR,
1054	    NULL, NULL, AHCI_WORK_SIZE, 1, AHCI_WORK_SIZE,
1055	    0, NULL, NULL, &ch->dma.work_tag))
1056		goto error;
1057	if (bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work, 0,
1058	    &ch->dma.work_map))
1059		goto error;
1060	if (bus_dmamap_load(ch->dma.work_tag, ch->dma.work_map, ch->dma.work,
1061	    AHCI_WORK_SIZE, ahci_dmasetupc_cb, &dcba, 0) || dcba.error) {
1062		bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
1063		goto error;
1064	}
1065	ch->dma.work_bus = dcba.maddr;
1066	/* FIS receive area. */
1067	if (ch->chcaps & AHCI_P_CMD_FBSCP)
1068	    rfsize = 4096;
1069	else
1070	    rfsize = 256;
1071	if (bus_dma_tag_create(bus_get_dma_tag(dev), rfsize, 0,
1072	    ch->dma.max_address, BUS_SPACE_MAXADDR,
1073	    NULL, NULL, rfsize, 1, rfsize,
1074	    0, NULL, NULL, &ch->dma.rfis_tag))
1075		goto error;
1076	if (bus_dmamem_alloc(ch->dma.rfis_tag, (void **)&ch->dma.rfis, 0,
1077	    &ch->dma.rfis_map))
1078		goto error;
1079	if (bus_dmamap_load(ch->dma.rfis_tag, ch->dma.rfis_map, ch->dma.rfis,
1080	    rfsize, ahci_dmasetupc_cb, &dcba, 0) || dcba.error) {
1081		bus_dmamem_free(ch->dma.rfis_tag, ch->dma.rfis, ch->dma.rfis_map);
1082		goto error;
1083	}
1084	ch->dma.rfis_bus = dcba.maddr;
1085	/* Data area. */
1086	if (bus_dma_tag_create(bus_get_dma_tag(dev), 2, 0,
1087	    ch->dma.max_address, BUS_SPACE_MAXADDR,
1088	    NULL, NULL,
1089	    AHCI_SG_ENTRIES * PAGE_SIZE * ch->numslots,
1090	    AHCI_SG_ENTRIES, AHCI_PRD_MAX,
1091	    0, busdma_lock_mutex, &ch->mtx, &ch->dma.data_tag)) {
1092		goto error;
1093	}
1094	return;
1095
1096error:
1097	device_printf(dev, "WARNING - DMA initialization failed\n");
1098	ahci_dmafini(dev);
1099}
1100
1101static void
1102ahci_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error)
1103{
1104	struct ahci_dc_cb_args *dcba = (struct ahci_dc_cb_args *)xsc;
1105
1106	if (!(dcba->error = error))
1107		dcba->maddr = segs[0].ds_addr;
1108}
1109
1110static void
1111ahci_dmafini(device_t dev)
1112{
1113	struct ahci_channel *ch = device_get_softc(dev);
1114
1115	if (ch->dma.data_tag) {
1116		bus_dma_tag_destroy(ch->dma.data_tag);
1117		ch->dma.data_tag = NULL;
1118	}
1119	if (ch->dma.rfis_bus) {
1120		bus_dmamap_unload(ch->dma.rfis_tag, ch->dma.rfis_map);
1121		bus_dmamem_free(ch->dma.rfis_tag, ch->dma.rfis, ch->dma.rfis_map);
1122		ch->dma.rfis_bus = 0;
1123		ch->dma.rfis_map = NULL;
1124		ch->dma.rfis = NULL;
1125	}
1126	if (ch->dma.work_bus) {
1127		bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map);
1128		bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
1129		ch->dma.work_bus = 0;
1130		ch->dma.work_map = NULL;
1131		ch->dma.work = NULL;
1132	}
1133	if (ch->dma.work_tag) {
1134		bus_dma_tag_destroy(ch->dma.work_tag);
1135		ch->dma.work_tag = NULL;
1136	}
1137}
1138
1139static void
1140ahci_slotsalloc(device_t dev)
1141{
1142	struct ahci_channel *ch = device_get_softc(dev);
1143	int i;
1144
1145	/* Alloc and setup command/dma slots */
1146	bzero(ch->slot, sizeof(ch->slot));
1147	for (i = 0; i < ch->numslots; i++) {
1148		struct ahci_slot *slot = &ch->slot[i];
1149
1150		slot->dev = dev;
1151		slot->slot = i;
1152		slot->state = AHCI_SLOT_EMPTY;
1153		slot->ccb = NULL;
1154		callout_init_mtx(&slot->timeout, &ch->mtx, 0);
1155
1156		if (bus_dmamap_create(ch->dma.data_tag, 0, &slot->dma.data_map))
1157			device_printf(ch->dev, "FAILURE - create data_map\n");
1158	}
1159}
1160
1161static void
1162ahci_slotsfree(device_t dev)
1163{
1164	struct ahci_channel *ch = device_get_softc(dev);
1165	int i;
1166
1167	/* Free all dma slots */
1168	for (i = 0; i < ch->numslots; i++) {
1169		struct ahci_slot *slot = &ch->slot[i];
1170
1171		callout_drain(&slot->timeout);
1172		if (slot->dma.data_map) {
1173			bus_dmamap_destroy(ch->dma.data_tag, slot->dma.data_map);
1174			slot->dma.data_map = NULL;
1175		}
1176	}
1177}
1178
1179static void
1180ahci_phy_check_events(device_t dev, u_int32_t serr)
1181{
1182	struct ahci_channel *ch = device_get_softc(dev);
1183
1184	if ((serr & ATA_SE_PHY_CHANGED) && (ch->pm_level == 0)) {
1185		u_int32_t status = ATA_INL(ch->r_mem, AHCI_P_SSTS);
1186		union ccb *ccb;
1187
1188		if (bootverbose) {
1189			if (((status & ATA_SS_DET_MASK) == ATA_SS_DET_PHY_ONLINE) &&
1190			    ((status & ATA_SS_SPD_MASK) != ATA_SS_SPD_NO_SPEED) &&
1191			    ((status & ATA_SS_IPM_MASK) == ATA_SS_IPM_ACTIVE)) {
1192				device_printf(dev, "CONNECT requested\n");
1193			} else
1194				device_printf(dev, "DISCONNECT requested\n");
1195		}
1196		ahci_reset(dev);
1197		if ((ccb = xpt_alloc_ccb_nowait()) == NULL)
1198			return;
1199		if (xpt_create_path(&ccb->ccb_h.path, NULL,
1200		    cam_sim_path(ch->sim),
1201		    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
1202			xpt_free_ccb(ccb);
1203			return;
1204		}
1205		xpt_rescan(ccb);
1206	}
1207}
1208
1209static void
1210ahci_notify_events(device_t dev, u_int32_t status)
1211{
1212	struct ahci_channel *ch = device_get_softc(dev);
1213	struct cam_path *dpath;
1214	int i;
1215
1216	if (ch->caps & AHCI_CAP_SSNTF)
1217		ATA_OUTL(ch->r_mem, AHCI_P_SNTF, status);
1218	if (bootverbose)
1219		device_printf(dev, "SNTF 0x%04x\n", status);
1220	for (i = 0; i < 16; i++) {
1221		if ((status & (1 << i)) == 0)
1222			continue;
1223		if (xpt_create_path(&dpath, NULL,
1224		    xpt_path_path_id(ch->path), i, 0) == CAM_REQ_CMP) {
1225			xpt_async(AC_SCSI_AEN, dpath, NULL);
1226			xpt_free_path(dpath);
1227		}
1228	}
1229}
1230
1231static void
1232ahci_ch_intr_locked(void *data)
1233{
1234	device_t dev = (device_t)data;
1235	struct ahci_channel *ch = device_get_softc(dev);
1236
1237	mtx_lock(&ch->mtx);
1238	ahci_ch_intr(data);
1239	mtx_unlock(&ch->mtx);
1240}
1241
1242static void
1243ahci_ch_pm(void *arg)
1244{
1245	device_t dev = (device_t)arg;
1246	struct ahci_channel *ch = device_get_softc(dev);
1247	uint32_t work;
1248
1249	if (ch->numrslots != 0)
1250		return;
1251	work = ATA_INL(ch->r_mem, AHCI_P_CMD);
1252	if (ch->pm_level == 4)
1253		work |= AHCI_P_CMD_PARTIAL;
1254	else
1255		work |= AHCI_P_CMD_SLUMBER;
1256	ATA_OUTL(ch->r_mem, AHCI_P_CMD, work);
1257}
1258
1259static void
1260ahci_ch_intr(void *data)
1261{
1262	device_t dev = (device_t)data;
1263	struct ahci_channel *ch = device_get_softc(dev);
1264	uint32_t istatus, sstatus, cstatus, serr = 0, sntf = 0, ok, err;
1265	enum ahci_err_type et;
1266	int i, ccs, port;
1267
1268	/* Read and clear interrupt statuses. */
1269	istatus = ATA_INL(ch->r_mem, AHCI_P_IS);
1270	if (istatus == 0)
1271		return;
1272	ATA_OUTL(ch->r_mem, AHCI_P_IS, istatus);
1273	/* Read command statuses. */
1274	sstatus = ATA_INL(ch->r_mem, AHCI_P_SACT);
1275	cstatus = ATA_INL(ch->r_mem, AHCI_P_CI);
1276	if (istatus & AHCI_P_IX_SDB) {
1277		if (ch->caps & AHCI_CAP_SSNTF)
1278			sntf = ATA_INL(ch->r_mem, AHCI_P_SNTF);
1279		else if (ch->fbs_enabled) {
1280			u_int8_t *fis = ch->dma.rfis + 0x58;
1281
1282			for (i = 0; i < 16; i++) {
1283				if (fis[1] & 0x80) {
1284					fis[1] &= 0x7f;
1285	    				sntf |= 1 << i;
1286	    			}
1287	    			fis += 256;
1288	    		}
1289		} else {
1290			u_int8_t *fis = ch->dma.rfis + 0x58;
1291
1292			if (fis[1] & 0x80)
1293				sntf = (1 << (fis[1] & 0x0f));
1294		}
1295	}
1296	/* Process PHY events */
1297	if (istatus & (AHCI_P_IX_PC | AHCI_P_IX_PRC | AHCI_P_IX_OF |
1298	    AHCI_P_IX_IF | AHCI_P_IX_HBD | AHCI_P_IX_HBF | AHCI_P_IX_TFE)) {
1299		serr = ATA_INL(ch->r_mem, AHCI_P_SERR);
1300		if (serr) {
1301			ATA_OUTL(ch->r_mem, AHCI_P_SERR, serr);
1302			ahci_phy_check_events(dev, serr);
1303		}
1304	}
1305	/* Process command errors */
1306	if (istatus & (AHCI_P_IX_OF | AHCI_P_IX_IF |
1307	    AHCI_P_IX_HBD | AHCI_P_IX_HBF | AHCI_P_IX_TFE)) {
1308		ccs = (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CCS_MASK)
1309		    >> AHCI_P_CMD_CCS_SHIFT;
1310//device_printf(dev, "%s ERROR is %08x cs %08x ss %08x rs %08x tfd %02x serr %08x fbs %08x ccs %d\n",
1311//    __func__, istatus, cstatus, sstatus, ch->rslots, ATA_INL(ch->r_mem, AHCI_P_TFD),
1312//    serr, ATA_INL(ch->r_mem, AHCI_P_FBS), ccs);
1313		port = -1;
1314		if (ch->fbs_enabled) {
1315			uint32_t fbs = ATA_INL(ch->r_mem, AHCI_P_FBS);
1316			if (fbs & AHCI_P_FBS_SDE) {
1317				port = (fbs & AHCI_P_FBS_DWE)
1318				    >> AHCI_P_FBS_DWE_SHIFT;
1319			} else {
1320				for (i = 0; i < 16; i++) {
1321					if (ch->numrslotspd[i] == 0)
1322						continue;
1323					if (port == -1)
1324						port = i;
1325					else if (port != i) {
1326						port = -2;
1327						break;
1328					}
1329				}
1330			}
1331		}
1332		err = ch->rslots & (cstatus | sstatus);
1333	} else {
1334		ccs = 0;
1335		err = 0;
1336		port = -1;
1337	}
1338	/* Complete all successfull commands. */
1339	ok = ch->rslots & ~(cstatus | sstatus);
1340	for (i = 0; i < ch->numslots; i++) {
1341		if ((ok >> i) & 1)
1342			ahci_end_transaction(&ch->slot[i], AHCI_ERR_NONE);
1343	}
1344	/* On error, complete the rest of commands with error statuses. */
1345	if (err) {
1346		if (ch->frozen) {
1347			union ccb *fccb = ch->frozen;
1348			ch->frozen = NULL;
1349			fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
1350			if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
1351				xpt_freeze_devq(fccb->ccb_h.path, 1);
1352				fccb->ccb_h.status |= CAM_DEV_QFRZN;
1353			}
1354			xpt_done(fccb);
1355		}
1356		for (i = 0; i < ch->numslots; i++) {
1357			/* XXX: reqests in loading state. */
1358			if (((err >> i) & 1) == 0)
1359				continue;
1360			if (port >= 0 &&
1361			    ch->slot[i].ccb->ccb_h.target_id != port)
1362				continue;
1363			if (istatus & AHCI_P_IX_TFE) {
1364			    if (port != -2) {
1365				/* Task File Error */
1366				if (ch->numtslotspd[
1367				    ch->slot[i].ccb->ccb_h.target_id] == 0) {
1368					/* Untagged operation. */
1369					if (i == ccs)
1370						et = AHCI_ERR_TFE;
1371					else
1372						et = AHCI_ERR_INNOCENT;
1373				} else {
1374					/* Tagged operation. */
1375					et = AHCI_ERR_NCQ;
1376				}
1377			    } else {
1378				et = AHCI_ERR_TFE;
1379				ch->fatalerr = 1;
1380			    }
1381			} else if (istatus & AHCI_P_IX_IF) {
1382				if (ch->numtslots == 0 && i != ccs && port != -2)
1383					et = AHCI_ERR_INNOCENT;
1384				else
1385					et = AHCI_ERR_SATA;
1386			} else
1387				et = AHCI_ERR_INVALID;
1388			ahci_end_transaction(&ch->slot[i], et);
1389		}
1390		/*
1391		 * We can't reinit port if there are some other
1392		 * commands active, use resume to complete them.
1393		 */
1394		if (ch->rslots != 0)
1395			ATA_OUTL(ch->r_mem, AHCI_P_FBS, AHCI_P_FBS_EN | AHCI_P_FBS_DEC);
1396	}
1397	/* Process NOTIFY events */
1398	if (sntf)
1399		ahci_notify_events(dev, sntf);
1400}
1401
1402/* Must be called with channel locked. */
1403static int
1404ahci_check_collision(device_t dev, union ccb *ccb)
1405{
1406	struct ahci_channel *ch = device_get_softc(dev);
1407	int t = ccb->ccb_h.target_id;
1408
1409	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1410	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
1411		/* Tagged command while we have no supported tag free. */
1412		if (((~ch->oslots) & (0xffffffff >> (32 -
1413		    ch->curr[t].tags))) == 0)
1414			return (1);
1415		/* If we have FBS */
1416		if (ch->fbs_enabled) {
1417			/* Tagged command while untagged are active. */
1418			if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] == 0)
1419				return (1);
1420		} else {
1421			/* Tagged command while untagged are active. */
1422			if (ch->numrslots != 0 && ch->numtslots == 0)
1423				return (1);
1424			/* Tagged command while tagged to other target is active. */
1425			if (ch->numtslots != 0 &&
1426			    ch->taggedtarget != ccb->ccb_h.target_id)
1427				return (1);
1428		}
1429	} else {
1430		/* If we have FBS */
1431		if (ch->fbs_enabled) {
1432			/* Untagged command while tagged are active. */
1433			if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] != 0)
1434				return (1);
1435		} else {
1436			/* Untagged command while tagged are active. */
1437			if (ch->numrslots != 0 && ch->numtslots != 0)
1438				return (1);
1439		}
1440	}
1441	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1442	    (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT))) {
1443		/* Atomic command while anything active. */
1444		if (ch->numrslots != 0)
1445			return (1);
1446	}
1447       /* We have some atomic command running. */
1448       if (ch->aslots != 0)
1449               return (1);
1450	return (0);
1451}
1452
1453/* Must be called with channel locked. */
1454static void
1455ahci_begin_transaction(device_t dev, union ccb *ccb)
1456{
1457	struct ahci_channel *ch = device_get_softc(dev);
1458	struct ahci_slot *slot;
1459	int tag, tags;
1460
1461	/* Choose empty slot. */
1462	tags = ch->numslots;
1463	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1464	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA))
1465		tags = ch->curr[ccb->ccb_h.target_id].tags;
1466	tag = ch->lastslot;
1467	while (1) {
1468		if (tag >= tags)
1469			tag = 0;
1470		if (ch->slot[tag].state == AHCI_SLOT_EMPTY)
1471			break;
1472		tag++;
1473	};
1474	ch->lastslot = tag;
1475	/* Occupy chosen slot. */
1476	slot = &ch->slot[tag];
1477	slot->ccb = ccb;
1478	/* Stop PM timer. */
1479	if (ch->numrslots == 0 && ch->pm_level > 3)
1480		callout_stop(&ch->pm_timer);
1481	/* Update channel stats. */
1482	ch->oslots |= (1 << slot->slot);
1483	ch->numrslots++;
1484	ch->numrslotspd[ccb->ccb_h.target_id]++;
1485	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1486	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
1487		ch->numtslots++;
1488		ch->numtslotspd[ccb->ccb_h.target_id]++;
1489		ch->taggedtarget = ccb->ccb_h.target_id;
1490	}
1491	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1492	    (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT)))
1493		ch->aslots |= (1 << slot->slot);
1494	slot->dma.nsegs = 0;
1495	/* If request moves data, setup and load SG list */
1496	if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
1497		void *buf;
1498		bus_size_t size;
1499
1500		slot->state = AHCI_SLOT_LOADING;
1501		if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1502			buf = ccb->ataio.data_ptr;
1503			size = ccb->ataio.dxfer_len;
1504		} else {
1505			buf = ccb->csio.data_ptr;
1506			size = ccb->csio.dxfer_len;
1507		}
1508		bus_dmamap_load(ch->dma.data_tag, slot->dma.data_map,
1509		    buf, size, ahci_dmasetprd, slot, 0);
1510	} else
1511		ahci_execute_transaction(slot);
1512}
1513
1514/* Locked by busdma engine. */
1515static void
1516ahci_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
1517{
1518	struct ahci_slot *slot = arg;
1519	struct ahci_channel *ch = device_get_softc(slot->dev);
1520	struct ahci_cmd_tab *ctp;
1521	struct ahci_dma_prd *prd;
1522	int i;
1523
1524	if (error) {
1525		device_printf(slot->dev, "DMA load error\n");
1526		ahci_end_transaction(slot, AHCI_ERR_INVALID);
1527		return;
1528	}
1529	KASSERT(nsegs <= AHCI_SG_ENTRIES, ("too many DMA segment entries\n"));
1530	/* Get a piece of the workspace for this request */
1531	ctp = (struct ahci_cmd_tab *)
1532		(ch->dma.work + AHCI_CT_OFFSET + (AHCI_CT_SIZE * slot->slot));
1533	/* Fill S/G table */
1534	prd = &ctp->prd_tab[0];
1535	for (i = 0; i < nsegs; i++) {
1536		prd[i].dba = htole64(segs[i].ds_addr);
1537		prd[i].dbc = htole32((segs[i].ds_len - 1) & AHCI_PRD_MASK);
1538	}
1539	slot->dma.nsegs = nsegs;
1540	bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
1541	    ((slot->ccb->ccb_h.flags & CAM_DIR_IN) ?
1542	    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE));
1543	ahci_execute_transaction(slot);
1544}
1545
1546/* Must be called with channel locked. */
1547static void
1548ahci_execute_transaction(struct ahci_slot *slot)
1549{
1550	device_t dev = slot->dev;
1551	struct ahci_channel *ch = device_get_softc(dev);
1552	struct ahci_cmd_tab *ctp;
1553	struct ahci_cmd_list *clp;
1554	union ccb *ccb = slot->ccb;
1555	int port = ccb->ccb_h.target_id & 0x0f;
1556	int fis_size, i;
1557	uint8_t *fis = ch->dma.rfis + 0x40;
1558	uint8_t val;
1559
1560	/* Get a piece of the workspace for this request */
1561	ctp = (struct ahci_cmd_tab *)
1562		(ch->dma.work + AHCI_CT_OFFSET + (AHCI_CT_SIZE * slot->slot));
1563	/* Setup the FIS for this request */
1564	if (!(fis_size = ahci_setup_fis(dev, ctp, ccb, slot->slot))) {
1565		device_printf(ch->dev, "Setting up SATA FIS failed\n");
1566		ahci_end_transaction(slot, AHCI_ERR_INVALID);
1567		return;
1568	}
1569	/* Setup the command list entry */
1570	clp = (struct ahci_cmd_list *)
1571	    (ch->dma.work + AHCI_CL_OFFSET + (AHCI_CL_SIZE * slot->slot));
1572	clp->prd_length = slot->dma.nsegs;
1573	clp->cmd_flags = (ccb->ccb_h.flags & CAM_DIR_OUT ? AHCI_CMD_WRITE : 0) |
1574		     (ccb->ccb_h.func_code == XPT_SCSI_IO ?
1575		      (AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH) : 0) |
1576		     (fis_size / sizeof(u_int32_t)) |
1577		     (port << 12);
1578	/* Special handling for Soft Reset command. */
1579	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1580	    (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL)) {
1581		if (ccb->ataio.cmd.control & ATA_A_RESET) {
1582			/* Kick controller into sane state */
1583			ahci_stop(dev);
1584			ahci_clo(dev);
1585			ahci_start(dev, 0);
1586			clp->cmd_flags |= AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY;
1587		} else {
1588			/* Prepare FIS receive area for check. */
1589			for (i = 0; i < 20; i++)
1590				fis[i] = 0xff;
1591		}
1592	}
1593	clp->bytecount = 0;
1594	clp->cmd_table_phys = htole64(ch->dma.work_bus + AHCI_CT_OFFSET +
1595				  (AHCI_CT_SIZE * slot->slot));
1596	bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map,
1597	    BUS_DMASYNC_PREWRITE);
1598	bus_dmamap_sync(ch->dma.rfis_tag, ch->dma.rfis_map,
1599	    BUS_DMASYNC_PREREAD);
1600	/* Set ACTIVE bit for NCQ commands. */
1601	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1602	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
1603		ATA_OUTL(ch->r_mem, AHCI_P_SACT, 1 << slot->slot);
1604	}
1605	/* If FBS is enabled, set PMP port. */
1606	if (ch->fbs_enabled) {
1607		ATA_OUTL(ch->r_mem, AHCI_P_FBS, AHCI_P_FBS_EN |
1608		    (port << AHCI_P_FBS_DEV_SHIFT));
1609	}
1610	/* Issue command to the controller. */
1611	slot->state = AHCI_SLOT_RUNNING;
1612	ch->rslots |= (1 << slot->slot);
1613	ATA_OUTL(ch->r_mem, AHCI_P_CI, (1 << slot->slot));
1614	/* Device reset commands doesn't interrupt. Poll them. */
1615	if (ccb->ccb_h.func_code == XPT_ATA_IO &&
1616	    (ccb->ataio.cmd.command == ATA_DEVICE_RESET ||
1617	    (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL))) {
1618		int count, timeout = ccb->ccb_h.timeout;
1619		enum ahci_err_type et = AHCI_ERR_NONE;
1620
1621		for (count = 0; count < timeout; count++) {
1622			DELAY(1000);
1623			if (!(ATA_INL(ch->r_mem, AHCI_P_CI) & (1 << slot->slot)))
1624				break;
1625			if (ATA_INL(ch->r_mem, AHCI_P_TFD) & ATA_S_ERROR) {
1626				device_printf(ch->dev,
1627				    "Poll error on slot %d, TFD: %04x\n",
1628				    slot->slot, ATA_INL(ch->r_mem, AHCI_P_TFD));
1629				et = AHCI_ERR_TFE;
1630				break;
1631			}
1632			/* Workaround for ATI SB600/SB700 chipsets. */
1633			if (ccb->ccb_h.target_id == 15 &&
1634			    pci_get_vendor(device_get_parent(dev)) == 0x1002 &&
1635			    (ATA_INL(ch->r_mem, AHCI_P_IS) & AHCI_P_IX_IPM)) {
1636				et = AHCI_ERR_TIMEOUT;
1637				break;
1638			}
1639		}
1640		if (timeout && (count >= timeout)) {
1641			device_printf(ch->dev,
1642			    "Poll timeout on slot %d\n", slot->slot);
1643			device_printf(dev, "is %08x cs %08x ss %08x "
1644			    "rs %08x tfd %02x serr %08x\n",
1645			    ATA_INL(ch->r_mem, AHCI_P_IS),
1646			    ATA_INL(ch->r_mem, AHCI_P_CI),
1647			    ATA_INL(ch->r_mem, AHCI_P_SACT), ch->rslots,
1648			    ATA_INL(ch->r_mem, AHCI_P_TFD),
1649			    ATA_INL(ch->r_mem, AHCI_P_SERR));
1650			et = AHCI_ERR_TIMEOUT;
1651		}
1652		/* Marvell controllers do not wait for readyness. */
1653		if ((ch->quirks & AHCI_Q_NOBSYRES) &&
1654		    (ccb->ccb_h.func_code == XPT_ATA_IO) &&
1655		    (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) &&
1656		    (ccb->ataio.cmd.control & ATA_A_RESET) == 0) {
1657			while ((val = fis[2]) & (ATA_S_BUSY | ATA_S_DRQ)) {
1658				DELAY(1000);
1659				if (count++ >= timeout) {
1660					device_printf(dev, "device is not "
1661					    "ready after soft-reset: "
1662					    "tfd = %08x\n", val);
1663	    				et = AHCI_ERR_TIMEOUT;
1664	    				break;
1665				}
1666			}
1667		}
1668		ahci_end_transaction(slot, et);
1669		/* Kick controller into sane state and enable FBS. */
1670		if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1671		    (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) &&
1672		    (ccb->ataio.cmd.control & ATA_A_RESET) == 0) {
1673			ahci_stop(ch->dev);
1674			ahci_start(ch->dev, 1);
1675		}
1676		return;
1677	}
1678	/* Start command execution timeout */
1679	callout_reset(&slot->timeout, (int)ccb->ccb_h.timeout * hz / 2000,
1680	    (timeout_t*)ahci_timeout, slot);
1681	return;
1682}
1683
1684/* Must be called with channel locked. */
1685static void
1686ahci_process_timeout(device_t dev)
1687{
1688	struct ahci_channel *ch = device_get_softc(dev);
1689	int i;
1690
1691	mtx_assert(&ch->mtx, MA_OWNED);
1692	/* Handle the rest of commands. */
1693	for (i = 0; i < ch->numslots; i++) {
1694		/* Do we have a running request on slot? */
1695		if (ch->slot[i].state < AHCI_SLOT_RUNNING)
1696			continue;
1697		ahci_end_transaction(&ch->slot[i], AHCI_ERR_TIMEOUT);
1698	}
1699}
1700
1701/* Must be called with channel locked. */
1702static void
1703ahci_rearm_timeout(device_t dev)
1704{
1705	struct ahci_channel *ch = device_get_softc(dev);
1706	int i;
1707
1708	mtx_assert(&ch->mtx, MA_OWNED);
1709	for (i = 0; i < ch->numslots; i++) {
1710		struct ahci_slot *slot = &ch->slot[i];
1711
1712		/* Do we have a running request on slot? */
1713		if (slot->state < AHCI_SLOT_RUNNING)
1714			continue;
1715		if ((ch->toslots & (1 << i)) == 0)
1716			continue;
1717		callout_reset(&slot->timeout,
1718		    (int)slot->ccb->ccb_h.timeout * hz / 2000,
1719		    (timeout_t*)ahci_timeout, slot);
1720	}
1721}
1722
1723/* Locked by callout mechanism. */
1724static void
1725ahci_timeout(struct ahci_slot *slot)
1726{
1727	device_t dev = slot->dev;
1728	struct ahci_channel *ch = device_get_softc(dev);
1729	uint32_t sstatus;
1730	int ccs;
1731	int i;
1732
1733	/* Check for stale timeout. */
1734	if (slot->state < AHCI_SLOT_RUNNING)
1735		return;
1736
1737	/* Check if slot was not being executed last time we checked. */
1738	if (slot->state < AHCI_SLOT_EXECUTING) {
1739		/* Check if slot started executing. */
1740		sstatus = ATA_INL(ch->r_mem, AHCI_P_SACT);
1741		ccs = (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CCS_MASK)
1742		    >> AHCI_P_CMD_CCS_SHIFT;
1743		if ((sstatus & (1 << slot->slot)) != 0 || ccs == slot->slot ||
1744		    ch->fbs_enabled)
1745			slot->state = AHCI_SLOT_EXECUTING;
1746
1747		callout_reset(&slot->timeout,
1748		    (int)slot->ccb->ccb_h.timeout * hz / 2000,
1749		    (timeout_t*)ahci_timeout, slot);
1750		return;
1751	}
1752
1753	device_printf(dev, "Timeout on slot %d\n", slot->slot);
1754	device_printf(dev, "is %08x cs %08x ss %08x rs %08x tfd %02x serr %08x\n",
1755	    ATA_INL(ch->r_mem, AHCI_P_IS), ATA_INL(ch->r_mem, AHCI_P_CI),
1756	    ATA_INL(ch->r_mem, AHCI_P_SACT), ch->rslots,
1757	    ATA_INL(ch->r_mem, AHCI_P_TFD), ATA_INL(ch->r_mem, AHCI_P_SERR));
1758
1759	/* Handle frozen command. */
1760	if (ch->frozen) {
1761		union ccb *fccb = ch->frozen;
1762		ch->frozen = NULL;
1763		fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
1764		if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
1765			xpt_freeze_devq(fccb->ccb_h.path, 1);
1766			fccb->ccb_h.status |= CAM_DEV_QFRZN;
1767		}
1768		xpt_done(fccb);
1769	}
1770	if (!ch->fbs_enabled) {
1771		/* Without FBS we know real timeout source. */
1772		ch->fatalerr = 1;
1773		/* Handle command with timeout. */
1774		ahci_end_transaction(&ch->slot[slot->slot], AHCI_ERR_TIMEOUT);
1775		/* Handle the rest of commands. */
1776		for (i = 0; i < ch->numslots; i++) {
1777			/* Do we have a running request on slot? */
1778			if (ch->slot[i].state < AHCI_SLOT_RUNNING)
1779				continue;
1780			ahci_end_transaction(&ch->slot[i], AHCI_ERR_INNOCENT);
1781		}
1782	} else {
1783		/* With FBS we wait for other commands timeout and pray. */
1784		if (ch->toslots == 0)
1785			xpt_freeze_simq(ch->sim, 1);
1786		ch->toslots |= (1 << slot->slot);
1787		if ((ch->rslots & ~ch->toslots) == 0)
1788			ahci_process_timeout(dev);
1789		else
1790			device_printf(dev, " ... waiting for slots %08x\n",
1791			    ch->rslots & ~ch->toslots);
1792	}
1793}
1794
1795/* Must be called with channel locked. */
1796static void
1797ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et)
1798{
1799	device_t dev = slot->dev;
1800	struct ahci_channel *ch = device_get_softc(dev);
1801	union ccb *ccb = slot->ccb;
1802
1803	bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map,
1804	    BUS_DMASYNC_POSTWRITE);
1805	/* Read result registers to the result struct
1806	 * May be incorrect if several commands finished same time,
1807	 * so read only when sure or have to.
1808	 */
1809	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1810		struct ata_res *res = &ccb->ataio.res;
1811
1812		if ((et == AHCI_ERR_TFE) ||
1813		    (ccb->ataio.cmd.flags & CAM_ATAIO_NEEDRESULT)) {
1814			u_int8_t *fis = ch->dma.rfis + 0x40;
1815
1816			bus_dmamap_sync(ch->dma.rfis_tag, ch->dma.rfis_map,
1817			    BUS_DMASYNC_POSTREAD);
1818			if (ch->fbs_enabled) {
1819				fis += ccb->ccb_h.target_id * 256;
1820				res->status = fis[2];
1821				res->error = fis[3];
1822			} else {
1823				uint16_t tfd = ATA_INL(ch->r_mem, AHCI_P_TFD);
1824
1825				res->status = tfd;
1826				res->error = tfd >> 8;
1827			}
1828			res->lba_low = fis[4];
1829			res->lba_mid = fis[5];
1830			res->lba_high = fis[6];
1831			res->device = fis[7];
1832			res->lba_low_exp = fis[8];
1833			res->lba_mid_exp = fis[9];
1834			res->lba_high_exp = fis[10];
1835			res->sector_count = fis[12];
1836			res->sector_count_exp = fis[13];
1837		} else
1838			bzero(res, sizeof(*res));
1839	}
1840	if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
1841		bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
1842		    (ccb->ccb_h.flags & CAM_DIR_IN) ?
1843		    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
1844		bus_dmamap_unload(ch->dma.data_tag, slot->dma.data_map);
1845	}
1846	if (et != AHCI_ERR_NONE)
1847		ch->eslots |= (1 << slot->slot);
1848	/* In case of error, freeze device for proper recovery. */
1849	if ((et != AHCI_ERR_NONE) && (!ch->readlog) &&
1850	    !(ccb->ccb_h.status & CAM_DEV_QFRZN)) {
1851		xpt_freeze_devq(ccb->ccb_h.path, 1);
1852		ccb->ccb_h.status |= CAM_DEV_QFRZN;
1853	}
1854	/* Set proper result status. */
1855	ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1856	switch (et) {
1857	case AHCI_ERR_NONE:
1858		ccb->ccb_h.status |= CAM_REQ_CMP;
1859		if (ccb->ccb_h.func_code == XPT_SCSI_IO)
1860			ccb->csio.scsi_status = SCSI_STATUS_OK;
1861		break;
1862	case AHCI_ERR_INVALID:
1863		ch->fatalerr = 1;
1864		ccb->ccb_h.status |= CAM_REQ_INVALID;
1865		break;
1866	case AHCI_ERR_INNOCENT:
1867		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1868		break;
1869	case AHCI_ERR_TFE:
1870	case AHCI_ERR_NCQ:
1871		if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
1872			ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
1873			ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
1874		} else {
1875			ccb->ccb_h.status |= CAM_ATA_STATUS_ERROR;
1876		}
1877		break;
1878	case AHCI_ERR_SATA:
1879		ch->fatalerr = 1;
1880		if (!ch->readlog) {
1881			xpt_freeze_simq(ch->sim, 1);
1882			ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1883			ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
1884		}
1885		ccb->ccb_h.status |= CAM_UNCOR_PARITY;
1886		break;
1887	case AHCI_ERR_TIMEOUT:
1888		if (!ch->readlog) {
1889			xpt_freeze_simq(ch->sim, 1);
1890			ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1891			ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
1892		}
1893		ccb->ccb_h.status |= CAM_CMD_TIMEOUT;
1894		break;
1895	default:
1896		ch->fatalerr = 1;
1897		ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
1898	}
1899	/* Free slot. */
1900	ch->oslots &= ~(1 << slot->slot);
1901	ch->rslots &= ~(1 << slot->slot);
1902	ch->aslots &= ~(1 << slot->slot);
1903	if (et != AHCI_ERR_TIMEOUT) {
1904		if (ch->toslots == (1 << slot->slot))
1905			xpt_release_simq(ch->sim, TRUE);
1906		ch->toslots &= ~(1 << slot->slot);
1907	}
1908	slot->state = AHCI_SLOT_EMPTY;
1909	slot->ccb = NULL;
1910	/* Update channel stats. */
1911	ch->numrslots--;
1912	ch->numrslotspd[ccb->ccb_h.target_id]--;
1913	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1914	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
1915		ch->numtslots--;
1916		ch->numtslotspd[ccb->ccb_h.target_id]--;
1917	}
1918	/* If it was first request of reset sequence and there is no error,
1919	 * proceed to second request. */
1920	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1921	    (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) &&
1922	    (ccb->ataio.cmd.control & ATA_A_RESET) &&
1923	    et == AHCI_ERR_NONE) {
1924		ccb->ataio.cmd.control &= ~ATA_A_RESET;
1925		ahci_begin_transaction(dev, ccb);
1926		return;
1927	}
1928	/* If it was our READ LOG command - process it. */
1929	if (ch->readlog) {
1930		ahci_process_read_log(dev, ccb);
1931	/* If it was NCQ command error, put result on hold. */
1932	} else if (et == AHCI_ERR_NCQ) {
1933		ch->hold[slot->slot] = ccb;
1934		ch->numhslots++;
1935	} else
1936		xpt_done(ccb);
1937	/* Unfreeze frozen command. */
1938	if (ch->frozen && !ahci_check_collision(dev, ch->frozen)) {
1939		union ccb *fccb = ch->frozen;
1940		ch->frozen = NULL;
1941		ahci_begin_transaction(dev, fccb);
1942		xpt_release_simq(ch->sim, TRUE);
1943	}
1944	/* If we have no other active commands, ... */
1945	if (ch->rslots == 0) {
1946		/* if there was fatal error - reset port. */
1947		if (ch->toslots != 0 || ch->fatalerr) {
1948			ahci_reset(dev);
1949		} else {
1950			/* if we have slots in error, we can reinit port. */
1951			if (ch->eslots != 0) {
1952				ahci_stop(dev);
1953				ahci_start(dev, 1);
1954			}
1955			/* if there commands on hold, we can do READ LOG. */
1956			if (!ch->readlog && ch->numhslots)
1957				ahci_issue_read_log(dev);
1958		}
1959	/* If all the rest of commands are in timeout - give them chance. */
1960	} else if ((ch->rslots & ~ch->toslots) == 0 &&
1961	    et != AHCI_ERR_TIMEOUT)
1962		ahci_rearm_timeout(dev);
1963	/* Start PM timer. */
1964	if (ch->numrslots == 0 && ch->pm_level > 3) {
1965		callout_schedule(&ch->pm_timer,
1966		    (ch->pm_level == 4) ? hz / 1000 : hz / 8);
1967	}
1968}
1969
1970static void
1971ahci_issue_read_log(device_t dev)
1972{
1973	struct ahci_channel *ch = device_get_softc(dev);
1974	union ccb *ccb;
1975	struct ccb_ataio *ataio;
1976	int i;
1977
1978	ch->readlog = 1;
1979	/* Find some holden command. */
1980	for (i = 0; i < ch->numslots; i++) {
1981		if (ch->hold[i])
1982			break;
1983	}
1984	ccb = xpt_alloc_ccb_nowait();
1985	if (ccb == NULL) {
1986		device_printf(dev, "Unable allocate READ LOG command");
1987		return; /* XXX */
1988	}
1989	ccb->ccb_h = ch->hold[i]->ccb_h;	/* Reuse old header. */
1990	ccb->ccb_h.func_code = XPT_ATA_IO;
1991	ccb->ccb_h.flags = CAM_DIR_IN;
1992	ccb->ccb_h.timeout = 1000;	/* 1s should be enough. */
1993	ataio = &ccb->ataio;
1994	ataio->data_ptr = malloc(512, M_AHCI, M_NOWAIT);
1995	if (ataio->data_ptr == NULL) {
1996		device_printf(dev, "Unable allocate memory for READ LOG command");
1997		return; /* XXX */
1998	}
1999	ataio->dxfer_len = 512;
2000	bzero(&ataio->cmd, sizeof(ataio->cmd));
2001	ataio->cmd.flags = CAM_ATAIO_48BIT;
2002	ataio->cmd.command = 0x2F;	/* READ LOG EXT */
2003	ataio->cmd.sector_count = 1;
2004	ataio->cmd.sector_count_exp = 0;
2005	ataio->cmd.lba_low = 0x10;
2006	ataio->cmd.lba_mid = 0;
2007	ataio->cmd.lba_mid_exp = 0;
2008	/* Freeze SIM while doing READ LOG EXT. */
2009	xpt_freeze_simq(ch->sim, 1);
2010	ahci_begin_transaction(dev, ccb);
2011}
2012
2013static void
2014ahci_process_read_log(device_t dev, union ccb *ccb)
2015{
2016	struct ahci_channel *ch = device_get_softc(dev);
2017	uint8_t *data;
2018	struct ata_res *res;
2019	int i;
2020
2021	ch->readlog = 0;
2022
2023	data = ccb->ataio.data_ptr;
2024	if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP &&
2025	    (data[0] & 0x80) == 0) {
2026		for (i = 0; i < ch->numslots; i++) {
2027			if (!ch->hold[i])
2028				continue;
2029			if ((data[0] & 0x1F) == i) {
2030				res = &ch->hold[i]->ataio.res;
2031				res->status = data[2];
2032				res->error = data[3];
2033				res->lba_low = data[4];
2034				res->lba_mid = data[5];
2035				res->lba_high = data[6];
2036				res->device = data[7];
2037				res->lba_low_exp = data[8];
2038				res->lba_mid_exp = data[9];
2039				res->lba_high_exp = data[10];
2040				res->sector_count = data[12];
2041				res->sector_count_exp = data[13];
2042			} else {
2043				ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
2044				ch->hold[i]->ccb_h.status |= CAM_REQUEUE_REQ;
2045			}
2046			xpt_done(ch->hold[i]);
2047			ch->hold[i] = NULL;
2048			ch->numhslots--;
2049		}
2050	} else {
2051		if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
2052			device_printf(dev, "Error while READ LOG EXT\n");
2053		else if ((data[0] & 0x80) == 0) {
2054			device_printf(dev, "Non-queued command error in READ LOG EXT\n");
2055		}
2056		for (i = 0; i < ch->numslots; i++) {
2057			if (!ch->hold[i])
2058				continue;
2059			xpt_done(ch->hold[i]);
2060			ch->hold[i] = NULL;
2061			ch->numhslots--;
2062		}
2063	}
2064	free(ccb->ataio.data_ptr, M_AHCI);
2065	xpt_free_ccb(ccb);
2066	xpt_release_simq(ch->sim, TRUE);
2067}
2068
2069static void
2070ahci_start(device_t dev, int fbs)
2071{
2072	struct ahci_channel *ch = device_get_softc(dev);
2073	u_int32_t cmd;
2074
2075	/* Clear SATA error register */
2076	ATA_OUTL(ch->r_mem, AHCI_P_SERR, 0xFFFFFFFF);
2077	/* Clear any interrupts pending on this channel */
2078	ATA_OUTL(ch->r_mem, AHCI_P_IS, 0xFFFFFFFF);
2079	/* Configure FIS-based switching if supported. */
2080	if (ch->chcaps & AHCI_P_CMD_FBSCP) {
2081		ch->fbs_enabled = (fbs && ch->pm_present) ? 1 : 0;
2082		ATA_OUTL(ch->r_mem, AHCI_P_FBS,
2083		    ch->fbs_enabled ? AHCI_P_FBS_EN : 0);
2084	}
2085	/* Start operations on this channel */
2086	cmd = ATA_INL(ch->r_mem, AHCI_P_CMD);
2087	ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd | AHCI_P_CMD_ST |
2088	    (ch->pm_present ? AHCI_P_CMD_PMA : 0));
2089}
2090
2091static void
2092ahci_stop(device_t dev)
2093{
2094	struct ahci_channel *ch = device_get_softc(dev);
2095	u_int32_t cmd;
2096	int timeout;
2097
2098	/* Kill all activity on this channel */
2099	cmd = ATA_INL(ch->r_mem, AHCI_P_CMD);
2100	ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd & ~AHCI_P_CMD_ST);
2101	/* Wait for activity stop. */
2102	timeout = 0;
2103	do {
2104		DELAY(1000);
2105		if (timeout++ > 1000) {
2106			device_printf(dev, "stopping AHCI engine failed\n");
2107			break;
2108		}
2109	} while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CR);
2110	ch->eslots = 0;
2111}
2112
2113static void
2114ahci_clo(device_t dev)
2115{
2116	struct ahci_channel *ch = device_get_softc(dev);
2117	u_int32_t cmd;
2118	int timeout;
2119
2120	/* Issue Command List Override if supported */
2121	if (ch->caps & AHCI_CAP_SCLO) {
2122		cmd = ATA_INL(ch->r_mem, AHCI_P_CMD);
2123		cmd |= AHCI_P_CMD_CLO;
2124		ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd);
2125		timeout = 0;
2126		do {
2127			DELAY(1000);
2128			if (timeout++ > 1000) {
2129			    device_printf(dev, "executing CLO failed\n");
2130			    break;
2131			}
2132		} while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CLO);
2133	}
2134}
2135
2136static void
2137ahci_stop_fr(device_t dev)
2138{
2139	struct ahci_channel *ch = device_get_softc(dev);
2140	u_int32_t cmd;
2141	int timeout;
2142
2143	/* Kill all FIS reception on this channel */
2144	cmd = ATA_INL(ch->r_mem, AHCI_P_CMD);
2145	ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd & ~AHCI_P_CMD_FRE);
2146	/* Wait for FIS reception stop. */
2147	timeout = 0;
2148	do {
2149		DELAY(1000);
2150		if (timeout++ > 1000) {
2151			device_printf(dev, "stopping AHCI FR engine failed\n");
2152			break;
2153		}
2154	} while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_FR);
2155}
2156
2157static void
2158ahci_start_fr(device_t dev)
2159{
2160	struct ahci_channel *ch = device_get_softc(dev);
2161	u_int32_t cmd;
2162
2163	/* Start FIS reception on this channel */
2164	cmd = ATA_INL(ch->r_mem, AHCI_P_CMD);
2165	ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd | AHCI_P_CMD_FRE);
2166}
2167
2168static int
2169ahci_wait_ready(device_t dev, int t)
2170{
2171	struct ahci_channel *ch = device_get_softc(dev);
2172	int timeout = 0;
2173	uint32_t val;
2174
2175	while ((val = ATA_INL(ch->r_mem, AHCI_P_TFD)) &
2176	    (ATA_S_BUSY | ATA_S_DRQ)) {
2177		DELAY(1000);
2178		if (timeout++ > t) {
2179			device_printf(dev, "device is not ready (timeout %dms) "
2180			    "tfd = %08x\n", t, val);
2181			return (EBUSY);
2182		}
2183	}
2184	if (bootverbose)
2185		device_printf(dev, "ready wait time=%dms\n", timeout);
2186	return (0);
2187}
2188
2189static void
2190ahci_reset(device_t dev)
2191{
2192	struct ahci_channel *ch = device_get_softc(dev);
2193	struct ahci_controller *ctlr = device_get_softc(device_get_parent(dev));
2194	int i;
2195
2196	xpt_freeze_simq(ch->sim, 1);
2197	if (bootverbose)
2198		device_printf(dev, "AHCI reset...\n");
2199	/* Requeue freezed command. */
2200	if (ch->frozen) {
2201		union ccb *fccb = ch->frozen;
2202		ch->frozen = NULL;
2203		fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
2204		if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
2205			xpt_freeze_devq(fccb->ccb_h.path, 1);
2206			fccb->ccb_h.status |= CAM_DEV_QFRZN;
2207		}
2208		xpt_done(fccb);
2209	}
2210	/* Kill the engine and requeue all running commands. */
2211	ahci_stop(dev);
2212	for (i = 0; i < ch->numslots; i++) {
2213		/* Do we have a running request on slot? */
2214		if (ch->slot[i].state < AHCI_SLOT_RUNNING)
2215			continue;
2216		/* XXX; Commands in loading state. */
2217		ahci_end_transaction(&ch->slot[i], AHCI_ERR_INNOCENT);
2218	}
2219	for (i = 0; i < ch->numslots; i++) {
2220		if (!ch->hold[i])
2221			continue;
2222		xpt_done(ch->hold[i]);
2223		ch->hold[i] = NULL;
2224		ch->numhslots--;
2225	}
2226	if (ch->toslots != 0)
2227		xpt_release_simq(ch->sim, TRUE);
2228	ch->eslots = 0;
2229	ch->toslots = 0;
2230	ch->fatalerr = 0;
2231	/* Tell the XPT about the event */
2232	xpt_async(AC_BUS_RESET, ch->path, NULL);
2233	/* Disable port interrupts */
2234	ATA_OUTL(ch->r_mem, AHCI_P_IE, 0);
2235	/* Reset and reconnect PHY, */
2236	if (!ahci_sata_phy_reset(dev)) {
2237		if (bootverbose)
2238			device_printf(dev,
2239			    "AHCI reset done: phy reset found no device\n");
2240		ch->devices = 0;
2241		/* Enable wanted port interrupts */
2242		ATA_OUTL(ch->r_mem, AHCI_P_IE,
2243		    (AHCI_P_IX_CPD | AHCI_P_IX_PRC | AHCI_P_IX_PC));
2244		xpt_release_simq(ch->sim, TRUE);
2245		return;
2246	}
2247	/* Wait for clearing busy status. */
2248	if (ahci_wait_ready(dev, 15000))
2249		ahci_clo(dev);
2250	ahci_start(dev, 1);
2251	ch->devices = 1;
2252	/* Enable wanted port interrupts */
2253	ATA_OUTL(ch->r_mem, AHCI_P_IE,
2254	     (AHCI_P_IX_CPD | AHCI_P_IX_TFE | AHCI_P_IX_HBF |
2255	      AHCI_P_IX_HBD | AHCI_P_IX_IF | AHCI_P_IX_OF |
2256	      ((ch->pm_level == 0) ? AHCI_P_IX_PRC | AHCI_P_IX_PC : 0) |
2257	      AHCI_P_IX_DP | AHCI_P_IX_UF | (ctlr->ccc ? 0 : AHCI_P_IX_SDB) |
2258	      AHCI_P_IX_DS | AHCI_P_IX_PS | (ctlr->ccc ? 0 : AHCI_P_IX_DHR)));
2259	if (bootverbose)
2260		device_printf(dev, "AHCI reset done: device found\n");
2261	xpt_release_simq(ch->sim, TRUE);
2262}
2263
2264static int
2265ahci_setup_fis(device_t dev, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag)
2266{
2267	struct ahci_channel *ch = device_get_softc(dev);
2268	u_int8_t *fis = &ctp->cfis[0];
2269
2270	bzero(ctp->cfis, 64);
2271	fis[0] = 0x27;  		/* host to device */
2272	fis[1] = (ccb->ccb_h.target_id & 0x0f);
2273	if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
2274		fis[1] |= 0x80;
2275		fis[2] = ATA_PACKET_CMD;
2276		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE &&
2277		    ch->curr[ccb->ccb_h.target_id].mode >= ATA_DMA)
2278			fis[3] = ATA_F_DMA;
2279		else {
2280			fis[5] = ccb->csio.dxfer_len;
2281		        fis[6] = ccb->csio.dxfer_len >> 8;
2282		}
2283		fis[7] = ATA_D_LBA;
2284		fis[15] = ATA_A_4BIT;
2285		bzero(ctp->acmd, 32);
2286		bcopy((ccb->ccb_h.flags & CAM_CDB_POINTER) ?
2287		    ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes,
2288		    ctp->acmd, ccb->csio.cdb_len);
2289	} else if ((ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) == 0) {
2290		fis[1] |= 0x80;
2291		fis[2] = ccb->ataio.cmd.command;
2292		fis[3] = ccb->ataio.cmd.features;
2293		fis[4] = ccb->ataio.cmd.lba_low;
2294		fis[5] = ccb->ataio.cmd.lba_mid;
2295		fis[6] = ccb->ataio.cmd.lba_high;
2296		fis[7] = ccb->ataio.cmd.device;
2297		fis[8] = ccb->ataio.cmd.lba_low_exp;
2298		fis[9] = ccb->ataio.cmd.lba_mid_exp;
2299		fis[10] = ccb->ataio.cmd.lba_high_exp;
2300		fis[11] = ccb->ataio.cmd.features_exp;
2301		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
2302			fis[12] = tag << 3;
2303			fis[13] = 0;
2304		} else {
2305			fis[12] = ccb->ataio.cmd.sector_count;
2306			fis[13] = ccb->ataio.cmd.sector_count_exp;
2307		}
2308		fis[15] = ATA_A_4BIT;
2309	} else {
2310		fis[15] = ccb->ataio.cmd.control;
2311	}
2312	return (20);
2313}
2314
2315static int
2316ahci_sata_connect(struct ahci_channel *ch)
2317{
2318	u_int32_t status;
2319	int timeout;
2320
2321	/* Wait up to 100ms for "connect well" */
2322	for (timeout = 0; timeout < 100 ; timeout++) {
2323		status = ATA_INL(ch->r_mem, AHCI_P_SSTS);
2324		if (((status & ATA_SS_DET_MASK) == ATA_SS_DET_PHY_ONLINE) &&
2325		    ((status & ATA_SS_SPD_MASK) != ATA_SS_SPD_NO_SPEED) &&
2326		    ((status & ATA_SS_IPM_MASK) == ATA_SS_IPM_ACTIVE))
2327			break;
2328		if ((status & ATA_SS_DET_MASK) == ATA_SS_DET_PHY_OFFLINE) {
2329			if (bootverbose) {
2330				device_printf(ch->dev, "SATA offline status=%08x\n",
2331				    status);
2332			}
2333			return (0);
2334		}
2335		DELAY(1000);
2336	}
2337	if (timeout >= 100) {
2338		if (bootverbose) {
2339			device_printf(ch->dev, "SATA connect timeout status=%08x\n",
2340			    status);
2341		}
2342		return (0);
2343	}
2344	if (bootverbose) {
2345		device_printf(ch->dev, "SATA connect time=%dms status=%08x\n",
2346		    timeout, status);
2347	}
2348	/* Clear SATA error register */
2349	ATA_OUTL(ch->r_mem, AHCI_P_SERR, 0xffffffff);
2350	return (1);
2351}
2352
2353static int
2354ahci_sata_phy_reset(device_t dev)
2355{
2356	struct ahci_channel *ch = device_get_softc(dev);
2357	int sata_rev;
2358	uint32_t val;
2359
2360	sata_rev = ch->user[ch->pm_present ? 15 : 0].revision;
2361	if (sata_rev == 1)
2362		val = ATA_SC_SPD_SPEED_GEN1;
2363	else if (sata_rev == 2)
2364		val = ATA_SC_SPD_SPEED_GEN2;
2365	else if (sata_rev == 3)
2366		val = ATA_SC_SPD_SPEED_GEN3;
2367	else
2368		val = 0;
2369	ATA_OUTL(ch->r_mem, AHCI_P_SCTL,
2370	    ATA_SC_DET_RESET | val |
2371	    ATA_SC_IPM_DIS_PARTIAL | ATA_SC_IPM_DIS_SLUMBER);
2372	DELAY(5000);
2373	ATA_OUTL(ch->r_mem, AHCI_P_SCTL,
2374	    ATA_SC_DET_IDLE | val | ((ch->pm_level > 0) ? 0 :
2375	    (ATA_SC_IPM_DIS_PARTIAL | ATA_SC_IPM_DIS_SLUMBER)));
2376	DELAY(5000);
2377	if (!ahci_sata_connect(ch)) {
2378		if (ch->pm_level > 0)
2379			ATA_OUTL(ch->r_mem, AHCI_P_SCTL, ATA_SC_DET_DISABLE);
2380		return (0);
2381	}
2382	return (1);
2383}
2384
2385static void
2386ahciaction(struct cam_sim *sim, union ccb *ccb)
2387{
2388	device_t dev;
2389	struct ahci_channel *ch;
2390
2391	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("ahciaction func_code=%x\n",
2392	    ccb->ccb_h.func_code));
2393
2394	ch = (struct ahci_channel *)cam_sim_softc(sim);
2395	dev = ch->dev;
2396	switch (ccb->ccb_h.func_code) {
2397	/* Common cases first */
2398	case XPT_ATA_IO:	/* Execute the requested I/O operation */
2399	case XPT_SCSI_IO:
2400		if (ch->devices == 0) {
2401			ccb->ccb_h.status = CAM_SEL_TIMEOUT;
2402			xpt_done(ccb);
2403			break;
2404		}
2405		/* Check for command collision. */
2406		if (ahci_check_collision(dev, ccb)) {
2407			/* Freeze command. */
2408			ch->frozen = ccb;
2409			/* We have only one frozen slot, so freeze simq also. */
2410			xpt_freeze_simq(ch->sim, 1);
2411			return;
2412		}
2413		ahci_begin_transaction(dev, ccb);
2414		break;
2415	case XPT_EN_LUN:		/* Enable LUN as a target */
2416	case XPT_TARGET_IO:		/* Execute target I/O request */
2417	case XPT_ACCEPT_TARGET_IO:	/* Accept Host Target Mode CDB */
2418	case XPT_CONT_TARGET_IO:	/* Continue Host Target I/O Connection*/
2419	case XPT_ABORT:			/* Abort the specified CCB */
2420		/* XXX Implement */
2421		ccb->ccb_h.status = CAM_REQ_INVALID;
2422		xpt_done(ccb);
2423		break;
2424	case XPT_SET_TRAN_SETTINGS:
2425	{
2426		struct	ccb_trans_settings *cts = &ccb->cts;
2427		struct	ahci_device *d;
2428
2429		if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
2430			d = &ch->curr[ccb->ccb_h.target_id];
2431		else
2432			d = &ch->user[ccb->ccb_h.target_id];
2433		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_REVISION)
2434			d->revision = cts->xport_specific.sata.revision;
2435		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_MODE)
2436			d->mode = cts->xport_specific.sata.mode;
2437		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT)
2438			d->bytecount = min(8192, cts->xport_specific.sata.bytecount);
2439		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_TAGS)
2440			d->tags = min(ch->numslots, cts->xport_specific.sata.tags);
2441		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_PM)
2442			ch->pm_present = cts->xport_specific.sata.pm_present;
2443		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_ATAPI)
2444			d->atapi = cts->xport_specific.sata.atapi;
2445		ccb->ccb_h.status = CAM_REQ_CMP;
2446		xpt_done(ccb);
2447		break;
2448	}
2449	case XPT_GET_TRAN_SETTINGS:
2450	/* Get default/user set transfer settings for the target */
2451	{
2452		struct	ccb_trans_settings *cts = &ccb->cts;
2453		struct  ahci_device *d;
2454		uint32_t status;
2455
2456		if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
2457			d = &ch->curr[ccb->ccb_h.target_id];
2458		else
2459			d = &ch->user[ccb->ccb_h.target_id];
2460		cts->protocol = PROTO_ATA;
2461		cts->protocol_version = PROTO_VERSION_UNSPECIFIED;
2462		cts->transport = XPORT_SATA;
2463		cts->transport_version = XPORT_VERSION_UNSPECIFIED;
2464		cts->proto_specific.valid = 0;
2465		cts->xport_specific.sata.valid = 0;
2466		if (cts->type == CTS_TYPE_CURRENT_SETTINGS &&
2467		    (ccb->ccb_h.target_id == 15 ||
2468		    (ccb->ccb_h.target_id == 0 && !ch->pm_present))) {
2469			status = ATA_INL(ch->r_mem, AHCI_P_SSTS) & ATA_SS_SPD_MASK;
2470			if (status & 0x0f0) {
2471				cts->xport_specific.sata.revision =
2472				    (status & 0x0f0) >> 4;
2473				cts->xport_specific.sata.valid |=
2474				    CTS_SATA_VALID_REVISION;
2475			}
2476		} else {
2477			cts->xport_specific.sata.revision = d->revision;
2478			cts->xport_specific.sata.valid |= CTS_SATA_VALID_REVISION;
2479		}
2480		cts->xport_specific.sata.mode = d->mode;
2481		cts->xport_specific.sata.valid |= CTS_SATA_VALID_MODE;
2482		cts->xport_specific.sata.bytecount = d->bytecount;
2483		cts->xport_specific.sata.valid |= CTS_SATA_VALID_BYTECOUNT;
2484		cts->xport_specific.sata.pm_present = ch->pm_present;
2485		cts->xport_specific.sata.valid |= CTS_SATA_VALID_PM;
2486		cts->xport_specific.sata.tags = d->tags;
2487		cts->xport_specific.sata.valid |= CTS_SATA_VALID_TAGS;
2488		cts->xport_specific.sata.atapi = d->atapi;
2489		cts->xport_specific.sata.valid |= CTS_SATA_VALID_ATAPI;
2490		ccb->ccb_h.status = CAM_REQ_CMP;
2491		xpt_done(ccb);
2492		break;
2493	}
2494#if 0
2495	case XPT_CALC_GEOMETRY:
2496	{
2497		struct	  ccb_calc_geometry *ccg;
2498		uint32_t size_mb;
2499		uint32_t secs_per_cylinder;
2500
2501		ccg = &ccb->ccg;
2502		size_mb = ccg->volume_size
2503			/ ((1024L * 1024L) / ccg->block_size);
2504		if (size_mb >= 1024 && (aha->extended_trans != 0)) {
2505			if (size_mb >= 2048) {
2506				ccg->heads = 255;
2507				ccg->secs_per_track = 63;
2508			} else {
2509				ccg->heads = 128;
2510				ccg->secs_per_track = 32;
2511			}
2512		} else {
2513			ccg->heads = 64;
2514			ccg->secs_per_track = 32;
2515		}
2516		secs_per_cylinder = ccg->heads * ccg->secs_per_track;
2517		ccg->cylinders = ccg->volume_size / secs_per_cylinder;
2518		ccb->ccb_h.status = CAM_REQ_CMP;
2519		xpt_done(ccb);
2520		break;
2521	}
2522#endif
2523	case XPT_RESET_BUS:		/* Reset the specified SCSI bus */
2524	case XPT_RESET_DEV:	/* Bus Device Reset the specified SCSI device */
2525		ahci_reset(dev);
2526		ccb->ccb_h.status = CAM_REQ_CMP;
2527		xpt_done(ccb);
2528		break;
2529	case XPT_TERM_IO:		/* Terminate the I/O process */
2530		/* XXX Implement */
2531		ccb->ccb_h.status = CAM_REQ_INVALID;
2532		xpt_done(ccb);
2533		break;
2534	case XPT_PATH_INQ:		/* Path routing inquiry */
2535	{
2536		struct ccb_pathinq *cpi = &ccb->cpi;
2537
2538		cpi->version_num = 1; /* XXX??? */
2539		cpi->hba_inquiry = PI_SDTR_ABLE;
2540		if (ch->caps & AHCI_CAP_SNCQ)
2541			cpi->hba_inquiry |= PI_TAG_ABLE;
2542		if (ch->caps & AHCI_CAP_SPM)
2543			cpi->hba_inquiry |= PI_SATAPM;
2544		cpi->target_sprt = 0;
2545		cpi->hba_misc = PIM_SEQSCAN;
2546		cpi->hba_eng_cnt = 0;
2547		if (ch->caps & AHCI_CAP_SPM)
2548			cpi->max_target = 15;
2549		else
2550			cpi->max_target = 0;
2551		cpi->max_lun = 0;
2552		cpi->initiator_id = 0;
2553		cpi->bus_id = cam_sim_bus(sim);
2554		cpi->base_transfer_speed = 150000;
2555		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2556		strncpy(cpi->hba_vid, "AHCI", HBA_IDLEN);
2557		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2558		cpi->unit_number = cam_sim_unit(sim);
2559		cpi->transport = XPORT_SATA;
2560		cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
2561		cpi->protocol = PROTO_ATA;
2562		cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
2563		cpi->maxio = MAXPHYS;
2564		/* ATI SB600 can't handle 256 sectors with FPDMA (NCQ). */
2565		if (pci_get_devid(device_get_parent(dev)) == 0x43801002)
2566			cpi->maxio = min(cpi->maxio, 128 * 512);
2567		cpi->ccb_h.status = CAM_REQ_CMP;
2568		xpt_done(ccb);
2569		break;
2570	}
2571	default:
2572		ccb->ccb_h.status = CAM_REQ_INVALID;
2573		xpt_done(ccb);
2574		break;
2575	}
2576}
2577
2578static void
2579ahcipoll(struct cam_sim *sim)
2580{
2581	struct ahci_channel *ch = (struct ahci_channel *)cam_sim_softc(sim);
2582
2583	ahci_ch_intr(ch->dev);
2584}
2585