ahci.c revision 3333:88329a0ff1be 
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#pragma ident	"%Z%%M%	%I%	%E% SMI"
28
29/*
30 * AHCI (Advanced Host Controller Interface) SATA HBA Driver
31 */
32
33#include <sys/scsi/scsi.h>
34#include <sys/pci.h>
35#include <sys/sata/sata_hba.h>
36#include <sys/sata/adapters/ahci/ahcireg.h>
37#include <sys/sata/adapters/ahci/ahcivar.h>
38
39/*
40 * Function prototypes for driver entry points
41 */
42static	int ahci_attach(dev_info_t *, ddi_attach_cmd_t);
43static	int ahci_detach(dev_info_t *, ddi_detach_cmd_t);
44static	int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
45
46/*
47 * Function prototypes for SATA Framework interfaces
48 */
49static	int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t);
50static	int ahci_unregister_sata_hba_tran(ahci_ctl_t *);
51
52static	int ahci_tran_probe_port(dev_info_t *, sata_device_t *);
53static	int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt);
54static	int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int);
55static	int ahci_tran_reset_dport(dev_info_t *, sata_device_t *);
56static	int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
57static	int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);
58#if defined(__lock_lint)
59static	int ahci_selftest(dev_info_t *, sata_device_t *);
60#endif
61
62/*
63 * Local function prototypes
64 */
65static	int ahci_initialize_controller(ahci_ctl_t *);
66static	void ahci_deallocate_controller(ahci_ctl_t *);
67
68
69static	int ahci_alloc_port_state(ahci_ctl_t *, uint8_t);
70static	void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t);
71static	int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *, uint8_t);
72static	void ahci_dealloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *);
73static	int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *, uint8_t);
74static	void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
75static  int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
76static  void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
77
78static	int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
79static	int ahci_start_port(ahci_ctl_t *, uint8_t);
80static	int ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *, uint8_t);
81static	void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *);
82static	int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *,
83    uint8_t, sata_pkt_t *);
84static	void ahci_poll_cmd(ahci_ctl_t *, ahci_port_t *, uint8_t,
85    int, sata_pkt_t *);
86static	int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *);
87static	void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *);
88static	void ahci_set_sense_data(sata_pkt_t *, int);
89
90static	int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, uint8_t);
91static	int ahci_hba_reset(ahci_ctl_t *);
92static	int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, uint8_t);
93static	void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
94static	int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
95static	int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
96static	int ahci_reset_hba_reject_pkts(ahci_ctl_t *);
97static	int ahci_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *,
98    uint8_t);
99static	int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *,
100    uint8_t, int);
101static	void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint8_t,
102    uint32_t, uint32_t, uint32_t, uint32_t, uint32_t);
103static	int ahci_recovery_fatal_error(ahci_ctl_t *,
104    ahci_port_t *, uint8_t, uint32_t);
105static	void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t, uint32_t);
106static	void ahci_watchdog_handler(ahci_ctl_t *);
107
108static	uint_t ahci_intr(caddr_t, caddr_t);
109static	int ahci_add_legacy_intrs(ahci_ctl_t *);
110static	int ahci_add_msi_intrs(ahci_ctl_t *);
111static	void ahci_rem_intrs(ahci_ctl_t *);
112static	void ahci_enable_all_intrs(ahci_ctl_t *);
113static	void ahci_disable_all_intrs(ahci_ctl_t *);
114static	void ahci_enable_port_intrs(ahci_ctl_t *, ahci_port_t *, uint8_t);
115static	void ahci_disable_port_intrs(ahci_ctl_t *, ahci_port_t *, uint8_t);
116
117static	int ahci_intr_d2h_register_fis(ahci_ctl_t *, ahci_port_t *, uint8_t);
118static	int ahci_intr_pio_setup_fis(ahci_ctl_t *, ahci_port_t *, uint8_t);
119static	int ahci_intr_dma_setup_fis(ahci_ctl_t *, ahci_port_t *, uint8_t);
120static	int ahci_intr_set_device_bits_fis(ahci_ctl_t *, ahci_port_t *, uint8_t);
121static	int ahci_intr_unknown_fis(ahci_ctl_t *, ahci_port_t *, uint8_t);
122static	int ahci_intr_descriptor_processed(ahci_ctl_t *, ahci_port_t *,
123    uint8_t);
124static	int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
125static	int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *,
126    ahci_port_t *, uint8_t);
127static	int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
128static	int ahci_intr_incorrect_port_multiplier(ahci_ctl_t *,
129    ahci_port_t *, uint8_t);
130static	int ahci_intr_overflow(ahci_ctl_t *, ahci_port_t *, uint8_t);
131static	int ahci_intr_interface_non_fatal_error(ahci_ctl_t *,
132    ahci_port_t *, uint8_t);
133static	int ahci_intr_interface_fatal_error(ahci_ctl_t *, ahci_port_t *,
134    uint8_t);
135static	int ahci_intr_host_bus_data_error(ahci_ctl_t *, ahci_port_t *, uint8_t);
136static	int ahci_intr_host_bus_fatal_error(ahci_ctl_t *, ahci_port_t *,
137    uint8_t);
138static	int ahci_intr_task_file_error(ahci_ctl_t *, ahci_port_t *, uint8_t);
139static	int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t);
140
141static	int ahci_get_num_implemented_ports(uint32_t);
142static	void ahci_log_error_message(ahci_ctl_t *, uint8_t, uint32_t);
143static	void ahci_log(ahci_ctl_t *, uint_t, char *, ...);
144
145
146/*
147 * DMA attributes for the data buffer
148 *
149 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
150 * does not support 64-bit addressing
151 */
152static ddi_dma_attr_t buffer_dma_attr = {
153	DMA_ATTR_V0,		/* dma_attr_version */
154	0,			/* dma_attr_addr_lo: lowest bus address */
155	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
156	0x3fffffull,		/* dma_attr_count_max i.e. for one cookie */
157	2,			/* dma_attr_align: word aligned */
158	1,			/* dma_attr_burstsizes */
159	1,			/* dma_attr_minxfer */
160	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
161	0xffffffffull,		/* dma_attr_seg */
162	AHCI_PRDT_NUMBER,	/* dma_attr_sgllen */
163	512,			/* dma_attr_granular */
164	0,			/* dma_attr_flags */
165};
166
167/*
168 * DMA attributes for the rcvd FIS
169 *
170 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
171 * does not support 64-bit addressing
172 */
173static ddi_dma_attr_t rcvd_fis_dma_attr = {
174	DMA_ATTR_V0,		/* dma_attr_version */
175	0,			/* dma_attr_addr_lo: lowest bus address */
176	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
177	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
178	0x100,			/* dma_attr_align: 256-byte aligned */
179	1,			/* dma_attr_burstsizes */
180	1,			/* dma_attr_minxfer */
181	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
182	0xffffffffull,		/* dma_attr_seg */
183	1,			/* dma_attr_sgllen */
184	1,			/* dma_attr_granular */
185	0,			/* dma_attr_flags */
186};
187
188/*
189 * DMA attributes for the command list
190 *
191 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
192 * does not support 64-bit addressing
193 */
194static ddi_dma_attr_t cmd_list_dma_attr = {
195	DMA_ATTR_V0,		/* dma_attr_version */
196	0,			/* dma_attr_addr_lo: lowest bus address */
197	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
198	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
199	0x400,			/* dma_attr_align: 1K-byte aligned */
200	1,			/* dma_attr_burstsizes */
201	1,			/* dma_attr_minxfer */
202	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
203	0xffffffffull,		/* dma_attr_seg */
204	1,			/* dma_attr_sgllen */
205	1,			/* dma_attr_granular */
206	0,			/* dma_attr_flags */
207};
208
209/*
210 * DMA attributes for cmd tables
211 *
212 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
213 * does not support 64-bit addressing
214 */
215static ddi_dma_attr_t cmd_table_dma_attr = {
216	DMA_ATTR_V0,		/* dma_attr_version */
217	0,			/* dma_attr_addr_lo: lowest bus address */
218	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
219	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
220	0x80,			/* dma_attr_align: 128-byte aligned */
221	1,			/* dma_attr_burstsizes */
222	1,			/* dma_attr_minxfer */
223	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
224	0xffffffffull,		/* dma_attr_seg */
225	1,			/* dma_attr_sgllen */
226	1,			/* dma_attr_granular */
227	0,			/* dma_attr_flags */
228};
229
230
231/* Device access attributes */
232static ddi_device_acc_attr_t accattr = {
233	DDI_DEVICE_ATTR_V0,
234	DDI_STRUCTURE_LE_ACC,
235	DDI_STRICTORDER_ACC
236};
237
238
239static struct dev_ops ahcictl_dev_ops = {
240	DEVO_REV,		/* devo_rev */
241	0,			/* refcnt  */
242	ahci_getinfo,		/* info */
243	nulldev,		/* identify */
244	nulldev,		/* probe */
245	ahci_attach,		/* attach */
246	ahci_detach,		/* detach */
247	nodev,			/* no reset */
248	(struct cb_ops *)0,	/* driver operations */
249	NULL,			/* bus operations */
250	NULL			/* power */
251};
252
253static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = {
254	SATA_TRAN_HOTPLUG_OPS_REV_1,
255	ahci_tran_hotplug_port_activate,
256	ahci_tran_hotplug_port_deactivate
257};
258
259extern struct mod_ops mod_driverops;
260
261static  struct modldrv modldrv = {
262	&mod_driverops,		/* driverops */
263	"ahci driver %I%",
264	&ahcictl_dev_ops,	/* driver ops */
265};
266
267static  struct modlinkage modlinkage = {
268	MODREV_1,
269	&modldrv,
270	NULL
271};
272
273static int ahci_watchdog_timeout = 5; /* 5 seconds */
274static int ahci_watchdog_tick;
275
276/* The following is needed for ahci_log() */
277static kmutex_t ahci_log_mutex;
278static char ahci_log_buf[512];
279
280static size_t ahci_cmd_table_size;
281
282/* The number of Physical Region Descriptor Table(PRDT) in Command Table */
283int ahci_dma_prdt_number = AHCI_PRDT_NUMBER;
284
285/*
286 * AHCI MSI tunable:
287 *
288 * MSI will be enabled in phase 2.
289 */
290boolean_t ahci_msi_enabled = B_FALSE;
291
292#if AHCI_DEBUG
293uint32_t ahci_debug_flags = 0;
294#endif
295
296/* Opaque state pointer initialized by ddi_soft_state_init() */
297static void *ahci_statep = NULL;
298
299/*
300 *  ahci module initialization.
301 */
302int
303_init(void)
304{
305	int	ret;
306
307	ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0);
308	if (ret != 0) {
309		goto err_out;
310	}
311
312	mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL);
313
314	if ((ret = sata_hba_init(&modlinkage)) != 0) {
315		mutex_destroy(&ahci_log_mutex);
316		ddi_soft_state_fini(&ahci_statep);
317		goto err_out;
318	}
319
320	ret = mod_install(&modlinkage);
321	if (ret != 0) {
322		sata_hba_fini(&modlinkage);
323		mutex_destroy(&ahci_log_mutex);
324		ddi_soft_state_fini(&ahci_statep);
325		goto err_out;
326	}
327
328	/* watchdog tick */
329	ahci_watchdog_tick = drv_usectohz(
330	    (clock_t)ahci_watchdog_timeout * 1000000);
331	return (ret);
332
333err_out:
334	cmn_err(CE_WARN, "!Module init failed");
335	return (ret);
336}
337
338/*
339 * ahci module uninitialize.
340 */
341int
342_fini(void)
343{
344	int	ret;
345
346	ret = mod_remove(&modlinkage);
347	if (ret != 0) {
348		return (ret);
349	}
350
351	/* Remove the resources allocated in _init(). */
352	sata_hba_fini(&modlinkage);
353	mutex_destroy(&ahci_log_mutex);
354	ddi_soft_state_fini(&ahci_statep);
355
356	return (ret);
357}
358
359/*
360 * _info entry point
361 */
362int
363_info(struct modinfo *modinfop)
364{
365	return (mod_info(&modlinkage, modinfop));
366}
367
368/*
369 * The attach entry point for dev_ops.
370 */
371static int
372ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
373{
374	ahci_ctl_t *ahci_ctlp;
375	int instance = ddi_get_instance(dip);
376	int status;
377	int attach_state;
378	uint32_t cap_status, ahci_version;
379	int intr_types;
380	ushort_t venid;
381	uint8_t revision;
382
383	AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, NULL, "ahci_attach enter");
384
385	switch (cmd) {
386	case DDI_ATTACH:
387		break;
388
389	case DDI_RESUME:
390		/* It will be implemented in Phase 2 */
391		return (DDI_FAILURE);
392
393	default:
394		return (DDI_FAILURE);
395	}
396
397	instance = ddi_get_instance(dip);
398	attach_state = AHCI_ATTACH_STATE_NONE;
399
400	/* Allocate soft state */
401	status = ddi_soft_state_zalloc(ahci_statep, instance);
402	if (status != DDI_SUCCESS) {
403		goto err_out;
404	}
405
406	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
407	ahci_ctlp->ahcictl_dip = dip;
408
409	attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC;
410
411	/*
412	 * Now map the AHCI base address; which includes global
413	 * registers and port control registers
414	 */
415	status = ddi_regs_map_setup(dip,
416	    AHCI_BASE_REG,
417	    (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr,
418	    0,
419	    0,
420	    &accattr,
421	    &ahci_ctlp->ahcictl_ahci_acc_handle);
422	if (status != DDI_SUCCESS) {
423		goto err_out;
424	}
425
426	attach_state |= AHCI_ATTACH_STATE_REG_MAP;
427
428	/* Get the AHCI version information */
429	ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
430	    (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp));
431
432	AHCIDBG2(AHCIDBG_INIT, ahci_ctlp, "hba AHCI version = %x.%x",
433	    (ahci_version & 0xffff0000) >> 16,
434	    ((ahci_version & 0x0000ff00) >> 4 |
435	    (ahci_version & 0x000000ff)));
436
437	/* We don't support controllers whose versions are lower than 1.0 */
438	if (!(ahci_version & 0xffff0000)) {
439		AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, "No support for lower than "
440		    "AHCI version 1.0");
441		goto err_out;
442	}
443
444	/* Get the HBA capabilities information */
445	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
446	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
447
448	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba capabilites = 0x%x",
449	    cap_status);
450
451	/* Get the max number of ports supported by the HBA */
452	ahci_ctlp->ahcictl_num_ports = (cap_status & AHCI_HBA_CAP_NP) + 1;
453
454	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: 0x%x",
455	    ahci_ctlp->ahcictl_num_ports);
456
457	/* Get the number of command slots supported by the HBA */
458	ahci_ctlp->ahcictl_num_cmd_slots =
459	    ((cap_status & AHCI_HBA_CAP_NCS) >>
460	    AHCI_HBA_CAP_NCS_SHIFT) + 1;
461
462	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: 0x%x",
463	    ahci_ctlp->ahcictl_num_cmd_slots);
464
465	/* Get the bit map which indicates ports implemented by the HBA */
466	ahci_ctlp->ahcictl_ports_implemented =
467	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
468	    (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp));
469
470	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x",
471	    ahci_ctlp->ahcictl_ports_implemented);
472
473	/* Get the number of implemented ports by the HBA */
474	ahci_ctlp->ahcictl_num_implemented_ports =
475	    ahci_get_num_implemented_ports(
476	    ahci_ctlp->ahcictl_ports_implemented);
477
478	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
479	    "hba number of implemented ports: 0x%x",
480	    ahci_ctlp->ahcictl_num_implemented_ports);
481
482	ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr;
483
484	if (!(cap_status & AHCI_HBA_CAP_S64A)) {
485		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
486		    "hba does not support 64-bit addressing");
487		ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi =
488		    0xffffffffull;
489	}
490
491	/*
492	 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those
493	 * controllers with 0x00 revision id work on 4-byte aligned buffer,
494	 * which is a bug and was fixed after 0x00 revision id controllers.
495	 */
496	if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle)
497	    != DDI_SUCCESS) {
498		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, "pci_config_setup failed");
499		goto err_out;
500	}
501
502	venid = pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
503	    PCI_CONF_VENID);
504
505	if (venid == VIA_VENID) {
506		revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
507		    PCI_CONF_REVID);
508		AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
509		    "revision id = 0x%x", revision);
510		if (revision == 0x00) {
511			ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4;
512			AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
513			    "change ddi_attr_align to 0x4");
514		}
515	}
516
517	pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
518
519	/*
520	 * Disable the whole controller interrupts before adding
521	 * interrupt handlers(s).
522	 */
523	ahci_disable_all_intrs(ahci_ctlp);
524
525	/* Get supported interrupt types */
526	if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
527		AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
528		    "ddi_intr_get_supported_types failed");
529		goto err_out;
530	}
531
532	AHCIDBG1(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
533	    "ddi_intr_get_supported_types() returned: 0x%x",
534	    intr_types);
535
536	if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) {
537		/*
538		 * Try MSI first, but fall back to FIXED if failed
539		 */
540		if (ahci_add_msi_intrs(ahci_ctlp) == DDI_SUCCESS) {
541			ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI;
542			AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
543			    "Using MSI interrupt type");
544			goto intr_done;
545		}
546
547		AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
548		    "MSI registration failed, "
549		    "trying FIXED interrupts");
550	}
551
552	if (intr_types & DDI_INTR_TYPE_FIXED) {
553		if (ahci_add_legacy_intrs(ahci_ctlp) == DDI_SUCCESS) {
554			ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED;
555			AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, NULL,
556			    "Using FIXED interrupt type");
557			goto intr_done;
558		}
559
560		AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
561		    "FIXED interrupt registration failed");
562	}
563
564	cmn_err(CE_WARN, "!Interrupt registration failed");
565
566	goto err_out;
567
568intr_done:
569
570	attach_state |= AHCI_ATTACH_STATE_INTR_ADDED;
571
572	/* Initialize the controller mutex */
573	mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER,
574	    (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
575
576	attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT;
577
578	if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) {
579		ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER;
580	} else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) {
581		ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER;
582	}
583
584	ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) +
585	    (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) *
586	    sizeof (ahci_prdt_item_t));
587
588	AHCIDBG2(AHCIDBG_INIT, ahci_ctlp,
589	    "ahci_attach: ahci_dma_prdt_number set by user is 0x%x,"
590	    " ahci_cmd_table_size is 0x%x",
591	    ahci_dma_prdt_number, ahci_cmd_table_size);
592
593	if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER)
594		ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen =
595		    ahci_dma_prdt_number;
596
597	/*
598	 * Initialize the controller and driver core.
599	 */
600	ahci_ctlp->ahcictl_flags |= AHCI_ATTACH;
601	status = ahci_initialize_controller(ahci_ctlp);
602	ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH;
603	if (status != AHCI_SUCCESS) {
604		goto err_out;
605	}
606
607	attach_state |= AHCI_ATTACH_STATE_HW_INIT;
608
609	/* Start one thread to check packet timeouts */
610	ahci_ctlp->ahcictl_timeout_id = timeout(
611	    (void (*)(void *))ahci_watchdog_handler,
612	    (caddr_t)ahci_ctlp, ahci_watchdog_tick);
613
614	attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED;
615
616	if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) {
617		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
618		    "ahci: setting sata hba tran failed");
619		goto err_out;
620	}
621
622	AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!");
623
624	return (DDI_SUCCESS);
625
626err_out:
627	if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) {
628		mutex_enter(&ahci_ctlp->ahcictl_mutex);
629		(void) untimeout(ahci_ctlp->ahcictl_timeout_id);
630		ahci_ctlp->ahcictl_timeout_id = 0;
631		mutex_exit(&ahci_ctlp->ahcictl_mutex);
632	}
633
634	if (attach_state & AHCI_ATTACH_STATE_HW_INIT) {
635		mutex_enter(&ahci_ctlp->ahcictl_mutex);
636		ahci_ctlp->ahcictl_flags |= AHCI_DETACH;
637		ahci_deallocate_controller(ahci_ctlp);
638		ahci_ctlp->ahcictl_flags &= ~AHCI_DETACH;
639		mutex_exit(&ahci_ctlp->ahcictl_mutex);
640	}
641
642	if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) {
643		mutex_destroy(&ahci_ctlp->ahcictl_mutex);
644	}
645
646	if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) {
647		ahci_rem_intrs(ahci_ctlp);
648	}
649
650	if (attach_state & AHCI_ATTACH_STATE_REG_MAP) {
651		ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
652	}
653
654	if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) {
655		ddi_soft_state_free(ahci_statep, instance);
656	}
657
658	return (DDI_FAILURE);
659}
660
661/*
662 * The detach entry point for dev_ops.
663 */
664static int
665ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
666{
667	ahci_ctl_t *ahci_ctlp;
668	int instance;
669	int ret;
670
671	instance = ddi_get_instance(dip);
672	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
673
674	AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter");
675
676	switch (cmd) {
677	case DDI_DETACH:
678		/* disable the interrupts for an uninterrupted detach */
679		mutex_enter(&ahci_ctlp->ahcictl_mutex);
680		ahci_disable_all_intrs(ahci_ctlp);
681		mutex_exit(&ahci_ctlp->ahcictl_mutex);
682
683		/* unregister from the sata framework. */
684		ret = ahci_unregister_sata_hba_tran(ahci_ctlp);
685		if (ret != AHCI_SUCCESS) {
686			mutex_enter(&ahci_ctlp->ahcictl_mutex);
687			ahci_enable_all_intrs(ahci_ctlp);
688			mutex_exit(&ahci_ctlp->ahcictl_mutex);
689			return (DDI_FAILURE);
690		}
691
692		mutex_enter(&ahci_ctlp->ahcictl_mutex);
693
694		/* stop the watchdog handler */
695		(void) untimeout(ahci_ctlp->ahcictl_timeout_id);
696		ahci_ctlp->ahcictl_timeout_id = 0;
697
698		/* deallocate the controller structures */
699		ahci_ctlp->ahcictl_flags |= AHCI_DETACH;
700		ahci_deallocate_controller(ahci_ctlp);
701		ahci_ctlp->ahcictl_flags &= ~AHCI_DETACH;
702
703		/* destroy any mutexes */
704		mutex_exit(&ahci_ctlp->ahcictl_mutex);
705
706		(void) pm_lower_power(ahci_ctlp->ahcictl_dip, 0,
707		    PM_LEVEL_D3);
708
709		mutex_destroy(&ahci_ctlp->ahcictl_mutex);
710
711		/* remove the interrupts */
712		ahci_rem_intrs(ahci_ctlp);
713
714		/* remove the reg maps. */
715		ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
716
717		/* free the soft state. */
718		ddi_soft_state_free(ahci_statep, instance);
719
720		return (DDI_SUCCESS);
721
722	case DDI_SUSPEND:
723		/* It will be implemented in Phase 2 */
724		return (DDI_FAILURE);
725
726	default:
727		return (DDI_FAILURE);
728	}
729}
730
731/*
732 * The info entry point for dev_ops.
733 *
734 */
735static int
736ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd,
737		    void *arg, void **result)
738{
739#ifndef __lock_lint
740	_NOTE(ARGUNUSED(dip))
741#endif /* __lock_lint */
742
743	ahci_ctl_t *ahci_ctlp;
744	int instance;
745	dev_t dev;
746
747	dev = (dev_t)arg;
748	instance = getminor(dev);
749
750	switch (infocmd) {
751		case DDI_INFO_DEVT2DEVINFO:
752			ahci_ctlp = ddi_get_soft_state(ahci_statep,  instance);
753			if (ahci_ctlp != NULL) {
754				*result = ahci_ctlp->ahcictl_dip;
755				return (DDI_SUCCESS);
756			} else {
757				*result = NULL;
758				return (DDI_FAILURE);
759			}
760		case DDI_INFO_DEVT2INSTANCE:
761			*(int *)result = instance;
762			break;
763		default:
764			break;
765	}
766
767	return (DDI_SUCCESS);
768}
769
770/*
771 * Registers the ahci with sata framework.
772 */
773static int
774ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status)
775{
776	struct 	sata_hba_tran	*sata_hba_tran;
777
778	AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
779	    "ahci_register_sata_hba_tran enter");
780
781	mutex_enter(&ahci_ctlp->ahcictl_mutex);
782
783	/* Allocate memory for the sata_hba_tran  */
784	sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);
785
786	sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV;
787	sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip;
788	sata_hba_tran->sata_tran_hba_dma_attr =
789	    &ahci_ctlp->ahcictl_buffer_dma_attr;
790
791	/* Report the number of implemented ports */
792	sata_hba_tran->sata_tran_hba_num_cports =
793	    ahci_ctlp->ahcictl_num_implemented_ports;
794
795	sata_hba_tran->sata_tran_hba_features_support = 0;
796
797	/* Get the data transfer capability for PIO command by the HBA */
798	if (cap_status & AHCI_HBA_CAP_PMD) {
799		ahci_ctlp->ahcictl_flags |= AHCI_PMD;
800		AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple "
801		    "DRQ block data transfer for PIO command protocol");
802	} else {
803		AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "HBA only supports single "
804		    "DRQ block data transfer for PIO command protocol");
805	}
806
807	/* Report the number of command slots */
808	sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots;
809
810	sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port;
811	sata_hba_tran->sata_tran_start = ahci_tran_start;
812	sata_hba_tran->sata_tran_abort = ahci_tran_abort;
813	sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport;
814	sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops;
815#ifdef __lock_lint
816	sata_hba_tran->sata_tran_selftest = ahci_selftest;
817#endif
818	/*
819	 * When SATA framework adds support for pwrmgt the
820	 * pwrmgt_ops needs to be updated
821	 */
822	sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
823	sata_hba_tran->sata_tran_ioctl = NULL;
824
825	ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran;
826
827	mutex_exit(&ahci_ctlp->ahcictl_mutex);
828
829	/* Attach it to SATA framework */
830	if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH)
831	    != DDI_SUCCESS) {
832		kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
833		mutex_enter(&ahci_ctlp->ahcictl_mutex);
834		ahci_ctlp->ahcictl_sata_hba_tran = NULL;
835		mutex_exit(&ahci_ctlp->ahcictl_mutex);
836		return (AHCI_FAILURE);
837	}
838
839	return (AHCI_SUCCESS);
840}
841
842/*
843 * Unregisters the ahci with sata framework.
844 */
845static int
846ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp)
847{
848	AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp,
849	    "ahci_unregister_sata_hba_tran enter");
850
851	/* Detach from the SATA framework. */
852	if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) !=
853	    DDI_SUCCESS) {
854		return (AHCI_FAILURE);
855	}
856
857	/* Deallocate sata_hba_tran. */
858	kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran,
859	    sizeof (sata_hba_tran_t));
860
861	mutex_enter(&ahci_ctlp->ahcictl_mutex);
862	ahci_ctlp->ahcictl_sata_hba_tran = NULL;
863	mutex_exit(&ahci_ctlp->ahcictl_mutex);
864
865	return (AHCI_SUCCESS);
866}
867
868/*
869 * ahci_tran_probe_port is called by SATA framework. It returns port state,
870 * port status registers and an attached device type via sata_device
871 * structure.
872 *
873 * We return the cached information from a previous hardware probe. The
874 * actual hardware probing itself was done either from within
875 * ahci_initialize_controller() during the driver attach or from a phy
876 * ready change interrupt handler.
877 */
878static int
879ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
880{
881	ahci_ctl_t *ahci_ctlp;
882	ahci_port_t *ahci_portp;
883	uint8_t	cport = sd->satadev_addr.cport;
884	uint8_t pmport = sd->satadev_addr.pmport;
885	uint8_t qual = sd->satadev_addr.qual;
886	uint8_t	device_type;
887	uint32_t port_state;
888	uint8_t port;
889
890	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
891	port = ahci_ctlp->ahcictl_cport_to_port[cport];
892
893	AHCIDBG3(AHCIDBG_ENTRY, ahci_ctlp,
894	    "ahci_tran_probe_port enter: cport: 0x%x, "
895	    "pmport: 0x%x, qual: 0x%x", cport, pmport, qual);
896
897	ahci_portp = ahci_ctlp->ahcictl_ports[port];
898
899	mutex_enter(&ahci_portp->ahciport_mutex);
900
901	port_state = ahci_portp->ahciport_port_state;
902	switch (port_state) {
903
904	case SATA_PSTATE_FAILED:
905		sd->satadev_state = SATA_PSTATE_FAILED;
906		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
907		    "ahci_tran_probe_port %d: PORT FAILED", port);
908		goto out;
909
910	case SATA_PSTATE_SHUTDOWN:
911		sd->satadev_state = SATA_PSTATE_SHUTDOWN;
912		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
913		    "ahci_tran_probe_port %d: PORT SHUTDOWN", port);
914		goto out;
915
916	case SATA_PSTATE_PWROFF:
917		sd->satadev_state = SATA_PSTATE_PWROFF;
918		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
919		    "ahci_tran_probe_port %d: PORT PWROFF", port);
920		goto out;
921
922	case SATA_PSTATE_PWRON:
923		sd->satadev_state = SATA_PSTATE_PWRON;
924		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
925		    "ahci_tran_probe_port %d: PORT PWRON", port);
926		break;
927
928	default:
929		sd->satadev_state = port_state;
930		AHCIDBG2(AHCIDBG_INFO, ahci_ctlp,
931		    "ahci_tran_probe_port %d: PORT NORMAL %x",
932		    port, port_state);
933		break;
934	}
935
936	device_type = ahci_portp->ahciport_device_type;
937
938	switch (device_type) {
939
940	case SATA_DTYPE_ATADISK:
941		sd->satadev_type = SATA_DTYPE_ATADISK;
942		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
943		    "ahci_tran_probe_port %d: DISK found", port);
944		break;
945
946	case SATA_DTYPE_ATAPICD:
947		sd->satadev_type = SATA_DTYPE_ATAPICD;
948		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
949		    "ahci_tran_probe_port %d: ATAPI found", port);
950		break;
951
952	case SATA_DTYPE_PMULT:
953		sd->satadev_type = SATA_DTYPE_PMULT;
954		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
955		    "ahci_tran_probe_port %d: Port Multiplier found",
956		    port);
957		break;
958
959	case SATA_DTYPE_UNKNOWN:
960		sd->satadev_type = SATA_DTYPE_UNKNOWN;
961		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
962		    "ahci_tran_probe_port %d: Unknown device found", port);
963		break;
964
965	default:
966		/* we don't support any other device types */
967		sd->satadev_type = SATA_DTYPE_NONE;
968		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
969		    "ahci_tran_probe_port %d: No device found", port);
970		break;
971	}
972
973out:
974	ahci_update_sata_registers(ahci_ctlp, port, sd);
975	mutex_exit(&ahci_portp->ahciport_mutex);
976
977	return (SATA_SUCCESS);
978}
979
980/*
981 * Called by sata framework to transport a sata packet down stream.
982 */
983static int
984ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
985{
986	ahci_ctl_t *ahci_ctlp;
987	ahci_port_t *ahci_portp;
988	uint8_t	cport = spkt->satapkt_device.satadev_addr.cport;
989	uint8_t port;
990	int slot;
991
992	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
993	port = ahci_ctlp->ahcictl_cport_to_port[cport];
994
995	AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp,
996	    "ahci_tran_start enter: cport %d satapkt 0x%p",
997	    cport, (void *)spkt);
998
999	ahci_portp = ahci_ctlp->ahcictl_ports[port];
1000
1001	mutex_enter(&ahci_portp->ahciport_mutex);
1002
1003	if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
1004	    ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
1005	    ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
1006		/*
1007		 * In case the targer driver would send the packet before
1008		 * sata framework can have the opportunity to process those
1009		 * event reports.
1010		 */
1011		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1012		spkt->satapkt_device.satadev_state =
1013		    ahci_portp->ahciport_port_state;
1014		ahci_update_sata_registers(ahci_ctlp, port,
1015		    &spkt->satapkt_device);
1016		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
1017		    "ahci_tran_start returning PORT_ERROR while "
1018		    "port in FAILED/SHUTDOWN/PWROFF state: "
1019		    "cport: 0x%x", port);
1020		mutex_exit(&ahci_portp->ahciport_mutex);
1021		return (SATA_TRAN_PORT_ERROR);
1022	}
1023
1024	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
1025		/*
1026		 * ahci_intr_phyrdy_change() may have rendered it to
1027		 * SATA_DTYPE_NONE.
1028		 */
1029		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1030		spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1031		spkt->satapkt_device.satadev_state =
1032		    ahci_portp->ahciport_port_state;
1033		ahci_update_sata_registers(ahci_ctlp, port,
1034		    &spkt->satapkt_device);
1035		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
1036		    "ahci_tran_start returning PORT_ERROR while "
1037		    "no device attached: cport: 0x%x", port);
1038		mutex_exit(&ahci_portp->ahciport_mutex);
1039		return (SATA_TRAN_PORT_ERROR);
1040	}
1041
1042	/*
1043	 * SATA HBA driver should remember that a device was reset and it
1044	 * is supposed to reject any packets which do not specify either
1045	 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE.
1046	 *
1047	 * This is to prevent a race condition when a device was arbitrarily
1048	 * reset by the HBA driver (and lost it's setting) and a target
1049	 * driver sending some commands to a device before the sata framework
1050	 * has a chance to restore the device setting (such as cache enable/
1051	 * disable or other resettable stuff).
1052	 */
1053	if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
1054		ahci_portp->ahciport_reset_in_progress = 0;
1055		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
1056		    "ahci_tran_start clearing the "
1057		    "reset_in_progress for port: 0x%x", port);
1058	}
1059
1060	if (ahci_portp->ahciport_reset_in_progress &&
1061	    ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
1062	    ! ddi_in_panic()) {
1063		spkt->satapkt_reason = SATA_PKT_BUSY;
1064		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
1065		    "ahci_tran_start returning BUSY while "
1066		    "reset in progress: cport: 0x%x", port);
1067		mutex_exit(&ahci_portp->ahciport_mutex);
1068		return (SATA_TRAN_BUSY);
1069	}
1070
1071	if (ahci_portp->ahciport_flags & AHCI_PORT_STATE_MOPPING) {
1072		spkt->satapkt_reason = SATA_PKT_BUSY;
1073		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
1074		    "ahci_tran_start returning BUSY while "
1075		    "mopping in progress: port: 0x%x", port);
1076		mutex_exit(&ahci_portp->ahciport_mutex);
1077		return (SATA_TRAN_BUSY);
1078	}
1079
1080	if ((slot = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, port, spkt))
1081	    == AHCI_FAILURE) {
1082		spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
1083		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
1084		    "returning QUEUE_FULL: port %d", port);
1085		mutex_exit(&ahci_portp->ahciport_mutex);
1086		return (SATA_TRAN_QUEUE_FULL);
1087	}
1088
1089	if (spkt->satapkt_op_mode &
1090	    (SATA_OPMODE_POLLING | SATA_OPMODE_SYNCH)) {
1091		/* we need to poll now */
1092		mutex_exit(&ahci_portp->ahciport_mutex);
1093		ahci_poll_cmd(ahci_ctlp, ahci_portp, port, slot, spkt);
1094		mutex_enter(&ahci_portp->ahciport_mutex);
1095	}
1096
1097	mutex_exit(&ahci_portp->ahciport_mutex);
1098
1099	AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
1100	    "sata tran accepted: port %d", port);
1101
1102	return (SATA_TRAN_ACCEPTED);
1103}
1104
1105#define	SENDUP_PACKET(ahci_portp, satapkt, reason)			\
1106	if (satapkt) {							\
1107		satapkt->satapkt_reason = reason;			\
1108		/*							\
1109		 * We set the satapkt_reason in both sync and		\
1110		 * non-sync cases.					\
1111		 */							\
1112	}								\
1113	if (satapkt &&							\
1114	    !(satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&		\
1115	    satapkt->satapkt_comp) {					\
1116		mutex_exit(&ahci_portp->ahciport_mutex);		\
1117		(*satapkt->satapkt_comp)(satapkt);			\
1118		mutex_enter(&ahci_portp->ahciport_mutex);		\
1119	}
1120
1121/*
1122 * Searches for and claims a free slot.
1123 *
1124 * Returns:     AHCI_FAILURE if no slots found
1125 *              claimed slot number if successful
1126 *
1127 * WARNING!!! ahciport_mutex should be acquired before the function
1128 * is called.
1129 */
1130/*ARGSUSED*/
1131static int
1132ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
1133{
1134	uint32_t free_slots;
1135	int slot;
1136
1137	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter "
1138	    "ahciport_pending_tags = 0x%x",
1139	    ahci_portp->ahciport_pending_tags);
1140
1141	free_slots = (~ahci_portp->ahciport_pending_tags)
1142	    & AHCI_SLOT_MASK(ahci_ctlp);
1143
1144	slot = ddi_ffs(free_slots) - 1;
1145	if (slot == -1) {
1146		AHCIDBG0(AHCIDBG_VERBOSE, ahci_ctlp,
1147		    "ahci_claim_free_slot: no empty slots");
1148		return (AHCI_FAILURE);
1149	}
1150
1151	ahci_portp->ahciport_pending_tags |= (0x1 << slot);
1152
1153	AHCIDBG1(AHCIDBG_VERBOSE, ahci_ctlp,
1154	    "ahci_claim_free_slot: found slot: 0x%x", slot);
1155
1156	return (slot);
1157}
1158
1159/*
1160 * Builds the Command Table for the sata packet and delivers it to controller.
1161 *
1162 * Returns:
1163 * 	slot number if we can obtain a slot successfully
1164 *	otherwise, return AHCI_FAILURE
1165 *
1166 * WARNING!!! ahciport_mutex should be acquired before the function is called.
1167 */
1168static int
1169ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1170    uint8_t port, sata_pkt_t *spkt)
1171{
1172	int slot;
1173	sata_cmd_t *cmd;
1174	ahci_fis_h2d_register_t *h2d_register_fisp;
1175	ahci_cmd_table_t *cmd_table;
1176	ahci_cmd_header_t *cmd_header;
1177	int ncookies;
1178	int i;
1179
1180	spkt->satapkt_reason = SATA_PKT_BUSY;
1181
1182	cmd = &spkt->satapkt_cmd;
1183
1184	/* Check if there is an empty command slot */
1185	slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp);
1186	if (slot == AHCI_FAILURE) {
1187		AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "no free slot");
1188		return (AHCI_FAILURE);
1189	}
1190
1191	AHCIDBG4(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp,
1192	    "ahci_deliver_satapkt enter: cmd_reg: 0x%x, slot: 0x%x, "
1193	    "port: %d, satapkt: 0x%p", cmd->satacmd_cmd_reg,
1194	    slot, port, (void *)spkt);
1195
1196	cmd_table = ahci_portp->ahciport_cmd_tables[slot];
1197	bzero((void *)cmd_table, ahci_cmd_table_size);
1198
1199	/* For data transfer operations, this is the H2D Register FIS */
1200	h2d_register_fisp =
1201	    &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
1202
1203	SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
1204	if ((spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_PMPORT) ||
1205	    (spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_DPMPORT)) {
1206		SET_FIS_PMP(h2d_register_fisp,
1207		    spkt->satapkt_device.satadev_addr.pmport);
1208	}
1209
1210	SET_FIS_CDMDEVCTL(h2d_register_fisp, 1);
1211	SET_FIS_COMMAND(h2d_register_fisp, cmd->satacmd_cmd_reg);
1212	SET_FIS_FEATURES(h2d_register_fisp, cmd->satacmd_features_reg);
1213	SET_FIS_SECTOR_COUNT(h2d_register_fisp, cmd->satacmd_sec_count_lsb);
1214
1215	switch (cmd->satacmd_addr_type) {
1216
1217	case ATA_ADDR_LBA:
1218		/* fallthrough */
1219
1220	case ATA_ADDR_LBA28:
1221		/* LBA[7:0] */
1222		SET_FIS_SECTOR(h2d_register_fisp, cmd->satacmd_lba_low_lsb);
1223
1224		/* LBA[15:8] */
1225		SET_FIS_CYL_LOW(h2d_register_fisp, cmd->satacmd_lba_mid_lsb);
1226
1227		/* LBA[23:16] */
1228		SET_FIS_CYL_HI(h2d_register_fisp, cmd->satacmd_lba_high_lsb);
1229
1230		/* LBA [27:24] (also called dev_head) */
1231		SET_FIS_DEV_HEAD(h2d_register_fisp, cmd->satacmd_device_reg);
1232
1233		break;
1234
1235	case ATA_ADDR_LBA48:
1236		/* LBA[7:0] */
1237		SET_FIS_SECTOR(h2d_register_fisp, cmd->satacmd_lba_low_lsb);
1238
1239		/* LBA[15:8] */
1240		SET_FIS_CYL_LOW(h2d_register_fisp, cmd->satacmd_lba_mid_lsb);
1241
1242		/* LBA[23:16] */
1243		SET_FIS_CYL_HI(h2d_register_fisp, cmd->satacmd_lba_high_lsb);
1244
1245		/* LBA [31:24] */
1246		SET_FIS_SECTOR_EXP(h2d_register_fisp,
1247		    cmd->satacmd_lba_low_msb);
1248
1249		/* LBA [39:32] */
1250		SET_FIS_CYL_LOW_EXP(h2d_register_fisp,
1251		    cmd->satacmd_lba_mid_msb);
1252
1253		/* LBA [47:40] */
1254		SET_FIS_CYL_HI_EXP(h2d_register_fisp,
1255		    cmd->satacmd_lba_high_msb);
1256
1257		/* Set dev_head */
1258		SET_FIS_DEV_HEAD(h2d_register_fisp,
1259		    cmd->satacmd_device_reg);
1260
1261		/* Set the extended sector count and features */
1262		SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp,
1263		    cmd->satacmd_sec_count_msb);
1264		SET_FIS_FEATURES_EXP(h2d_register_fisp,
1265		    cmd->satacmd_features_reg_ext);
1266		break;
1267	}
1268
1269	ncookies = cmd->satacmd_num_dma_cookies;
1270	AHCIDBG2(AHCIDBG_INFO, ahci_ctlp,
1271	    "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x",
1272	    ncookies, ahci_dma_prdt_number);
1273
1274	ASSERT(ncookies <= ahci_dma_prdt_number);
1275
1276	/* *** now fill the scatter gather list ******* */
1277	for (i = 0; i < ncookies; i++) {
1278		cmd_table->ahcict_prdt[i].ahcipi_data_base_addr =
1279		    cmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0];
1280		cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper =
1281		    cmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1];
1282		cmd_table->ahcict_prdt[i].ahcipi_descr_info =
1283		    cmd->satacmd_dma_cookie_list[i].dmac_size - 1;
1284	}
1285
1286	/* Set Command Header in Command List */
1287	cmd_header = &ahci_portp->ahciport_cmd_list[slot];
1288	BZERO_DESCR_INFO(cmd_header);
1289	BZERO_PRD_BYTE_COUNT(cmd_header);
1290	SET_PRD_TABLE_LENGTH(cmd_header, ncookies);
1291	SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH);
1292
1293	AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "command data direction is "
1294	    "sata_data_direction = 0x%x",
1295	    cmd->satacmd_flags.sata_data_direction);
1296
1297	if (cmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE)
1298		SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE);
1299
1300	SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE);
1301
1302	/* Now remember the sata packet in ahciport_slot_pkts[]. */
1303	ahci_portp->ahciport_slot_pkts[slot] = spkt;
1304
1305	/*
1306	 * We are overloading satapkt_hba_driver_private with
1307	 * watched_cycle count.
1308	 */
1309	spkt->satapkt_hba_driver_private = (void *)(intptr_t)0;
1310
1311	/* *** finished filling the scatter gather list , sync******* */
1312	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
1313	    0,
1314	    ahci_cmd_table_size,
1315	    DDI_DMA_SYNC_FORDEV);
1316
1317	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
1318	    slot * sizeof (ahci_cmd_header_t),
1319	    sizeof (ahci_cmd_header_t),
1320	    DDI_DMA_SYNC_FORDEV);
1321
1322	/* Indicate to the HBA that a command is active. */
1323	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
1324	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
1325	    (0x1 << slot));
1326
1327	AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt "
1328	    "exit: port %d", port);
1329
1330	return (slot);
1331}
1332
1333/*
1334 * Polls for the completion of the command. This is safe with both
1335 * interrupts enabled or disabled.
1336 */
1337static void
1338ahci_poll_cmd(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1339    uint8_t port, int slot, sata_pkt_t *spkt)
1340{
1341	uint32_t slot_status;
1342	int pkt_timeout_ticks;
1343	int in_panic = ddi_in_panic();
1344
1345	AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp,
1346	    "ahci_poll_cmd entered, port: %x", port);
1347
1348	pkt_timeout_ticks =
1349	    drv_usectohz((clock_t)spkt->satapkt_time * 1000000);
1350
1351	mutex_enter(&ahci_portp->ahciport_mutex);
1352
1353	/* we start out with SATA_PKT_COMPLETED as the satapkt_reason */
1354	spkt->satapkt_reason = SATA_PKT_COMPLETED;
1355
1356	/*
1357	 * The interrupt handler will return directly for poll commands in case
1358	 * the interrupt has been generated before interrupts were disabled.
1359	 */
1360	ahci_disable_port_intrs(ahci_ctlp, ahci_portp, port);
1361
1362	do {
1363		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
1364		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
1365
1366		if (slot_status & (0x1 << slot)) {
1367			if (in_panic) {
1368				/*
1369				 * If we are in panic, we can't rely on
1370				 * timers; so, busy wait instead of delay().
1371				 */
1372				mutex_exit(&ahci_portp->ahciport_mutex);
1373				drv_usecwait(AHCI_1MS_USECS);
1374				mutex_enter(&ahci_portp->ahciport_mutex);
1375			} else {
1376				mutex_exit(&ahci_portp->ahciport_mutex);
1377#ifndef __lock_lint
1378				delay(AHCI_1MS_TICKS);
1379#endif /* __lock_lint */
1380				mutex_enter(&ahci_portp->ahciport_mutex);
1381			}
1382		} else {
1383			AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "poll_cmd finished");
1384			break;
1385		}
1386
1387		pkt_timeout_ticks -= AHCI_1MS_TICKS;
1388
1389	} while (pkt_timeout_ticks > 0);
1390
1391	if (spkt->satapkt_reason != SATA_PKT_COMPLETED) {
1392		goto poll_done;
1393	}
1394
1395	if (slot_status & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)) {
1396		spkt->satapkt_reason = SATA_PKT_TIMEOUT;
1397	}
1398
1399	CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot);
1400	ahci_portp->ahciport_slot_pkts[slot] = NULL;
1401
1402poll_done:
1403	ahci_enable_port_intrs(ahci_ctlp, ahci_portp, port);
1404	mutex_exit(&ahci_portp->ahciport_mutex);
1405}
1406
1407/*
1408 * Called by the sata framework to abort the previously sent packet(s).
1409 *
1410 * Reset device to abort commands.
1411 */
1412static int
1413ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag)
1414{
1415	ahci_ctl_t *ahci_ctlp;
1416	ahci_port_t *ahci_portp;
1417	uint32_t slot_status;
1418	uint32_t aborted_tags, finished_tags;
1419	uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
1420	uint8_t port;
1421	int tmp_slot;
1422
1423	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1424	port = ahci_ctlp->ahcictl_cport_to_port[cport];
1425
1426	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
1427	    "ahci_tran_abort on cport %d", cport);
1428
1429	ahci_portp = ahci_ctlp->ahcictl_ports[port];
1430
1431	mutex_enter(&ahci_portp->ahciport_mutex);
1432
1433	/*
1434	 * To prevent recursive enter to ahci_mop_commands, we need
1435	 * check AHCI_PORT_STATE_MOPPING flag.
1436	 */
1437	if (ahci_portp->ahciport_flags & AHCI_PORT_STATE_MOPPING) {
1438		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
1439		    "ahci_tran_abort: port %d is in "
1440		    "mopping process, so just return directly ", port);
1441		mutex_exit(&ahci_portp->ahciport_mutex);
1442		return (SATA_SUCCESS);
1443	}
1444
1445	if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
1446	    ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
1447	    ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
1448		/*
1449		 * In case the targer driver would send the request before
1450		 * sata framework can have the opportunity to process those
1451		 * event reports.
1452		 */
1453		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1454		spkt->satapkt_device.satadev_state =
1455		    ahci_portp->ahciport_port_state;
1456		ahci_update_sata_registers(ahci_ctlp, port,
1457		    &spkt->satapkt_device);
1458		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
1459		    "ahci_tran_abort returning SATA_FAILURE while "
1460		    "port in FAILED/SHUTDOWN/PWROFF state: "
1461		    "cport: 0x%x", port);
1462		mutex_exit(&ahci_portp->ahciport_mutex);
1463		return (SATA_FAILURE);
1464	}
1465
1466	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
1467		/*
1468		 * ahci_intr_phyrdy_change() may have rendered it to
1469		 * AHCI_PORT_TYPE_NODEV.
1470		 */
1471		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1472		spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1473		spkt->satapkt_device.satadev_state =
1474		    ahci_portp->ahciport_port_state;
1475		ahci_update_sata_registers(ahci_ctlp, port,
1476		    &spkt->satapkt_device);
1477		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
1478		    "ahci_tran_abort returning SATA_FAILURE while "
1479		    "no device attached: cport: 0x%x", port);
1480		mutex_exit(&ahci_portp->ahciport_mutex);
1481		return (SATA_FAILURE);
1482	}
1483
1484	if (flag == SATA_ABORT_ALL_PACKETS) {
1485		aborted_tags = ahci_portp->ahciport_pending_tags;
1486		cmn_err(CE_NOTE, "!ahci port %d abort all packets", port);
1487	} else {
1488		aborted_tags = 0xffffffff;
1489		/*
1490		 * Aborting one specific packet, first search our
1491		 * ahciport_slot_pkts[] list for matching spkt.
1492		 */
1493		for (tmp_slot = 0;
1494		    tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) {
1495			if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) {
1496				aborted_tags = (0x1 << tmp_slot);
1497				break;
1498			}
1499		}
1500
1501		if (aborted_tags == 0xffffffff) {
1502			/* request packet is not on the pending list */
1503			AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
1504			    "Cannot find the aborting pkt 0x%p on the "
1505			    "pending list", (void *)spkt);
1506			ahci_update_sata_registers(ahci_ctlp, port,
1507			    &spkt->satapkt_device);
1508			mutex_exit(&ahci_portp->ahciport_mutex);
1509			return (SATA_FAILURE);
1510		}
1511		cmn_err(CE_NOTE, "!ahci port %d abort satapkt 0x%p",
1512		    port, (void *)spkt);
1513	}
1514
1515	slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
1516	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
1517
1518	ahci_portp->ahciport_flags |= AHCI_PORT_STATE_MOPPING;
1519
1520	/*
1521	 * To abort the packet(s), first we are trying to clear PxCMD.ST
1522	 * and PxCMD.FRE to stop the port, and if the port can be stopped
1523	 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0',
1524	 * then we just send back the aborted packet(s) with ABORTED flag
1525	 * and then restart the port by setting PxCMD.ST and PxCMD.FRE.
1526	 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we
1527	 * perform a COMRESET.
1528	 */
1529	(void) ahci_restart_port_wait_till_ready(ahci_ctlp,
1530	    ahci_portp, port, NULL);
1531
1532	/*
1533	 * Compute which have finished and which need to be retried.
1534	 *
1535	 * The finished tags are ahciport_pending_tags minus the slot_status.
1536	 * The aborted_tags have to be reduced by finished_tags since we
1537	 * can't possibly abort a tag which had finished already.
1538	 */
1539	finished_tags = ahci_portp->ahciport_pending_tags &
1540	    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
1541
1542	aborted_tags &= ~finished_tags;
1543
1544	mutex_exit(&ahci_portp->ahciport_mutex);
1545	ahci_mop_commands(ahci_ctlp,
1546	    ahci_portp,
1547	    port,
1548	    slot_status,
1549	    0, /* failed tags */
1550	    0, /* timeout tags */
1551	    aborted_tags,
1552	    0); /* reset tags */
1553
1554	mutex_enter(&ahci_portp->ahciport_mutex);
1555	ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device);
1556	mutex_exit(&ahci_portp->ahciport_mutex);
1557
1558	return (SATA_SUCCESS);
1559}
1560
1561/*
1562 * Used to do device reset and reject all the pending packets on a device
1563 * during the reset operation.
1564 *
1565 * WARNING!!! ahciport_mutex should be acquired before the function is called.
1566 */
1567static int
1568ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp,
1569    ahci_port_t *ahci_portp, uint8_t port)
1570{
1571	uint32_t slot_status;
1572	uint32_t reset_tags, finished_tags;
1573	sata_device_t sdevice;
1574	int ret;
1575
1576	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
1577	    "ahci_reset_device_reject_pkts on port: %d", port);
1578
1579	/*
1580	 * To prevent recursive enter to ahci_mop_commands, we need
1581	 * check AHCI_PORT_STATE_MOPPING flag.
1582	 */
1583	if (ahci_portp->ahciport_flags & AHCI_PORT_STATE_MOPPING) {
1584		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
1585		    "ahci_reset_device_reject_pkts: port %d is in "
1586		    "mopping process, so return directly ", port);
1587		return (SATA_SUCCESS);
1588	}
1589
1590	slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
1591	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
1592
1593	reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
1594
1595	if (ahci_software_reset(ahci_ctlp, ahci_portp, port)
1596	    != AHCI_SUCCESS) {
1597		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
1598		    "Try to do a port reset after software "
1599		    "reset failed", port);
1600		ret = ahci_port_reset(ahci_ctlp, ahci_portp, port);
1601		if (ret != AHCI_SUCCESS) {
1602			ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
1603			AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
1604			    "ahci_reset_device_reject_pkts: port %d "
1605			    "failed", port);
1606			return (SATA_FAILURE);
1607		}
1608	}
1609	/* Set the reset in progress flag */
1610	ahci_portp->ahciport_reset_in_progress = 1;
1611
1612	ahci_portp->ahciport_flags |= AHCI_PORT_STATE_MOPPING;
1613
1614	/* Indicate to the framework that a reset has happened */
1615	bzero((void *)&sdevice, sizeof (sata_device_t));
1616	sdevice.satadev_addr.cport = port;
1617	sdevice.satadev_addr.pmport = AHCI_PORTMULT_CONTROL_PORT;
1618
1619	if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
1620		sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT;
1621	} else {
1622		sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
1623	}
1624
1625	sdevice.satadev_state = SATA_DSTATE_RESET |
1626	    SATA_DSTATE_PWR_ACTIVE;
1627	mutex_exit(&ahci_portp->ahciport_mutex);
1628	sata_hba_event_notify(
1629	    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
1630	    &sdevice,
1631	    SATA_EVNT_DEVICE_RESET);
1632	mutex_enter(&ahci_portp->ahciport_mutex);
1633
1634	AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp,
1635	    "port %d sending event up: SATA_EVNT_RESET", port);
1636
1637	/* Next try to mop the pending commands */
1638	finished_tags = ahci_portp->ahciport_pending_tags &
1639	    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
1640
1641	reset_tags &= ~finished_tags;
1642
1643	mutex_exit(&ahci_portp->ahciport_mutex);
1644	ahci_mop_commands(ahci_ctlp,
1645	    ahci_portp,
1646	    port,
1647	    slot_status,
1648	    0, /* failed tags */
1649	    0, /* timeout tags */
1650	    0, /* aborted tags */
1651	    reset_tags); /* reset tags */
1652	mutex_enter(&ahci_portp->ahciport_mutex);
1653
1654	return (SATA_SUCCESS);
1655}
1656
1657/*
1658 * Used to do port reset and reject all the pending packets on a port during
1659 * the reset operation.
1660 *
1661 * WARNING!!! ahciport_mutex should be acquired before the function is called.
1662 */
1663static int
1664ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp,
1665    ahci_port_t *ahci_portp, uint8_t port)
1666{
1667	uint32_t slot_status;
1668	uint32_t reset_tags, finished_tags;
1669
1670	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
1671	    "ahci_reset_port_reject_pkts on port: %d", port);
1672
1673	/*
1674	 * To prevent recursive enter to ahci_mop_commands, we need
1675	 * check AHCI_PORT_STATE_MOPPING flag.
1676	 */
1677	if (ahci_portp->ahciport_flags & AHCI_PORT_STATE_MOPPING) {
1678		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
1679		    "ahci_reset_port_reject_pkts: port %d is in "
1680		    "mopping process, so return directly ", port);
1681		return (SATA_SUCCESS);
1682	}
1683
1684	ahci_portp->ahciport_flags |= AHCI_PORT_STATE_MOPPING;
1685
1686	slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
1687	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
1688
1689	reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
1690
1691	if (ahci_restart_port_wait_till_ready(ahci_ctlp,
1692	    ahci_portp, port, AHCI_PORT_RESET) != AHCI_SUCCESS)
1693		return (SATA_FAILURE);
1694
1695	finished_tags = ahci_portp->ahciport_pending_tags &
1696	    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
1697
1698	reset_tags &= ~finished_tags;
1699
1700	mutex_exit(&ahci_portp->ahciport_mutex);
1701	ahci_mop_commands(ahci_ctlp,
1702	    ahci_portp,
1703	    port,
1704	    slot_status,
1705	    0, /* failed tags */
1706	    0, /* timeout tags */
1707	    0, /* aborted tags */
1708	    reset_tags); /* reset tags */
1709	mutex_enter(&ahci_portp->ahciport_mutex);
1710
1711	return (SATA_SUCCESS);
1712}
1713
1714/*
1715 * Used to do hba reset and reject all the pending packets on all ports
1716 * during the reset operation.
1717 */
1718static int
1719ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp)
1720{
1721	ahci_port_t *ahci_portp;
1722	uint32_t slot_status[AHCI_MAX_PORTS];
1723	uint32_t reset_tags[AHCI_MAX_PORTS];
1724	uint32_t finished_tags[AHCI_MAX_PORTS];
1725	sata_device_t sdevice[AHCI_MAX_PORTS];
1726	uint8_t port;
1727	int ret = SATA_SUCCESS;
1728
1729	AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp,
1730	    "ahci_reset_hba_reject_pkts enter");
1731
1732	mutex_enter(&ahci_ctlp->ahcictl_mutex);
1733
1734	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
1735		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
1736			continue;
1737		}
1738
1739		ahci_portp = ahci_ctlp->ahcictl_ports[port];
1740
1741		mutex_enter(&ahci_portp->ahciport_mutex);
1742		slot_status[port] = ddi_get32(
1743		    ahci_ctlp->ahcictl_ahci_acc_handle,
1744		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
1745		reset_tags[port] = slot_status[port]
1746		    & AHCI_SLOT_MASK(ahci_ctlp);
1747		mutex_exit(&ahci_portp->ahciport_mutex);
1748	}
1749
1750	if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) {
1751		ret = SATA_FAILURE;
1752		goto out;
1753	}
1754
1755	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
1756		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
1757			continue;
1758		}
1759
1760		ahci_portp = ahci_ctlp->ahcictl_ports[port];
1761
1762		mutex_enter(&ahci_portp->ahciport_mutex);
1763		/*
1764		 * To prevent recursive enter to ahci_mop_commands, we need
1765		 * check AHCI_PORT_STATE_MOPPING flag.
1766		 */
1767		if (ahci_portp->ahciport_flags & AHCI_PORT_STATE_MOPPING) {
1768			AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
1769			    "ahci_reset_hba_reject_pkts: port %d is in "
1770			    "mopping process, so return directly ", port);
1771			mutex_exit(&ahci_portp->ahciport_mutex);
1772			continue;
1773		}
1774
1775		ahci_portp->ahciport_flags |= AHCI_PORT_STATE_MOPPING;
1776
1777		/* Indicate to the framework that a reset has happened */
1778		bzero((void *)&sdevice[port], sizeof (sata_device_t));
1779		sdevice[port].satadev_addr.cport = port;
1780		sdevice[port].satadev_addr.pmport = AHCI_PORTMULT_CONTROL_PORT;
1781
1782		if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
1783			sdevice[port].satadev_addr.qual = SATA_ADDR_DPMPORT;
1784		} else {
1785			sdevice[port].satadev_addr.qual = SATA_ADDR_DCPORT;
1786		}
1787		sdevice[port].satadev_state = SATA_DSTATE_RESET |
1788		    SATA_DSTATE_PWR_ACTIVE;
1789		mutex_exit(&ahci_portp->ahciport_mutex);
1790		sata_hba_event_notify(
1791		    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
1792		    &sdevice[port],
1793		    SATA_EVNT_DEVICE_RESET);
1794		mutex_enter(&ahci_portp->ahciport_mutex);
1795
1796		AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp,
1797		    "port %d sending event up: SATA_EVNT_RESET",
1798		    port);
1799
1800		finished_tags[port]  = ahci_portp->ahciport_pending_tags &
1801		    ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp);
1802
1803		reset_tags[port] &= ~finished_tags[port];
1804
1805		mutex_exit(&ahci_portp->ahciport_mutex);
1806		ahci_mop_commands(ahci_ctlp,
1807		    ahci_portp,
1808		    port,
1809		    slot_status[port],
1810		    0, /* failed tags */
1811		    0, /* timeout tags */
1812		    0, /* aborted tags */
1813		    reset_tags[port]); /* reset tags */
1814	}
1815out:
1816	mutex_exit(&ahci_ctlp->ahcictl_mutex);
1817
1818	return (ret);
1819}
1820
1821/*
1822 * Called by sata framework to reset a port(s) or device.
1823 */
1824static int
1825ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd)
1826{
1827	ahci_ctl_t *ahci_ctlp;
1828	ahci_port_t *ahci_portp;
1829	uint8_t cport = sd->satadev_addr.cport;
1830	uint8_t port;
1831	int ret = SATA_SUCCESS;
1832
1833	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1834	port = ahci_ctlp->ahcictl_cport_to_port[cport];
1835
1836	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
1837	    "ahci_tran_reset_port enter: cport: 0x%x", cport);
1838
1839	switch (sd->satadev_addr.qual) {
1840	case SATA_ADDR_CPORT:
1841		/* Port reset */
1842		ahci_portp = ahci_ctlp->ahcictl_ports[port];
1843		cmn_err(CE_NOTE, "!ahci port %d reset port", port);
1844
1845		mutex_enter(&ahci_portp->ahciport_mutex);
1846		ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, port);
1847		mutex_exit(&ahci_portp->ahciport_mutex);
1848
1849		break;
1850
1851	case SATA_ADDR_DCPORT:
1852		/* Device reset */
1853		ahci_portp = ahci_ctlp->ahcictl_ports[port];
1854		cmn_err(CE_NOTE, "!ahci port %d reset device", port);
1855
1856		mutex_enter(&ahci_portp->ahciport_mutex);
1857		if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
1858		    ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
1859		    ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
1860			/*
1861			 * In case the targer driver would send the request
1862			 * before sata framework can have the opportunity to
1863			 * process those event reports.
1864			 */
1865			sd->satadev_state = ahci_portp->ahciport_port_state;
1866			ahci_update_sata_registers(ahci_ctlp, port, sd);
1867			AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
1868			    "ahci_tran_reset_dport returning SATA_FAILURE "
1869			    "while port in FAILED/SHUTDOWN/PWROFF state: "
1870			    "cport: 0x%x", port);
1871			mutex_exit(&ahci_portp->ahciport_mutex);
1872			ret = SATA_FAILURE;
1873			break;
1874		}
1875
1876		if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
1877			/*
1878			 * ahci_intr_phyrdy_change() may have rendered it to
1879			 * AHCI_PORT_TYPE_NODEV.
1880			 */
1881			sd->satadev_type = SATA_DTYPE_NONE;
1882			sd->satadev_state = ahci_portp->ahciport_port_state;
1883			ahci_update_sata_registers(ahci_ctlp, port, sd);
1884			AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
1885			    "ahci_tran_reset_dport returning SATA_FAILURE "
1886			    "while no device attached: cport: 0x%x", port);
1887			mutex_exit(&ahci_portp->ahciport_mutex);
1888			ret = SATA_FAILURE;
1889			break;
1890		}
1891
1892		ret = ahci_reset_device_reject_pkts(ahci_ctlp,
1893		    ahci_portp, port);
1894		mutex_exit(&ahci_portp->ahciport_mutex);
1895		break;
1896
1897	case SATA_ADDR_CNTRL:
1898		/* Reset the whole controller */
1899		cmn_err(CE_NOTE, "!ahci port %d reset the whole hba", port);
1900		ret = ahci_reset_hba_reject_pkts(ahci_ctlp);
1901		break;
1902
1903	case SATA_ADDR_PMPORT:
1904	case SATA_ADDR_DPMPORT:
1905		AHCIDBG0(AHCIDBG_INFO, ahci_ctlp,
1906		    "port multiplier will be supported later");
1907		/* FALLSTHROUGH */
1908	default:
1909		ret = SATA_FAILURE;
1910	}
1911
1912	return (ret);
1913}
1914
1915/*
1916 * Called by sata framework to activate a port as part of hotplug.
1917 * (cfgadm -c connect satax/y)
1918 * Note: Not port-mult aware.
1919 */
1920static int
1921ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev)
1922{
1923	ahci_ctl_t *ahci_ctlp;
1924	ahci_port_t *ahci_portp;
1925	uint8_t	cport = satadev->satadev_addr.cport;
1926	uint8_t port;
1927
1928	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1929	port = ahci_ctlp->ahcictl_cport_to_port[cport];
1930
1931	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
1932	    "ahci_tran_hotplug_port_activate cport %d enter", cport);
1933
1934	ahci_portp = ahci_ctlp->ahcictl_ports[port];
1935
1936	mutex_enter(&ahci_portp->ahciport_mutex);
1937	cmn_err(CE_NOTE, "!ahci port %d activate", port);
1938
1939	/* Enable the interrupts on the port. */
1940	ahci_enable_port_intrs(ahci_ctlp, ahci_portp, port);
1941
1942	/*
1943	 * Reset the port so that the PHY communication would be re-established.
1944	 * But this reset is an internal operation; the sata framework does
1945	 * not need to know about it.
1946	 */
1947	(void) ahci_restart_port_wait_till_ready(ahci_ctlp,
1948	    ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP);
1949
1950	/*
1951	 * Need to check the link status and device status of the port
1952	 * and consider raising power if the port was in D3 state
1953	 */
1954	ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON;
1955	satadev->satadev_state = SATA_PSTATE_PWRON;
1956
1957	ahci_update_sata_registers(ahci_ctlp, port, satadev);
1958
1959	mutex_exit(&ahci_portp->ahciport_mutex);
1960	return (SATA_SUCCESS);
1961}
1962
1963/*
1964 * Called by sata framework to deactivate a port as part of hotplug.
1965 * (cfgadm -c disconnect satax/y)
1966 * Note: Not port-mult aware.
1967 */
1968static int
1969ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev)
1970{
1971	ahci_ctl_t *ahci_ctlp;
1972	ahci_port_t *ahci_portp;
1973	uint8_t cport = satadev->satadev_addr.cport;
1974	uint8_t port;
1975
1976	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1977	port = ahci_ctlp->ahcictl_cport_to_port[cport];
1978
1979	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
1980	    "ahci_tran_hotplug_port_deactivate cport %d enter", cport);
1981
1982	ahci_portp = ahci_ctlp->ahcictl_ports[port];
1983
1984	mutex_enter(&ahci_portp->ahciport_mutex);
1985	cmn_err(CE_NOTE, "!ahci port %d deactivate", port);
1986
1987	/* Disable the interrupts on the port. */
1988	ahci_disable_port_intrs(ahci_ctlp, ahci_portp, port);
1989
1990	/* First to abort all the pending commands */
1991	ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
1992
1993	/* Then shut down the port */
1994	(void) ahci_port_into_notrunning_state(ahci_ctlp,
1995	    ahci_portp, port);
1996
1997	/* Update ahciport_port_state */
1998	ahci_portp->ahciport_port_state = SATA_PSTATE_SHUTDOWN;
1999
2000	satadev->satadev_state = SATA_PSTATE_SHUTDOWN;
2001
2002	ahci_update_sata_registers(ahci_ctlp, port, satadev);
2003
2004	mutex_exit(&ahci_portp->ahciport_mutex);
2005	return (SATA_SUCCESS);
2006}
2007
2008/*
2009 * To be used to mark all the pending pkts with ABORTED
2010 * when a device is unplugged or a port is deactivated.
2011 *
2012 * WARNING!!! ahciport_mutex should be acquired before the function is called.
2013 */
2014static void
2015ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp,
2016    ahci_port_t *ahci_portp, uint8_t port)
2017{
2018	uint32_t slot_status;
2019	uint32_t abort_tags;
2020
2021	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
2022	    "ahci_reject_all_abort_pkts on port: %d", port);
2023
2024	/*
2025	 * To prevent recursive enter to ahci_mop_commands, we need
2026	 * check AHCI_PORT_STATE_MOPPING flag.
2027	 */
2028	if (ahci_portp->ahciport_flags & AHCI_PORT_STATE_MOPPING) {
2029		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
2030		    "ahci_reject_all_abort_pkts: port %d is in "
2031		    "mopping process, so return directly ", port);
2032		return;
2033	}
2034
2035	ahci_portp->ahciport_flags |= AHCI_PORT_STATE_MOPPING;
2036
2037	slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2038	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2039
2040	abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2041
2042	mutex_exit(&ahci_portp->ahciport_mutex);
2043	ahci_mop_commands(ahci_ctlp,
2044	    ahci_portp,
2045	    port,
2046	    slot_status,
2047	    0, /* failed tags */
2048	    0, /* timeout tags */
2049	    abort_tags, /* aborting tags */
2050	    0); /* reset tags */
2051	mutex_enter(&ahci_portp->ahciport_mutex);
2052}
2053
2054#if defined(__lock_lint)
2055static int
2056ahci_selftest(dev_info_t *dip, sata_device_t *device)
2057{
2058	return (SATA_SUCCESS);
2059}
2060#endif
2061
2062/*
2063 * Initialize the controller and set up driver data structures.
2064 *
2065 * This routine can be called from three seperate cases: DDI_ATTACH,
2066 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from
2067 * other two cases; the memory allocation and device signature probing
2068 * are attempted only during DDI_ATTACH case.
2069 *
2070 * WARNING!!! Disable the whole controller's interrupts before calling and
2071 * the interrupts will be enabled upon successfully return.
2072 */
2073static int
2074ahci_initialize_controller(ahci_ctl_t *ahci_ctlp)
2075{
2076	ahci_port_t *ahci_portp;
2077	uint32_t ghc_control;
2078	int port, cport = 0;
2079
2080	AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
2081	    "ahci_initialize_controller enter");
2082
2083	mutex_enter(&ahci_ctlp->ahcictl_mutex);
2084
2085	/*
2086	 * Indicate that system software is AHCI aware by setting
2087	 * GHC.AE to 1
2088	 */
2089	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2090	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
2091
2092	ghc_control |= AHCI_HBA_GHC_AE;
2093	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2094	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp),
2095	    ghc_control);
2096
2097	/* Initialize the implemented ports and structures */
2098	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2099		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2100			AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
2101			    "hba port %d not implemented", port);
2102			continue;
2103		}
2104
2105		/* only allocate port during attach */
2106		if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) {
2107#ifndef __lock_lint
2108			ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port;
2109			ahci_ctlp->ahcictl_port_to_cport[port] =
2110			    (uint8_t)cport++;
2111#endif /* __lock_lint */
2112			if (ahci_alloc_port_state(ahci_ctlp, port) !=
2113			    AHCI_SUCCESS) {
2114				goto err_out;
2115			}
2116		}
2117
2118		ahci_portp = ahci_ctlp->ahcictl_ports[port];
2119		mutex_enter(&ahci_portp->ahciport_mutex);
2120
2121		/*
2122		 * Ensure that the controller is not in the running state
2123		 * by checking every implemented port's PxCMD register
2124		 */
2125		if (ahci_initialize_port(ahci_ctlp, ahci_portp, port)
2126		    != AHCI_SUCCESS) {
2127			AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
2128			    "ahci_initialize_controller: failed to "
2129			    "initialize port %d", port);
2130			/*
2131			 * Set the port state to SATA_PSTATE_FAILED if
2132			 * failed to initialize it.
2133			 */
2134			ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED;
2135		}
2136
2137		mutex_exit(&ahci_portp->ahciport_mutex);
2138	}
2139
2140	/* Enable the whole controller interrupts */
2141	ahci_enable_all_intrs(ahci_ctlp);
2142	mutex_exit(&ahci_ctlp->ahcictl_mutex);
2143
2144	return (AHCI_SUCCESS);
2145err_out:
2146
2147	for (port--; port >= 0; port--) {
2148		if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2149			ahci_dealloc_port_state(ahci_ctlp, port);
2150		}
2151	}
2152
2153	mutex_exit(&ahci_ctlp->ahcictl_mutex);
2154
2155	return (AHCI_FAILURE);
2156}
2157
2158/*
2159 * Reverse of ahci_initialize_controller(), only need to de-allocate
2160 * all ports' state structures for AHCI_DETACH case.
2161 *
2162 * WARNING!!! ahcictl_mutex should be acquired before the function is called.
2163 */
2164static void
2165ahci_deallocate_controller(ahci_ctl_t *ahci_ctlp)
2166{
2167	uint8_t port;
2168
2169	AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
2170	    "ahci_deallocate_controller enter");
2171
2172	/* disable all the interrupts. */
2173	ahci_disable_all_intrs(ahci_ctlp);
2174
2175	if (ahci_ctlp->ahcictl_flags & AHCI_DETACH) {
2176		for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2177
2178			/* if this port is implemented by the HBA */
2179			if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
2180				ahci_dealloc_port_state(ahci_ctlp, port);
2181		}
2182	}
2183}
2184
2185/*
2186 * The routine is to initialize the port. First put the port in NOTRunning
2187 * state, then enable port interrupt and clear Serror register. And under
2188 * AHCI_ATTACH case, find device signature and then try to start the port.
2189 *
2190 * WARNING!!! ahcictl_mutex and ahciport_mutex should be acquired before
2191 * the function is called.
2192 */
2193static int
2194ahci_initialize_port(ahci_ctl_t *ahci_ctlp,
2195    ahci_port_t *ahci_portp, uint8_t port)
2196{
2197	uint32_t port_cmd_status;
2198
2199	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2200	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
2201
2202	AHCIDBG2(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
2203	    "ahci_initialize_port: port %d "
2204	    "port_cmd_status = 0x%x", port, port_cmd_status);
2205	/*
2206	 * Check whether the port is in NotRunning state, if not,
2207	 * put the port in NotRunning state
2208	 */
2209	if (!(port_cmd_status &
2210	    (AHCI_CMD_STATUS_ST |
2211	    AHCI_CMD_STATUS_CR |
2212	    AHCI_CMD_STATUS_FRE |
2213	    AHCI_CMD_STATUS_FR))) {
2214
2215		goto done_out;
2216	}
2217
2218	if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
2219	    port, AHCI_RESET_NO_EVENTS_UP|AHCI_PORT_INIT) != AHCI_SUCCESS)
2220		return (AHCI_FAILURE);
2221
2222done_out:
2223	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
2224	    "port %d is in NotRunning state", port);
2225
2226	/* Enable port interrupts */
2227	ahci_enable_port_intrs(ahci_ctlp, ahci_portp, port);
2228
2229	/*
2230	 * Only probe ports/devices and get the types of attached
2231	 * devices during attach.
2232	 */
2233	if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) {
2234		if (ahci_find_dev_signature(ahci_ctlp,
2235		    ahci_portp, port) != AHCI_SUCCESS) {
2236
2237			return (AHCI_FAILURE);
2238		}
2239
2240		/* Try to start the port */
2241		if ((ahci_portp->ahciport_device_type
2242		    == SATA_DTYPE_ATADISK) &&
2243		    (ahci_start_port(ahci_ctlp, port)
2244		    != AHCI_SUCCESS)) {
2245
2246			return (AHCI_FAILURE);
2247		}
2248	}
2249
2250	return (AHCI_SUCCESS);
2251}
2252
2253/*
2254 * AHCI device reset ...; a single device on one of the ports is reset,
2255 * but the HBA and physical communication remain intact. This is the
2256 * least intrusive.
2257 *
2258 * When issuing a software reset sequence, there should not be other
2259 * commands in the command list, so we will first clear and then re-set
2260 * PxCMD.ST to clear PxCI. And before issuing the software reset,
2261 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be
2262 * cleared.
2263 *
2264 * WARNING!!! ahciport_mutex should be acquired and PxCMD.FRE should be
2265 * set before the function is called.
2266 */
2267static int
2268ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
2269    uint8_t port)
2270{
2271	ahci_fis_h2d_register_t *h2d_register_fisp;
2272	ahci_cmd_table_t *cmd_table;
2273	ahci_cmd_header_t *cmd_header;
2274	int32_t port_cmd_status, port_cmd_issue, port_task_file;
2275	int slot, loop_count;
2276
2277	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
2278	    "Port %d device resetting", port);
2279
2280	/* First to clear PxCMD.ST */
2281	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2282	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
2283
2284	port_cmd_status &= ~AHCI_CMD_STATUS_ST;
2285
2286	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2287	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
2288	    port_cmd_status|AHCI_CMD_STATUS_ST);
2289
2290	/* And then to re-set PxCMD.ST */
2291	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2292	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
2293
2294	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2295	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
2296	    port_cmd_status|AHCI_CMD_STATUS_ST);
2297
2298	/* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */
2299	port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2300	    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
2301
2302	if (port_task_file & AHCI_TFD_STS_BSY ||
2303	    port_task_file & AHCI_TFD_STS_DRQ) {
2304		if (!(port_cmd_status & AHCI_CMD_STATUS_CLO)) {
2305			AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp,
2306			    "PxTFD.STS.BSY or PxTFD.STS.DRQ is still set, "
2307			    "but PxCMD.CLO isn't supported, so a port "
2308			    "reset is needed.");
2309			return (AHCI_FAILURE);
2310		}
2311	}
2312
2313	slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp);
2314	if (slot == AHCI_FAILURE) {
2315		AHCIDBG0(AHCIDBG_INFO, ahci_ctlp,
2316		    "ahci_software_reset: no free slot");
2317		return (AHCI_FAILURE);
2318	}
2319
2320	/* Now send the first R2H FIS with SRST set to 1 */
2321	cmd_table = ahci_portp->ahciport_cmd_tables[slot];
2322	bzero((void *)cmd_table, ahci_cmd_table_size);
2323
2324	h2d_register_fisp =
2325	    &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
2326
2327	SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
2328	SET_FIS_PMP(h2d_register_fisp, AHCI_PORTMULT_CONTROL_PORT);
2329	SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST);
2330
2331	/* Set Command Header in Command List */
2332	cmd_header = &ahci_portp->ahciport_cmd_list[slot];
2333	BZERO_DESCR_INFO(cmd_header);
2334	BZERO_PRD_BYTE_COUNT(cmd_header);
2335	SET_COMMAND_FIS_LENGTH(cmd_header, 5);
2336
2337	SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1);
2338	SET_RESET(cmd_header, 1);
2339	SET_WRITE(cmd_header, 1);
2340
2341	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
2342	    0,
2343	    ahci_cmd_table_size,
2344	    DDI_DMA_SYNC_FORDEV);
2345
2346	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
2347	    slot * sizeof (ahci_cmd_header_t),
2348	    sizeof (ahci_cmd_header_t),
2349	    DDI_DMA_SYNC_FORDEV);
2350
2351	/* Indicate to the HBA that a command is active. */
2352	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2353	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
2354	    (0x1 << slot));
2355
2356	loop_count = 0;
2357
2358	/* Loop till the first command is finished */
2359	do {
2360		port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2361		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2362
2363		if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
2364			/* We are effectively timing out after 0.1 sec. */
2365			break;
2366		}
2367		/* Wait for 10 millisec */
2368#ifndef __lock_lint
2369		delay(AHCI_10MS_TICKS);
2370#endif /* __lock_lint */
2371	} while (port_cmd_issue	& AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
2372
2373	AHCIDBG3(AHCIDBG_POLL_LOOP, ahci_ctlp,
2374	    "ahci_software_reset: 1st loop count: %d, "
2375	    "port_cmd_issue = 0x%x, slot = 0x%x",
2376	    loop_count, port_cmd_issue, slot);
2377
2378	CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot);
2379	ahci_portp->ahciport_slot_pkts[slot] = NULL;
2380
2381	/* Now send the second R2H FIS with SRST cleard to zero */
2382	cmd_table = ahci_portp->ahciport_cmd_tables[slot];
2383	bzero((void *)cmd_table, ahci_cmd_table_size);
2384
2385	h2d_register_fisp =
2386	    &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
2387
2388	SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
2389	SET_FIS_PMP(h2d_register_fisp, AHCI_PORTMULT_CONTROL_PORT);
2390
2391	/* Set Command Header in Command List */
2392	cmd_header = &ahci_portp->ahciport_cmd_list[slot];
2393	BZERO_DESCR_INFO(cmd_header);
2394	BZERO_PRD_BYTE_COUNT(cmd_header);
2395	SET_COMMAND_FIS_LENGTH(cmd_header, 5);
2396
2397	SET_WRITE(cmd_header, 1);
2398
2399	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
2400	    0,
2401	    ahci_cmd_table_size,
2402	    DDI_DMA_SYNC_FORDEV);
2403
2404	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
2405	    slot * sizeof (ahci_cmd_header_t),
2406	    sizeof (ahci_cmd_header_t),
2407	    DDI_DMA_SYNC_FORDEV);
2408
2409	/* Indicate to the HBA that a command is active. */
2410	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2411	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
2412	    (0x1 << slot));
2413
2414	loop_count = 0;
2415
2416	/* Loop till the second command is finished */
2417	do {
2418		port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2419		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2420
2421		if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
2422			/* We are effectively timing out after 0.1 sec. */
2423			break;
2424		}
2425		/* Wait for 10 millisec */
2426#ifndef __lock_lint
2427		delay(AHCI_10MS_TICKS);
2428#endif /* __lock_lint */
2429	} while (port_cmd_issue	& AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
2430
2431	AHCIDBG3(AHCIDBG_POLL_LOOP, ahci_ctlp,
2432	    "ahci_software_reset: 2nd loop count: %d, "
2433	    "port_cmd_issue = 0x%x, slot = 0x%x",
2434	    loop_count, port_cmd_issue, slot);
2435
2436	CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot);
2437	ahci_portp->ahciport_slot_pkts[slot] = NULL;
2438
2439	return (AHCI_SUCCESS);
2440}
2441
2442/*
2443 * AHCI port reset ...; the physical communication between the HBA and device
2444 * on a port are disabled. This is more intrusive.
2445 *
2446 * When an HBA or port reset occurs, Phy communication shall
2447 * be re-established with the device through a COMRESET followed by the
2448 * normal out-of-band communication sequence defined in Serial ATA. AT
2449 * the end of reset, the device, if working properly, will send a D2H
2450 * Register FIS, which contains the device signature. When the HBA receives
2451 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates
2452 * the PxSIG register with the signature.
2453 *
2454 * Staggered spin-up is an optional feature in SATA II, and it enables an HBA
2455 * to individually spin-up attached devices. Please refer to chapter 10.9 of
2456 * AHCI 1.1 spec.
2457 */
2458/*
2459 * WARNING!!! ahciport_mutex should be acquired, intr should be disabled,
2460 * and PxCMD.ST and PxCMD.FRE should be also cleared before the function
2461 * is called.
2462 */
2463static int
2464ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
2465{
2466	uint32_t cap_status, port_cmd_status;
2467	uint32_t port_scontrol, port_sstatus;
2468	uint32_t port_signature, port_intr_status, port_task_file;
2469	int loop_count;
2470
2471	AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
2472	    "Port %d port resetting...", port);
2473
2474	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2475	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
2476
2477	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2478	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
2479
2480	if (cap_status & AHCI_HBA_CAP_SSS) {
2481		/*
2482		 * HBA support staggered spin-up, if the port has
2483		 * not spin up yet, then force it to do spin-up
2484		 */
2485		if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) {
2486			if (!(ahci_portp->ahciport_flags
2487			    & AHCI_PORT_STATE_SPINUP)) {
2488				AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
2489				    "Port %d PxCMD.SUD is zero, force "
2490				    "it to do spin-up", port);
2491				ahci_portp->ahciport_flags |=
2492				    AHCI_PORT_STATE_SPINUP;
2493			}
2494		}
2495	} else {
2496		/*
2497		 * HBA doesn't support stagger spin-up, force it
2498		 * to do normal COMRESET
2499		 */
2500		ASSERT(port_cmd_status & AHCI_CMD_STATUS_SUD);
2501		if (ahci_portp->ahciport_flags &
2502		    AHCI_PORT_STATE_SPINUP) {
2503			AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
2504			    "HBA does not support staggered spin-up "
2505			    "force it to do normal COMRESET");
2506			ahci_portp->ahciport_flags &=
2507			    ~AHCI_PORT_STATE_SPINUP;
2508		}
2509	}
2510
2511	if (!(ahci_portp->ahciport_flags & AHCI_PORT_STATE_SPINUP)) {
2512		/* Do normal COMRESET */
2513		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
2514		    "ahci_port_reset: do normal COMRESET", port);
2515
2516		ASSERT(port_cmd_status & AHCI_CMD_STATUS_SUD);
2517
2518		port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2519		    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
2520		AHCI_SCONTROL_SET_DET(port_scontrol,
2521		    AHCI_SCONTROL_DET_COMRESET);
2522
2523		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2524		    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
2525		    port_scontrol);
2526
2527		/* Enable PxCMD.FRE to read device */
2528		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2529		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
2530		    port_cmd_status|AHCI_CMD_STATUS_FRE);
2531
2532		/* give time for COMRESET to percolate */
2533#ifndef __lock_lint
2534		delay(AHCI_1MS_TICKS*2);
2535#endif /* __lock_lint */
2536
2537		/* Fetch the SCONTROL again and rewrite the DET part with 0 */
2538		port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2539		    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
2540		AHCI_SCONTROL_SET_DET(port_scontrol,
2541		    AHCI_SCONTROL_DET_NOACTION);
2542		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2543		    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
2544		    port_scontrol);
2545	} else {
2546		/* Do staggered spin-up */
2547		port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2548		    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
2549		AHCI_SCONTROL_SET_DET(port_scontrol,
2550		    AHCI_SCONTROL_DET_NOACTION);
2551
2552		/* PxSCTL.DET must be 0 */
2553		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2554		    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
2555		    port_scontrol);
2556
2557		port_cmd_status &= ~AHCI_CMD_STATUS_SUD;
2558		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2559		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
2560		    port_cmd_status);
2561
2562		/* 0 -> 1 edge */
2563#ifndef __lock_lint
2564		delay(AHCI_1MS_TICKS*2);
2565#endif /* __lock_lint */
2566
2567		/* Set PxCMD.SUD to 1 */
2568		port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2569		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
2570		port_cmd_status |= AHCI_CMD_STATUS_SUD;
2571		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2572		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
2573		    port_cmd_status);
2574
2575		/* Enable PxCMD.FRE to read device */
2576		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2577		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
2578		    port_cmd_status|AHCI_CMD_STATUS_FRE);
2579	}
2580
2581	/*
2582	 * After PxSCTL.DET is set to 0h, software should wait for
2583	 * communication to be re-established as indicated by bit 0
2584	 * of PxSSTS.DET being set to '1'.
2585	 */
2586	loop_count = 0;
2587	do {
2588		port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2589		    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
2590
2591		if (port_intr_status & AHCI_INTR_STATUS_PCS) {
2592			AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
2593			    "COMINIT signal is received", port);
2594			/*
2595			 * Clear PxSERR.DIAG.X to update PxTFD by the D2H FIS
2596			 * received by HBA
2597			 */
2598			ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2599			    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
2600			    AHCI_SERROR_DIAG_X);
2601		}
2602
2603		if (loop_count++ > AHCI_POLLRATE_PORT_COMRESET) {
2604			/*
2605			 * We are effectively timing out after 0.1 sec.
2606			 */
2607			break;
2608		}
2609
2610		/* Wait for 10 millisec */
2611#ifndef __lock_lint
2612		delay(AHCI_10MS_TICKS);
2613#endif /* __lock_lint */
2614
2615	} while (!(port_intr_status & AHCI_INTR_STATUS_PCS));
2616
2617	AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
2618	    "ahci_port_reset: 1st loop count: %d, "
2619	    "port_intr_status = 0x%x", loop_count, port_intr_status);
2620
2621	/*
2622	 * The DET field is valid only if IPM field indicates
2623	 * that the interface is in active state.
2624	 */
2625	loop_count = 0;
2626	do {
2627		port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2628		    (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
2629
2630		if (AHCI_SSTATUS_GET_IPM(port_sstatus) !=
2631		    AHCI_SSTATUS_IPM_INTERFACE_ACTIVE) {
2632			/*
2633			 * If the interface is not active, the DET field
2634			 * is considered not accurate. So we want to
2635			 * continue looping.
2636			 */
2637			AHCI_SSTATUS_SET_DET(port_sstatus,
2638			    AHCI_SSTATUS_DET_NODEV_NOPHY);
2639		}
2640
2641		if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
2642			/*
2643			 * We are effectively timing out after 0.1 sec.
2644			 */
2645			break;
2646		}
2647
2648		/* Wait for 10 millisec */
2649#ifndef __lock_lint
2650		delay(AHCI_10MS_TICKS);
2651#endif /* __lock_lint */
2652
2653	} while (AHCI_SSTATUS_GET_DET(port_sstatus) !=
2654	    AHCI_SSTATUS_DET_DEVPRESENT_PHYONLINE);
2655
2656	AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
2657	    "ahci_port_reset: 2nd loop count: %d, "
2658	    "port_sstatus = 0x%x", loop_count, port_sstatus);
2659
2660	if ((AHCI_SSTATUS_GET_IPM(port_sstatus) !=
2661	    AHCI_SSTATUS_IPM_INTERFACE_ACTIVE) ||
2662	    (AHCI_SSTATUS_GET_DET(port_sstatus) !=
2663	    AHCI_SSTATUS_DET_DEVPRESENT_PHYONLINE)) {
2664		/*
2665		 * Either the port is not active or there
2666		 * is no device present.
2667		 */
2668		ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
2669
2670		(void) ahci_port_into_notrunning_state(ahci_ctlp,
2671		    ahci_portp, port);
2672	}
2673
2674	if (AHCI_SSTATUS_GET_DET(port_sstatus) ==
2675	    AHCI_SSTATUS_DET_DEVPRESENT_PHYONLINE) {
2676		/*
2677		 * If device exist, then first check PxTFD.STS.BSY
2678		 */
2679		loop_count = 0;
2680		do {
2681			port_task_file =
2682			    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2683			    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
2684
2685			if (loop_count++ > AHCI_POLLRATE_PORT_TFD_BSY) {
2686				/*
2687				 * We are effectively timing out after 11 sec.
2688				 */
2689				break;
2690			}
2691
2692			port_intr_status =
2693			    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2694			    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
2695
2696			if (port_intr_status & AHCI_INTR_STATUS_PCS) {
2697				/*
2698				 * Clear PxSERR.DIAG.X to update PxTFD by
2699				 * the D2H FIS received by HBA.
2700				 */
2701				ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2702				    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp,
2703				    port), AHCI_SERROR_DIAG_X);
2704			}
2705
2706			/* Wait for 10 millisec */
2707#ifndef __lock_lint
2708			delay(AHCI_10MS_TICKS);
2709#endif /* __lock_lint */
2710
2711		} while (port_task_file & AHCI_TFD_STS_BSY);
2712
2713		AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
2714		    "ahci_port_reset: 3rd loop count: %d, "
2715		    "port_task_file = 0x%x",
2716		    loop_count, port_task_file);
2717
2718		/*
2719		 * Next check COMRESET is completed successfully
2720		 */
2721		loop_count = 0;
2722		do {
2723			port_task_file =
2724			    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2725			    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
2726
2727			if (((port_task_file & AHCI_TFD_ERR_MASK)
2728			    >> AHCI_TFD_ERR_SHIFT) == 0x1) {
2729				AHCIDBG0(AHCIDBG_INFO, ahci_ctlp,
2730				    "COMRESET success, D2H register FIS "
2731				    "post to received FIS structure");
2732				break;
2733			}
2734
2735			if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
2736				/*
2737				 * We are effectively timing out after 0.1 sec.
2738				 */
2739				break;
2740			}
2741
2742			/* Wait for 10 millisec */
2743#ifndef __lock_lint
2744			delay(AHCI_10MS_TICKS);
2745#endif /* __lock_lint */
2746
2747		} while (((port_task_file & AHCI_TFD_ERR_MASK)
2748		    >> AHCI_TFD_ERR_SHIFT) != 0x1);
2749
2750		AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
2751		    "ahci_port_reset: 4th loop count: %d, "
2752		    "port_task_file = 0x%x",
2753		    loop_count, port_task_file);
2754
2755		port_signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2756		    (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port));
2757		AHCIDBG2(AHCIDBG_INFO, ahci_ctlp, "Port %d signature = 0x%x",
2758		    port, port_signature);
2759
2760		/*
2761		 * Check device status, if keep busy or COMRESET error
2762		 * do device reset to patch some SATA disks' issue
2763		 *
2764		 * For VT8251, sometimes need to do the device reset
2765		 */
2766		if (port_task_file & AHCI_TFD_STS_BSY) {
2767			AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "port %d keep busy "
2768			    "need to do device reset", port);
2769
2770			if (ahci_software_reset(ahci_ctlp, ahci_portp, port)
2771			    != AHCI_SUCCESS) {
2772				AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
2773				    "port %d device reset failed", port);
2774				return (AHCI_FAILURE);
2775			}
2776
2777			port_task_file =
2778			    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2779			    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
2780
2781			if (port_task_file & AHCI_TFD_STS_BSY) {
2782				AHCIDBG2(AHCIDBG_INFO, ahci_ctlp,
2783				    "port %d keep busy after device reset "
2784				    "port_task_file = 0x%x",
2785				    port, port_task_file);
2786				return (AHCI_FAILURE);
2787			}
2788		}
2789	}
2790
2791	/* Clear port serror register for each implemented port. */
2792	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2793	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
2794	    AHCI_SERROR_CLEAR_ALL);
2795
2796	return (AHCI_SUCCESS);
2797}
2798
2799/*
2800 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled.
2801 * This is the most intrusive.
2802 *
2803 * When an HBA reset occurs, Phy communication shall
2804 * be re-established with the device through a COMRESET followed by the
2805 * normal out-of-band communication sequence defined in Serial ATA. AT
2806 * the end of reset, the device, if working properly, will send a D2H
2807 * Register FIS, which contains the device signature. When the HBA receives
2808 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates
2809 * the PxSIG register with the signature.
2810 *
2811 * Remember to set GHC.AE to 1 before calling ahci_hba_reset.
2812 *
2813 * WARNING!!! ahcictl_mutex should be already held before the function
2814 * is called.
2815 */
2816static int
2817ahci_hba_reset(ahci_ctl_t *ahci_ctlp)
2818{
2819	ahci_port_t *ahci_portp;
2820	uint32_t ghc_control;
2821	uint8_t port;
2822	int loop_count;
2823	int rval = AHCI_SUCCESS;
2824
2825	AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting");
2826
2827	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2828	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
2829
2830	/* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */
2831	ghc_control |= AHCI_HBA_GHC_HR;
2832	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2833	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
2834
2835	/*
2836	 * Wait until HBA Reset complete or timeout
2837	 */
2838	loop_count = 0;
2839	do {
2840		ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2841		    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
2842
2843		if (loop_count++ > AHCI_POLLRATE_HBA_RESET) {
2844			AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
2845			    "ahci hba reset is timing out, "
2846			    "ghc_control = 0x%x", ghc_control);
2847			/* We are effectively timing out after 1 sec. */
2848			break;
2849		}
2850
2851		/* Wait for 10 millisec */
2852#ifndef __lock_lint
2853		delay(AHCI_10MS_TICKS);
2854#endif /* __lock_lint */
2855
2856	} while (ghc_control & AHCI_HBA_GHC_HR);
2857
2858	AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
2859	    "ahci_hba_reset: 1st loop count: %d, "
2860	    "ghc_control = 0x%x", loop_count, ghc_control);
2861
2862	if (ghc_control & AHCI_HBA_GHC_HR) {
2863		/* The hba is not reset for some reasons */
2864		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
2865		    "hba reset failed: HBA in a hung or locked state");
2866		return (AHCI_FAILURE);
2867	}
2868
2869	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2870		/* Only check implemented ports */
2871		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2872			continue;
2873		}
2874
2875		ahci_portp = ahci_ctlp->ahcictl_ports[port];
2876		mutex_enter(&ahci_portp->ahciport_mutex);
2877
2878		if (ahci_port_reset(ahci_ctlp, ahci_portp, port)
2879		    != AHCI_SUCCESS) {
2880			ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
2881			rval = AHCI_FAILURE;
2882			AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
2883			    "ahci_hba_reset: port %d failed", port);
2884		}
2885
2886		mutex_exit(&ahci_portp->ahciport_mutex);
2887	}
2888
2889	/*
2890	 * Indicate that system software is AHCI aware by setting
2891	 * GHC.AE to 1
2892	 */
2893	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2894	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
2895
2896	ghc_control |= AHCI_HBA_GHC_AE;
2897	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2898	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
2899
2900	return (rval);
2901}
2902
2903/*
2904 * This routine is only called from AHCI_ATTACH or phyrdy change
2905 * case. It first calls port reset to initialize port, probe port and probe
2906 * device, then read PxSIG register to find the type of device attached to
2907 * the port.
2908 *
2909 * WARNING!!! ahciport_mutex should be acquired before the function
2910 * is called.
2911 */
2912static int
2913ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp,
2914    ahci_port_t *ahci_portp, uint8_t port)
2915{
2916	uint32_t signature;
2917	int ret = AHCI_SUCCESS;
2918
2919	AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
2920	    "port %d: ahci_find_dev_signature enter", port);
2921
2922	ahci_disable_port_intrs(ahci_ctlp, ahci_portp, port);
2923
2924	/* Call port reset to check link status and get device signature */
2925	if (ahci_port_reset(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) {
2926		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
2927		    "ahci_find_dev_signature failed for port %d: ", port);
2928		ahci_enable_port_intrs(ahci_ctlp, ahci_portp, port);
2929		return (AHCI_FAILURE);
2930	}
2931
2932	ahci_enable_port_intrs(ahci_ctlp, ahci_portp, port);
2933
2934	signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2935	    (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port));
2936
2937	AHCIDBG2(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
2938	    "port %d signature = 0x%x", port, signature);
2939
2940	switch (signature) {
2941
2942	case AHCI_SIGNATURE_DISK:
2943		ahci_portp->ahciport_device_type = SATA_DTYPE_ATADISK;
2944		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
2945		    "Disk is found at port: %d", port);
2946		break;
2947
2948	case AHCI_SIGNATURE_ATAPI:
2949		ahci_portp->ahciport_device_type = SATA_DTYPE_ATAPICD;
2950		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
2951		    "ATAPI device is found at port: %d", port);
2952		break;
2953
2954	case AHCI_SIGNATURE_PORT_MULTIPLIER:
2955		ahci_portp->ahciport_device_type = SATA_DTYPE_PMULT;
2956		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
2957		    "Port Multiplier is found at port: %d", port);
2958		break;
2959
2960	case AHCI_SIGNATURE_NONE:
2961		ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
2962		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
2963		    "No device is found at port: %d", port);
2964		break;
2965
2966	default:
2967		ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN;
2968		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
2969		    "Unknown device is found at port: %d", port);
2970	}
2971
2972	return (ret);
2973}
2974
2975/*
2976 * Try to start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set
2977 * to 1, then set it.
2978 *
2979 * WARNING!!! ahciport_mutex should be acquired before the function
2980 * is called.
2981 */
2982static int
2983ahci_start_port(ahci_ctl_t *ahci_ctlp, uint8_t port)
2984{
2985	uint32_t port_task_file, port_cmd_status;
2986
2987	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
2988	    "ahci_start_port: %d enter", port);
2989
2990	port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2991	    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
2992
2993	if (port_task_file & AHCI_TFD_STS_BSY ||
2994	    port_task_file & AHCI_TFD_STS_DRQ) {
2995
2996		AHCIDBG2(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
2997		    "Port %d cannot start!!! port_task_file = 0x%x",
2998		    port, port_task_file);
2999		return (AHCI_FAILURE);
3000
3001	} else {
3002		AHCIDBG2(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
3003		    "Port %d start!!! port_task_file = 0x%x",
3004		    port, port_task_file);
3005
3006		/* First to set PxCMD.FRE before setting PxCMD.ST. */
3007		port_cmd_status =
3008		    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3009		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3010
3011		if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) {
3012			port_cmd_status |= AHCI_CMD_STATUS_FRE;
3013			ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3014			    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3015			    port_cmd_status);
3016		}
3017
3018		port_cmd_status =
3019		    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3020		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3021
3022		port_cmd_status |= AHCI_CMD_STATUS_ST;
3023
3024		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3025		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3026		    port_cmd_status);
3027
3028		return (AHCI_SUCCESS);
3029	}
3030}
3031
3032/*
3033 * Allocate the ahci_port_t including Received FIS and Command List.
3034 * The argument - port is the physical port number, and not logical
3035 * port number seen by the SATA framework.
3036 *
3037 * WARNING!!! ahcictl_mutex should be acquired before the function
3038 * is called.
3039 */
3040static int
3041ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
3042{
3043	ahci_port_t *ahci_portp;
3044
3045	ahci_portp =
3046	    (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP);
3047
3048#ifndef __lock_lint
3049	ahci_ctlp->ahcictl_ports[port] = ahci_portp;
3050#endif /* __lock_lint */
3051
3052	ahci_portp->ahciport_port_num = port;
3053
3054	mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER,
3055	    (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
3056	mutex_enter(&ahci_portp->ahciport_mutex);
3057
3058	/*
3059	 * Allocate memory for received FIS structure and
3060	 * command list for this port
3061	 */
3062	if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) {
3063		goto err_case1;
3064	}
3065
3066	if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) {
3067		goto err_case2;
3068	}
3069
3070	mutex_exit(&ahci_portp->ahciport_mutex);
3071
3072	return (AHCI_SUCCESS);
3073
3074err_case2:
3075	ahci_dealloc_rcvd_fis(ahci_ctlp, ahci_portp);
3076
3077err_case1:
3078	mutex_exit(&ahci_portp->ahciport_mutex);
3079
3080	mutex_destroy(&ahci_portp->ahciport_mutex);
3081
3082	kmem_free(ahci_portp, sizeof (ahci_port_t));
3083
3084	return (AHCI_FAILURE);
3085}
3086
3087/*
3088 * Reverse of ahci_dealloc_port_state().
3089 *
3090 * WARNING!!! ahcictl_mutex should be acquired before the function
3091 * is called.
3092 */
3093static void
3094ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
3095{
3096	ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
3097
3098	ASSERT(ahci_portp != NULL);
3099
3100	mutex_enter(&ahci_portp->ahciport_mutex);
3101	ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
3102	ahci_dealloc_rcvd_fis(ahci_ctlp, ahci_portp);
3103	mutex_exit(&ahci_portp->ahciport_mutex);
3104
3105	mutex_destroy(&ahci_portp->ahciport_mutex);
3106
3107	kmem_free(ahci_portp, sizeof (ahci_port_t));
3108
3109#ifndef __lock_lint
3110	ahci_ctlp->ahcictl_ports[port] = NULL;
3111#endif /* __lock_lint */
3112}
3113
3114/*
3115 * Allocates memory for the Received FIS Structure
3116 *
3117 * WARNING!!! ahciport_mutex should be acquired before the function
3118 * is called.
3119 */
3120static int
3121ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
3122    uint8_t port)
3123{
3124	size_t rcvd_fis_size;
3125	size_t ret_len;
3126	ddi_dma_cookie_t rcvd_fis_dma_cookie;
3127	uint_t cookie_count;
3128	uint32_t cap_status;
3129
3130	rcvd_fis_size = sizeof (ahci_rcvd_fis_t);
3131
3132	/* Check whether the HBA can access 64-bit data structures */
3133	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3134	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
3135
3136	ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr;
3137
3138	/*
3139	 * If 64-bit addressing is not supported,
3140	 * change dma_attr_addr_hi of ahcictl_rcvd_fis_dma_attr
3141	 */
3142	if (!(cap_status & AHCI_HBA_CAP_S64A)) {
3143		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
3144		    "hba does not support 64-bit addressing");
3145		ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi =
3146		    0xffffffffull;
3147	}
3148
3149	/* allocate rcvd FIS dma handle. */
3150	if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
3151	    &ahci_ctlp->ahcictl_rcvd_fis_dma_attr,
3152	    DDI_DMA_SLEEP,
3153	    NULL,
3154	    &ahci_portp->ahciport_rcvd_fis_dma_handle) !=
3155	    DDI_SUCCESS) {
3156		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
3157		    "rcvd FIS dma handle alloc failed");
3158
3159		return (AHCI_FAILURE);
3160	}
3161
3162	if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle,
3163	    rcvd_fis_size,
3164	    &accattr,
3165	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
3166	    DDI_DMA_SLEEP,
3167	    NULL,
3168	    (caddr_t *)&ahci_portp->ahciport_rcvd_fis,
3169	    &ret_len,
3170	    &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) {
3171
3172		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
3173		    "rcvd FIS dma mem alloc fail");
3174		/* error.. free the dma handle. */
3175		ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
3176		return (AHCI_FAILURE);
3177	}
3178
3179	if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle,
3180	    NULL,
3181	    (caddr_t)ahci_portp->ahciport_rcvd_fis,
3182	    rcvd_fis_size,
3183	    DDI_DMA_CONSISTENT,
3184	    DDI_DMA_SLEEP,
3185	    NULL,
3186	    &rcvd_fis_dma_cookie,
3187	    &cookie_count) !=  DDI_DMA_MAPPED) {
3188
3189		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
3190		    "rcvd FIS dma handle bind fail");
3191		/*  error.. free the dma handle & free the memory. */
3192		ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
3193		ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
3194		return (AHCI_FAILURE);
3195	}
3196
3197	bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size);
3198
3199	/* Config Port Received FIS Base Address */
3200	ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle,
3201	    (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port),
3202	    rcvd_fis_dma_cookie.dmac_laddress);
3203
3204	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
3205	    rcvd_fis_dma_cookie.dmac_laddress);
3206	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
3207	    rcvd_fis_dma_cookie.dmac_address);
3208
3209	return (AHCI_SUCCESS);
3210}
3211
3212/*
3213 * Deallocates the Received FIS Structure
3214 *
3215 * WARNING!!! ahciport_mutex should be acquired before the function
3216 * is called.
3217 */
3218static void
3219ahci_dealloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3220{
3221	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
3222	    "ahci_dealloc_rcvd_fis: port %d enter",
3223	    ahci_portp->ahciport_port_num);
3224
3225	/* Unbind the cmd list dma handle first. */
3226	(void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle);
3227
3228	/* Then free the underlying memory. */
3229	ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
3230
3231	/* Now free the handle itself. */
3232	ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
3233}
3234
3235/*
3236 * Allocates memory for the Command List, which contains up to 32 entries.
3237 * Each entry contains a command header, which is a 32-byte structure that
3238 * includes the pointer to the command table.
3239 *
3240 * WARNING!!! ahciport_mutex should be acquired before the function
3241 * is called.
3242 */
3243static int
3244ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
3245    uint8_t port)
3246{
3247	size_t cmd_list_size;
3248	size_t ret_len;
3249	ddi_dma_cookie_t cmd_list_dma_cookie;
3250	uint_t cookie_count;
3251	uint32_t cap_status;
3252
3253	cmd_list_size =
3254	    ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t);
3255
3256	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3257	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
3258
3259	ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr;
3260
3261	/*
3262	 * If 64-bit addressing is not supported,
3263	 * change dma_attr_addr_hi of ahcictl_cmd_list_dma_attr
3264	 */
3265	if (!(cap_status & AHCI_HBA_CAP_S64A)) {
3266		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
3267		    "hba does not support 64-bit addressing");
3268		ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi =
3269		    0xffffffffull;
3270	}
3271
3272	/* allocate cmd list dma handle. */
3273	if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
3274	    &ahci_ctlp->ahcictl_cmd_list_dma_attr,
3275	    DDI_DMA_SLEEP,
3276	    NULL,
3277	    &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) {
3278
3279		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
3280		    "cmd list dma handle alloc failed");
3281		return (AHCI_FAILURE);
3282	}
3283
3284	if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle,
3285	    cmd_list_size,
3286	    &accattr,
3287	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
3288	    DDI_DMA_SLEEP,
3289	    NULL,
3290	    (caddr_t *)&ahci_portp->ahciport_cmd_list,
3291	    &ret_len,
3292	    &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) {
3293
3294		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
3295		    "cmd list dma mem alloc fail");
3296		/* error.. free the dma handle. */
3297		ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
3298		return (AHCI_FAILURE);
3299	}
3300
3301	if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle,
3302	    NULL,
3303	    (caddr_t)ahci_portp->ahciport_cmd_list,
3304	    cmd_list_size,
3305	    DDI_DMA_CONSISTENT,
3306	    DDI_DMA_SLEEP,
3307	    NULL,
3308	    &cmd_list_dma_cookie,
3309	    &cookie_count) !=  DDI_DMA_MAPPED) {
3310
3311		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
3312		    "cmd list dma handle bind fail");
3313		/*  error.. free the dma handle & free the memory. */
3314		ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
3315		ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
3316		return (AHCI_FAILURE);
3317	}
3318
3319	bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size);
3320
3321	/* Config Port Command List Base Address */
3322	ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle,
3323	    (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port),
3324	    cmd_list_dma_cookie.dmac_laddress);
3325
3326	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
3327	    cmd_list_dma_cookie.dmac_laddress);
3328
3329	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
3330	    cmd_list_dma_cookie.dmac_address);
3331
3332	if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
3333		ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
3334		return (AHCI_FAILURE);
3335	}
3336
3337	return (AHCI_SUCCESS);
3338}
3339
3340/*
3341 * Deallocates the Command List
3342 *
3343 * WARNING!!! ahciport_mutex should be acquired before the function
3344 * is called.
3345 */
3346static void
3347ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3348{
3349	/* First dealloc command table */
3350	ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp);
3351
3352	/* Unbind the cmd list dma handle first. */
3353	(void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
3354
3355	/* Then free the underlying memory. */
3356	ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
3357
3358	/* Now free the handle itself. */
3359	ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
3360}
3361
3362/*
3363 * Allocates memory for all Command Tables, which contains Command FIS,
3364 * ATAPI Command and Physical Region Descriptor Table.
3365 *
3366 * WARNING!!! ahciport_mutex should be acquired before the function
3367 * is called.
3368 */
3369static int
3370ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3371{
3372	size_t ret_len;
3373	ddi_dma_cookie_t cmd_table_dma_cookie;
3374	uint_t cookie_count;
3375	uint32_t cap_status;
3376	int slot;
3377
3378	AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3379	    "ahci_alloc_cmd_tables: port %d enter",
3380	    ahci_portp->ahciport_port_num);
3381
3382	/* Check whether the HBA can access 64-bit data structures */
3383	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3384	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
3385
3386	ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr;
3387
3388	/*
3389	 * If 64-bit addressing is not supported,
3390	 * change dma_attr_addr_hi of ahcictl_cmd_table_dma_attr
3391	 */
3392	if (!(cap_status & AHCI_HBA_CAP_S64A)) {
3393		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
3394		    "hba does not support 64-bit addressing");
3395		ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi =
3396		    0xffffffffull;
3397	}
3398
3399	for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
3400		/* Allocate cmd table dma handle. */
3401		if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
3402		    &ahci_ctlp->ahcictl_cmd_table_dma_attr,
3403		    DDI_DMA_SLEEP,
3404		    NULL,
3405		    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) !=
3406		    DDI_SUCCESS) {
3407
3408			AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
3409			    "cmd table dma handle alloc failed");
3410
3411			goto err_out;
3412		}
3413
3414		if (ddi_dma_mem_alloc(
3415		    ahci_portp->ahciport_cmd_tables_dma_handle[slot],
3416		    ahci_cmd_table_size,
3417		    &accattr,
3418		    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
3419		    DDI_DMA_SLEEP,
3420		    NULL,
3421		    (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot],
3422		    &ret_len,
3423		    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) !=
3424		    NULL) {
3425
3426			AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
3427			    "cmd table dma mem alloc fail");
3428
3429			/* error.. free the dma handle. */
3430			ddi_dma_free_handle(
3431			    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
3432			goto err_out;
3433		}
3434
3435		if (ddi_dma_addr_bind_handle(
3436		    ahci_portp->ahciport_cmd_tables_dma_handle[slot],
3437		    NULL,
3438		    (caddr_t)ahci_portp->ahciport_cmd_tables[slot],
3439		    ahci_cmd_table_size,
3440		    DDI_DMA_CONSISTENT,
3441		    DDI_DMA_SLEEP,
3442		    NULL,
3443		    &cmd_table_dma_cookie,
3444		    &cookie_count) !=  DDI_DMA_MAPPED) {
3445
3446			AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
3447			    "cmd table dma handle bind fail");
3448			/*  error.. free the dma handle & free the memory. */
3449			ddi_dma_mem_free(
3450			    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
3451			ddi_dma_free_handle(
3452			    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
3453			goto err_out;
3454		}
3455
3456		bzero((void *)ahci_portp->ahciport_cmd_tables[slot],
3457		    ahci_cmd_table_size);
3458
3459		/* Config Port Command Table Base Address */
3460		SET_COMMAND_TABLE_BASE_ADDR(
3461		    (&ahci_portp->ahciport_cmd_list[slot]),
3462		    cmd_table_dma_cookie.dmac_laddress & 0xffffffffull);
3463
3464#ifndef __lock_lint
3465		SET_COMMAND_TABLE_BASE_ADDR_UPPER(
3466		    (&ahci_portp->ahciport_cmd_list[slot]),
3467		    cmd_table_dma_cookie.dmac_laddress >> 32);
3468#endif /* __lock_lint */
3469	}
3470
3471	return (AHCI_SUCCESS);
3472err_out:
3473
3474	for (slot--; slot >= 0; slot--) {
3475		/* Unbind the cmd table dma handle first */
3476		(void) ddi_dma_unbind_handle(
3477		    ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
3478
3479		/* Then free the underlying memory */
3480		ddi_dma_mem_free(
3481		    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
3482
3483		/* Now free the handle itself */
3484		ddi_dma_free_handle(
3485		    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
3486	}
3487
3488	return (AHCI_FAILURE);
3489}
3490
3491/*
3492 * Deallocates memory for all Command Tables.
3493 *
3494 * WARNING!!! ahciport_mutex should be acquired before the function
3495 * is called.
3496 */
3497static void
3498ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3499{
3500	int slot;
3501
3502	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
3503	    "ahci_dealloc_cmd_tables: %d enter",
3504	    ahci_portp->ahciport_port_num);
3505
3506	for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
3507		/* Unbind the cmd table dma handle first. */
3508		(void) ddi_dma_unbind_handle(
3509		    ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
3510
3511		/* Then free the underlying memory. */
3512		ddi_dma_mem_free(
3513		    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
3514
3515		/* Now free the handle itself. */
3516		ddi_dma_free_handle(
3517		    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
3518	}
3519}
3520
3521/*
3522 * WARNING!!! ahciport_mutex should be acquired before the function
3523 * is called.
3524 */
3525static void
3526ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port,
3527    sata_device_t *sd)
3528{
3529	sd->satadev_scr.sstatus =
3530	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3531	    (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port)));
3532	sd->satadev_scr.serror =
3533	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3534	    (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port)));
3535	sd->satadev_scr.scontrol =
3536	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3537	    (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port)));
3538	sd->satadev_scr.sactive =
3539	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3540	    (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port)));
3541}
3542
3543/*
3544 * Set the auto sense data for ATAPI devices.
3545 *
3546 * Note: Currently the sense data is simulated; this code will be enhanced
3547 * in second phase to fetch the real sense data from the atapi device.
3548 */
3549static void
3550ahci_set_sense_data(sata_pkt_t *satapkt, int reason)
3551{
3552	struct scsi_extended_sense *sense;
3553
3554	sense = (struct scsi_extended_sense *)
3555	    satapkt->satapkt_cmd.satacmd_rqsense;
3556	bzero(sense, sizeof (struct scsi_extended_sense));
3557	sense->es_valid = 1;		/* Valid sense */
3558	sense->es_class = 7;		/* Response code 0x70 - current err */
3559	sense->es_key = 0;
3560	sense->es_info_1 = 0;
3561	sense->es_info_2 = 0;
3562	sense->es_info_3 = 0;
3563	sense->es_info_4 = 0;
3564	sense->es_add_len = 6;		/* Additional length */
3565	sense->es_cmd_info[0] = 0;
3566	sense->es_cmd_info[1] = 0;
3567	sense->es_cmd_info[2] = 0;
3568	sense->es_cmd_info[3] = 0;
3569	sense->es_add_code = 0;
3570	sense->es_qual_code = 0;
3571
3572	if ((reason == SATA_PKT_DEV_ERROR) || (reason == SATA_PKT_TIMEOUT)) {
3573		sense->es_key = KEY_HARDWARE_ERROR;
3574	}
3575}
3576
3577/*
3578 * Interrupt service handler
3579 */
3580/* ARGSUSED1 */
3581static uint_t
3582ahci_intr(caddr_t arg1, caddr_t arg2)
3583{
3584	ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1;
3585	ahci_port_t *ahci_portp;
3586	int32_t global_intr_status, port_intr_status;
3587	int32_t port_intr_enable;
3588	uint8_t port;
3589
3590	/*
3591	 * global_intr_status indicates that the corresponding port has
3592	 * an interrupt pending.
3593	 */
3594	global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3595	    (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp));
3596
3597	AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
3598	    "ahci_intr enter: global_intr_status = 0x%x", global_intr_status);
3599
3600	if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) {
3601		/* The interrupt is not ours */
3602		return (DDI_INTR_UNCLAIMED);
3603	}
3604
3605	/* Loop for all the ports */
3606	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3607		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3608			continue;
3609		}
3610
3611		if (!((0x1 << port) & global_intr_status)) {
3612			continue;
3613		}
3614
3615		ahci_portp = ahci_ctlp->ahcictl_ports[port];
3616
3617		/*
3618		 * port_intr_stats indicates that the corresponding interrupt
3619		 * condition is active.
3620		 */
3621		port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3622		    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
3623
3624		/*
3625		 * port_intr_enable indicates that the corresponding interrrupt
3626		 * reporting is enabled.
3627		 */
3628		port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3629		    (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
3630
3631		AHCIDBG3(AHCIDBG_INTR, ahci_ctlp,
3632		    "ahci_intr: port %d, port_intr_status = 0x%x, "
3633		    "port_intr_enable = 0x%x", port,
3634		    port_intr_status, port_intr_enable);
3635
3636		port_intr_status &= port_intr_enable;
3637
3638		/* First clear the port interrupts status */
3639		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3640		    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
3641		    port_intr_status);
3642
3643		if (port_intr_status & AHCI_INTR_STATUS_DHRS) {
3644			(void) ahci_intr_d2h_register_fis(ahci_ctlp, ahci_portp,
3645			    port);
3646		}
3647
3648		if (port_intr_status & AHCI_INTR_STATUS_PSS) {
3649			(void) ahci_intr_pio_setup_fis(ahci_ctlp, ahci_portp,
3650			    port);
3651		}
3652
3653		if (port_intr_status & AHCI_INTR_STATUS_DSS) {
3654			(void) ahci_intr_dma_setup_fis(ahci_ctlp, ahci_portp,
3655			    port);
3656		}
3657
3658		if (port_intr_status & AHCI_INTR_STATUS_SDBS) {
3659			(void) ahci_intr_set_device_bits_fis(ahci_ctlp,
3660			    ahci_portp, port);
3661		}
3662
3663		if (port_intr_status & AHCI_INTR_STATUS_UFS) {
3664			(void) ahci_intr_unknown_fis(ahci_ctlp, ahci_portp,
3665			    port);
3666		}
3667
3668		if (port_intr_status & AHCI_INTR_STATUS_DPS) {
3669			(void) ahci_intr_descriptor_processed(ahci_ctlp,
3670			    ahci_portp, port);
3671		}
3672
3673		if (port_intr_status & AHCI_INTR_STATUS_PCS) {
3674			(void) ahci_intr_port_connect_change(ahci_ctlp,
3675			    ahci_portp, port);
3676		}
3677
3678		if (port_intr_status & AHCI_INTR_STATUS_DMPS) {
3679			(void) ahci_intr_device_mechanical_presence_status(
3680			    ahci_ctlp, ahci_portp, port);
3681		}
3682
3683		if (port_intr_status & AHCI_INTR_STATUS_PRCS) {
3684			(void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp,
3685			    port);
3686		}
3687
3688		if (port_intr_status & AHCI_INTR_STATUS_IPMS) {
3689			(void) ahci_intr_incorrect_port_multiplier(ahci_ctlp,
3690			    ahci_portp, port);
3691		}
3692
3693		if (port_intr_status & AHCI_INTR_STATUS_OFS) {
3694			(void) ahci_intr_overflow(ahci_ctlp, ahci_portp, port);
3695		}
3696
3697		if (port_intr_status & AHCI_INTR_STATUS_INFS) {
3698			(void) ahci_intr_interface_non_fatal_error(ahci_ctlp,
3699			    ahci_portp, port);
3700		}
3701
3702		if (port_intr_status & AHCI_INTR_STATUS_IFS) {
3703			(void) ahci_intr_interface_fatal_error(ahci_ctlp,
3704			    ahci_portp, port);
3705		}
3706
3707		if (port_intr_status & AHCI_INTR_STATUS_HBDS) {
3708			(void) ahci_intr_host_bus_data_error(ahci_ctlp,
3709			    ahci_portp, port);
3710		}
3711
3712		if (port_intr_status & AHCI_INTR_STATUS_HBFS) {
3713			(void) ahci_intr_host_bus_fatal_error(ahci_ctlp,
3714			    ahci_portp, port);
3715		}
3716
3717		if (port_intr_status & AHCI_INTR_STATUS_TFES) {
3718			(void) ahci_intr_task_file_error(ahci_ctlp, ahci_portp,
3719			    port);
3720		}
3721
3722		if (port_intr_status & AHCI_INTR_STATUS_CPDS) {
3723			(void) ahci_intr_cold_port_detect(ahci_ctlp,
3724			    ahci_portp, port);
3725		}
3726
3727		/* Second clear the corresponding bit in IS.IPS */
3728		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3729		    (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port));
3730	}
3731
3732	return (DDI_INTR_CLAIMED);
3733}
3734
3735/*
3736 * A D2H Register FIS has been received with the 'I' bit set.
3737 *
3738 * The following commands will send this FIS upon the successful completion:
3739 * 	1. Non-data commands
3740 * 	2. DMA data-in command
3741 * 	3. DMA data-out command
3742 * 	4. PIO data-out command
3743 */
3744static int
3745ahci_intr_d2h_register_fis(ahci_ctl_t *ahci_ctlp,
3746    ahci_port_t *ahci_portp, uint8_t port)
3747{
3748	uint32_t port_cmd_issue;
3749	uint32_t finished_tags;
3750	int finished_slot, i, num = 0;
3751	sata_pkt_t *satapkt;
3752	ahci_fis_d2h_register_t *rcvd_fisp;
3753
3754	AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
3755	    "ahci_intr_d2h_register_fis enter, port %d", port);
3756
3757	mutex_enter(&ahci_portp->ahciport_mutex);
3758
3759	port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3760	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3761
3762	if (!ahci_portp->ahciport_pending_tags) {
3763		/*
3764		 * Spurious interrupt. Nothing to be done.
3765		 */
3766		mutex_exit(&ahci_portp->ahciport_mutex);
3767		return (AHCI_SUCCESS);
3768	}
3769
3770	finished_tags = ahci_portp->ahciport_pending_tags &
3771	    ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
3772
3773	AHCIDBG3(AHCIDBG_INTR, ahci_ctlp,
3774	    "ahci_intr_d2h_reigster_fis: pending_tags = 0x%x, "
3775	    "port_cmd_issue = 0x%x, finished_tags = 0x%x,",
3776	    ahci_portp->ahciport_pending_tags, port_cmd_issue,
3777	    finished_tags);
3778
3779	for (i = 0; i < ahci_ctlp->ahcictl_num_cmd_slots; i++) {
3780		if ((0x1 << i) & finished_tags)
3781			num++;
3782	}
3783
3784	ASSERT(num <= 1);
3785
3786	finished_slot = ddi_ffs(finished_tags) - 1;
3787	if (finished_slot == -1) {
3788		goto out;
3789	}
3790
3791	satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
3792	ASSERT(satapkt != NULL);
3793
3794	/*
3795	 * In case the interrupt generates before disabling for poll commands.
3796	 */
3797	if (satapkt->satapkt_op_mode &
3798	    (SATA_OPMODE_POLLING | SATA_OPMODE_SYNCH)) {
3799		goto out;
3800	}
3801
3802	rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->ahcirf_d2h_register_fis);
3803
3804	/* Return the status */
3805	satapkt->satapkt_cmd.satacmd_status_reg = GET_RFIS_STATUS(rcvd_fisp);
3806
3807	if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs)
3808		ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
3809
3810	AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
3811	    "ahci_intr_d2h_register_fis: sending up pkt 0x%p "
3812	    "with SATA_PKT_COMPLETED", (void *)satapkt);
3813
3814	CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot);
3815	ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
3816
3817	SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED);
3818out:
3819	AHCIDBG1(AHCIDBG_PKTCOMP, ahci_ctlp,
3820	    "ahci_intr_d2h_register_fis pending_tags = 0x%x",
3821	    ahci_portp->ahciport_pending_tags);
3822
3823	mutex_exit(&ahci_portp->ahciport_mutex);
3824
3825	return (AHCI_SUCCESS);
3826}
3827
3828/*
3829 * A PIO Setup FIS has been received with the 'I' bit set.
3830 *
3831 * The following commands will send this FIS upon the successful completion:
3832 * 	PIO data-in command
3833 */
3834static int
3835ahci_intr_pio_setup_fis(ahci_ctl_t *ahci_ctlp,
3836    ahci_port_t *ahci_portp, uint8_t port)
3837{
3838	uint32_t port_cmd_issue;
3839	uint32_t finished_tags;
3840	int finished_slot, i, num = 0;
3841	sata_pkt_t *satapkt;
3842	ahci_fis_d2h_register_t *rcvd_fisp;
3843
3844	AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
3845	    "ahci_intr_pio_setup_fis enter, port %d", port);
3846
3847	mutex_enter(&ahci_portp->ahciport_mutex);
3848
3849	port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3850	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3851
3852	if (!ahci_portp->ahciport_pending_tags) {
3853		/*
3854		 * Spurious interrupt. Nothing to be done.
3855		 */
3856		mutex_exit(&ahci_portp->ahciport_mutex);
3857		return (AHCI_SUCCESS);
3858	}
3859
3860	finished_tags = ahci_portp->ahciport_pending_tags &
3861	    ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
3862
3863	AHCIDBG3(AHCIDBG_INTR, ahci_ctlp,
3864	    "ahci_intr_pio_setup_fis: pending_tags = 0x%x, "
3865	    "port_cmd_issue = 0x%x, finished_tags = 0x%x,",
3866	    ahci_portp->ahciport_pending_tags, port_cmd_issue,
3867	    finished_tags);
3868
3869	for (i = 0; i < ahci_ctlp->ahcictl_num_cmd_slots; i++) {
3870		if ((0x1 << i) & finished_tags)
3871			num++;
3872	}
3873
3874	ASSERT(num <= 1);
3875
3876	finished_slot = ddi_ffs(finished_tags) - 1;
3877	if (finished_slot == -1) {
3878		goto out;
3879	}
3880
3881	satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
3882	ASSERT(satapkt != NULL);
3883
3884	/*
3885	 * In case the interrupt generates before disabling.
3886	 */
3887	if (satapkt->satapkt_op_mode &
3888	    (SATA_OPMODE_POLLING | SATA_OPMODE_SYNCH)) {
3889		goto out;
3890	}
3891
3892	rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->ahcirf_d2h_register_fis);
3893
3894	/* Return the status */
3895	satapkt->satapkt_cmd.satacmd_status_reg = GET_RFIS_STATUS(rcvd_fisp);
3896
3897	if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs)
3898		ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
3899
3900	AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
3901	    "ahci_intr_pio_setup_fis: sending up pkt 0x%p "
3902	    "with SATA_PKT_COMPLETED", (void *)satapkt);
3903
3904	CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot);
3905	ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
3906
3907	SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED);
3908out:
3909	AHCIDBG1(AHCIDBG_PKTCOMP, ahci_ctlp,
3910	    "ahci_intr_pio_setup_fis pending_tags = 0x%x",
3911	    ahci_portp->ahciport_pending_tags);
3912
3913	mutex_exit(&ahci_portp->ahciport_mutex);
3914
3915	return (AHCI_SUCCESS);
3916}
3917
3918/*
3919 * A DMA Setup FIS has been received with the 'I' bit set
3920 *
3921 * DMA Setup FIS will be used during NCQ transfers to activate data
3922 * transfer.
3923 *
3924 * For the first phase, this interrupt is disabled and we just
3925 * log a debug message.
3926 */
3927/* ARGSUSED */
3928static int
3929ahci_intr_dma_setup_fis(ahci_ctl_t *ahci_ctlp,
3930    ahci_port_t *ahci_portp, uint8_t port)
3931{
3932	AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
3933	    "ahci_intr_dma_setup_fis enter, port %d", port);
3934
3935	return (AHCI_SUCCESS);
3936}
3937
3938/*
3939 * A Set Device Bits FIS has been received with the 'I' bit set
3940 *
3941 * Set Device Bits FIS will be sent during NCQ completion.
3942 *
3943 * For the first phase, this interrupt is disabled and we just log
3944 * the debug message.
3945 */
3946/* ARGSUSED */
3947static int
3948ahci_intr_set_device_bits_fis(ahci_ctl_t *ahci_ctlp,
3949    ahci_port_t *ahci_portp, uint8_t port)
3950{
3951	AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
3952	    "ahci_intr_set_device_bits_fis enter, port %d", port);
3953
3954	return (AHCI_SUCCESS);
3955}
3956
3957/*
3958 * An unknown FIS was received and has been copied into system memory.
3959 *
3960 * An unknown FIS is not considered an illegal FIS, unless the length
3961 * received is more than 64 bytes. If an unknown FIS arrives with length
3962 * <= 64 bytes, it is posted and the HBA continues normal operation.
3963 *
3964 * Therefore we just need to clear PxSERR.DIAG.F to clear the intr bit
3965 * and log the debug message.
3966 */
3967static int
3968ahci_intr_unknown_fis(ahci_ctl_t *ahci_ctlp,
3969    ahci_port_t *ahci_portp, uint8_t port)
3970{
3971	uint32_t port_serror;
3972
3973	mutex_enter(&ahci_portp->ahciport_mutex);
3974
3975	port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3976	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
3977
3978	AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
3979	    "ahci_intr_unknown_fis: port %d, port_serror = 0x%x",
3980	    port, port_serror);
3981
3982	/* Clear the interrupt bit by clearing PxSERR.DIAG.F */
3983	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3984	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
3985	    AHCI_SERROR_DIAG_F);
3986
3987	mutex_exit(&ahci_portp->ahciport_mutex);
3988
3989	return (AHCI_SUCCESS);
3990}
3991
3992/*
3993 * A PRD with the 'I' bit set has tranfered all of its data.
3994 *
3995 * The PRD Interrupt is an opportunistic interrupt. And it should not
3996 * be used to definitively indicate the end of a transfer since two
3997 * PRD interrupts could happen close in time together such that the
3998 * second interrupt is missed when the first PRD interrupt is being
3999 * cleared.
4000 *
4001 * For the first phase, this interrupt is disabled and we just
4002 * log the debug message.
4003 */
4004/* ARGSUSED */
4005static int
4006ahci_intr_descriptor_processed(ahci_ctl_t *ahci_ctlp,
4007    ahci_port_t *ahci_portp, uint8_t port)
4008{
4009	AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
4010	    "ahci_intr_descriptor_processed enter, port %d", port);
4011
4012	return (AHCI_SUCCESS);
4013}
4014
4015/*
4016 * 1=Change in Current Connect Status. 0=No change in Current Connect Status.
4017 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared
4018 * when PxSERR.DIAG.X is cleared.
4019 *
4020 * When PxSERR.DIAG.X is set to one, it indicates a COMINIT signal was received.
4021 *
4022 * For the first phase, this interrupt is disabled.
4023 */
4024static int
4025ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp,
4026    ahci_port_t *ahci_portp, uint8_t port)
4027{
4028	uint32_t port_serror, port_cmd_status;
4029
4030	mutex_enter(&ahci_portp->ahciport_mutex);
4031
4032	port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4033	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
4034
4035	AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
4036	    "ahci_intr_port_connect_change: port %d, "
4037	    "port_serror = 0x%x", port, port_serror);
4038
4039	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4040	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4041
4042	AHCIDBG2(AHCIDBG_INTR, ahci_ctlp,
4043	    "ahci_intr_port_connect_change: port %d, "
4044	    "port_command_status = 0x%x", port, port_cmd_status);
4045
4046	/* Clear PxSERR.DIAG.X to clear the interrupt bit */
4047	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4048	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
4049	    AHCI_SERROR_DIAG_X);
4050
4051	mutex_exit(&ahci_portp->ahciport_mutex);
4052
4053	return (AHCI_SUCCESS);
4054}
4055
4056/*
4057 * Hot Plug Operation for platforms that support Mechanical Presence
4058 * Switches.
4059 *
4060 * When set, it indicates that a mechanical presence switch attached to this
4061 * port has been opened or closed, which may lead to a change in the connection
4062 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP
4063 * are set to '1'.
4064 *
4065 * For the first phase, this interrupt is disabled and we just log
4066 * the debug message.
4067 */
4068/* ARGSUSED */
4069static int
4070ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp,
4071    ahci_port_t *ahci_portp, uint8_t port)
4072{
4073	uint32_t cap_status, port_cmd_status;
4074
4075	AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
4076	    "ahci_intr_device_mechanical_presence_status enter, "
4077	    "port %d", port);
4078
4079	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4080	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
4081
4082	mutex_enter(&ahci_portp->ahciport_mutex);
4083	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4084	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4085
4086	if (!(cap_status & AHCI_HBA_CAP_SMPS) ||
4087	    !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) {
4088		AHCIDBG2(AHCIDBG_INTR, ahci_ctlp,
4089		    "CAP.SMPS or PxCMD.MPSP is not set, so just ignore "
4090		    "the interrupt: cap_status = 0x%x, "
4091		    "port_cmd_status = 0x%x", cap_status, port_cmd_status);
4092		mutex_exit(&ahci_portp->ahciport_mutex);
4093
4094		return (AHCI_SUCCESS);
4095	}
4096
4097	if (port_cmd_status & AHCI_CMD_STATUS_MPSS) {
4098		AHCIDBG2(AHCIDBG_INTR, ahci_ctlp,
4099		    "The mechanical presence switch is open: "
4100		    "port %d, port_cmd_status = 0x%x",
4101		    port, port_cmd_status);
4102	} else {
4103		AHCIDBG2(AHCIDBG_INTR, ahci_ctlp,
4104		    "The mechanical presence switch is close: "
4105		    "port %d, port_cmd_status = 0x%x",
4106		    port, port_cmd_status);
4107	}
4108
4109	mutex_exit(&ahci_portp->ahciport_mutex);
4110
4111	return (AHCI_SUCCESS);
4112}
4113
4114/*
4115 * Native Hot Plug Support.
4116 *
4117 * When set, it indicates that the internal PHYRDY signal changed state.
4118 * This bit reflects the state of PxSERR.DIAG.N.
4119 */
4120static int
4121ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp,
4122    ahci_port_t *ahci_portp, uint8_t port)
4123{
4124	uint32_t port_sstatus = 0; /* No dev present & PHY not established. */
4125	sata_device_t sdevice;
4126	int dev_exists_now = 0;
4127	int dev_existed_previously = 0;
4128
4129	AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
4130	    "ahci_intr_phyrdy_change enter, port %d", port);
4131
4132	/* Clear PxSERR.DIAG.N to clear the interrupt bit */
4133	mutex_enter(&ahci_portp->ahciport_mutex);
4134	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4135	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
4136	    AHCI_SERROR_DIAG_N);
4137	mutex_exit(&ahci_portp->ahciport_mutex);
4138
4139	mutex_enter(&ahci_ctlp->ahcictl_mutex);
4140	if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) ||
4141	    (ahci_portp == NULL)) {
4142		/* The whole controller setup is not yet done. */
4143		mutex_exit(&ahci_ctlp->ahcictl_mutex);
4144		return (AHCI_SUCCESS);
4145	}
4146	mutex_exit(&ahci_ctlp->ahcictl_mutex);
4147
4148	mutex_enter(&ahci_portp->ahciport_mutex);
4149
4150	/* SStatus tells the presence of device. */
4151	port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4152	    (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
4153
4154	dev_exists_now = (AHCI_SSTATUS_GET_DET(port_sstatus) ==
4155	    AHCI_SSTATUS_DET_DEVPRESENT_PHYONLINE);
4156
4157	if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
4158		dev_existed_previously = 1;
4159	}
4160
4161	bzero((void *)&sdevice, sizeof (sata_device_t));
4162	sdevice.satadev_addr.cport = port;
4163	sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
4164	sdevice.satadev_addr.pmport = 0;
4165	sdevice.satadev_state = SATA_PSTATE_PWRON;
4166	ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON;
4167
4168	if (dev_exists_now) {
4169		if (dev_existed_previously) {
4170			/* Things are fine now. The loss was temporary. */
4171			cmn_err(CE_NOTE, "!ahci phyrdy: port %d "
4172			    "device link lost/established", port);
4173
4174			mutex_exit(&ahci_portp->ahciport_mutex);
4175			sata_hba_event_notify(
4176			    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
4177			    &sdevice,
4178			    SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED);
4179			mutex_enter(&ahci_portp->ahciport_mutex);
4180
4181		} else {
4182			cmn_err(CE_NOTE, "!ahci phyrdy: port %d "
4183			    "device link established", port);
4184
4185			/* A new device has been detected. */
4186			if (ahci_find_dev_signature(ahci_ctlp, ahci_portp, port)
4187			    != AHCI_SUCCESS) {
4188				ahci_portp->ahciport_port_state =
4189				    SATA_PSTATE_FAILED;
4190				AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
4191				    "phyrdy: port %d failed "
4192				    "at find dev signature", port);
4193			}
4194
4195			if ((ahci_portp->ahciport_device_type
4196			    == SATA_DTYPE_ATADISK) &&
4197			    (ahci_start_port(ahci_ctlp, port)
4198			    != AHCI_SUCCESS)) {
4199				ahci_portp->ahciport_port_state =
4200				    SATA_PSTATE_FAILED;
4201				AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
4202				    "phyrdy: port %d failed "
4203				    "at start port", port);
4204			}
4205
4206			mutex_exit(&ahci_portp->ahciport_mutex);
4207			sata_hba_event_notify(
4208			    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
4209			    &sdevice,
4210			    SATA_EVNT_LINK_ESTABLISHED);
4211			mutex_enter(&ahci_portp->ahciport_mutex);
4212
4213		}
4214	} else { /* No device exists now */
4215
4216		if (dev_existed_previously) {
4217
4218			cmn_err(CE_NOTE, "!ahci phyrdy: port %d "
4219			    "device link lost", port);
4220
4221			ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
4222			(void) ahci_port_into_notrunning_state(ahci_ctlp,
4223			    ahci_portp, port);
4224
4225			/* An existing device is lost. */
4226			ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
4227
4228			mutex_exit(&ahci_portp->ahciport_mutex);
4229			sata_hba_event_notify(
4230			    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
4231			    &sdevice,
4232			    SATA_EVNT_LINK_LOST);
4233			mutex_enter(&ahci_portp->ahciport_mutex);
4234
4235		} else {
4236
4237			/* Spurious interrupt */
4238			AHCIDBG0(AHCIDBG_INTR, ahci_ctlp,
4239			    "spurious phy ready interrupt");
4240		}
4241	}
4242
4243	mutex_exit(&ahci_portp->ahciport_mutex);
4244
4245	return (AHCI_SUCCESS);
4246}
4247
4248/*
4249 * PxIS.IPMS - Port Multiplier Errors.
4250 *
4251 * Which indicates that the HBA received a FIS from a device whose Port
4252 * Multiplier field did not match what was expected.
4253 *
4254 * For the first phase, this interrupt is disabled, and we just log
4255 * the debug message.
4256 */
4257/* ARGSUSED */
4258static int
4259ahci_intr_incorrect_port_multiplier(ahci_ctl_t *ahci_ctlp,
4260    ahci_port_t *ahci_portp, uint8_t port)
4261{
4262	AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
4263	    "ahci_intr_incorrect_port_multiplier enter, port %d", port);
4264
4265	return (AHCI_SUCCESS);
4266}
4267
4268/*
4269 * PxIS.OFS - Fatal Error
4270 *
4271 * Which indicates that the HBA received more bytes from a device than
4272 * was specified in the PRD table for the command.
4273 *
4274 * Overflow is a serious error, thus software should perform a fatal error
4275 * recovery procedure to ensure that the HBA is brought back to a known
4276 * condition before continuing.
4277 */
4278static int
4279ahci_intr_overflow(ahci_ctl_t *ahci_ctlp,
4280    ahci_port_t *ahci_portp, uint8_t port)
4281{
4282	AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
4283	    "ahci_intr_overflow enter, port %d", port);
4284
4285	mutex_enter(&ahci_portp->ahciport_mutex);
4286
4287	cmn_err(CE_NOTE, "!ahci port %d has overflow error", port);
4288
4289	(void) ahci_recovery_fatal_error(ahci_ctlp, ahci_portp,
4290	    port, AHCI_INTR_STATUS_OFS);
4291
4292	mutex_exit(&ahci_portp->ahciport_mutex);
4293
4294	return (AHCI_SUCCESS);
4295}
4296
4297/*
4298 * PxIS.IFS and PxIS.INFS are errors that occur due to eletrical issues
4299 * on the interface, or protocol miscommunication between the device
4300 * and HBA.
4301 *
4302 * The conditions that causes PxIS.IFS/PxIS.INFS to be set are
4303 * 	1. in PxSERR.ERR, P bit is set to '1'
4304 *	2. in PxSERR.DIAG, C or H bit is set to '1'
4305 *	3. PhyRdy drop unexpectly, N bit is set to '1'
4306 */
4307/*
4308 * PxIS.INFS - Interface Error
4309 *
4310 * Which indicates that the HBA encountered an error on the Serial ATA interface
4311 * but was able to continue operation. It's non-fatal error, so we just log
4312 * the debug message.
4313 *
4314 * The error occurred during a non-Data FIS, and the FIS will be
4315 * re-transmitted by HBA automatically.
4316 */
4317static int
4318ahci_intr_interface_non_fatal_error(ahci_ctl_t *ahci_ctlp,
4319    ahci_port_t *ahci_portp, uint8_t port)
4320{
4321	sata_pkt_t *satapkt;
4322	uint32_t port_serror;
4323	uint32_t port_cmd_status;
4324	int current_slot;
4325
4326	mutex_enter(&ahci_portp->ahciport_mutex);
4327
4328	port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4329	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
4330
4331	AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
4332	    "ahci_intr_interface_non_fatal_error: port %d, "
4333	    "port_serror = 0x%x", port, port_serror);
4334
4335	ahci_log_error_message(ahci_ctlp, port, port_serror);
4336
4337	cmn_err(CE_NOTE, "!ahci port %d has interface non fatal error", port);
4338
4339	/*
4340	 * Record the error occurred command's slot.
4341	 */
4342#if AHCI_DEBUG
4343	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4344	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4345
4346	current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
4347	    AHCI_CMD_STATUS_CCS_SHIFT;
4348
4349	satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
4350
4351	if (satapkt != NULL) {
4352		AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
4353		    "ahci_intr_interface_non_fatal_error: "
4354		    "pending_tags= 0x%x ",
4355		    ahci_portp->ahciport_pending_tags);
4356
4357		AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
4358		    "ahci_intr_interface_non_fatal_error: cmd= 0x%x ",
4359		    satapkt->satapkt_cmd.satacmd_cmd_reg);
4360
4361		AHCIDBG2(AHCIDBG_INTR, ahci_ctlp,
4362		    "ahci_intr_interface_non_fatal_error: port %d, "
4363		    "satapkt = %p will be re-transmitted",
4364		    port, (void *)satapkt);
4365	}
4366#endif
4367	mutex_exit(&ahci_portp->ahciport_mutex);
4368
4369	return (AHCI_SUCCESS);
4370}
4371
4372/*
4373 * PxIS.IFS - Interface Error and Fatal Error
4374 *
4375 * Which indicates that the HBA encountered an error on the Serial ATA interface
4376 * which caused the transfer to stop.
4377 *
4378 * The error occurred during a Data FIS, the transfer shall stop.
4379 */
4380static int
4381ahci_intr_interface_fatal_error(ahci_ctl_t *ahci_ctlp,
4382    ahci_port_t *ahci_portp, uint8_t port)
4383{
4384	uint32_t port_serror;
4385
4386	mutex_enter(&ahci_portp->ahciport_mutex);
4387
4388	port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4389	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
4390
4391	AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
4392	    "ahci_intr_interface_fatal_error: port %d, "
4393	    "port_serror = 0x%x", port, port_serror);
4394
4395	ahci_log_error_message(ahci_ctlp, port, port_serror);
4396
4397	cmn_err(CE_NOTE, "!ahci port %d has interface fatal error", port);
4398
4399	(void) ahci_recovery_fatal_error(ahci_ctlp, ahci_portp,
4400	    port, AHCI_INTR_STATUS_IFS);
4401
4402	mutex_exit(&ahci_portp->ahciport_mutex);
4403
4404	return (AHCI_SUCCESS);
4405}
4406
4407/*
4408 * PxIS.HBDS - System Memory Error and Fatal Error
4409 *
4410 * Which indicates that the HBA encountered a data error
4411 * (uncorrectable ECC/parity) when reading from or writing to system memory.
4412 */
4413static int
4414ahci_intr_host_bus_data_error(ahci_ctl_t *ahci_ctlp,
4415    ahci_port_t *ahci_portp, uint8_t port)
4416{
4417	uint32_t port_serror;
4418
4419	mutex_enter(&ahci_portp->ahciport_mutex);
4420
4421	port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4422	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
4423
4424	AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
4425	    "ahci_intr_host_bus_data_error: port %d, "
4426	    "port_serror = 0x%x", port, port_serror);
4427
4428	ahci_log_error_message(ahci_ctlp, port, port_serror);
4429
4430	cmn_err(CE_NOTE, "!ahci port %d has bus data error", port);
4431
4432	(void) ahci_recovery_fatal_error(ahci_ctlp, ahci_portp,
4433	    port, AHCI_INTR_STATUS_HBDS);
4434
4435	mutex_exit(&ahci_portp->ahciport_mutex);
4436
4437	return (AHCI_SUCCESS);
4438}
4439
4440/*
4441 * PxIS.HBFS - System Memory Error and Fatal Error
4442 *
4443 * Which indicates that the HBA encountered a host bus error that it cannot
4444 * recover from, such as a bad software pointer.
4445 */
4446static int
4447ahci_intr_host_bus_fatal_error(ahci_ctl_t *ahci_ctlp,
4448    ahci_port_t *ahci_portp, uint8_t port)
4449{
4450	uint32_t port_serror;
4451
4452	mutex_enter(&ahci_portp->ahciport_mutex);
4453
4454	port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4455	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
4456
4457	AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
4458	    "ahci_intr_host_bus_fatal_error: port %d, "
4459	    "port_serror = 0x%x", port, port_serror);
4460
4461	ahci_log_error_message(ahci_ctlp, port, port_serror);
4462
4463	cmn_err(CE_NOTE, "!ahci port %d has bus fatal error", port);
4464
4465	(void) ahci_recovery_fatal_error(ahci_ctlp, ahci_portp,
4466	    port, AHCI_INTR_STATUS_HBFS);
4467
4468	mutex_exit(&ahci_portp->ahciport_mutex);
4469
4470	return (AHCI_SUCCESS);
4471}
4472
4473/*
4474 * PxIS.TFES - Device Error and Fatal Error
4475 *
4476 * This bit is set whenever the status register is updated by the device
4477 * and the error bit (bit 0) is set.
4478 */
4479static int
4480ahci_intr_task_file_error(ahci_ctl_t *ahci_ctlp,
4481    ahci_port_t *ahci_portp, uint8_t port)
4482{
4483	uint32_t task_file_status;
4484
4485	mutex_enter(&ahci_portp->ahciport_mutex);
4486
4487	task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4488	    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
4489
4490	AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
4491	    "ahci_intr_task_file_error: port %d: "
4492	    "task_file_status = 0x%x", port, task_file_status);
4493
4494	cmn_err(CE_NOTE, "!ahci port %d has task file error", port);
4495
4496	(void) ahci_recovery_fatal_error(ahci_ctlp, ahci_portp,
4497	    port, AHCI_INTR_STATUS_TFES);
4498
4499	mutex_exit(&ahci_portp->ahciport_mutex);
4500
4501	return (AHCI_SUCCESS);
4502}
4503
4504/*
4505 * Hot Plug Operation for platforms that support Cold Presence Detect.
4506 *
4507 * When set, a device status has changed as detected by the cold presence
4508 * detect logic. This bit can either be set due to a non-connected port
4509 * receiving a device, or a connected port having its device removed.
4510 * This bit is only valid if the port supports cold presence detect as
4511 * indicated by PxCMD.CPD set to '1'.
4512 *
4513 * For the first phase, this interrupt is disabled and we just log
4514 * the debug message.
4515 */
4516/* ARGSUSED */
4517static int
4518ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp,
4519    ahci_port_t *ahci_portp, uint8_t port)
4520{
4521	uint32_t port_cmd_status;
4522	sata_device_t sdevice;
4523
4524	AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
4525	    "ahci_intr_cold_port_detect enter, port %d", port);
4526
4527	mutex_enter(&ahci_portp->ahciport_mutex);
4528
4529	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4530	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4531	if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) {
4532		AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
4533		    "port %d does not support cold presence detect, so "
4534		    "we just ignore this interrupt", port);
4535		mutex_exit(&ahci_portp->ahciport_mutex);
4536		return (AHCI_SUCCESS);
4537	}
4538
4539	AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
4540	    "port %d device status has changed", port);
4541
4542	bzero((void *)&sdevice, sizeof (sata_device_t));
4543	sdevice.satadev_addr.cport = port;
4544	sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
4545	sdevice.satadev_addr.pmport = 0;
4546	sdevice.satadev_state = SATA_PSTATE_PWRON;
4547
4548	if (port_cmd_status & AHCI_CMD_STATUS_CPS) {
4549		AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
4550		    "port %d: a device is hot plugged", port);
4551		mutex_exit(&ahci_portp->ahciport_mutex);
4552		sata_hba_event_notify(
4553		    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
4554		    &sdevice,
4555		    SATA_EVNT_DEVICE_ATTACHED);
4556		mutex_enter(&ahci_portp->ahciport_mutex);
4557
4558	} else {
4559		AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
4560		    "port %d: a device is hot unplugged", port);
4561		mutex_exit(&ahci_portp->ahciport_mutex);
4562		sata_hba_event_notify(
4563		    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
4564		    &sdevice,
4565		    SATA_EVNT_DEVICE_DETACHED);
4566		mutex_enter(&ahci_portp->ahciport_mutex);
4567	}
4568
4569	mutex_exit(&ahci_portp->ahciport_mutex);
4570
4571	return (AHCI_SUCCESS);
4572}
4573
4574/*
4575 * Enable the interrupts for a particular port.
4576 *
4577 * WARNING!!! ahciport_mutex should be acquired before the function
4578 * is called.
4579 */
4580/* ARGSUSED */
4581static void
4582ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp,
4583    ahci_port_t *ahci_portp, uint8_t port)
4584{
4585	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
4586	    "ahci_enable_port_intrs enter, port %d", port);
4587
4588	/*
4589	 * Clear port interrupt status before enabling interrupt
4590	 */
4591	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4592	    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
4593	    AHCI_PORT_INTR_MASK);
4594
4595	/*
4596	 * Clear the pending bit from IS.IPS
4597	 */
4598	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4599	    (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port));
4600
4601	/*
4602	 * Enable the following interrupts:
4603	 *	Device to Host Register FIS Interrupt (DHRS)
4604	 *	PIO Setup FIS Interrupt (PSS)
4605	 *	Unknown FIS Interrupt (UFS)
4606	 *	Port Connect Change Status (PCS)
4607	 *	PhyRdy Change Status (PRCS)
4608	 *	Overflow Status (OFS)
4609	 *	Interface Non-fatal Error Status (INFS)
4610	 *	Interface Fatal Error Status (IFS)
4611	 *	Host Bus Data Error Status (HBDS)
4612	 *	Host Bus Fatal Error Status (HBFS)
4613	 *	Task File Error Status (TFES)
4614	 */
4615	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4616	    (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port),
4617	    (AHCI_INTR_STATUS_DHRS |
4618	    AHCI_INTR_STATUS_PSS |
4619	    AHCI_INTR_STATUS_UFS |
4620	    AHCI_INTR_STATUS_PCS |
4621	    AHCI_INTR_STATUS_PRCS |
4622	    AHCI_INTR_STATUS_OFS |
4623	    AHCI_INTR_STATUS_INFS |
4624	    AHCI_INTR_STATUS_IFS |
4625	    AHCI_INTR_STATUS_HBDS |
4626	    AHCI_INTR_STATUS_HBFS |
4627	    AHCI_INTR_STATUS_TFES));
4628}
4629
4630/*
4631 * Enable interrupts for all the ports.
4632 *
4633 * WARNING!!! ahcictl_mutex should be acquired before the function
4634 * is called.
4635 */
4636static void
4637ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp)
4638{
4639	uint32_t ghc_control;
4640
4641	AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter");
4642
4643	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4644	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
4645
4646	ghc_control |= AHCI_HBA_GHC_IE;
4647
4648	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4649	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
4650}
4651
4652/*
4653 * Disable interrupts for a particular port.
4654 *
4655 * WARNING!!! ahciport_mutex should be acquired before the function
4656 * is called.
4657 */
4658/* ARGSUSED */
4659static void
4660ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp,
4661    ahci_port_t *ahci_portp, uint8_t port)
4662{
4663	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
4664	    "ahci_disable_port_intrs enter, port %d", port);
4665
4666	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4667	    (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
4668}
4669
4670/*
4671 * Disable interrupts for the whole HBA.
4672 *
4673 * The global bit is cleared, then all interrupt sources from all
4674 * ports are disabled.
4675 *
4676 * WARNING!!! ahcictl_mutex should be acquired before the function
4677 * is called.
4678 */
4679static void
4680ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp)
4681{
4682	uint32_t ghc_control;
4683
4684	AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter");
4685
4686	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4687	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
4688
4689	ghc_control &= ~ AHCI_HBA_GHC_IE;
4690
4691	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4692	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
4693}
4694
4695/*
4696 * Handle INTx and legacy interrupts.
4697 */
4698static int
4699ahci_add_legacy_intrs(ahci_ctl_t *ahci_ctlp)
4700{
4701	dev_info_t	*dip = ahci_ctlp->ahcictl_dip;
4702	int		actual, count = 0;
4703	int		x, y, rc, inum = 0;
4704
4705	AHCIDBG0(AHCIDBG_ENTRY|AHCIDBG_INIT, ahci_ctlp,
4706	    "ahci_add_legacy_intrs enter");
4707
4708	/* get number of interrupts. */
4709	rc = ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_FIXED, &count);
4710	if ((rc != DDI_SUCCESS) || (count == 0)) {
4711		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
4712		    "ddi_intr_get_nintrs() failed, "
4713		    "rc %d count %d\n", rc, count);
4714		return (DDI_FAILURE);
4715	}
4716
4717	/* Allocate an array of interrupt handles. */
4718	ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t);
4719	ahci_ctlp->ahcictl_intr_htable =
4720	    kmem_zalloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP);
4721
4722	/* call ddi_intr_alloc(). */
4723	rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable,
4724	    DDI_INTR_TYPE_FIXED,
4725	    inum, count, &actual, DDI_INTR_ALLOC_STRICT);
4726
4727	if ((rc != DDI_SUCCESS) || (actual == 0)) {
4728		AHCIDBG1(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
4729		    "ddi_intr_alloc() failed, rc %d\n", rc);
4730		kmem_free(ahci_ctlp->ahcictl_intr_htable,
4731		    ahci_ctlp->ahcictl_intr_size);
4732		return (DDI_FAILURE);
4733	}
4734
4735	if (actual < count) {
4736		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
4737		    "Requested: %d, Received: %d", count, actual);
4738
4739		for (x = 0; x < actual; x++) {
4740			(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
4741		}
4742
4743		kmem_free(ahci_ctlp->ahcictl_intr_htable,
4744		    ahci_ctlp->ahcictl_intr_size);
4745		return (DDI_FAILURE);
4746	}
4747
4748	ahci_ctlp->ahcictl_intr_cnt = actual;
4749
4750	/* Get intr priority. */
4751	if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0],
4752	    &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) {
4753		AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
4754		    "ddi_intr_get_pri() failed");
4755
4756		for (x = 0; x < actual; x++) {
4757			(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
4758		}
4759
4760		kmem_free(ahci_ctlp->ahcictl_intr_htable,
4761		    ahci_ctlp->ahcictl_intr_size);
4762		return (DDI_FAILURE);
4763	}
4764
4765	/* Test for high level interrupt. */
4766	if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
4767		AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
4768		    "ahci_add_legacy_intrs: Hi level intr not supported");
4769
4770		for (x = 0; x < actual; x++) {
4771			(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
4772		}
4773
4774		kmem_free(ahci_ctlp->ahcictl_intr_htable,
4775		    sizeof (ddi_intr_handle_t));
4776
4777		return (DDI_FAILURE);
4778	}
4779
4780	/* Call ddi_intr_add_handler(). */
4781	for (x = 0; x < actual; x++) {
4782		if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[x],
4783		    ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) {
4784			AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
4785			    "ddi_intr_add_handler() failed");
4786
4787			for (y = 0; y < actual; y++) {
4788				(void) ddi_intr_free(
4789				    ahci_ctlp->ahcictl_intr_htable[y]);
4790			}
4791
4792			kmem_free(ahci_ctlp->ahcictl_intr_htable,
4793			    ahci_ctlp->ahcictl_intr_size);
4794			return (DDI_FAILURE);
4795		}
4796	}
4797
4798	/* Call ddi_intr_enable() for legacy interrupts. */
4799	for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
4800		(void) ddi_intr_enable(ahci_ctlp->ahcictl_intr_htable[x]);
4801	}
4802
4803	return (DDI_SUCCESS);
4804}
4805
4806/*
4807 * Handle MSI interrupts.
4808 */
4809static int
4810ahci_add_msi_intrs(ahci_ctl_t *ahci_ctlp)
4811{
4812	dev_info_t *dip = ahci_ctlp->ahcictl_dip;
4813	int		count, avail, actual;
4814	int		x, y, rc, inum = 0;
4815
4816	AHCIDBG0(AHCIDBG_ENTRY|AHCIDBG_INIT, ahci_ctlp,
4817	    "ahci_add_msi_intrs enter");
4818
4819	/* get number of interrupts. */
4820	rc = ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_MSI, &count);
4821	if ((rc != DDI_SUCCESS) || (count == 0)) {
4822		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
4823		    "ddi_intr_get_nintrs() failed, "
4824		    "rc %d count %d\n", rc, count);
4825		return (DDI_FAILURE);
4826	}
4827
4828	/* get number of available interrupts. */
4829	rc = ddi_intr_get_navail(dip, DDI_INTR_TYPE_MSI, &avail);
4830	if ((rc != DDI_SUCCESS) || (avail == 0)) {
4831		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
4832		    "ddi_intr_get_navail() failed, "
4833		    "rc %d avail %d\n", rc, avail);
4834		return (DDI_FAILURE);
4835	}
4836
4837	if (avail < count) {
4838		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
4839		    "ddi_intr_get_nvail returned %d, navail() returned %d",
4840		    count, avail);
4841	}
4842
4843	/* Allocate an array of interrupt handles. */
4844	ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t);
4845	ahci_ctlp->ahcictl_intr_htable =
4846	    kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP);
4847
4848	/* call ddi_intr_alloc(). */
4849	rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable,
4850	    DDI_INTR_TYPE_MSI, inum, count, &actual, DDI_INTR_ALLOC_NORMAL);
4851
4852	if ((rc != DDI_SUCCESS) || (actual == 0)) {
4853		AHCIDBG1(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
4854		    "ddi_intr_alloc() failed, rc %d\n", rc);
4855		kmem_free(ahci_ctlp->ahcictl_intr_htable,
4856		    ahci_ctlp->ahcictl_intr_size);
4857		return (DDI_FAILURE);
4858	}
4859
4860	/* use interrupt count returned */
4861	if (actual < count) {
4862		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
4863		    "Requested: %d, Received: %d", count, actual);
4864	}
4865
4866	ahci_ctlp->ahcictl_intr_cnt = actual;
4867
4868	/*
4869	 * Get priority for first msi, assume remaining are all the same.
4870	 */
4871	if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0],
4872	    &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) {
4873		AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
4874		    "ddi_intr_get_pri() failed");
4875
4876		/* Free already allocated intr. */
4877		for (y = 0; y < actual; y++) {
4878			(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[y]);
4879		}
4880
4881		kmem_free(ahci_ctlp->ahcictl_intr_htable,
4882		    ahci_ctlp->ahcictl_intr_size);
4883		return (DDI_FAILURE);
4884	}
4885
4886	/* Test for high level interrupt. */
4887	if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
4888		AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
4889		    "ahci_add_msi_intrs: Hi level intr not supported");
4890
4891		/* Free already allocated intr. */
4892		for (y = 0; y < actual; y++) {
4893			(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[y]);
4894		}
4895
4896		kmem_free(ahci_ctlp->ahcictl_intr_htable,
4897		    sizeof (ddi_intr_handle_t));
4898
4899		return (DDI_FAILURE);
4900	}
4901
4902	/* Call ddi_intr_add_handler(). */
4903	for (x = 0; x < actual; x++) {
4904		if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[x],
4905		    ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) {
4906			AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
4907			    "ddi_intr_add_handler() failed");
4908
4909			/* Free already allocated intr. */
4910			for (y = 0; y < actual; y++) {
4911				(void) ddi_intr_free(
4912				    ahci_ctlp->ahcictl_intr_htable[y]);
4913			}
4914
4915			kmem_free(ahci_ctlp->ahcictl_intr_htable,
4916			    ahci_ctlp->ahcictl_intr_size);
4917			return (DDI_FAILURE);
4918		}
4919	}
4920
4921
4922	(void) ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0],
4923	    &ahci_ctlp->ahcictl_intr_cap);
4924
4925	if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) {
4926		/* Call ddi_intr_block_enable() for MSI. */
4927		(void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable,
4928		    ahci_ctlp->ahcictl_intr_cnt);
4929	} else {
4930		/* Call ddi_intr_enable() for MSI non block enable. */
4931		for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
4932			(void) ddi_intr_enable(
4933			    ahci_ctlp->ahcictl_intr_htable[x]);
4934		}
4935	}
4936
4937	return (DDI_SUCCESS);
4938}
4939
4940/*
4941 * Removes the registered interrupts irrespective of whether they
4942 * were legacy or MSI.
4943 *
4944 * WARNING!!! The controller interrupts must be disabled before calling
4945 * this routine.
4946 */
4947static void
4948ahci_rem_intrs(ahci_ctl_t *ahci_ctlp)
4949{
4950	int x;
4951
4952	AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered");
4953
4954	/* Disable all interrupts. */
4955	if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) &&
4956	    (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) {
4957		/* Call ddi_intr_block_disable(). */
4958		(void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable,
4959		    ahci_ctlp->ahcictl_intr_cnt);
4960	} else {
4961		for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
4962			(void) ddi_intr_disable(
4963			    ahci_ctlp->ahcictl_intr_htable[x]);
4964		}
4965	}
4966
4967	/* Call ddi_intr_remove_handler(). */
4968	for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
4969		(void) ddi_intr_remove_handler(
4970		    ahci_ctlp->ahcictl_intr_htable[x]);
4971		(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
4972	}
4973
4974	kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size);
4975}
4976
4977/*
4978 * This routine tries to put port into NotRunning state by clearing
4979 * PxCMD.ST and PxCMD.FRE. Remember to clear PxCMD.FRE before clearing
4980 * PxCMD.ST.
4981 *
4982 * WARNING!!! ahciport_mutex should be acquired before the function
4983 * is called.
4984 */
4985/* ARGSUSED */
4986static int
4987ahci_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp,
4988    ahci_port_t *ahci_port, uint8_t port)
4989{
4990	uint32_t port_cmd_status;
4991	int loop_count;
4992
4993	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
4994	    "ahci_port_into_notrunning_state enter: port %d", port);
4995
4996	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4997	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4998
4999	if (!(port_cmd_status & AHCI_CMD_STATUS_ST))
5000		goto clear_fre;
5001
5002	port_cmd_status &= ~AHCI_CMD_STATUS_ST;
5003	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5004	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
5005
5006	/* Wait until PxCMD.CR is cleared */
5007	loop_count = 0;
5008	do {
5009		port_cmd_status =
5010		    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5011		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5012
5013		if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) {
5014			AHCIDBG2(AHCIDBG_INIT, ahci_ctlp,
5015			    "clearing port %d CMD.CR timeout, "
5016			    "port_cmd_status = 0x%x", port,
5017			    port_cmd_status);
5018			/*
5019			 * We are effectively timing out after 0.5 sec.
5020			 * This value is specified in AHCI spec.
5021			 */
5022			break;
5023		}
5024
5025	/* Wait for 10 millisec */
5026#ifndef __lock_lint
5027		delay(AHCI_10MS_TICKS);
5028#endif /* __lock_lint */
5029
5030	} while (port_cmd_status & AHCI_CMD_STATUS_CR);
5031
5032	AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5033	    "ahci_port_into_notrunning_state: 1st loop count: %d, "
5034	    "port_cmd_status = 0x%x", loop_count, port_cmd_status);
5035
5036clear_fre:
5037	if (port_cmd_status & AHCI_CMD_STATUS_FRE) {
5038
5039		port_cmd_status &= ~AHCI_CMD_STATUS_FRE;
5040		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5041		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5042		    port_cmd_status);
5043
5044		/* Wait until PxCMD.FR is cleared */
5045		loop_count = 0;
5046		do {
5047			port_cmd_status =
5048			    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5049			    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5050
5051			if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) {
5052				AHCIDBG2(AHCIDBG_INIT, ahci_ctlp,
5053				    "clearing port %d CMD.FR "
5054				    "timeout, port_cmd_status = 0x%x",
5055				    port, port_cmd_status);
5056				/*
5057				 * We are effectively timing out after 0.5 sec.
5058				 * This value is specified in AHCI spec.
5059				 */
5060				break;
5061			}
5062
5063			/* Wait for 10 millisec */
5064#ifndef __lock_lint
5065			delay(AHCI_10MS_TICKS);
5066#endif /* __lock_lint */
5067		} while (port_cmd_status & AHCI_CMD_STATUS_FR);
5068
5069		AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5070		    "ahci_port_into_notrunning_state: 2nd loop count: "
5071		    "%d, port_cmd_status = 0x%x",
5072		    loop_count, port_cmd_status);
5073	}
5074
5075	if (port_cmd_status &
5076	    (AHCI_CMD_STATUS_ST |
5077	    AHCI_CMD_STATUS_CR |
5078	    AHCI_CMD_STATUS_FRE |
5079	    AHCI_CMD_STATUS_FR)) {
5080
5081		AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
5082		    "port %d cannot be put in NotRunning state", port);
5083		return (AHCI_FAILURE);
5084
5085	} else {
5086
5087		AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
5088		    "port %d is put in NotRunning state", port);
5089		return (AHCI_SUCCESS);
5090	}
5091}
5092
5093/*
5094 * This routine will be called under five scenarios:
5095 *	1. Initialize the port
5096 *	2. To abort the packet(s)
5097 *	3. To reset the port
5098 *	4. Fatal error recovery
5099 *	5. To abort the timeout packet(s)
5100 *
5101 * WARNING!!! ahciport_mutex should be acquired before the function
5102 * is called.
5103 */
5104static int
5105ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp,
5106    ahci_port_t *ahci_portp, uint8_t port, int flag)
5107{
5108	uint32_t task_file_status;
5109	sata_device_t sdevice;
5110	int rval;
5111
5112	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
5113	    "ahci_restart_port_wait_till_ready: port %d enter", port);
5114
5115	ahci_disable_port_intrs(ahci_ctlp, ahci_portp, port);
5116
5117	rval = ahci_port_into_notrunning_state(ahci_ctlp, ahci_portp, port);
5118
5119	if (rval == AHCI_SUCCESS) {
5120		task_file_status = ddi_get32(
5121		    ahci_ctlp->ahcictl_ahci_acc_handle,
5122		    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5123
5124		if (task_file_status & AHCI_TFD_STS_BSY ||
5125		    task_file_status & AHCI_TFD_STS_DRQ)
5126			rval = AHCI_FAILURE;
5127	}
5128
5129	if (!(flag & AHCI_PORT_RESET) && (rval == AHCI_SUCCESS))
5130		goto out;
5131
5132	/* Set the reset in progress flag */
5133	if (!(flag & AHCI_RESET_NO_EVENTS_UP)) {
5134		ahci_portp->ahciport_reset_in_progress = 1;
5135	}
5136
5137	rval = ahci_port_reset(ahci_ctlp, ahci_portp, port);
5138	if (rval != AHCI_SUCCESS) {
5139		ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5140		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
5141		    "ahci_restart_port_wait_till_ready: port %d failed",
5142		    port);
5143	}
5144
5145	/* Indicate to the framework that a reset has happened. */
5146	if (!(flag & AHCI_RESET_NO_EVENTS_UP)) {
5147
5148		bzero((void *)&sdevice, sizeof (sata_device_t));
5149		sdevice.satadev_addr.cport = port;
5150		sdevice.satadev_addr.pmport = AHCI_PORTMULT_CONTROL_PORT;
5151
5152		if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
5153			sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT;
5154		} else {
5155			sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
5156		}
5157		sdevice.satadev_state = SATA_DSTATE_RESET |
5158		    SATA_DSTATE_PWR_ACTIVE;
5159		if (ahci_ctlp->ahcictl_sata_hba_tran) {
5160			mutex_exit(&ahci_portp->ahciport_mutex);
5161			sata_hba_event_notify(
5162			    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
5163			    &sdevice,
5164			    SATA_EVNT_DEVICE_RESET);
5165			mutex_enter(&ahci_portp->ahciport_mutex);
5166		}
5167
5168		AHCIDBG0(AHCIDBG_EVENT, ahci_ctlp,
5169		    "sending event up: SATA_EVNT_RESET");
5170	}
5171out:
5172	if (!(flag & AHCI_PORT_INIT)) {
5173		(void) ahci_start_port(ahci_ctlp, port);
5174		ahci_enable_port_intrs(ahci_ctlp, ahci_portp, port);
5175	}
5176
5177	return (rval);
5178}
5179
5180/*
5181 * This routine is called under four scenarios:
5182 *	a) some commands failed with errors
5183 *	b) or we need to timeout some commands
5184 *	c) or we need to abort some commands
5185 *	d) or we need reset device/port/controller
5186 *
5187 * In all these scenarios, we need to send any pending unfinished
5188 * commands up to sata framework.
5189 *
5190 * Only one mopping process at a time is allowed; this is achieved
5191 * by checking AHCI_PORT_STATE_MOPPING by the caller.
5192 */
5193static void
5194ahci_mop_commands(ahci_ctl_t *ahci_ctlp,
5195    ahci_port_t *ahci_portp,
5196    uint8_t port,
5197    uint32_t slot_status,
5198    uint32_t failed_tags,
5199    uint32_t timeout_tags,
5200    uint32_t aborted_tags,
5201    uint32_t reset_tags)
5202{
5203	uint32_t finished_tags, unfinished_tags;
5204	int tmp_slot;
5205	sata_pkt_t *satapkt;
5206	int i, num = 0;
5207	ahci_fis_d2h_register_t *rcvd_fisp;
5208
5209	AHCIDBG2(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
5210	    "ahci_mop_commands entered: port: %d slot_status: 0x%x",
5211	    port, slot_status);
5212
5213	AHCIDBG4(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
5214	    "ahci_mop_commands: failed_tags: 0x%x, "
5215	    "timeout_tags: 0x%x aborted_tags: 0x%x, "
5216	    "reset_tags: 0x%x", failed_tags,
5217	    timeout_tags, aborted_tags, reset_tags);
5218
5219	/*
5220	 * We could be here for four reasons: abort, reset,
5221	 * timeout or error handling. Only one such mopping
5222	 * is allowed at a time.
5223	 */
5224	mutex_enter(&ahci_portp->ahciport_mutex);
5225
5226	finished_tags = ahci_portp->ahciport_pending_tags &
5227	    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
5228
5229	unfinished_tags = slot_status &
5230	    AHCI_SLOT_MASK(ahci_ctlp) &
5231	    ~failed_tags &
5232	    ~aborted_tags &
5233	    ~reset_tags &
5234	    ~timeout_tags;
5235
5236	/*
5237	 * Try to check if there are more than one command which have
5238	 * been finished.
5239	 */
5240	for (i = 0; i < ahci_ctlp->ahcictl_num_cmd_slots; i++) {
5241		if ((0x1 << i) & finished_tags)
5242			num++;
5243	}
5244	ASSERT(num <= 1);
5245
5246	/* Set finished packets with SATA_PKT_COMPLETED */
5247	while (finished_tags) {
5248		tmp_slot = ddi_ffs(finished_tags) - 1;
5249		if (tmp_slot == -1) {
5250			break;
5251		}
5252
5253		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
5254		ASSERT(satapkt != NULL);
5255
5256		rcvd_fisp =
5257		    &(ahci_portp->ahciport_rcvd_fis->ahcirf_d2h_register_fis);
5258
5259		/* Return the status */
5260		satapkt->satapkt_cmd.satacmd_status_reg =
5261		    GET_RFIS_STATUS(rcvd_fisp);
5262
5263		if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs)
5264			ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
5265
5266		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: "
5267		    "sending up pkt 0x%p with SATA_PKT_COMPLETED",
5268		    (void *)satapkt);
5269
5270		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
5271		CLEAR_BIT(finished_tags, tmp_slot);
5272		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
5273
5274		SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED);
5275	}
5276
5277	ASSERT(finished_tags == 0);
5278
5279	/* Send up failed_tags with SATA_PKT_DEV_ERROR. */
5280	while (failed_tags) {
5281		tmp_slot = ddi_ffs(failed_tags) - 1;
5282		if (tmp_slot == -1) {
5283			break;
5284		}
5285
5286		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
5287		ASSERT(satapkt != NULL);
5288
5289		if (satapkt->satapkt_device.satadev_type ==
5290		    SATA_DTYPE_ATAPICD) {
5291			ahci_set_sense_data(satapkt, SATA_PKT_ABORTED);
5292		}
5293
5294		rcvd_fisp =
5295		    &(ahci_portp->ahciport_rcvd_fis->ahcirf_d2h_register_fis);
5296
5297		/* Return the status */
5298		satapkt->satapkt_cmd.satacmd_status_reg =
5299		    GET_RFIS_STATUS(rcvd_fisp);
5300
5301		if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs)
5302			ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
5303
5304		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
5305		    "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
5306		    (void *)satapkt);
5307
5308		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
5309		CLEAR_BIT(failed_tags, tmp_slot);
5310		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
5311
5312		SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
5313	}
5314
5315	ASSERT(failed_tags == 0);
5316
5317	/* Send up timeout_tags with SATA_PKT_TIMEOUT. */
5318	while (timeout_tags) {
5319		tmp_slot = ddi_ffs(timeout_tags) - 1;
5320		if (tmp_slot == -1) {
5321			break;
5322		}
5323
5324		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
5325		ASSERT(satapkt != NULL);
5326
5327		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
5328		    "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
5329		    (void *)satapkt);
5330
5331		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
5332		CLEAR_BIT(timeout_tags, tmp_slot);
5333		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
5334
5335		SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
5336	}
5337
5338	ASSERT(timeout_tags == 0);
5339
5340	/* Set aborted packets with SATA_PKT_ABORTED */
5341	while (aborted_tags) {
5342		tmp_slot = ddi_ffs(aborted_tags) - 1;
5343		if (tmp_slot == -1) {
5344			break;
5345		}
5346
5347		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
5348		ASSERT(satapkt != NULL);
5349
5350		if (satapkt->satapkt_device.satadev_type ==
5351		    SATA_DTYPE_ATAPICD) {
5352			ahci_set_sense_data(satapkt, SATA_PKT_ABORTED);
5353		}
5354
5355		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
5356		    "sending up pkt 0x%p with SATA_PKT_ABORTED",
5357		    (void *)satapkt);
5358
5359		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
5360		CLEAR_BIT(aborted_tags, tmp_slot);
5361		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
5362
5363		SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED);
5364	}
5365
5366	ASSERT(aborted_tags == 0);
5367
5368	/* Reset tags are sent up to framework with SATA_PKT_RESET. */
5369	while (reset_tags) {
5370		tmp_slot = ddi_ffs(reset_tags) - 1;
5371		if (tmp_slot == -1) {
5372			break;
5373		}
5374
5375		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
5376		ASSERT(satapkt != NULL);
5377
5378		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
5379		    "sending up pkt 0x%p with SATA_PKT_RESET",
5380		    (void *)satapkt);
5381
5382		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
5383		CLEAR_BIT(reset_tags, tmp_slot);
5384		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
5385
5386		SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET);
5387	}
5388
5389	ASSERT(reset_tags == 0);
5390
5391	/* Set unfinished packets with SATA_PKT_BUSY */
5392	while (unfinished_tags) {
5393		tmp_slot = ddi_ffs(unfinished_tags) - 1;
5394		if (tmp_slot == -1) {
5395			break;
5396		}
5397
5398		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
5399		ASSERT(satapkt != NULL);
5400
5401		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
5402		    "sending up pkt 0x%p with SATA_PKT_BUSY",
5403		    (void *)satapkt);
5404
5405		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
5406		CLEAR_BIT(unfinished_tags, tmp_slot);
5407		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
5408
5409		SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_BUSY);
5410	}
5411
5412	ASSERT(unfinished_tags == 0);
5413
5414	ahci_portp->ahciport_flags &= ~AHCI_PORT_STATE_MOPPING;
5415
5416	mutex_exit(&ahci_portp->ahciport_mutex);
5417}
5418
5419/*
5420 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover,
5421 * the port must be restarted.
5422 *
5423 * Fatal errors include the following:
5424 * 	PxIS.OFS - Overflow Status
5425 * 	PxIS.IFS - Interface Fatal Error Status
5426 * 	PxIS.HBDS - Host Bus Data Error Status
5427 * 	PxIS.HBFS - Host Bus Fatal Error Status
5428 *	PxIS.TFES - Task File Error Status
5429 *
5430 * WARNING!!! ahciport_mutex should be acquired before the function is called.
5431 */
5432static int
5433ahci_recovery_fatal_error(ahci_ctl_t *ahci_ctlp,
5434    ahci_port_t *ahci_portp, uint8_t port, uint32_t error)
5435{
5436	uint32_t port_cmd_status, slot_status;
5437	uint32_t failed_tags;
5438	int current_slot;
5439	sata_pkt_t *spkt;
5440	ahci_fis_d2h_register_t *ahci_rcvd_fisp;
5441
5442	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
5443	    "ahci_recovery_fatal_error enter port %d", port);
5444
5445	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
5446		/*
5447		 * ahci_intr_phyrdy_change() may have rendered it to
5448		 * SATA_DTYPE_NONE.
5449		 */
5450		AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
5451		    "ahci_recovery_fatal_error: no device attached, and just "
5452		    "return without doing anything", port);
5453		return (SATA_SUCCESS);
5454	}
5455
5456	/* Record the error occurred command slot */
5457	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5458	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5459
5460	current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
5461	    AHCI_CMD_STATUS_CCS_SHIFT;
5462
5463	spkt = ahci_portp->ahciport_slot_pkts[current_slot];
5464	if (spkt == NULL) {
5465		/*
5466		 * For example, sometimes PxIS.OFS and PxIS.IFS can be set
5467		 * concurrently.
5468		 */
5469		AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
5470		    "ahci_recovery_fatal_error: port %d has done "
5471		    "fatal error recovery, so return directory", port);
5472		return (SATA_SUCCESS);
5473	}
5474
5475	/*
5476	 * To prevent recursive enter to ahci_mop_commands, we need
5477	 * check AHCI_PORT_STATE_MOPPING flag.
5478	 */
5479	if (ahci_portp->ahciport_flags & AHCI_PORT_STATE_MOPPING) {
5480		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
5481		    "ahci_recovery_fatal_error: port %d is in "
5482		    "mopping process, so return directly", port);
5483		return (SATA_SUCCESS);
5484	}
5485
5486	ahci_portp->ahciport_flags |= AHCI_PORT_STATE_MOPPING;
5487
5488	failed_tags = 0x1 << current_slot;
5489
5490	switch (error) {
5491	case AHCI_INTR_STATUS_IFS:
5492	case AHCI_INTR_STATUS_HBFS:
5493	case AHCI_INTR_STATUS_HBDS:
5494		/*
5495		 * For PxIS.IFS, PxIS.HBFS and PxIS.HBDS, we need to return
5496		 * PxSERR in satapkt.
5497		 */
5498		ahci_update_sata_registers(ahci_ctlp, port,
5499		    &spkt->satapkt_device);
5500		break;
5501
5502	case AHCI_INTR_STATUS_TFES:
5503		/*
5504		 * For PxIS.TFES, we need to return the status and error
5505		 * registers in sata_cmd structure
5506		 */
5507		ahci_rcvd_fisp =
5508		    &(ahci_portp->ahciport_rcvd_fis->ahcirf_d2h_register_fis);
5509
5510		spkt->satapkt_cmd.satacmd_status_reg =
5511		    GET_RFIS_STATUS(ahci_rcvd_fisp);
5512
5513		spkt->satapkt_cmd.satacmd_error_reg =
5514		    GET_RFIS_ERROR(ahci_rcvd_fisp);
5515		break;
5516
5517	case AHCI_INTR_STATUS_OFS:
5518		/*
5519		 * For PxIS.OFS, until now have no way to send the error
5520		 * to sata framework.
5521		 */
5522		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
5523		    "ahci_recovery_fatal_error: "
5524		    "PxOFS occured for port %d", port);
5525
5526		break;
5527
5528	default:
5529		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5530		    "Not supported error 0x%x for port %d",
5531		    error, port);
5532		break;
5533	}
5534
5535	slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5536	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5537
5538	AHCIDBG2(AHCIDBG_INFO, ahci_ctlp,
5539	    "ahci_recovery_fatal_error: current_slot = 0x%x "
5540	    "slot_status = 0x%x", current_slot, slot_status);
5541
5542	(void) ahci_restart_port_wait_till_ready(ahci_ctlp,
5543	    ahci_portp, port, NULL);
5544
5545	mutex_exit(&ahci_portp->ahciport_mutex);
5546	ahci_mop_commands(ahci_ctlp,
5547	    ahci_portp,
5548	    port,
5549	    slot_status,
5550	    failed_tags, /* failed tags */
5551	    0, /* timeout tags */
5552	    0, /* aborted tags */
5553	    0); /* reset tags */
5554	mutex_enter(&ahci_portp->ahciport_mutex);
5555
5556	return (SATA_SUCCESS);
5557}
5558
5559/*
5560 * ahci_watchdog_handler() calls us if it detects that there are some
5561 * commands which timed out.
5562 */
5563static void
5564ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5565    uint8_t port, uint32_t tmp_timeout_tags)
5566{
5567	uint32_t slot_status;
5568	uint32_t finished_tags, timeout_tags;
5569
5570	AHCIDBG0(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp,
5571	    "ahci_timeout_pkts entered");
5572
5573	mutex_enter(&ahci_portp->ahciport_mutex);
5574
5575	/*
5576	 * To prevent recursive enter to ahci_mop_commands, we need
5577	 * check AHCI_PORT_STATE_MOPPING flag.
5578	 */
5579	if (ahci_portp->ahciport_flags & AHCI_PORT_STATE_MOPPING) {
5580		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
5581		    "ahci_timeout_pkts: port %d is in "
5582		    "mopping process, so return directly ", port);
5583		mutex_exit(&ahci_portp->ahciport_mutex);
5584		return;
5585	}
5586
5587	ahci_portp->ahciport_flags |= AHCI_PORT_STATE_MOPPING;
5588
5589	slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5590	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5591
5592	(void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
5593	    port, NULL);
5594	/*
5595	 * Re-identify timeout tags because some previously checked commands
5596	 * could already complete.
5597	 */
5598	finished_tags = ahci_portp->ahciport_pending_tags &
5599	    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
5600
5601	timeout_tags = tmp_timeout_tags & ~finished_tags;
5602
5603	AHCIDBG3(AHCIDBG_TIMEOUT, ahci_ctlp,
5604	    "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
5605	    "timeout_tags = 0x%x", port,
5606	    finished_tags, timeout_tags);
5607
5608	mutex_exit(&ahci_portp->ahciport_mutex);
5609	ahci_mop_commands(ahci_ctlp,
5610	    ahci_portp,
5611	    port,
5612	    slot_status,
5613	    0, /* failed tags */
5614	    timeout_tags, /* timeout tags */
5615	    0, /* aborted tags */
5616	    0); /* reset tags */
5617}
5618
5619/*
5620 * Watchdog handler kicks in every 5 seconds to timeout any commands pending
5621 * for long time.
5622 */
5623static void
5624ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp)
5625{
5626	ahci_port_t *ahci_portp;
5627	sata_pkt_t *satapkt;
5628	uint32_t pending_tags = 0;
5629	uint32_t timeout_tags = 0;
5630	uint8_t port;
5631	int tmp_slot;
5632	/* max number of cycles this packet should survive */
5633	int max_life_cycles;
5634
5635	/* how many cycles this packet survived so far */
5636	int watched_cycles;
5637
5638	mutex_enter(&ahci_ctlp->ahcictl_mutex);
5639
5640	AHCIDBG0(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp,
5641	    "ahci_watchdog_handler entered");
5642
5643	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
5644		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
5645			continue;
5646		}
5647
5648		ahci_portp = ahci_ctlp->ahcictl_ports[port];
5649
5650		mutex_enter(&ahci_portp->ahciport_mutex);
5651		if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
5652			mutex_exit(&ahci_portp->ahciport_mutex);
5653			continue;
5654		}
5655
5656		pending_tags = ahci_portp->ahciport_pending_tags;
5657		timeout_tags = 0;
5658		while (pending_tags) {
5659			tmp_slot = ddi_ffs(pending_tags) - 1;
5660			if (tmp_slot == -1) {
5661				break;
5662			}
5663
5664			satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
5665			if ((satapkt != NULL) && satapkt->satapkt_time) {
5666				/*
5667				 * We are overloading satapkt_hba_driver_private
5668				 * with watched_cycle count.
5669				 *
5670				 * If a packet has survived for more than it's
5671				 * max life cycles, it is a candidate for time
5672				 * out.
5673				 */
5674				watched_cycles = (int)(intptr_t)
5675				    satapkt->satapkt_hba_driver_private;
5676				watched_cycles++;
5677				max_life_cycles = (satapkt->satapkt_time +
5678				    ahci_watchdog_timeout - 1) /
5679				    ahci_watchdog_timeout;
5680				if (watched_cycles > max_life_cycles) {
5681					timeout_tags |= (0x1 << tmp_slot);
5682					cmn_err(CE_NOTE, "!ahci watchdog: "
5683					    "port %d satapkt 0x%p timed out\n",
5684					    port, (void *)satapkt);
5685				}
5686				satapkt->satapkt_hba_driver_private =
5687				    (void *)(intptr_t)watched_cycles;
5688			}
5689
5690			CLEAR_BIT(pending_tags, tmp_slot);
5691		}
5692
5693		if (timeout_tags) {
5694			mutex_exit(&ahci_portp->ahciport_mutex);
5695			mutex_exit(&ahci_ctlp->ahcictl_mutex);
5696			ahci_timeout_pkts(ahci_ctlp, ahci_portp,
5697			    port, timeout_tags);
5698			mutex_enter(&ahci_ctlp->ahcictl_mutex);
5699			mutex_enter(&ahci_portp->ahciport_mutex);
5700		}
5701
5702		mutex_exit(&ahci_portp->ahciport_mutex);
5703	}
5704
5705	/* Re-install the watchdog timeout handler */
5706	if (ahci_ctlp->ahcictl_timeout_id != 0) {
5707		ahci_ctlp->ahcictl_timeout_id =
5708		    timeout((void (*)(void *))ahci_watchdog_handler,
5709		    (caddr_t)ahci_ctlp, ahci_watchdog_tick);
5710	}
5711
5712	mutex_exit(&ahci_ctlp->ahcictl_mutex);
5713}
5714
5715/*
5716 * Put a copy back to sata_cmd_t.
5717 */
5718static void
5719ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
5720{
5721	if (scmd->satacmd_flags.sata_copy_out_sec_count_msb)
5722		scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
5723	if (scmd->satacmd_flags.sata_copy_out_lba_low_msb)
5724		scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
5725	if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb)
5726		scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
5727	if (scmd->satacmd_flags.sata_copy_out_lba_high_msb)
5728		scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
5729	if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb)
5730		scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
5731	if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb)
5732		scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
5733	if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb)
5734		scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
5735	if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb)
5736		scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
5737	if (scmd->satacmd_flags.sata_copy_out_device_reg)
5738		scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
5739	if (scmd->satacmd_flags.sata_copy_out_error_reg)
5740		scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
5741}
5742
5743/*
5744 * Dump the serror message to the log.
5745 */
5746static void
5747ahci_log_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
5748    uint32_t port_serror)
5749{
5750	char *err_str;
5751
5752	if (port_serror & AHCI_SERROR_ERR_I) {
5753		err_str = "Recovered Data Integrity Error (I)";
5754		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5755		    "command error: port: %d, error: %s",
5756		    port, err_str);
5757	}
5758
5759	if (port_serror & AHCI_SERROR_ERR_M) {
5760		err_str = "Recovered Communication Error (M)";
5761		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5762		    "command error: port: %d, error: %s",
5763		    port, err_str);
5764	}
5765
5766	if (port_serror & AHCI_SERROR_ERR_T) {
5767		err_str = "Transient Data Integrity Error (T)";
5768		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5769		    "command error: port: %d, error: %s",
5770		    port, err_str);
5771	}
5772
5773	if (port_serror & AHCI_SERROR_ERR_C) {
5774		err_str =
5775		    "Persistent Communication or Data Integrity Error (C)";
5776		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5777		    "command error: port: %d, error: %s",
5778		    port, err_str);
5779	}
5780
5781	if (port_serror & AHCI_SERROR_ERR_P) {
5782		err_str = "Protocol Error (P)";
5783		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5784		    "command error: port: %d, error: %s",
5785		    port, err_str);
5786	}
5787
5788	if (port_serror & AHCI_SERROR_ERR_E) {
5789		err_str = "Internal Error (E)";
5790		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5791		    "command error: port: %d, error: %s",
5792		    port, err_str);
5793	}
5794
5795	if (port_serror & AHCI_SERROR_DIAG_N) {
5796		err_str = "PhyRdy Change (N)";
5797		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5798		    "command error: port: %d, error: %s",
5799		    port, err_str);
5800	}
5801
5802	if (port_serror & AHCI_SERROR_DIAG_I) {
5803		err_str = "Phy Internal Error (I)";
5804		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5805		    "command error: port: %d, error: %s",
5806		    port, err_str);
5807	}
5808
5809	if (port_serror & AHCI_SERROR_DIAG_W) {
5810		err_str = "Comm Wake (W)";
5811		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5812		    "command error: port: %d, error: %s",
5813		    port, err_str);
5814	}
5815
5816	if (port_serror & AHCI_SERROR_DIAG_B) {
5817		err_str = "10B to 8B Decode Error (B)";
5818		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5819		    "command error: port: %d, error: %s",
5820		    port, err_str);
5821	}
5822
5823	if (port_serror & AHCI_SERROR_DIAG_D) {
5824		err_str = "Disparity Error (D)";
5825		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5826		    "command error: port: %d, error: %s",
5827		    port, err_str);
5828	}
5829
5830	if (port_serror & AHCI_SERROR_DIAG_C) {
5831		err_str = "CRC Error (C)";
5832		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5833		    "command error: port: %d, error: %s",
5834		    port, err_str);
5835	}
5836
5837	if (port_serror & AHCI_SERROR_DIAG_H) {
5838		err_str = "Handshake Error (H)";
5839		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5840		    "command error: port: %d, error: %s",
5841		    port, err_str);
5842	}
5843
5844	if (port_serror & AHCI_SERROR_DIAG_S) {
5845		err_str = "Link Sequence Error (S)";
5846		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5847		    "command error: port: %d, error: %s",
5848		    port, err_str);
5849	}
5850
5851	if (port_serror & AHCI_SERROR_DIAG_T) {
5852		err_str = "Transport state transition error (T)";
5853		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5854		    "command error: port: %d, error: %s",
5855		    port, err_str);
5856	}
5857
5858	if (port_serror & AHCI_SERROR_DIAG_F) {
5859		err_str = "Unknown FIS Type (F)";
5860		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5861		    "command error: port: %d, error: %s",
5862		    port, err_str);
5863	}
5864
5865	if (port_serror & AHCI_SERROR_DIAG_X) {
5866		err_str = "Exchanged (X)";
5867		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5868		    "command error: port: %d, error: %s",
5869		    port, err_str);
5870	}
5871}
5872
5873/*
5874 * This routine is to calculate the total number of ports implemented
5875 * by the HBA.
5876 */
5877static int
5878ahci_get_num_implemented_ports(uint32_t ports_implemented)
5879{
5880	uint8_t i;
5881	int num = 0;
5882
5883	for (i = 0; i < AHCI_MAX_PORTS; i++) {
5884		if (((uint32_t)0x1 << i) & ports_implemented)
5885			num++;
5886	}
5887
5888	return (num);
5889}
5890
5891static void
5892ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...)
5893{
5894	va_list ap;
5895
5896	mutex_enter(&ahci_log_mutex);
5897
5898	va_start(ap, fmt);
5899	if (ahci_ctlp) {
5900		(void) sprintf(ahci_log_buf, "%s-[%d]:",
5901		    ddi_get_name(ahci_ctlp->ahcictl_dip),
5902		    ddi_get_instance(ahci_ctlp->ahcictl_dip));
5903	} else {
5904		(void) sprintf(ahci_log_buf, "ahci:");
5905	}
5906
5907	(void) vsprintf(ahci_log_buf, fmt, ap);
5908	va_end(ap);
5909
5910	cmn_err(level, "%s", ahci_log_buf);
5911
5912	mutex_exit(&ahci_log_mutex);
5913}
5914