ahci_generic.c revision 219341
151078Speter/*- 251078Speter * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org> 351078Speter * All rights reserved. 451078Speter * 551078Speter * Redistribution and use in source and binary forms, with or without 651078Speter * modification, are permitted provided that the following conditions 751078Speter * are met: 851078Speter * 1. Redistributions of source code must retain the above copyright 951078Speter * notice, this list of conditions and the following disclaimer, 1051078Speter * without modification, immediately at the beginning of the file. 1151078Speter * 2. Redistributions in binary form must reproduce the above copyright 1251078Speter * notice, this list of conditions and the following disclaimer in the 1351078Speter * documentation and/or other materials provided with the distribution. 1451078Speter * 1551078Speter * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1651078Speter * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1751078Speter * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 1851078Speter * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 1951078Speter * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2051078Speter * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2151078Speter * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2251078Speter * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2351078Speter * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2451078Speter * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2551078Speter */ 2651078Speter 2751078Speter#include <sys/cdefs.h> 2851078Speter__FBSDID("$FreeBSD: head/sys/dev/ahci/ahci.c 219341 2011-03-06 16:10:39Z mav $"); 2951078Speter 3051078Speter#include <sys/param.h> 3151078Speter#include <sys/module.h> 3251078Speter#include <sys/systm.h> 33119419Sobrien#include <sys/kernel.h> 34119419Sobrien#include <sys/ata.h> 35119419Sobrien#include <sys/bus.h> 3651078Speter#include <sys/endian.h> 37131939Smarcel#include <sys/malloc.h> 38131939Smarcel#include <sys/lock.h> 3951078Speter#include <sys/mutex.h> 4051078Speter#include <sys/sema.h> 4151078Speter#include <sys/taskqueue.h> 4251078Speter#include <vm/uma.h> 4351078Speter#include <machine/stdarg.h> 4451078Speter#include <machine/resource.h> 4551078Speter#include <machine/bus.h> 4651078Speter#include <sys/rman.h> 47150460Simp#include <dev/pci/pcivar.h> 48150460Simp#include <dev/pci/pcireg.h> 4951078Speter#include "ahci.h" 5051078Speter 5176166Smarkm#include <cam/cam.h> 5265822Sjhb#include <cam/cam_ccb.h> 5351078Speter#include <cam/cam_sim.h> 5451078Speter#include <cam/cam_xpt_sim.h> 5551078Speter#include <cam/cam_debug.h> 56131939Smarcel 5751078Speter/* local prototypes */ 58114216Skanstatic int ahci_setup_interrupt(device_t dev); 5976166Smarkmstatic void ahci_intr(void *data); 6076166Smarkmstatic void ahci_intr_one(void *data); 6176166Smarkmstatic int ahci_suspend(device_t dev); 6276166Smarkmstatic int ahci_resume(device_t dev); 6376166Smarkmstatic int ahci_ch_init(device_t dev); 6476166Smarkmstatic int ahci_ch_deinit(device_t dev); 65131094Sphkstatic int ahci_ch_suspend(device_t dev); 6676166Smarkmstatic int ahci_ch_resume(device_t dev); 6751078Speterstatic void ahci_ch_pm(void *arg); 6876166Smarkmstatic void ahci_ch_intr_locked(void *data); 6951078Speterstatic void ahci_ch_intr(void *data); 7051078Speterstatic int ahci_ctlr_reset(device_t dev); 7151078Speterstatic int ahci_ctlr_setup(device_t dev); 7293466Sbdestatic void ahci_begin_transaction(device_t dev, union ccb *ccb); 73119485Snjlstatic void ahci_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error); 7451078Speterstatic void ahci_execute_transaction(struct ahci_slot *slot); 7586909Simpstatic void ahci_timeout(struct ahci_slot *slot); 7686909Simpstatic void ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et); 7751078Speterstatic int ahci_setup_fis(device_t dev, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag); 7851078Speterstatic void ahci_dmainit(device_t dev); 7985302Simpstatic void ahci_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error); 8085365Simpstatic void ahci_dmafini(device_t dev); 8151078Speterstatic void ahci_slotsalloc(device_t dev); 8251078Speterstatic void ahci_slotsfree(device_t dev); 8377726Sjoergstatic void ahci_reset(device_t dev); 8451078Speterstatic void ahci_start(device_t dev, int fbs); 8577726Sjoergstatic void ahci_stop(device_t dev); 8651078Speterstatic void ahci_clo(device_t dev); 8751078Speterstatic void ahci_start_fr(device_t dev); 8851078Speterstatic void ahci_stop_fr(device_t dev); 8951078Speter 9051078Speterstatic int ahci_sata_connect(struct ahci_channel *ch); 9151078Speterstatic int ahci_sata_phy_reset(device_t dev); 9251078Speterstatic int ahci_wait_ready(device_t dev, int t); 9351078Speter 9451078Speterstatic void ahci_issue_read_log(device_t dev); 9551078Speterstatic void ahci_process_read_log(device_t dev, union ccb *ccb); 96104067Sphk 97104067Sphkstatic void ahciaction(struct cam_sim *sim, union ccb *ccb); 9851078Speterstatic void ahcipoll(struct cam_sim *sim); 9951078Speter 100120175SbdeMALLOC_DEFINE(M_AHCI, "AHCI driver", "AHCI driver data buffers"); 10151078Speter 102120175Sbdestatic struct { 103120175Sbde uint32_t id; 10451078Speter uint8_t rev; 105120175Sbde const char *name; 10651078Speter int quirks; 10751078Speter#define AHCI_Q_NOFORCE 1 108120175Sbde#define AHCI_Q_NOPMP 2 109120175Sbde#define AHCI_Q_NONCQ 4 110120175Sbde#define AHCI_Q_1CH 8 111111613Sphk#define AHCI_Q_2CH 16 112120175Sbde#define AHCI_Q_4CH 32 113112384Ssobomax#define AHCI_Q_EDGEIS 64 11451078Speter#define AHCI_Q_SATA2 128 11560471Snyan#define AHCI_Q_NOBSYRES 256 11660471Snyan#define AHCI_Q_NOAA 512 11760471Snyan#define AHCI_Q_NOCOUNT 1024 11860471Snyan} ahci_ids[] = { 11960471Snyan {0x43801002, 0x00, "ATI IXP600", 0}, 12051078Speter {0x43901002, 0x00, "ATI IXP700", 0}, 12151078Speter {0x43911002, 0x00, "ATI IXP700", 0}, 12251078Speter {0x43921002, 0x00, "ATI IXP700", 0}, 12351078Speter {0x43931002, 0x00, "ATI IXP700", 0}, 12451078Speter {0x43941002, 0x00, "ATI IXP800", 0}, 12551078Speter {0x43951002, 0x00, "ATI IXP800", 0}, 12651078Speter {0x26528086, 0x00, "Intel ICH6", AHCI_Q_NOFORCE}, 12751078Speter {0x26538086, 0x00, "Intel ICH6M", AHCI_Q_NOFORCE}, 12853344Speter {0x26818086, 0x00, "Intel ESB2", 0}, 12951078Speter {0x26828086, 0x00, "Intel ESB2", 0}, 13051078Speter {0x26838086, 0x00, "Intel ESB2", 0}, 13151078Speter {0x27c18086, 0x00, "Intel ICH7", 0}, 13251078Speter {0x27c38086, 0x00, "Intel ICH7", 0}, 13351078Speter {0x27c58086, 0x00, "Intel ICH7M", 0}, 13451078Speter {0x27c68086, 0x00, "Intel ICH7M", 0}, 13551078Speter {0x28218086, 0x00, "Intel ICH8", 0}, 13651078Speter {0x28228086, 0x00, "Intel ICH8", 0}, 13751078Speter {0x28248086, 0x00, "Intel ICH8", 0}, 13851078Speter {0x28298086, 0x00, "Intel ICH8M", 0}, 13951078Speter {0x282a8086, 0x00, "Intel ICH8M", 0}, 14051078Speter {0x29228086, 0x00, "Intel ICH9", 0}, 14151078Speter {0x29238086, 0x00, "Intel ICH9", 0}, 14251078Speter {0x29248086, 0x00, "Intel ICH9", 0}, 14351078Speter {0x29258086, 0x00, "Intel ICH9", 0}, 14451078Speter {0x29278086, 0x00, "Intel ICH9", 0}, 14551078Speter {0x29298086, 0x00, "Intel ICH9M", 0}, 14651078Speter {0x292a8086, 0x00, "Intel ICH9M", 0}, 14751078Speter {0x292b8086, 0x00, "Intel ICH9M", 0}, 14851078Speter {0x292c8086, 0x00, "Intel ICH9M", 0}, 14951078Speter {0x292f8086, 0x00, "Intel ICH9M", 0}, 15051078Speter {0x294d8086, 0x00, "Intel ICH9", 0}, 15151078Speter {0x294e8086, 0x00, "Intel ICH9M", 0}, 15251078Speter {0x3a058086, 0x00, "Intel ICH10", 0}, 15351078Speter {0x3a228086, 0x00, "Intel ICH10", 0}, 15486909Simp {0x3a258086, 0x00, "Intel ICH10", 0}, 15551078Speter {0x3b228086, 0x00, "Intel 5 Series/3400 Series", 0}, 15651078Speter {0x3b238086, 0x00, "Intel 5 Series/3400 Series", 0}, 15786909Simp {0x3b258086, 0x00, "Intel 5 Series/3400 Series", 0}, 15886909Simp {0x3b298086, 0x00, "Intel 5 Series/3400 Series", 0}, 15986909Simp {0x3b2c8086, 0x00, "Intel 5 Series/3400 Series", 0}, 16086909Simp {0x3b2f8086, 0x00, "Intel 5 Series/3400 Series", 0}, 16186909Simp {0x1c028086, 0x00, "Intel Cougar Point", 0}, 16286909Simp {0x1c038086, 0x00, "Intel Cougar Point", 0}, 16386909Simp {0x1c048086, 0x00, "Intel Cougar Point", 0}, 16486909Simp {0x1c058086, 0x00, "Intel Cougar Point", 0}, 16586909Simp {0x23238086, 0x00, "Intel DH89xxCC", 0}, 16686909Simp {0x1d028086, 0x00, "Intel Patsburg", 0}, 16786909Simp {0x1d048086, 0x00, "Intel Patsburg", 0}, 16886909Simp {0x1d068086, 0x00, "Intel Patsburg", 0}, 16986909Simp {0x2361197b, 0x00, "JMicron JMB361", AHCI_Q_NOFORCE}, 17086909Simp {0x2363197b, 0x00, "JMicron JMB363", AHCI_Q_NOFORCE}, 17186909Simp {0x2365197b, 0x00, "JMicron JMB365", AHCI_Q_NOFORCE}, 17286909Simp {0x2366197b, 0x00, "JMicron JMB366", AHCI_Q_NOFORCE}, 17351078Speter {0x2368197b, 0x00, "JMicron JMB368", AHCI_Q_NOFORCE}, 17486909Simp {0x611111ab, 0x00, "Marvell 88SX6111", AHCI_Q_NOFORCE | AHCI_Q_1CH | 17586909Simp AHCI_Q_EDGEIS}, 17686909Simp {0x612111ab, 0x00, "Marvell 88SX6121", AHCI_Q_NOFORCE | AHCI_Q_2CH | 17786909Simp AHCI_Q_EDGEIS | AHCI_Q_NONCQ | AHCI_Q_NOCOUNT}, 17886909Simp {0x614111ab, 0x00, "Marvell 88SX6141", AHCI_Q_NOFORCE | AHCI_Q_4CH | 17986909Simp AHCI_Q_EDGEIS | AHCI_Q_NONCQ | AHCI_Q_NOCOUNT}, 18086909Simp {0x614511ab, 0x00, "Marvell 88SX6145", AHCI_Q_NOFORCE | AHCI_Q_4CH | 18186909Simp AHCI_Q_EDGEIS | AHCI_Q_NONCQ | AHCI_Q_NOCOUNT}, 18286909Simp {0x91231b4b, 0x11, "Marvell 88SE912x", AHCI_Q_NOBSYRES}, 18386909Simp {0x91231b4b, 0x00, "Marvell 88SE912x", AHCI_Q_EDGEIS|AHCI_Q_SATA2|AHCI_Q_NOBSYRES}, 18486909Simp {0x06201103, 0x00, "HighPoint RocketRAID 620", AHCI_Q_NOBSYRES}, 18586909Simp {0x06201b4b, 0x00, "HighPoint RocketRAID 620", AHCI_Q_NOBSYRES}, 18686909Simp {0x06221103, 0x00, "HighPoint RocketRAID 622", AHCI_Q_NOBSYRES}, 18786909Simp {0x06221b4b, 0x00, "HighPoint RocketRAID 622", AHCI_Q_NOBSYRES}, 188120175Sbde {0x06401103, 0x00, "HighPoint RocketRAID 640", AHCI_Q_NOBSYRES}, 18986909Simp {0x06401b4b, 0x00, "HighPoint RocketRAID 640", AHCI_Q_NOBSYRES}, 190120189Sbde {0x06441103, 0x00, "HighPoint RocketRAID 644", AHCI_Q_NOBSYRES}, 19186909Simp {0x06441b4b, 0x00, "HighPoint RocketRAID 644", AHCI_Q_NOBSYRES}, 19286909Simp {0x044c10de, 0x00, "NVIDIA MCP65", AHCI_Q_NOAA}, 19386909Simp {0x044d10de, 0x00, "NVIDIA MCP65", AHCI_Q_NOAA}, 19486909Simp {0x044e10de, 0x00, "NVIDIA MCP65", AHCI_Q_NOAA}, 19586909Simp {0x044f10de, 0x00, "NVIDIA MCP65", AHCI_Q_NOAA}, 19686909Simp {0x045c10de, 0x00, "NVIDIA MCP65", AHCI_Q_NOAA}, 19786909Simp {0x045d10de, 0x00, "NVIDIA MCP65", AHCI_Q_NOAA}, 19886909Simp {0x045e10de, 0x00, "NVIDIA MCP65", AHCI_Q_NOAA}, 19986909Simp {0x045f10de, 0x00, "NVIDIA MCP65", AHCI_Q_NOAA}, 20086909Simp {0x055010de, 0x00, "NVIDIA MCP67", AHCI_Q_NOAA}, 20186909Simp {0x055110de, 0x00, "NVIDIA MCP67", AHCI_Q_NOAA}, 20286909Simp {0x055210de, 0x00, "NVIDIA MCP67", AHCI_Q_NOAA}, 20386909Simp {0x055310de, 0x00, "NVIDIA MCP67", AHCI_Q_NOAA}, 20486909Simp {0x055410de, 0x00, "NVIDIA MCP67", AHCI_Q_NOAA}, 20586909Simp {0x055510de, 0x00, "NVIDIA MCP67", AHCI_Q_NOAA}, 20686909Simp {0x055610de, 0x00, "NVIDIA MCP67", AHCI_Q_NOAA}, 20786909Simp {0x055710de, 0x00, "NVIDIA MCP67", AHCI_Q_NOAA}, 20886909Simp {0x055810de, 0x00, "NVIDIA MCP67", AHCI_Q_NOAA}, 20986909Simp {0x055910de, 0x00, "NVIDIA MCP67", AHCI_Q_NOAA}, 21086909Simp {0x055A10de, 0x00, "NVIDIA MCP67", AHCI_Q_NOAA}, 21186909Simp {0x055B10de, 0x00, "NVIDIA MCP67", AHCI_Q_NOAA}, 21286909Simp {0x058410de, 0x00, "NVIDIA MCP67", AHCI_Q_NOAA}, 21386909Simp {0x07f010de, 0x00, "NVIDIA MCP73", AHCI_Q_NOAA}, 21486909Simp {0x07f110de, 0x00, "NVIDIA MCP73", AHCI_Q_NOAA}, 21586909Simp {0x07f210de, 0x00, "NVIDIA MCP73", AHCI_Q_NOAA}, 21686909Simp {0x07f310de, 0x00, "NVIDIA MCP73", AHCI_Q_NOAA}, 21786909Simp {0x07f410de, 0x00, "NVIDIA MCP73", AHCI_Q_NOAA}, 21886909Simp {0x07f510de, 0x00, "NVIDIA MCP73", AHCI_Q_NOAA}, 21986909Simp {0x07f610de, 0x00, "NVIDIA MCP73", AHCI_Q_NOAA}, 220120189Sbde {0x07f710de, 0x00, "NVIDIA MCP73", AHCI_Q_NOAA}, 22186909Simp {0x07f810de, 0x00, "NVIDIA MCP73", AHCI_Q_NOAA}, 22286909Simp {0x07f910de, 0x00, "NVIDIA MCP73", AHCI_Q_NOAA}, 22386909Simp {0x07fa10de, 0x00, "NVIDIA MCP73", AHCI_Q_NOAA}, 22486909Simp {0x07fb10de, 0x00, "NVIDIA MCP73", AHCI_Q_NOAA}, 22586909Simp {0x0ad010de, 0x00, "NVIDIA MCP77", AHCI_Q_NOAA}, 22686909Simp {0x0ad110de, 0x00, "NVIDIA MCP77", AHCI_Q_NOAA}, 22786909Simp {0x0ad210de, 0x00, "NVIDIA MCP77", AHCI_Q_NOAA}, 22886909Simp {0x0ad310de, 0x00, "NVIDIA MCP77", AHCI_Q_NOAA}, 229111613Sphk {0x0ad410de, 0x00, "NVIDIA MCP77", AHCI_Q_NOAA}, 230225203Srwatson {0x0ad510de, 0x00, "NVIDIA MCP77", AHCI_Q_NOAA}, 231119485Snjl {0x0ad610de, 0x00, "NVIDIA MCP77", AHCI_Q_NOAA}, 232119485Snjl {0x0ad710de, 0x00, "NVIDIA MCP77", AHCI_Q_NOAA}, 23386909Simp {0x0ad810de, 0x00, "NVIDIA MCP77", AHCI_Q_NOAA}, 23486909Simp {0x0ad910de, 0x00, "NVIDIA MCP77", AHCI_Q_NOAA}, 23586909Simp {0x0ada10de, 0x00, "NVIDIA MCP77", AHCI_Q_NOAA}, 23686909Simp {0x0adb10de, 0x00, "NVIDIA MCP77", AHCI_Q_NOAA}, 23786909Simp {0x0ab410de, 0x00, "NVIDIA MCP79", AHCI_Q_NOAA}, 23886909Simp {0x0ab510de, 0x00, "NVIDIA MCP79", AHCI_Q_NOAA}, 23989986Sjhay {0x0ab610de, 0x00, "NVIDIA MCP79", AHCI_Q_NOAA}, 24089986Sjhay {0x0ab710de, 0x00, "NVIDIA MCP79", AHCI_Q_NOAA}, 24186909Simp {0x0ab810de, 0x00, "NVIDIA MCP79", AHCI_Q_NOAA}, 24286909Simp {0x0ab910de, 0x00, "NVIDIA MCP79", AHCI_Q_NOAA}, 243116120Sscottl {0x0aba10de, 0x00, "NVIDIA MCP79", AHCI_Q_NOAA}, 244116120Sscottl {0x0abb10de, 0x00, "NVIDIA MCP79", AHCI_Q_NOAA}, 24586909Simp {0x0abc10de, 0x00, "NVIDIA MCP79", AHCI_Q_NOAA}, 24686909Simp {0x0abd10de, 0x00, "NVIDIA MCP79", AHCI_Q_NOAA}, 24786909Simp {0x0abe10de, 0x00, "NVIDIA MCP79", AHCI_Q_NOAA}, 24886909Simp {0x0abf10de, 0x00, "NVIDIA MCP79", AHCI_Q_NOAA}, 24986909Simp {0x0d8410de, 0x00, "NVIDIA MCP89", AHCI_Q_NOAA}, 25086909Simp {0x0d8510de, 0x00, "NVIDIA MCP89", AHCI_Q_NOAA}, 25186909Simp {0x0d8610de, 0x00, "NVIDIA MCP89", AHCI_Q_NOAA}, 25286909Simp {0x0d8710de, 0x00, "NVIDIA MCP89", AHCI_Q_NOAA}, 25386909Simp {0x0d8810de, 0x00, "NVIDIA MCP89", AHCI_Q_NOAA}, 25486909Simp {0x0d8910de, 0x00, "NVIDIA MCP89", AHCI_Q_NOAA}, 25593010Sbde {0x0d8a10de, 0x00, "NVIDIA MCP89", AHCI_Q_NOAA}, 25651078Speter {0x0d8b10de, 0x00, "NVIDIA MCP89", AHCI_Q_NOAA}, 25751078Speter {0x0d8c10de, 0x00, "NVIDIA MCP89", AHCI_Q_NOAA}, 258131373Sphk {0x0d8d10de, 0x00, "NVIDIA MCP89", AHCI_Q_NOAA}, 25951078Speter {0x0d8e10de, 0x00, "NVIDIA MCP89", AHCI_Q_NOAA}, 26093010Sbde {0x0d8f10de, 0x00, "NVIDIA MCP89", AHCI_Q_NOAA}, 261136478Sphk {0x33491106, 0x00, "VIA VT8251", AHCI_Q_NOPMP|AHCI_Q_NONCQ}, 262136478Sphk {0x62871106, 0x00, "VIA VT8251", AHCI_Q_NOPMP|AHCI_Q_NONCQ}, 26393010Sbde {0x11841039, 0x00, "SiS 966", 0}, 26493010Sbde {0x11851039, 0x00, "SiS 968", 0}, 265166901Spiso {0x01861039, 0x00, "SiS 968", 0}, 266131094Sphk {0x00000000, 0x00, NULL, 0} 26793010Sbde}; 26893010Sbde 26993010Sbdestatic int 27093010Sbdeahci_probe(device_t dev) 27193010Sbde{ 27293010Sbde char buf[64]; 27351078Speter int i, valid = 0; 27451078Speter uint32_t devid = pci_get_devid(dev); 27585365Simp uint8_t revid = pci_get_revid(dev); 27670174Sjhb 27770174Sjhb /* Is this a possible AHCI candidate? */ 27851078Speter if (pci_get_class(dev) == PCIC_STORAGE && 27951078Speter pci_get_subclass(dev) == PCIS_STORAGE_SATA && 28085365Simp pci_get_progif(dev) == PCIP_STORAGE_SATA_AHCI_1_0) 28151078Speter valid = 1; 28286909Simp /* Is this a known AHCI chip? */ 28351078Speter for (i = 0; ahci_ids[i].id != 0; i++) { 284114722Sobrien if (ahci_ids[i].id == devid && 28551078Speter ahci_ids[i].rev <= revid && 28689986Sjhay (valid || !(ahci_ids[i].quirks & AHCI_Q_NOFORCE))) { 28789986Sjhay /* Do not attach JMicrons with single PCI function. */ 28898401Sn_hibma if (pci_get_vendor(dev) == 0x197b && 28998401Sn_hibma (pci_read_config(dev, 0xdf, 1) & 0x40) == 0) 29098401Sn_hibma return (ENXIO); 29151078Speter snprintf(buf, sizeof(buf), "%s AHCI SATA controller", 29251078Speter ahci_ids[i].name); 29398401Sn_hibma device_set_desc_copy(dev, buf); 29472238Sjhb return (BUS_PROBE_VENDOR); 29572238Sjhb } 29651078Speter } 29751078Speter if (!valid) 29851078Speter return (ENXIO); 29951078Speter device_set_desc_copy(dev, "AHCI SATA controller"); 30053344Speter return (BUS_PROBE_VENDOR); 30151078Speter} 302131939Smarcel 303131939Smarcelstatic int 304131939Smarcelahci_ata_probe(device_t dev) 305131939Smarcel{ 30651078Speter char buf[64]; 30751078Speter int i; 30886909Simp uint32_t devid = pci_get_devid(dev); 30951078Speter uint8_t revid = pci_get_revid(dev); 31051078Speter 31151078Speter if ((intptr_t)device_get_ivars(dev) >= 0) 31251078Speter return (ENXIO); 31351078Speter /* Is this a known AHCI chip? */ 31451078Speter for (i = 0; ahci_ids[i].id != 0; i++) { 31551078Speter if (ahci_ids[i].id == devid && 31651078Speter ahci_ids[i].rev <= revid) { 31751078Speter snprintf(buf, sizeof(buf), "%s AHCI SATA controller", 31851078Speter ahci_ids[i].name); 31951078Speter device_set_desc_copy(dev, buf); 32051078Speter return (BUS_PROBE_VENDOR); 32151078Speter } 32251078Speter } 32351078Speter device_set_desc_copy(dev, "AHCI SATA controller"); 32462573Sphk return (BUS_PROBE_VENDOR); 32551078Speter} 32651078Speter 32751078Speterstatic int 32851078Speterahci_attach(device_t dev) 32951078Speter{ 33051078Speter struct ahci_controller *ctlr = device_get_softc(dev); 33151078Speter device_t child; 33251078Speter int error, unit, speed, i; 33351078Speter uint32_t devid = pci_get_devid(dev); 33451078Speter uint8_t revid = pci_get_revid(dev); 33551078Speter u_int32_t version; 33651078Speter 33751078Speter ctlr->dev = dev; 33851078Speter i = 0; 33951078Speter while (ahci_ids[i].id != 0 && 34051078Speter (ahci_ids[i].id != devid || 34151078Speter ahci_ids[i].rev > revid)) 34251078Speter i++; 34357915Simp ctlr->quirks = ahci_ids[i].quirks; 34451078Speter resource_int_value(device_get_name(dev), 34551078Speter device_get_unit(dev), "ccc", &ctlr->ccc); 346135329Sphk /* if we have a memory BAR(5) we are likely on an AHCI part */ 347135516Snyan ctlr->r_rid = PCIR_BAR(5); 348135516Snyan if (!(ctlr->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 349135516Snyan &ctlr->r_rid, RF_ACTIVE))) 35051078Speter return ENXIO; 35151078Speter /* Setup our own memory management for channels. */ 35251078Speter ctlr->sc_iomem.rm_start = rman_get_start(ctlr->r_mem); 35351078Speter ctlr->sc_iomem.rm_end = rman_get_end(ctlr->r_mem); 35451078Speter ctlr->sc_iomem.rm_type = RMAN_ARRAY; 355135329Sphk ctlr->sc_iomem.rm_descr = "I/O memory addresses"; 356135329Sphk if ((error = rman_init(&ctlr->sc_iomem)) != 0) { 357135329Sphk bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem); 358135329Sphk return (error); 35951078Speter } 360135516Snyan if ((error = rman_manage_region(&ctlr->sc_iomem, 36151078Speter rman_get_start(ctlr->r_mem), rman_get_end(ctlr->r_mem))) != 0) { 36251078Speter bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem); 36351078Speter rman_fini(&ctlr->sc_iomem); 36451078Speter return (error); 36551078Speter } 36651078Speter pci_enable_busmaster(dev); 36751078Speter /* Reset controller */ 36851078Speter if ((error = ahci_ctlr_reset(dev)) != 0) { 36951078Speter bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem); 37051078Speter rman_fini(&ctlr->sc_iomem); 37151078Speter return (error); 372153195Simp }; 37351078Speter /* Get the HW capabilities */ 37486909Simp version = ATA_INL(ctlr->r_mem, AHCI_VS); 37586909Simp ctlr->caps = ATA_INL(ctlr->r_mem, AHCI_CAP); 37686909Simp if (version >= 0x00010020) 37786909Simp ctlr->caps2 = ATA_INL(ctlr->r_mem, AHCI_CAP2); 37886909Simp if (ctlr->caps & AHCI_CAP_EMS) 37986909Simp ctlr->capsem = ATA_INL(ctlr->r_mem, AHCI_EM_CTL); 38086909Simp ctlr->ichannels = ATA_INL(ctlr->r_mem, AHCI_PI); 38186909Simp if (ctlr->quirks & AHCI_Q_1CH) { 382104933Simp ctlr->caps &= ~AHCI_CAP_NPMASK; 38386909Simp ctlr->ichannels &= 0x01; 38486909Simp } 38586909Simp if (ctlr->quirks & AHCI_Q_2CH) { 38686909Simp ctlr->caps &= ~AHCI_CAP_NPMASK; 38785365Simp ctlr->caps |= 1; 388136478Sphk ctlr->ichannels &= 0x03; 38951078Speter } 39051078Speter if (ctlr->quirks & AHCI_Q_4CH) { 39151078Speter ctlr->caps &= ~AHCI_CAP_NPMASK; 39252471Simp ctlr->caps |= 3; 39357915Simp ctlr->ichannels &= 0x0f; 39452471Simp } 39554386Simp ctlr->channels = MAX(flsl(ctlr->ichannels), 39651078Speter (ctlr->caps & AHCI_CAP_NPMASK) + 1); 397120175Sbde if (ctlr->quirks & AHCI_Q_NOPMP) 398132561Simp ctlr->caps &= ~AHCI_CAP_SPM; 399136478Sphk if (ctlr->quirks & AHCI_Q_NONCQ) 40054386Simp ctlr->caps &= ~AHCI_CAP_SNCQ; 40154386Simp if ((ctlr->caps & AHCI_CAP_CCCS) == 0) 40254386Simp ctlr->ccc = 0; 40354386Simp ahci_ctlr_setup(dev); 40454386Simp /* Setup interrupts. */ 405116120Sscottl if (ahci_setup_interrupt(dev)) { 406116120Sscottl bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem); 407136478Sphk rman_fini(&ctlr->sc_iomem); 408136478Sphk return ENXIO; 409136478Sphk } 410136478Sphk /* Announce HW capabilities. */ 411136478Sphk speed = (ctlr->caps & AHCI_CAP_ISS) >> AHCI_CAP_ISS_SHIFT; 41253978Simp device_printf(dev, 41351078Speter "AHCI v%x.%02x with %d %sGbps ports, Port Multiplier %s%s\n", 41451078Speter ((version >> 20) & 0xf0) + ((version >> 16) & 0x0f), 41585365Simp ((version >> 4) & 0xf0) + (version & 0x0f), 41689986Sjhay (ctlr->caps & AHCI_CAP_NPMASK) + 1, 41758885Simp ((speed == 1) ? "1.5":((speed == 2) ? "3": 41858885Simp ((speed == 3) ? "6":"?"))), 41989986Sjhay (ctlr->caps & AHCI_CAP_SPM) ? 42085365Simp "supported" : "not supported", 42151078Speter (ctlr->caps & AHCI_CAP_FBSS) ? 42253344Speter " with FBS" : ""); 42351078Speter if (bootverbose) { 42453344Speter device_printf(dev, "Caps:%s%s%s%s%s%s%s%s %sGbps", 42553344Speter (ctlr->caps & AHCI_CAP_64BIT) ? " 64bit":"", 42660471Snyan (ctlr->caps & AHCI_CAP_SNCQ) ? " NCQ":"", 42789986Sjhay (ctlr->caps & AHCI_CAP_SSNTF) ? " SNTF":"", 42851078Speter (ctlr->caps & AHCI_CAP_SMPS) ? " MPS":"", 42951078Speter (ctlr->caps & AHCI_CAP_SSS) ? " SS":"", 43051078Speter (ctlr->caps & AHCI_CAP_SALP) ? " ALP":"", 43151078Speter (ctlr->caps & AHCI_CAP_SAL) ? " AL":"", 43251078Speter (ctlr->caps & AHCI_CAP_SCLO) ? " CLO":"", 43351078Speter ((speed == 1) ? "1.5":((speed == 2) ? "3": 43451078Speter ((speed == 3) ? "6":"?")))); 43551078Speter printf("%s%s%s%s%s%s %dcmd%s%s%s %dports\n", 43654206Speter (ctlr->caps & AHCI_CAP_SAM) ? " AM":"", 43751088Speter (ctlr->caps & AHCI_CAP_SPM) ? " PM":"", 43851078Speter (ctlr->caps & AHCI_CAP_FBSS) ? " FBS":"", 43951078Speter (ctlr->caps & AHCI_CAP_PMD) ? " PMD":"", 44051078Speter (ctlr->caps & AHCI_CAP_SSC) ? " SSC":"", 44158885Simp (ctlr->caps & AHCI_CAP_PSC) ? " PSC":"", 44251078Speter ((ctlr->caps & AHCI_CAP_NCS) >> AHCI_CAP_NCS_SHIFT) + 1, 44351078Speter (ctlr->caps & AHCI_CAP_CCCS) ? " CCC":"", 44451078Speter (ctlr->caps & AHCI_CAP_EMS) ? " EM":"", 44557915Simp (ctlr->caps & AHCI_CAP_SXS) ? " eSATA":"", 44651078Speter (ctlr->caps & AHCI_CAP_NPMASK) + 1); 44786909Simp } 448124669Sru if (bootverbose && version >= 0x00010020) { 449124669Sru device_printf(dev, "Caps2:%s%s%s\n", 45086909Simp (ctlr->caps2 & AHCI_CAP2_APST) ? " APST":"", 451124669Sru (ctlr->caps2 & AHCI_CAP2_NVMP) ? " NVMP":"", 45286909Simp (ctlr->caps2 & AHCI_CAP2_BOH) ? " BOH":""); 45360471Snyan } 45460471Snyan if (bootverbose && (ctlr->caps & AHCI_CAP_EMS)) { 45589986Sjhay device_printf(dev, "EM Caps:%s%s%s%s%s%s%s%s\n", 45689986Sjhay (ctlr->capsem & AHCI_EM_PM) ? " PM":"", 45789986Sjhay (ctlr->capsem & AHCI_EM_ALHD) ? " ALHD":"", 45860471Snyan (ctlr->capsem & AHCI_EM_XMT) ? " XMT":"", 45985209Sjhb (ctlr->capsem & AHCI_EM_SMB) ? " SMB":"", 46085209Sjhb (ctlr->capsem & AHCI_EM_SGPIO) ? " SGPIO":"", 46193818Sjhb (ctlr->capsem & AHCI_EM_SES2) ? " SES-2":"", 46293818Sjhb (ctlr->capsem & AHCI_EM_SAFTE) ? " SAF-TE":"", 46385209Sjhb (ctlr->capsem & AHCI_EM_LED) ? " LED":""); 46485209Sjhb } 46585209Sjhb /* Attach all channels on this controller */ 46670174Sjhb for (unit = 0; unit < ctlr->channels; unit++) { 46753344Speter if ((ctlr->ichannels & (1 << unit)) == 0) 46853344Speter continue; 46953344Speter child = device_add_child(dev, "ahcich", -1); 47053344Speter if (child == NULL) 47153344Speter device_printf(dev, "failed to add channel device\n"); 47253344Speter else 47353344Speter device_set_ivars(child, (void *)(intptr_t)unit); 47453344Speter } 47551078Speter bus_generic_attach(dev); 47651078Speter return 0; 47751078Speter} 47851078Speter 47951078Speterstatic int 48051078Speterahci_detach(device_t dev) 48151078Speter{ 48251078Speter struct ahci_controller *ctlr = device_get_softc(dev); 48353344Speter device_t *children; 48451078Speter int nchildren, i; 48551078Speter 48651078Speter /* Detach & delete all children */ 48751078Speter if (!device_get_children(dev, &children, &nchildren)) { 48854194Speter for (i = 0; i < nchildren; i++) 48954194Speter device_delete_child(dev, children[i]); 49054194Speter free(children, M_TEMP); 49153344Speter } 49251078Speter /* Free interrupts. */ 49351078Speter for (i = 0; i < ctlr->numirqs; i++) { 49451078Speter if (ctlr->irqs[i].r_irq) { 49551078Speter bus_teardown_intr(dev, ctlr->irqs[i].r_irq, 49653344Speter ctlr->irqs[i].handle); 49751078Speter bus_release_resource(dev, SYS_RES_IRQ, 49851078Speter ctlr->irqs[i].r_irq_rid, ctlr->irqs[i].r_irq); 49951078Speter } 50051078Speter } 50156788Sbde pci_release_msi(dev); 50286909Simp /* Free memory. */ 50386909Simp rman_fini(&ctlr->sc_iomem); 50451078Speter if (ctlr->r_mem) 50551078Speter bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem); 50651078Speter return (0); 50751078Speter} 50851078Speter 50951078Speterstatic int 51051078Speterahci_ctlr_reset(device_t dev) 51151078Speter{ 51251078Speter struct ahci_controller *ctlr = device_get_softc(dev); 51351078Speter int timeout; 51451078Speter 51551078Speter if (pci_read_config(dev, 0x00, 4) == 0x28298086 && 51651078Speter (pci_read_config(dev, 0x92, 1) & 0xfe) == 0x04) 51751078Speter pci_write_config(dev, 0x92, 0x01, 1); 51857234Sbde /* Enable AHCI mode */ 51954206Speter ATA_OUTL(ctlr->r_mem, AHCI_GHC, AHCI_GHC_AE); 52054206Speter /* Reset AHCI controller */ 52154206Speter ATA_OUTL(ctlr->r_mem, AHCI_GHC, AHCI_GHC_AE|AHCI_GHC_HR); 52251078Speter for (timeout = 1000; timeout > 0; timeout--) { 52351078Speter DELAY(1000); 52451078Speter if ((ATA_INL(ctlr->r_mem, AHCI_GHC) & AHCI_GHC_HR) == 0) 52551078Speter break; 52651078Speter } 52751078Speter if (timeout == 0) { 52857234Sbde device_printf(dev, "AHCI controller reset failure\n"); 52957234Sbde return ENXIO; 53057234Sbde } 53157234Sbde /* Reenable AHCI mode */ 53257234Sbde ATA_OUTL(ctlr->r_mem, AHCI_GHC, AHCI_GHC_AE); 53357234Sbde return (0); 53457234Sbde} 53557234Sbde 53657234Sbdestatic int 53757234Sbdeahci_ctlr_setup(device_t dev) 53857234Sbde{ 53951078Speter struct ahci_controller *ctlr = device_get_softc(dev); 54051078Speter /* Clear interrupts */ 54151078Speter ATA_OUTL(ctlr->r_mem, AHCI_IS, ATA_INL(ctlr->r_mem, AHCI_IS)); 54254194Speter /* Configure CCC */ 54351078Speter if (ctlr->ccc) { 54451078Speter ATA_OUTL(ctlr->r_mem, AHCI_CCCP, ATA_INL(ctlr->r_mem, AHCI_PI)); 54551078Speter ATA_OUTL(ctlr->r_mem, AHCI_CCCC, 54651078Speter (ctlr->ccc << AHCI_CCCC_TV_SHIFT) | 54751078Speter (4 << AHCI_CCCC_CC_SHIFT) | 54851078Speter AHCI_CCCC_EN); 54951078Speter ctlr->cccv = (ATA_INL(ctlr->r_mem, AHCI_CCCC) & 55051078Speter AHCI_CCCC_INT_MASK) >> AHCI_CCCC_INT_SHIFT; 55151078Speter if (bootverbose) { 55251078Speter device_printf(dev, 55351078Speter "CCC with %dms/4cmd enabled on vector %d\n", 55472200Sbmilekic ctlr->ccc, ctlr->cccv); 55551078Speter } 55651078Speter } 55751078Speter /* Enable AHCI interrupts */ 558112384Ssobomax ATA_OUTL(ctlr->r_mem, AHCI_GHC, 559112384Ssobomax ATA_INL(ctlr->r_mem, AHCI_GHC) | AHCI_GHC_IE); 560112270Ssobomax return (0); 561112270Ssobomax} 562112270Ssobomax 563112384Ssobomaxstatic int 564112270Ssobomaxahci_suspend(device_t dev) 565112384Ssobomax{ 566112384Ssobomax struct ahci_controller *ctlr = device_get_softc(dev); 567112384Ssobomax 568112384Ssobomax bus_generic_suspend(dev); 569112384Ssobomax /* Disable interupts, so the state change(s) doesn't trigger */ 570112384Ssobomax ATA_OUTL(ctlr->r_mem, AHCI_GHC, 571112384Ssobomax ATA_INL(ctlr->r_mem, AHCI_GHC) & (~AHCI_GHC_IE)); 572112384Ssobomax return 0; 573112384Ssobomax} 574112384Ssobomax 575112384Ssobomaxstatic int 576112384Ssobomaxahci_resume(device_t dev) 577112384Ssobomax{ 578112270Ssobomax int res; 579112384Ssobomax 580112270Ssobomax if ((res = ahci_ctlr_reset(dev)) != 0) 58151078Speter return (res); 58251078Speter ahci_ctlr_setup(dev); 58351078Speter return (bus_generic_resume(dev)); 58451078Speter} 58551078Speter 58651078Speterstatic int 58751078Speterahci_setup_interrupt(device_t dev) 58851078Speter{ 58951078Speter struct ahci_controller *ctlr = device_get_softc(dev); 59051078Speter int i, msi = 1; 59151078Speter 59251078Speter /* Process hints. */ 59360471Snyan resource_int_value(device_get_name(dev), 59489986Sjhay device_get_unit(dev), "msi", &msi); 59589986Sjhay if (msi < 0) 59689986Sjhay msi = 0; 59760471Snyan else if (msi == 1) 59851078Speter msi = min(1, pci_msi_count(dev)); 59951078Speter else if (msi > 1) 60051078Speter msi = pci_msi_count(dev); 60151078Speter /* Allocate MSI if needed/present. */ 602172568Skevlo if (msi && pci_alloc_msi(dev, &msi) == 0) { 60351078Speter ctlr->numirqs = msi; 60451078Speter } else { 60551078Speter msi = 0; 60651078Speter ctlr->numirqs = 1; 60760471Snyan } 60860471Snyan /* Check for single MSI vector fallback. */ 60951078Speter if (ctlr->numirqs > 1 && 61051078Speter (ATA_INL(ctlr->r_mem, AHCI_GHC) & AHCI_GHC_MRSM) != 0) { 61151078Speter device_printf(dev, "Falling back to one MSI\n"); 61251078Speter ctlr->numirqs = 1; 61351078Speter } 61451078Speter /* Allocate all IRQs. */ 61551078Speter for (i = 0; i < ctlr->numirqs; i++) { 61651078Speter ctlr->irqs[i].ctlr = ctlr; 61760471Snyan ctlr->irqs[i].r_irq_rid = i + (msi ? 1 : 0); 61851078Speter if (ctlr->numirqs == 1 || i >= ctlr->channels || 61951078Speter (ctlr->ccc && i == ctlr->cccv)) 62051078Speter ctlr->irqs[i].mode = AHCI_IRQ_MODE_ALL; 62151078Speter else if (i == ctlr->numirqs - 1) 62251078Speter ctlr->irqs[i].mode = AHCI_IRQ_MODE_AFTER; 62351078Speter else 62451078Speter ctlr->irqs[i].mode = AHCI_IRQ_MODE_ONE; 62551078Speter if (!(ctlr->irqs[i].r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, 62651078Speter &ctlr->irqs[i].r_irq_rid, RF_SHAREABLE | RF_ACTIVE))) { 62760471Snyan device_printf(dev, "unable to map interrupt\n"); 62851078Speter return ENXIO; 62951078Speter } 63051078Speter if ((bus_setup_intr(dev, ctlr->irqs[i].r_irq, ATA_INTR_FLAGS, NULL, 63151078Speter (ctlr->irqs[i].mode == AHCI_IRQ_MODE_ONE) ? ahci_intr_one : ahci_intr, 63251078Speter &ctlr->irqs[i], &ctlr->irqs[i].handle))) { 63351078Speter /* SOS XXX release r_irq */ 63451078Speter device_printf(dev, "unable to setup interrupt\n"); 63560471Snyan return ENXIO; 636120468Sbde } 637120468Sbde if (ctlr->numirqs > 1) { 638120468Sbde bus_describe_intr(dev, ctlr->irqs[i].r_irq, 639120468Sbde ctlr->irqs[i].handle, 64051078Speter ctlr->irqs[i].mode == AHCI_IRQ_MODE_ONE ? 64151078Speter "ch%d" : "%d", i); 64251078Speter } 64351078Speter } 64451078Speter return (0); 64551078Speter} 64651078Speter 64751078Speter/* 64851078Speter * Common case interrupt handler. 64960471Snyan */ 65092401Simpstatic void 65192401Simpahci_intr(void *data) 65292401Simp{ 65392401Simp struct ahci_controller_irq *irq = data; 65492401Simp struct ahci_controller *ctlr = irq->ctlr; 65592401Simp u_int32_t is, ise = 0; 65692401Simp void *arg; 65751078Speter int unit; 65851078Speter 659120189Sbde if (irq->mode == AHCI_IRQ_MODE_ALL) { 660120189Sbde unit = 0; 661120189Sbde if (ctlr->ccc) 662120189Sbde is = ctlr->ichannels; 663120189Sbde else 66451078Speter is = ATA_INL(ctlr->r_mem, AHCI_IS); 66585365Simp } else { /* AHCI_IRQ_MODE_AFTER */ 666120189Sbde unit = irq->r_irq_rid - 1; 66753370Speter is = ATA_INL(ctlr->r_mem, AHCI_IS); 66853370Speter } 66960471Snyan /* CCC interrupt is edge triggered. */ 67053370Speter if (ctlr->ccc) 671120189Sbde ise = 1 << ctlr->cccv; 672120189Sbde /* Some controllers have edge triggered IS. */ 67353370Speter if (ctlr->quirks & AHCI_Q_EDGEIS) 67453370Speter ise |= is; 675120189Sbde if (ise != 0) 676120189Sbde ATA_OUTL(ctlr->r_mem, AHCI_IS, ise); 677120189Sbde for (; unit < ctlr->channels; unit++) { 678120189Sbde if ((is & (1 << unit)) != 0 && 67960471Snyan (arg = ctlr->interrupt[unit].argument)) { 68060471Snyan ctlr->interrupt[unit].function(arg); 681120189Sbde } 68253370Speter } 68353370Speter /* AHCI declares level triggered IS. */ 684120189Sbde if (!(ctlr->quirks & AHCI_Q_EDGEIS)) 68553370Speter ATA_OUTL(ctlr->r_mem, AHCI_IS, is); 68681793Simp} 68753370Speter 68851078Speter/* 689120189Sbde * Simplified interrupt handler for multivector MSI mode. 69053370Speter */ 69151078Speterstatic void 69281793Simpahci_intr_one(void *data) 69360471Snyan{ 69472200Sbmilekic struct ahci_controller_irq *irq = data; 69553344Speter struct ahci_controller *ctlr = irq->ctlr; 696128899Sambrisko void *arg; 69786909Simp int unit; 698183692Simp 699183692Simp unit = irq->r_irq_rid - 1; 700183692Simp /* Some controllers have edge triggered IS. */ 701183692Simp if (ctlr->quirks & AHCI_Q_EDGEIS) 702183692Simp ATA_OUTL(ctlr->r_mem, AHCI_IS, 1 << unit); 703183692Simp if ((arg = ctlr->interrupt[unit].argument)) 704183692Simp ctlr->interrupt[unit].function(arg); 705183692Simp /* AHCI declares level triggered IS. */ 70686909Simp if (!(ctlr->quirks & AHCI_Q_EDGEIS)) 70786909Simp ATA_OUTL(ctlr->r_mem, AHCI_IS, 1 << unit); 70886909Simp} 70986909Simp 710160420Simpstatic struct resource * 71153344Speterahci_alloc_resource(device_t dev, device_t child, int type, int *rid, 71253344Speter u_long start, u_long end, u_long count, u_int flags) 71351078Speter{ 71451078Speter struct ahci_controller *ctlr = device_get_softc(dev); 71551078Speter int unit = ((struct ahci_channel *)device_get_softc(child))->unit; 71651078Speter struct resource *res = NULL; 71751078Speter int offset = AHCI_OFFSET + (unit << 7); 71851078Speter long st; 71951078Speter 72051078Speter switch (type) { 72151078Speter case SYS_RES_MEMORY: 72260471Snyan st = rman_get_start(ctlr->r_mem); 72360471Snyan res = rman_reserve_resource(&ctlr->sc_iomem, st + offset, 724112270Ssobomax st + offset + 127, 128, RF_ACTIVE, child); 72551078Speter if (res) { 72651078Speter bus_space_handle_t bsh; 72760471Snyan bus_space_tag_t bst; 72851078Speter bsh = rman_get_bushandle(ctlr->r_mem); 72951078Speter bst = rman_get_bustag(ctlr->r_mem); 73060471Snyan bus_space_subregion(bst, bsh, offset, 128, &bsh); 73151078Speter rman_set_bushandle(res, bsh); 73251078Speter rman_set_bustag(res, bst); 73351078Speter } 73451078Speter break; 73551078Speter case SYS_RES_IRQ: 73651078Speter if (*rid == ATA_IRQ_RID) 73751078Speter res = ctlr->irqs[0].r_irq; 73851078Speter break; 73951078Speter } 74051078Speter return (res); 74160471Snyan} 74260471Snyan 74360471Snyanstatic int 74451078Speterahci_release_resource(device_t dev, device_t child, int type, int rid, 74551078Speter struct resource *r) 74660471Snyan{ 74751078Speter 74872200Sbmilekic switch (type) { 74951078Speter case SYS_RES_MEMORY: 75051078Speter rman_release_resource(r); 75154194Speter return (0); 75289463Simp case SYS_RES_IRQ: 75351078Speter if (rid != ATA_IRQ_RID) 75454206Speter return ENOENT; 75553344Speter return (0); 75689447Sbmah } 75789470Sbmah return (EINVAL); 75889447Sbmah} 75989463Simp 76051078Speterstatic int 76151078Speterahci_setup_intr(device_t dev, device_t child, struct resource *irq, 76251078Speter int flags, driver_filter_t *filter, driver_intr_t *function, 76351078Speter void *argument, void **cookiep) 76451078Speter{ 765128899Sambrisko struct ahci_controller *ctlr = device_get_softc(dev); 766128899Sambrisko int unit = (intptr_t)device_get_ivars(child); 76751078Speter 76860471Snyan if (filter != NULL) { 76951078Speter printf("ahci.c: we cannot use a filter here\n"); 77051078Speter return (EINVAL); 77151078Speter } 77251078Speter ctlr->interrupt[unit].function = function; 77351078Speter ctlr->interrupt[unit].argument = argument; 77451078Speter return (0); 77551078Speter} 77651078Speter 77751078Speterstatic int 77851078Speterahci_teardown_intr(device_t dev, device_t child, struct resource *irq, 77951078Speter void *cookie) 78051078Speter{ 781128899Sambrisko struct ahci_controller *ctlr = device_get_softc(dev); 78286909Simp int unit = (intptr_t)device_get_ivars(child); 783183692Simp 784183692Simp ctlr->interrupt[unit].function = NULL; 785183692Simp ctlr->interrupt[unit].argument = NULL; 786183692Simp return (0); 787183692Simp} 788183692Simp 789183692Simpstatic int 79086909Simpahci_print_child(device_t dev, device_t child) 79186909Simp{ 79286909Simp int retval; 79386909Simp 794160420Simp retval = bus_print_child_header(dev, child); 79551078Speter retval += printf(" at channel %d", 79651078Speter (int)(intptr_t)device_get_ivars(child)); 79751078Speter retval += bus_print_child_footer(dev, child); 79851078Speter 79951078Speter return (retval); 80051078Speter} 80151078Speter 80251078Speterstatic int 80351078Speterahci_child_location_str(device_t dev, device_t child, char *buf, 80451078Speter size_t buflen) 80551078Speter{ 80651078Speter 80751078Speter snprintf(buf, buflen, "channel=%d", 80851078Speter (int)(intptr_t)device_get_ivars(child)); 80951078Speter return (0); 81051078Speter} 81151078Speter 81251078Speterdevclass_t ahci_devclass; 81351078Speterstatic device_method_t ahci_methods[] = { 81451078Speter DEVMETHOD(device_probe, ahci_probe), 81551078Speter DEVMETHOD(device_attach, ahci_attach), 81651078Speter DEVMETHOD(device_detach, ahci_detach), 81751078Speter DEVMETHOD(device_suspend, ahci_suspend), 81851078Speter DEVMETHOD(device_resume, ahci_resume), 81951078Speter DEVMETHOD(bus_print_child, ahci_print_child), 82051078Speter DEVMETHOD(bus_alloc_resource, ahci_alloc_resource), 82151078Speter DEVMETHOD(bus_release_resource, ahci_release_resource), 82251078Speter DEVMETHOD(bus_setup_intr, ahci_setup_intr), 82351078Speter DEVMETHOD(bus_teardown_intr,ahci_teardown_intr), 82451078Speter DEVMETHOD(bus_child_location_str, ahci_child_location_str), 82551078Speter { 0, 0 } 82651078Speter}; 82760471Snyanstatic driver_t ahci_driver = { 82851078Speter "ahci", 82951078Speter ahci_methods, 83051078Speter sizeof(struct ahci_controller) 83151078Speter}; 83251078SpeterDRIVER_MODULE(ahci, pci, ahci_driver, ahci_devclass, 0, 0); 83351078Speterstatic device_method_t ahci_ata_methods[] = { 83451078Speter DEVMETHOD(device_probe, ahci_ata_probe), 83551078Speter DEVMETHOD(device_attach, ahci_attach), 83651078Speter DEVMETHOD(device_detach, ahci_detach), 83751078Speter DEVMETHOD(device_suspend, ahci_suspend), 83851078Speter DEVMETHOD(device_resume, ahci_resume), 83951078Speter DEVMETHOD(bus_print_child, ahci_print_child), 84051078Speter DEVMETHOD(bus_alloc_resource, ahci_alloc_resource), 84151078Speter DEVMETHOD(bus_release_resource, ahci_release_resource), 84251078Speter DEVMETHOD(bus_setup_intr, ahci_setup_intr), 84351078Speter DEVMETHOD(bus_teardown_intr,ahci_teardown_intr), 84451078Speter DEVMETHOD(bus_child_location_str, ahci_child_location_str), 84551078Speter { 0, 0 } 84651078Speter}; 84751078Speterstatic driver_t ahci_ata_driver = { 84851078Speter "ahci", 84951078Speter ahci_ata_methods, 85051078Speter sizeof(struct ahci_controller) 85151078Speter}; 85251078SpeterDRIVER_MODULE(ahci, atapci, ahci_ata_driver, ahci_devclass, 0, 0); 85351078SpeterMODULE_VERSION(ahci, 1); 85451078SpeterMODULE_DEPEND(ahci, cam, 1, 1, 1); 85551078Speter 85651078Speterstatic int 85751078Speterahci_ch_probe(device_t dev) 85851078Speter{ 85951078Speter 86051078Speter device_set_desc_copy(dev, "AHCI channel"); 86151078Speter return (0); 86285365Simp} 86389986Sjhay 86451078Speterstatic int 86558885Simpahci_ch_attach(device_t dev) 86689986Sjhay{ 86751078Speter struct ahci_controller *ctlr = device_get_softc(device_get_parent(dev)); 86851078Speter struct ahci_channel *ch = device_get_softc(dev); 86951078Speter struct cam_devq *devq; 87051078Speter int rid, error, i, sata_rev = 0; 87151078Speter u_int32_t version; 87251078Speter 87351078Speter ch->dev = dev; 87453344Speter ch->unit = (intptr_t)device_get_ivars(dev); 87551078Speter ch->caps = ctlr->caps; 87651078Speter ch->caps2 = ctlr->caps2; 87753344Speter ch->quirks = ctlr->quirks; 878136478Sphk ch->numslots = ((ch->caps & AHCI_CAP_NCS) >> AHCI_CAP_NCS_SHIFT) + 1; 879135329Sphk mtx_init(&ch->mtx, "AHCI channel lock", NULL, MTX_DEF); 88051078Speter resource_int_value(device_get_name(dev), 88158885Simp device_get_unit(dev), "pm_level", &ch->pm_level); 88251078Speter if (ch->pm_level > 3) 88351078Speter callout_init_mtx(&ch->pm_timer, &ch->mtx, 0); 88451078Speter /* Limit speed for my onboard JMicron external port. 88557915Simp * It is not eSATA really. */ 88651078Speter if (pci_get_devid(ctlr->dev) == 0x2363197b && 88751078Speter pci_get_subvendor(ctlr->dev) == 0x1043 && 88851078Speter pci_get_subdevice(ctlr->dev) == 0x81e4 && 88951078Speter ch->unit == 0) 89053344Speter sata_rev = 1; 89151078Speter if (ch->quirks & AHCI_Q_SATA2) 89253344Speter sata_rev = 2; 89353344Speter resource_int_value(device_get_name(dev), 89451078Speter device_get_unit(dev), "sata_rev", &sata_rev); 89551078Speter for (i = 0; i < 16; i++) { 89651078Speter ch->user[i].revision = sata_rev; 89751078Speter ch->user[i].mode = 0; 89851078Speter ch->user[i].bytecount = 8192; 89951078Speter ch->user[i].tags = ch->numslots; 90051078Speter ch->user[i].caps = 0; 90151078Speter ch->curr[i] = ch->user[i]; 90251078Speter if (ch->pm_level) { 90351078Speter ch->user[i].caps = CTS_SATA_CAPS_H_PMREQ | 90451078Speter CTS_SATA_CAPS_H_APST | 90551078Speter CTS_SATA_CAPS_D_PMREQ | CTS_SATA_CAPS_D_APST; 90651078Speter } 90751078Speter ch->user[i].caps |= CTS_SATA_CAPS_H_DMAAA; 90851078Speter } 909116120Sscottl rid = ch->unit; 91060471Snyan if (!(ch->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 91160471Snyan &rid, RF_ACTIVE))) 91251078Speter return (ENXIO); 91351078Speter ahci_dmainit(dev); 91457234Sbde ahci_slotsalloc(dev); 91551078Speter ahci_ch_init(dev); 91651078Speter mtx_lock(&ch->mtx); 91751078Speter rid = ATA_IRQ_RID; 91851078Speter if (!(ch->r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, 91951078Speter &rid, RF_SHAREABLE | RF_ACTIVE))) { 920120189Sbde device_printf(dev, "Unable to map interrupt\n"); 92151078Speter error = ENXIO; 92251078Speter goto err0; 92351078Speter } 92451078Speter if ((bus_setup_intr(dev, ch->r_irq, ATA_INTR_FLAGS, NULL, 92551078Speter ahci_ch_intr_locked, dev, &ch->ih))) { 92651078Speter device_printf(dev, "Unable to setup interrupt\n"); 927135367Sphk error = ENXIO; 928135329Sphk goto err1; 929135329Sphk } 930135329Sphk ch->chcaps = ATA_INL(ch->r_mem, AHCI_P_CMD); 931135329Sphk version = ATA_INL(ctlr->r_mem, AHCI_VS); 932135329Sphk if (version < 0x00010020 && (ctlr->caps & AHCI_CAP_FBSS)) 933136478Sphk ch->chcaps |= AHCI_P_CMD_FBSCP; 934136478Sphk if (bootverbose) { 935135329Sphk device_printf(dev, "Caps:%s%s%s%s%s\n", 936135329Sphk (ch->chcaps & AHCI_P_CMD_HPCP) ? " HPCP":"", 93789986Sjhay (ch->chcaps & AHCI_P_CMD_MPSP) ? " MPSP":"", 93889986Sjhay (ch->chcaps & AHCI_P_CMD_CPD) ? " CPD":"", 93989986Sjhay (ch->chcaps & AHCI_P_CMD_ESP) ? " ESP":"", 94089986Sjhay (ch->chcaps & AHCI_P_CMD_FBSCP) ? " FBSCP":""); 941136478Sphk } 942136450Sphk /* Create the device queue for our SIM. */ 943136478Sphk devq = cam_simq_alloc(ch->numslots); 944136478Sphk if (devq == NULL) { 945136478Sphk device_printf(dev, "Unable to allocate simq\n"); 94656788Sbde error = ENOMEM; 94756788Sbde goto err1; 94856788Sbde } 94956788Sbde /* Construct SIM entry */ 95056788Sbde ch->sim = cam_sim_alloc(ahciaction, ahcipoll, "ahcich", ch, 95156788Sbde device_get_unit(dev), &ch->mtx, 95256788Sbde min(2, ch->numslots), 95351078Speter (ch->caps & AHCI_CAP_SNCQ) ? ch->numslots : 0, 95451078Speter devq); 95551078Speter if (ch->sim == NULL) { 95651078Speter cam_simq_free(devq); 95751078Speter device_printf(dev, "unable to allocate sim\n"); 958104067Sphk error = ENOMEM; 959104067Sphk goto err1; 96051078Speter } 96151078Speter if (xpt_bus_register(ch->sim, dev, 0) != CAM_SUCCESS) { 96251078Speter device_printf(dev, "unable to register xpt bus\n"); 96351078Speter error = ENXIO; 96460471Snyan goto err2; 96560471Snyan } 96660471Snyan if (xpt_create_path(&ch->path, /*periph*/NULL, cam_sim_path(ch->sim), 96760471Snyan CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 96860471Snyan device_printf(dev, "unable to create path\n"); 96960471Snyan error = ENXIO; 97051078Speter goto err3; 97189447Sbmah } 97251078Speter if (ch->pm_level > 3) { 97351078Speter callout_reset(&ch->pm_timer, 97451078Speter (ch->pm_level == 4) ? hz / 1000 : hz / 8, 97560471Snyan ahci_ch_pm, dev); 97651078Speter } 97751078Speter mtx_unlock(&ch->mtx); 97851078Speter return (0); 97951078Speter 98051078Spetererr3: 98151078Speter xpt_bus_deregister(cam_sim_path(ch->sim)); 98251078Spetererr2: 98351078Speter cam_sim_free(ch->sim, /*free_devq*/TRUE); 98451078Spetererr1: 98551078Speter bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq); 98651078Spetererr0: 98751078Speter bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem); 98851078Speter mtx_unlock(&ch->mtx); 98951078Speter mtx_destroy(&ch->mtx); 990120173Sbde return (error); 99151078Speter} 992120173Sbde 993120173Sbdestatic int 994120173Sbdeahci_ch_detach(device_t dev) 995120173Sbde{ 996120173Sbde struct ahci_channel *ch = device_get_softc(dev); 997120173Sbde 998120173Sbde mtx_lock(&ch->mtx); 999120173Sbde xpt_async(AC_LOST_DEVICE, ch->path, NULL); 1000120173Sbde xpt_free_path(ch->path); 1001120173Sbde xpt_bus_deregister(cam_sim_path(ch->sim)); 1002120173Sbde cam_sim_free(ch->sim, /*free_devq*/TRUE); 1003120173Sbde mtx_unlock(&ch->mtx); 1004120173Sbde 100551078Speter if (ch->pm_level > 3) 100651078Speter callout_drain(&ch->pm_timer); 100751078Speter bus_teardown_intr(dev, ch->r_irq, ch->ih); 100851078Speter bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq); 100951078Speter 101051078Speter ahci_ch_deinit(dev); 1011123796Sbde ahci_slotsfree(dev); 1012120173Sbde ahci_dmafini(dev); 101351078Speter 1014120173Sbde bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem); 1015120173Sbde mtx_destroy(&ch->mtx); 1016120173Sbde return (0); 1017120173Sbde} 1018120173Sbde 1019120173Sbdestatic int 102051078Speterahci_ch_init(device_t dev) 102151078Speter{ 102251078Speter struct ahci_channel *ch = device_get_softc(dev); 1023123796Sbde uint64_t work; 102451078Speter 102551078Speter /* Disable port interrupts */ 102651078Speter ATA_OUTL(ch->r_mem, AHCI_P_IE, 0); 102751078Speter /* Setup work areas */ 102851078Speter work = ch->dma.work_bus + AHCI_CL_OFFSET; 102951078Speter ATA_OUTL(ch->r_mem, AHCI_P_CLB, work & 0xffffffff); 103051078Speter ATA_OUTL(ch->r_mem, AHCI_P_CLBU, work >> 32); 103151078Speter work = ch->dma.rfis_bus; 103251078Speter ATA_OUTL(ch->r_mem, AHCI_P_FB, work & 0xffffffff); 103351078Speter ATA_OUTL(ch->r_mem, AHCI_P_FBU, work >> 32); 103451078Speter /* Activate the channel and power/spin up device */ 103551078Speter ATA_OUTL(ch->r_mem, AHCI_P_CMD, 103651078Speter (AHCI_P_CMD_ACTIVE | AHCI_P_CMD_POD | AHCI_P_CMD_SUD | 103751078Speter ((ch->pm_level == 2 || ch->pm_level == 3) ? AHCI_P_CMD_ALPE : 0) | 103851078Speter ((ch->pm_level > 2) ? AHCI_P_CMD_ASP : 0 ))); 103951078Speter ahci_start_fr(dev); 104051078Speter ahci_start(dev, 1); 104151078Speter return (0); 104251078Speter} 104351078Speter 104451078Speterstatic int 104551078Speterahci_ch_deinit(device_t dev) 104651078Speter{ 104760471Snyan struct ahci_channel *ch = device_get_softc(dev); 104851078Speter 104951078Speter /* Disable port interrupts. */ 105051078Speter ATA_OUTL(ch->r_mem, AHCI_P_IE, 0); 105151078Speter /* Reset command register. */ 105253344Speter ahci_stop(dev); 105353344Speter ahci_stop_fr(dev); 105451078Speter ATA_OUTL(ch->r_mem, AHCI_P_CMD, 0); 105551078Speter /* Allow everything, including partial and slumber modes. */ 105651078Speter ATA_OUTL(ch->r_mem, AHCI_P_SCTL, 0); 105751078Speter /* Request slumber mode transition and give some time to get there. */ 105851078Speter ATA_OUTL(ch->r_mem, AHCI_P_CMD, AHCI_P_CMD_SLUMBER); 105953344Speter DELAY(100); 106053344Speter /* Disable PHY. */ 106157234Sbde ATA_OUTL(ch->r_mem, AHCI_P_SCTL, ATA_SC_DET_DISABLE); 106257234Sbde return (0); 106351078Speter} 106451078Speter 106551078Speterstatic int 106651078Speterahci_ch_suspend(device_t dev) 106753344Speter{ 1068120189Sbde struct ahci_channel *ch = device_get_softc(dev); 106951078Speter 107051078Speter mtx_lock(&ch->mtx); 107167551Sjhb xpt_freeze_simq(ch->sim, 1); 1072151658Sjhb while (ch->oslots) 107372238Sjhb msleep(ch, &ch->mtx, PRIBIO, "ahcisusp", hz/100); 1074151658Sjhb ahci_ch_deinit(dev); 107572238Sjhb mtx_unlock(&ch->mtx); 107651078Speter return (0); 1077136478Sphk} 107851078Speter 107951078Speterstatic int 1080136478Sphkahci_ch_resume(device_t dev) 1081111613Sphk{ 1082111613Sphk struct ahci_channel *ch = device_get_softc(dev); 1083111613Sphk 1084111613Sphk mtx_lock(&ch->mtx); 1085111613Sphk ahci_ch_init(dev); 108651078Speter ahci_reset(dev); 108751078Speter xpt_release_simq(ch->sim, TRUE); 108851078Speter mtx_unlock(&ch->mtx); 1089127135Snjl return (0); 109053344Speter} 1091155921Sjhb 1092166901Spisodevclass_t ahcich_devclass; 1093166901Spisostatic device_method_t ahcich_methods[] = { 1094166901Spiso DEVMETHOD(device_probe, ahci_ch_probe), 109554194Speter DEVMETHOD(device_attach, ahci_ch_attach), 1096155921Sjhb DEVMETHOD(device_detach, ahci_ch_detach), 109754194Speter DEVMETHOD(device_suspend, ahci_ch_suspend), 1098166901Spiso DEVMETHOD(device_resume, ahci_ch_resume), 109954194Speter { 0, 0 } 110083246Sdd}; 110154194Speterstatic driver_t ahcich_driver = { 110253344Speter "ahcich", 110353344Speter ahcich_methods, 1104225203Srwatson sizeof(struct ahci_channel) 110578504Siedowse}; 110678504SiedowseDRIVER_MODULE(ahcich, ahci, ahcich_driver, ahcich_devclass, 0, 0); 1107127157Snjl 110878504Siedowsestruct ahci_dc_cb_args { 1109127157Snjl bus_addr_t maddr; 111078504Siedowse int error; 111178504Siedowse}; 111278504Siedowse 111353344Speterstatic void 111451078Speterahci_dmainit(device_t dev) 1115136478Sphk{ 1116151383Sphk struct ahci_channel *ch = device_get_softc(dev); 111751078Speter struct ahci_dc_cb_args dcba; 1118130872Sphk size_t rfsize; 1119130872Sphk 1120130872Sphk if (ch->caps & AHCI_CAP_64BIT) 1121130872Sphk ch->dma.max_address = BUS_SPACE_MAXADDR; 1122136478Sphk else 112351078Speter ch->dma.max_address = BUS_SPACE_MAXADDR_32BIT; 112451078Speter /* Command area. */ 1125136478Sphk if (bus_dma_tag_create(bus_get_dma_tag(dev), 1024, 0, 112651078Speter ch->dma.max_address, BUS_SPACE_MAXADDR, 1127136478Sphk NULL, NULL, AHCI_WORK_SIZE, 1, AHCI_WORK_SIZE, 1128136478Sphk 0, NULL, NULL, &ch->dma.work_tag)) 1129136478Sphk goto error; 1130136478Sphk if (bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work, 0, 113151078Speter &ch->dma.work_map)) 1132136478Sphk goto error; 1133136478Sphk if (bus_dmamap_load(ch->dma.work_tag, ch->dma.work_map, ch->dma.work, 1134136478Sphk AHCI_WORK_SIZE, ahci_dmasetupc_cb, &dcba, 0) || dcba.error) { 1135136478Sphk bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map); 1136136478Sphk goto error; 1137136478Sphk } 1138136478Sphk ch->dma.work_bus = dcba.maddr; 1139136478Sphk /* FIS receive area. */ 1140136478Sphk if (ch->chcaps & AHCI_P_CMD_FBSCP) 114151078Speter rfsize = 4096; 1142136478Sphk else 1143136478Sphk rfsize = 256; 1144136478Sphk if (bus_dma_tag_create(bus_get_dma_tag(dev), rfsize, 0, 1145136478Sphk ch->dma.max_address, BUS_SPACE_MAXADDR, 114651078Speter NULL, NULL, rfsize, 1, rfsize, 1147136478Sphk 0, NULL, NULL, &ch->dma.rfis_tag)) 1148136478Sphk goto error; 1149136478Sphk if (bus_dmamem_alloc(ch->dma.rfis_tag, (void **)&ch->dma.rfis, 0, 1150136478Sphk &ch->dma.rfis_map)) 1151136478Sphk goto error; 1152136478Sphk if (bus_dmamap_load(ch->dma.rfis_tag, ch->dma.rfis_map, ch->dma.rfis, 1153136478Sphk rfsize, ahci_dmasetupc_cb, &dcba, 0) || dcba.error) { 1154136478Sphk bus_dmamem_free(ch->dma.rfis_tag, ch->dma.rfis, ch->dma.rfis_map); 1155136478Sphk goto error; 115651078Speter } 1157136478Sphk ch->dma.rfis_bus = dcba.maddr; 1158136478Sphk /* Data area. */ 1159136478Sphk if (bus_dma_tag_create(bus_get_dma_tag(dev), 2, 0, 1160136478Sphk ch->dma.max_address, BUS_SPACE_MAXADDR, 1161136478Sphk NULL, NULL, 1162136478Sphk AHCI_SG_ENTRIES * PAGE_SIZE * ch->numslots, 116351078Speter AHCI_SG_ENTRIES, AHCI_PRD_MAX, 1164136478Sphk 0, busdma_lock_mutex, &ch->mtx, &ch->dma.data_tag)) { 1165136478Sphk goto error; 116651078Speter } 116751078Speter return; 1168136478Sphk 1169136478Sphkerror: 1170136478Sphk device_printf(dev, "WARNING - DMA initialization failed\n"); 1171136478Sphk ahci_dmafini(dev); 1172136478Sphk} 1173136478Sphk 1174136478Sphkstatic void 1175136478Sphkahci_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) 1176136478Sphk{ 117751078Speter struct ahci_dc_cb_args *dcba = (struct ahci_dc_cb_args *)xsc; 1178136478Sphk 1179136478Sphk if (!(dcba->error = error)) 1180136478Sphk dcba->maddr = segs[0].ds_addr; 118151078Speter} 118251078Speter 118351078Speterstatic void 118451078Speterahci_dmafini(device_t dev) 1185136478Sphk{ 1186136478Sphk struct ahci_channel *ch = device_get_softc(dev); 118751078Speter 118851078Speter if (ch->dma.data_tag) { 1189136478Sphk bus_dma_tag_destroy(ch->dma.data_tag); 119051078Speter ch->dma.data_tag = NULL; 119151078Speter } 1192136478Sphk if (ch->dma.rfis_bus) { 119351078Speter bus_dmamap_unload(ch->dma.rfis_tag, ch->dma.rfis_map); 119451078Speter bus_dmamem_free(ch->dma.rfis_tag, ch->dma.rfis, ch->dma.rfis_map); 119560471Snyan ch->dma.rfis_bus = 0; 119678504Siedowse ch->dma.rfis_map = NULL; 1197225203Srwatson ch->dma.rfis = NULL; 119878504Siedowse } 119978504Siedowse if (ch->dma.work_bus) { 120078504Siedowse bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map); 120178504Siedowse bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map); 120278504Siedowse ch->dma.work_bus = 0; 1203127157Snjl ch->dma.work_map = NULL; 120478504Siedowse ch->dma.work = NULL; 120551078Speter } 120660471Snyan if (ch->dma.work_tag) { 120751078Speter bus_dma_tag_destroy(ch->dma.work_tag); 120851078Speter ch->dma.work_tag = NULL; 120951078Speter } 121051078Speter} 121151078Speter 121251078Speterstatic void 121351078Speterahci_slotsalloc(device_t dev) 121451078Speter{ 1215136478Sphk struct ahci_channel *ch = device_get_softc(dev); 121651078Speter int i; 1217135329Sphk 121851078Speter /* Alloc and setup command/dma slots */ 1219131094Sphk bzero(ch->slot, sizeof(ch->slot)); 1220131981Sphk for (i = 0; i < ch->numslots; i++) { 122151078Speter struct ahci_slot *slot = &ch->slot[i]; 122251078Speter 122351078Speter slot->dev = dev; 122451078Speter slot->slot = i; 122551078Speter slot->state = AHCI_SLOT_EMPTY; 122651078Speter slot->ccb = NULL; 122751078Speter callout_init_mtx(&slot->timeout, &ch->mtx, 0); 122851078Speter 122960471Snyan if (bus_dmamap_create(ch->dma.data_tag, 0, &slot->dma.data_map)) 123051078Speter device_printf(ch->dev, "FAILURE - create data_map\n"); 1231136478Sphk } 1232136478Sphk} 123351078Speter 1234136478Sphkstatic void 123551078Speterahci_slotsfree(device_t dev) 123651078Speter{ 123751078Speter struct ahci_channel *ch = device_get_softc(dev); 123851078Speter int i; 123951078Speter 124051078Speter /* Free all dma slots */ 124151078Speter for (i = 0; i < ch->numslots; i++) { 124251078Speter struct ahci_slot *slot = &ch->slot[i]; 124351078Speter 124451078Speter callout_drain(&slot->timeout); 124551078Speter if (slot->dma.data_map) { 124651078Speter bus_dmamap_destroy(ch->dma.data_tag, slot->dma.data_map); 124751078Speter slot->dma.data_map = NULL; 124851078Speter } 124951078Speter } 125051078Speter} 125151078Speter 125251078Speterstatic void 125351078Speterahci_phy_check_events(device_t dev, u_int32_t serr) 125451078Speter{ 125551078Speter struct ahci_channel *ch = device_get_softc(dev); 125651078Speter 125751078Speter if ((serr & ATA_SE_PHY_CHANGED) && (ch->pm_level == 0)) { 125851078Speter u_int32_t status = ATA_INL(ch->r_mem, AHCI_P_SSTS); 125951078Speter union ccb *ccb; 126051078Speter 126151078Speter if (bootverbose) { 126251078Speter if (((status & ATA_SS_DET_MASK) == ATA_SS_DET_PHY_ONLINE) && 126351078Speter ((status & ATA_SS_SPD_MASK) != ATA_SS_SPD_NO_SPEED) && 126451078Speter ((status & ATA_SS_IPM_MASK) == ATA_SS_IPM_ACTIVE)) { 126551078Speter device_printf(dev, "CONNECT requested\n"); 126651078Speter } else 126751078Speter device_printf(dev, "DISCONNECT requested\n"); 126889986Sjhay } 126989986Sjhay ahci_reset(dev); 127089986Sjhay if ((ccb = xpt_alloc_ccb_nowait()) == NULL) 127189986Sjhay return; 127289986Sjhay if (xpt_create_path(&ccb->ccb_h.path, NULL, 127389986Sjhay cam_sim_path(ch->sim), 127489986Sjhay CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 127589986Sjhay xpt_free_ccb(ccb); 127689986Sjhay return; 1277114334Speter } 127889986Sjhay xpt_rescan(ccb); 1279114334Speter } 1280114334Speter} 1281114334Speter 1282114334Speterstatic void 128389986Sjhayahci_notify_events(device_t dev, u_int32_t status) 128489986Sjhay{ 128589986Sjhay struct ahci_channel *ch = device_get_softc(dev); 128689986Sjhay struct cam_path *dpath; 128789986Sjhay int i; 128889986Sjhay 128989986Sjhay if (ch->caps & AHCI_CAP_SSNTF) 129089986Sjhay ATA_OUTL(ch->r_mem, AHCI_P_SNTF, status); 129189986Sjhay if (bootverbose) 129289986Sjhay device_printf(dev, "SNTF 0x%04x\n", status); 129389986Sjhay for (i = 0; i < 16; i++) { 129489986Sjhay if ((status & (1 << i)) == 0) 129589986Sjhay continue; 129689986Sjhay if (xpt_create_path(&dpath, NULL, 129789986Sjhay xpt_path_path_id(ch->path), i, 0) == CAM_REQ_CMP) { 129865557Sjasone xpt_async(AC_SCSI_AEN, dpath, NULL); 129970174Sjhb xpt_free_path(dpath); 130070174Sjhb } 130165557Sjasone } 130251078Speter} 130351078Speter 130451078Speterstatic void 130551078Speterahci_ch_intr_locked(void *data) 130651078Speter{ 130751078Speter device_t dev = (device_t)data; 130851078Speter struct ahci_channel *ch = device_get_softc(dev); 130951078Speter 131051078Speter mtx_lock(&ch->mtx); 131151078Speter ahci_ch_intr(data); 131251078Speter mtx_unlock(&ch->mtx); 131351078Speter} 131451078Speter 131551078Speterstatic void 131651078Speterahci_ch_pm(void *arg) 131751078Speter{ 131851078Speter device_t dev = (device_t)arg; 131951078Speter struct ahci_channel *ch = device_get_softc(dev); 132051078Speter uint32_t work; 132151078Speter 132251078Speter if (ch->numrslots != 0) 132351078Speter return; 132451078Speter work = ATA_INL(ch->r_mem, AHCI_P_CMD); 132551078Speter if (ch->pm_level == 4) 132651078Speter work |= AHCI_P_CMD_PARTIAL; 132751078Speter else 132865557Sjasone work |= AHCI_P_CMD_SLUMBER; 132965557Sjasone ATA_OUTL(ch->r_mem, AHCI_P_CMD, work); 133065557Sjasone} 133165557Sjasone 133265557Sjasonestatic void 133372200Sbmilekicahci_ch_intr(void *data) 133451078Speter{ 133551078Speter device_t dev = (device_t)data; 133651078Speter struct ahci_channel *ch = device_get_softc(dev); 133751078Speter uint32_t istatus, sstatus, cstatus, serr = 0, sntf = 0, ok, err; 133851078Speter enum ahci_err_type et; 133951078Speter int i, ccs, port; 134051078Speter 134151078Speter /* Read and clear interrupt statuses. */ 134251078Speter istatus = ATA_INL(ch->r_mem, AHCI_P_IS); 134351078Speter if (istatus == 0) 134451078Speter return; 134551078Speter ATA_OUTL(ch->r_mem, AHCI_P_IS, istatus); 134651078Speter /* Read command statuses. */ 134751078Speter sstatus = ATA_INL(ch->r_mem, AHCI_P_SACT); 134851078Speter cstatus = ATA_INL(ch->r_mem, AHCI_P_CI); 134951078Speter if (istatus & AHCI_P_IX_SDB) { 135051078Speter if (ch->caps & AHCI_CAP_SSNTF) 135151078Speter sntf = ATA_INL(ch->r_mem, AHCI_P_SNTF); 135251078Speter else if (ch->fbs_enabled) { 135351078Speter u_int8_t *fis = ch->dma.rfis + 0x58; 135472200Sbmilekic 135551078Speter for (i = 0; i < 16; i++) { 135651078Speter if (fis[1] & 0x80) { 135751078Speter fis[1] &= 0x7f; 135865557Sjasone sntf |= 1 << i; 135965557Sjasone } 136065557Sjasone fis += 256; 136165557Sjasone } 136265557Sjasone } else { 136372200Sbmilekic u_int8_t *fis = ch->dma.rfis + 0x58; 136451078Speter 136551078Speter if (fis[1] & 0x80) 136651078Speter sntf = (1 << (fis[1] & 0x0f)); 136751078Speter } 136851078Speter } 136951078Speter /* Process PHY events */ 137051078Speter if (istatus & (AHCI_P_IX_PC | AHCI_P_IX_PRC | AHCI_P_IX_OF | 137151078Speter AHCI_P_IX_IF | AHCI_P_IX_HBD | AHCI_P_IX_HBF | AHCI_P_IX_TFE)) { 137251078Speter serr = ATA_INL(ch->r_mem, AHCI_P_SERR); 137351078Speter if (serr) { 137451078Speter ATA_OUTL(ch->r_mem, AHCI_P_SERR, serr); 137551078Speter ahci_phy_check_events(dev, serr); 137651078Speter } 1377130077Sphk } 137872200Sbmilekic /* Process command errors */ 137951078Speter if (istatus & (AHCI_P_IX_OF | AHCI_P_IX_IF | 138051078Speter AHCI_P_IX_HBD | AHCI_P_IX_HBF | AHCI_P_IX_TFE)) { 138151078Speter ccs = (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CCS_MASK) 138251078Speter >> AHCI_P_CMD_CCS_SHIFT; 138351078Speter//device_printf(dev, "%s ERROR is %08x cs %08x ss %08x rs %08x tfd %02x serr %08x fbs %08x ccs %d\n", 138451078Speter// __func__, istatus, cstatus, sstatus, ch->rslots, ATA_INL(ch->r_mem, AHCI_P_TFD), 138551078Speter// serr, ATA_INL(ch->r_mem, AHCI_P_FBS), ccs); 138651078Speter port = -1; 138751078Speter if (ch->fbs_enabled) { 138851078Speter uint32_t fbs = ATA_INL(ch->r_mem, AHCI_P_FBS); 138951078Speter if (fbs & AHCI_P_FBS_SDE) { 139051078Speter port = (fbs & AHCI_P_FBS_DWE) 139151078Speter >> AHCI_P_FBS_DWE_SHIFT; 139251078Speter } else { 139351078Speter for (i = 0; i < 16; i++) { 1394166901Spiso if (ch->numrslotspd[i] == 0) 139551078Speter continue; 139651078Speter if (port == -1) 139751078Speter port = i; 139870174Sjhb else if (port != i) { 139970174Sjhb port = -2; 140051078Speter break; 140170174Sjhb } 140270174Sjhb } 140372200Sbmilekic } 140470174Sjhb } 140572200Sbmilekic err = ch->rslots & (cstatus | sstatus); 140651078Speter } else { 140751078Speter ccs = 0; 140851078Speter err = 0; 140951078Speter port = -1; 141051078Speter } 141151078Speter /* Complete all successfull commands. */ 141251078Speter ok = ch->rslots & ~(cstatus | sstatus); 141351078Speter for (i = 0; i < ch->numslots; i++) { 141451078Speter if ((ok >> i) & 1) 141551078Speter ahci_end_transaction(&ch->slot[i], AHCI_ERR_NONE); 141651078Speter } 141772200Sbmilekic /* On error, complete the rest of commands with error statuses. */ 141851078Speter if (err) { 141951078Speter if (ch->frozen) { 142053344Speter union ccb *fccb = ch->frozen; 142151078Speter ch->frozen = NULL; 142251078Speter fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ; 142351078Speter if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) { 142451078Speter xpt_freeze_devq(fccb->ccb_h.path, 1); 142551078Speter fccb->ccb_h.status |= CAM_DEV_QFRZN; 142651078Speter } 142751078Speter xpt_done(fccb); 142851078Speter } 142951078Speter for (i = 0; i < ch->numslots; i++) { 143051078Speter /* XXX: reqests in loading state. */ 143151078Speter if (((err >> i) & 1) == 0) 143251078Speter continue; 143351078Speter if (port >= 0 && 143451078Speter ch->slot[i].ccb->ccb_h.target_id != port) 143551078Speter continue; 143672200Sbmilekic if (istatus & AHCI_P_IX_TFE) { 143751078Speter if (port != -2) { 1438166901Spiso /* Task File Error */ 143951078Speter if (ch->numtslotspd[ 144051078Speter ch->slot[i].ccb->ccb_h.target_id] == 0) { 1441122819Sbde /* Untagged operation. */ 144293466Sbde if (i == ccs) 144393466Sbde et = AHCI_ERR_TFE; 144493466Sbde else 144593466Sbde et = AHCI_ERR_INNOCENT; 144693466Sbde } else { 144793466Sbde /* Tagged operation. */ 144893466Sbde et = AHCI_ERR_NCQ; 144993466Sbde } 145093466Sbde } else { 1451122819Sbde et = AHCI_ERR_TFE; 145293466Sbde ch->fatalerr = 1; 1453122819Sbde } 145493466Sbde } else if (istatus & AHCI_P_IX_IF) { 145593466Sbde if (ch->numtslots == 0 && i != ccs && port != -2) 145693466Sbde et = AHCI_ERR_INNOCENT; 145793466Sbde else 145893466Sbde et = AHCI_ERR_SATA; 145993466Sbde } else 1460122819Sbde et = AHCI_ERR_INVALID; 1461122819Sbde ahci_end_transaction(&ch->slot[i], et); 146293466Sbde } 146393466Sbde /* 146493466Sbde * We can't reinit port if there are some other 146593466Sbde * commands active, use resume to complete them. 146693466Sbde */ 146793466Sbde if (ch->rslots != 0) 146893466Sbde ATA_OUTL(ch->r_mem, AHCI_P_FBS, AHCI_P_FBS_EN | AHCI_P_FBS_DEC); 146993466Sbde } 147093466Sbde /* Process NOTIFY events */ 147193466Sbde if (sntf) 147293466Sbde ahci_notify_events(dev, sntf); 147351078Speter} 147451078Speter 147551078Speter/* Must be called with channel locked. */ 147651078Speterstatic int 1477120189Sbdeahci_check_collision(device_t dev, union ccb *ccb) 1478120189Sbde{ 147951078Speter struct ahci_channel *ch = device_get_softc(dev); 148051078Speter int t = ccb->ccb_h.target_id; 148151078Speter 148251078Speter if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 148351078Speter (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 1484225203Srwatson /* Tagged command while we have no supported tag free. */ 1485178766Speter if (((~ch->oslots) & (0xffffffff >> (32 - 1486178766Speter ch->curr[t].tags))) == 0) 1487178766Speter return (1); 1488120091Sbde /* If we have FBS */ 1489120189Sbde if (ch->fbs_enabled) { 1490120091Sbde /* Tagged command while untagged are active. */ 1491120091Sbde if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] == 0) 1492120091Sbde return (1); 1493120091Sbde } else { 1494120091Sbde /* Tagged command while untagged are active. */ 149551078Speter if (ch->numrslots != 0 && ch->numtslots == 0) 149651078Speter return (1); 149751078Speter /* Tagged command while tagged to other target is active. */ 149851078Speter if (ch->numtslots != 0 && 1499111613Sphk ch->taggedtarget != ccb->ccb_h.target_id) 1500111613Sphk return (1); 150195523Sphk } 1502111613Sphk } else { 1503111616Sphk /* If we have FBS */ 150451078Speter if (ch->fbs_enabled) { 150551078Speter /* Untagged command while tagged are active. */ 150651078Speter if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] != 0) 150751078Speter return (1); 1508128020Simp } else { 150951078Speter /* Untagged command while tagged are active. */ 151051078Speter if (ch->numrslots != 0 && ch->numtslots != 0) 151151078Speter return (1); 151251078Speter } 151351078Speter } 151451078Speter if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 151551078Speter (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT))) { 1516131939Smarcel /* Atomic command while anything active. */ 1517119485Snjl if (ch->numrslots != 0) 1518225203Srwatson return (1); 1519178766Speter } 1520131939Smarcel /* We have some atomic command running. */ 152151078Speter if (ch->aslots != 0) 152251078Speter return (1); 152351078Speter return (0); 152451078Speter} 152551078Speter 152651078Speter/* Must be called with channel locked. */ 152751078Speterstatic void 152851078Speterahci_begin_transaction(device_t dev, union ccb *ccb) 152951078Speter{ 153051078Speter struct ahci_channel *ch = device_get_softc(dev); 153151078Speter struct ahci_slot *slot; 153251078Speter int tag, tags; 153351078Speter 153451078Speter /* Choose empty slot. */ 1535225203Srwatson tags = ch->numslots; 153651078Speter if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1537225203Srwatson (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) 153851078Speter tags = ch->curr[ccb->ccb_h.target_id].tags; 153951078Speter tag = ch->lastslot; 154051078Speter while (1) { 154151078Speter if (tag >= tags) 154251078Speter tag = 0; 154351078Speter if (ch->slot[tag].state == AHCI_SLOT_EMPTY) 154451078Speter break; 154551078Speter tag++; 154651078Speter }; 154751078Speter ch->lastslot = tag; 154851078Speter /* Occupy chosen slot. */ 154951078Speter slot = &ch->slot[tag]; 155051078Speter slot->ccb = ccb; 155151078Speter /* Stop PM timer. */ 155251078Speter if (ch->numrslots == 0 && ch->pm_level > 3) 155351078Speter callout_stop(&ch->pm_timer); 155451078Speter /* Update channel stats. */ 1555131185Sphk ch->oslots |= (1 << slot->slot); 1556131185Sphk ch->numrslots++; 155788900Sjhb ch->numrslotspd[ccb->ccb_h.target_id]++; 155851078Speter if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 155951078Speter (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 156051078Speter ch->numtslots++; 156151078Speter ch->numtslotspd[ccb->ccb_h.target_id]++; 1562136478Sphk ch->taggedtarget = ccb->ccb_h.target_id; 1563136478Sphk } 156451078Speter if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 156572238Sjhb (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT))) 156651078Speter ch->aslots |= (1 << slot->slot); 156751078Speter slot->dma.nsegs = 0; 156888900Sjhb /* If request moves data, setup and load SG list */ 156951078Speter if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) { 157051078Speter void *buf; 157151078Speter bus_size_t size; 157251078Speter 157351078Speter slot->state = AHCI_SLOT_LOADING; 157451078Speter if (ccb->ccb_h.func_code == XPT_ATA_IO) { 157551078Speter buf = ccb->ataio.data_ptr; 157651078Speter size = ccb->ataio.dxfer_len; 157751078Speter } else { 157851078Speter buf = ccb->csio.data_ptr; 157951078Speter size = ccb->csio.dxfer_len; 158051078Speter } 1581122844Sbde bus_dmamap_load(ch->dma.data_tag, slot->dma.data_map, 1582122844Sbde buf, size, ahci_dmasetprd, slot, 0); 1583122844Sbde } else 1584122844Sbde ahci_execute_transaction(slot); 158551078Speter} 158651078Speter 158751078Speter/* Locked by busdma engine. */ 158851078Speterstatic void 158951078Speterahci_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 159051078Speter{ 159151078Speter struct ahci_slot *slot = arg; 159251078Speter struct ahci_channel *ch = device_get_softc(slot->dev); 159351078Speter struct ahci_cmd_tab *ctp; 159451078Speter struct ahci_dma_prd *prd; 159551078Speter int i; 159651078Speter 159751078Speter if (error) { 159851078Speter device_printf(slot->dev, "DMA load error\n"); 159951078Speter ahci_end_transaction(slot, AHCI_ERR_INVALID); 160051078Speter return; 160151078Speter } 160251078Speter KASSERT(nsegs <= AHCI_SG_ENTRIES, ("too many DMA segment entries\n")); 160351078Speter /* Get a piece of the workspace for this request */ 160451078Speter ctp = (struct ahci_cmd_tab *) 160588900Sjhb (ch->dma.work + AHCI_CT_OFFSET + (AHCI_CT_SIZE * slot->slot)); 160651078Speter /* Fill S/G table */ 160751078Speter prd = &ctp->prd_tab[0]; 160851078Speter for (i = 0; i < nsegs; i++) { 160951078Speter prd[i].dba = htole64(segs[i].ds_addr); 161051078Speter prd[i].dbc = htole32((segs[i].ds_len - 1) & AHCI_PRD_MASK); 161151078Speter } 161251078Speter slot->dma.nsegs = nsegs; 161351078Speter bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map, 161451078Speter ((slot->ccb->ccb_h.flags & CAM_DIR_IN) ? 161551078Speter BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE)); 161651078Speter ahci_execute_transaction(slot); 1617122844Sbde} 161851078Speter 161951078Speter/* Must be called with channel locked. */ 162051078Speterstatic void 162151078Speterahci_execute_transaction(struct ahci_slot *slot) 162293466Sbde{ 162351078Speter device_t dev = slot->dev; 162451078Speter struct ahci_channel *ch = device_get_softc(dev); 162551078Speter struct ahci_cmd_tab *ctp; 162651078Speter struct ahci_cmd_list *clp; 162751078Speter union ccb *ccb = slot->ccb; 162851078Speter int port = ccb->ccb_h.target_id & 0x0f; 162951078Speter int fis_size, i; 163051078Speter uint8_t *fis = ch->dma.rfis + 0x40; 163151078Speter uint8_t val; 163251078Speter 163351078Speter /* Get a piece of the workspace for this request */ 163451078Speter ctp = (struct ahci_cmd_tab *) 1635122819Sbde (ch->dma.work + AHCI_CT_OFFSET + (AHCI_CT_SIZE * slot->slot)); 1636122819Sbde /* Setup the FIS for this request */ 1637122819Sbde if (!(fis_size = ahci_setup_fis(dev, ctp, ccb, slot->slot))) { 163851078Speter device_printf(ch->dev, "Setting up SATA FIS failed\n"); 163951078Speter ahci_end_transaction(slot, AHCI_ERR_INVALID); 1640120189Sbde return; 164151078Speter } 164251078Speter /* Setup the command list entry */ 164351078Speter clp = (struct ahci_cmd_list *) 164451078Speter (ch->dma.work + AHCI_CL_OFFSET + (AHCI_CL_SIZE * slot->slot)); 164551078Speter clp->cmd_flags = htole16( 164651078Speter (ccb->ccb_h.flags & CAM_DIR_OUT ? AHCI_CMD_WRITE : 0) | 164751078Speter (ccb->ccb_h.func_code == XPT_SCSI_IO ? 164851078Speter (AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH) : 0) | 164951078Speter (fis_size / sizeof(u_int32_t)) | 165051078Speter (port << 12)); 165151078Speter clp->prd_length = htole16(slot->dma.nsegs); 165251078Speter /* Special handling for Soft Reset command. */ 165351078Speter if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1654120189Sbde (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL)) { 1655120189Sbde if (ccb->ataio.cmd.control & ATA_A_RESET) { 1656120189Sbde /* Kick controller into sane state */ 165751078Speter ahci_stop(dev); 165851078Speter ahci_clo(dev); 165951078Speter ahci_start(dev, 0); 166051078Speter clp->cmd_flags |= AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY; 166151078Speter } else { 166267551Sjhb /* Prepare FIS receive area for check. */ 166388900Sjhb for (i = 0; i < 20; i++) 166451078Speter fis[i] = 0xff; 166551078Speter } 1666120189Sbde } 1667120189Sbde clp->bytecount = 0; 1668120189Sbde clp->cmd_table_phys = htole64(ch->dma.work_bus + AHCI_CT_OFFSET + 166951078Speter (AHCI_CT_SIZE * slot->slot)); 167051078Speter bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 167151078Speter BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 167251078Speter bus_dmamap_sync(ch->dma.rfis_tag, ch->dma.rfis_map, 167351078Speter BUS_DMASYNC_PREREAD); 167451078Speter /* Set ACTIVE bit for NCQ commands. */ 167551078Speter if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 167651078Speter (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 167751078Speter ATA_OUTL(ch->r_mem, AHCI_P_SACT, 1 << slot->slot); 167851078Speter } 167965557Sjasone /* If FBS is enabled, set PMP port. */ 168051078Speter if (ch->fbs_enabled) { 168167551Sjhb ATA_OUTL(ch->r_mem, AHCI_P_FBS, AHCI_P_FBS_EN | 168251078Speter (port << AHCI_P_FBS_DEV_SHIFT)); 168351078Speter } 168451078Speter /* Issue command to the controller. */ 168551078Speter slot->state = AHCI_SLOT_RUNNING; 168651078Speter ch->rslots |= (1 << slot->slot); 168751078Speter ATA_OUTL(ch->r_mem, AHCI_P_CI, (1 << slot->slot)); 168853344Speter /* Device reset commands doesn't interrupt. Poll them. */ 168951078Speter if (ccb->ccb_h.func_code == XPT_ATA_IO && 169051078Speter (ccb->ataio.cmd.command == ATA_DEVICE_RESET || 169151078Speter (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL))) { 169251078Speter int count, timeout = ccb->ccb_h.timeout; 169351078Speter enum ahci_err_type et = AHCI_ERR_NONE; 169451078Speter 169551078Speter for (count = 0; count < timeout; count++) { 169651078Speter DELAY(1000); 169751078Speter if (!(ATA_INL(ch->r_mem, AHCI_P_CI) & (1 << slot->slot))) 169851078Speter break; 169951078Speter if (ATA_INL(ch->r_mem, AHCI_P_TFD) & ATA_S_ERROR) { 170051078Speter device_printf(ch->dev, 170151078Speter "Poll error on slot %d, TFD: %04x\n", 170272200Sbmilekic slot->slot, ATA_INL(ch->r_mem, AHCI_P_TFD)); 170351078Speter et = AHCI_ERR_TFE; 170451078Speter break; 170551078Speter } 170651078Speter /* Workaround for ATI SB600/SB700 chipsets. */ 170751078Speter if (ccb->ccb_h.target_id == 15 && 170851078Speter pci_get_vendor(device_get_parent(dev)) == 0x1002 && 170951078Speter (ATA_INL(ch->r_mem, AHCI_P_IS) & AHCI_P_IX_IPM)) { 171072200Sbmilekic et = AHCI_ERR_TIMEOUT; 171151078Speter break; 171251078Speter } 171351078Speter } 171472200Sbmilekic if (timeout && (count >= timeout)) { 171551078Speter device_printf(ch->dev, 171672200Sbmilekic "Poll timeout on slot %d\n", slot->slot); 171751078Speter device_printf(dev, "is %08x cs %08x ss %08x " 171851078Speter "rs %08x tfd %02x serr %08x\n", 171951078Speter ATA_INL(ch->r_mem, AHCI_P_IS), 172051078Speter ATA_INL(ch->r_mem, AHCI_P_CI), 172172200Sbmilekic ATA_INL(ch->r_mem, AHCI_P_SACT), ch->rslots, 172251078Speter ATA_INL(ch->r_mem, AHCI_P_TFD), 172351078Speter ATA_INL(ch->r_mem, AHCI_P_SERR)); 172451078Speter et = AHCI_ERR_TIMEOUT; 172551078Speter } 172651078Speter /* Marvell controllers do not wait for readyness. */ 172772200Sbmilekic if ((ch->quirks & AHCI_Q_NOBSYRES) && 172851078Speter (ccb->ccb_h.func_code == XPT_ATA_IO) && 1729130095Sphk (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) && 1730130095Sphk (ccb->ataio.cmd.control & ATA_A_RESET) == 0) { 173151078Speter while ((val = fis[2]) & (ATA_S_BUSY | ATA_S_DRQ)) { 173251078Speter DELAY(1000); 173372200Sbmilekic if (count++ >= timeout) { 173451078Speter device_printf(dev, "device is not " 173551078Speter "ready after soft-reset: " 173672200Sbmilekic "tfd = %08x\n", val); 173751078Speter et = AHCI_ERR_TIMEOUT; 173851078Speter break; 173951078Speter } 174051078Speter } 174151078Speter } 1742130077Sphk ahci_end_transaction(slot, et); 174351078Speter /* Kick controller into sane state and enable FBS. */ 174451078Speter if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 174551078Speter (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) && 174651078Speter (ccb->ataio.cmd.control & ATA_A_RESET) == 0) { 174751078Speter ahci_stop(ch->dev); 174851078Speter ahci_start(ch->dev, 1); 174951078Speter } 175051078Speter return; 1751131373Sphk } 1752131094Sphk /* Start command execution timeout */ 1753131094Sphk callout_reset(&slot->timeout, (int)ccb->ccb_h.timeout * hz / 2000, 1754131094Sphk (timeout_t*)ahci_timeout, slot); 1755131094Sphk return; 1756131094Sphk} 1757131094Sphk 1758135329Sphk/* Must be called with channel locked. */ 1759131094Sphkstatic void 1760131094Sphkahci_process_timeout(device_t dev) 1761131094Sphk{ 1762131094Sphk struct ahci_channel *ch = device_get_softc(dev); 1763131094Sphk int i; 1764131094Sphk 1765131094Sphk mtx_assert(&ch->mtx, MA_OWNED); 1766131094Sphk /* Handle the rest of commands. */ 176751078Speter for (i = 0; i < ch->numslots; i++) { 176851078Speter /* Do we have a running request on slot? */ 176951078Speter if (ch->slot[i].state < AHCI_SLOT_RUNNING) 177051078Speter continue; 177151078Speter ahci_end_transaction(&ch->slot[i], AHCI_ERR_TIMEOUT); 177251078Speter } 177351078Speter} 177489986Sjhay 177551078Speter/* Must be called with channel locked. */ 177651078Speterstatic void 1777120159Sbdeahci_rearm_timeout(device_t dev) 177851078Speter{ 177951078Speter struct ahci_channel *ch = device_get_softc(dev); 1780135329Sphk int i; 178189986Sjhay 178289986Sjhay mtx_assert(&ch->mtx, MA_OWNED); 178389986Sjhay for (i = 0; i < ch->numslots; i++) { 178451078Speter struct ahci_slot *slot = &ch->slot[i]; 1785120505Sbde 1786120505Sbde /* Do we have a running request on slot? */ 1787120505Sbde if (slot->state < AHCI_SLOT_RUNNING) 1788120505Sbde continue; 1789120505Sbde if ((ch->toslots & (1 << i)) == 0) 179051078Speter continue; 179151078Speter callout_reset(&slot->timeout, 179251078Speter (int)slot->ccb->ccb_h.timeout * hz / 2000, 1793120505Sbde (timeout_t*)ahci_timeout, slot); 1794131094Sphk } 179551078Speter} 1796131094Sphk 179751078Speter/* Locked by callout mechanism. */ 179851078Speterstatic void 179951078Speterahci_timeout(struct ahci_slot *slot) 180051078Speter{ 180151078Speter device_t dev = slot->dev; 180251078Speter struct ahci_channel *ch = device_get_softc(dev); 180351078Speter uint32_t sstatus; 180451078Speter int ccs; 180551078Speter int i; 180651078Speter 180751078Speter /* Check for stale timeout. */ 180851078Speter if (slot->state < AHCI_SLOT_RUNNING) 180951078Speter return; 181051078Speter 181151078Speter /* Check if slot was not being executed last time we checked. */ 181251078Speter if (slot->state < AHCI_SLOT_EXECUTING) { 181351078Speter /* Check if slot started executing. */ 181451078Speter sstatus = ATA_INL(ch->r_mem, AHCI_P_SACT); 181551078Speter ccs = (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CCS_MASK) 181651078Speter >> AHCI_P_CMD_CCS_SHIFT; 181751078Speter if ((sstatus & (1 << slot->slot)) != 0 || ccs == slot->slot || 181851078Speter ch->fbs_enabled) 181951078Speter slot->state = AHCI_SLOT_EXECUTING; 1820120505Sbde 182151078Speter callout_reset(&slot->timeout, 182251078Speter (int)slot->ccb->ccb_h.timeout * hz / 2000, 182351078Speter (timeout_t*)ahci_timeout, slot); 182451078Speter return; 182551078Speter } 182651078Speter 182751078Speter device_printf(dev, "Timeout on slot %d\n", slot->slot); 182888433Sdillon device_printf(dev, "is %08x cs %08x ss %08x rs %08x tfd %02x serr %08x\n", 182988433Sdillon ATA_INL(ch->r_mem, AHCI_P_IS), ATA_INL(ch->r_mem, AHCI_P_CI), 183088433Sdillon ATA_INL(ch->r_mem, AHCI_P_SACT), ch->rslots, 183188433Sdillon ATA_INL(ch->r_mem, AHCI_P_TFD), ATA_INL(ch->r_mem, AHCI_P_SERR)); 183288433Sdillon 183351078Speter /* Handle frozen command. */ 183493466Sbde if (ch->frozen) { 1835120505Sbde union ccb *fccb = ch->frozen; 183688451Stanimura ch->frozen = NULL; 183751078Speter fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ; 183851078Speter if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) { 183951078Speter xpt_freeze_devq(fccb->ccb_h.path, 1); 184051078Speter fccb->ccb_h.status |= CAM_DEV_QFRZN; 184151078Speter } 184251078Speter xpt_done(fccb); 184351078Speter } 184451078Speter if (!ch->fbs_enabled) { 184551078Speter /* Without FBS we know real timeout source. */ 184660471Snyan ch->fatalerr = 1; 184751078Speter /* Handle command with timeout. */ 184851078Speter ahci_end_transaction(&ch->slot[slot->slot], AHCI_ERR_TIMEOUT); 184965605Sjhb /* Handle the rest of commands. */ 185065605Sjhb for (i = 0; i < ch->numslots; i++) { 185165605Sjhb /* Do we have a running request on slot? */ 185265605Sjhb if (ch->slot[i].state < AHCI_SLOT_RUNNING) 185365605Sjhb continue; 185465605Sjhb ahci_end_transaction(&ch->slot[i], AHCI_ERR_INNOCENT); 185565557Sjasone } 1856120505Sbde } else { 1857120505Sbde /* With FBS we wait for other commands timeout and pray. */ 1858120505Sbde if (ch->toslots == 0) 1859120505Sbde xpt_freeze_simq(ch->sim, 1); 1860120505Sbde ch->toslots |= (1 << slot->slot); 1861120505Sbde if ((ch->rslots & ~ch->toslots) == 0) 1862120505Sbde ahci_process_timeout(dev); 1863120505Sbde else 1864120505Sbde device_printf(dev, " ... waiting for slots %08x\n", 1865120505Sbde ch->rslots & ~ch->toslots); 1866120505Sbde } 1867120505Sbde} 186851078Speter 1869120159Sbde/* Must be called with channel locked. */ 187051078Speterstatic void 187151078Speterahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et) 187251078Speter{ 1873120159Sbde device_t dev = slot->dev; 187451078Speter struct ahci_channel *ch = device_get_softc(dev); 187551078Speter union ccb *ccb = slot->ccb; 187651078Speter struct ahci_cmd_list *clp; 187751078Speter int lastto; 187851078Speter 187951078Speter bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 188051078Speter BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 188151078Speter clp = (struct ahci_cmd_list *) 188251078Speter (ch->dma.work + AHCI_CL_OFFSET + (AHCI_CL_SIZE * slot->slot)); 188351078Speter /* Read result registers to the result struct 188451078Speter * May be incorrect if several commands finished same time, 188551078Speter * so read only when sure or have to. 188651078Speter */ 188751078Speter if (ccb->ccb_h.func_code == XPT_ATA_IO) { 188851078Speter struct ata_res *res = &ccb->ataio.res; 188951078Speter 189051078Speter if ((et == AHCI_ERR_TFE) || 189151078Speter (ccb->ataio.cmd.flags & CAM_ATAIO_NEEDRESULT)) { 189251078Speter u_int8_t *fis = ch->dma.rfis + 0x40; 189351078Speter 189451078Speter bus_dmamap_sync(ch->dma.rfis_tag, ch->dma.rfis_map, 189551078Speter BUS_DMASYNC_POSTREAD); 189651078Speter if (ch->fbs_enabled) { 189751078Speter fis += ccb->ccb_h.target_id * 256; 1898120159Sbde res->status = fis[2]; 189951078Speter res->error = fis[3]; 190051078Speter } else { 190151078Speter uint16_t tfd = ATA_INL(ch->r_mem, AHCI_P_TFD); 190251078Speter 1903120159Sbde res->status = tfd; 1904120159Sbde res->error = tfd >> 8; 1905120159Sbde } 1906120159Sbde res->lba_low = fis[4]; 1907120159Sbde res->lba_mid = fis[5]; 1908120159Sbde res->lba_high = fis[6]; 1909120159Sbde res->device = fis[7]; 191051078Speter res->lba_low_exp = fis[8]; 191151078Speter res->lba_mid_exp = fis[9]; 1912131134Sphk res->lba_high_exp = fis[10]; 191351078Speter res->sector_count = fis[12]; 191472200Sbmilekic res->sector_count_exp = fis[13]; 191551078Speter } else 191651078Speter bzero(res, sizeof(*res)); 191751078Speter if ((ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) == 0 && 191851078Speter (ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE && 191951078Speter (ch->quirks & AHCI_Q_NOCOUNT) == 0) { 192051078Speter ccb->ataio.resid = 192151078Speter ccb->ataio.dxfer_len - le32toh(clp->bytecount); 192251078Speter } 192351078Speter } else { 192465605Sjhb if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE && 192570174Sjhb (ch->quirks & AHCI_Q_NOCOUNT) == 0) { 192670174Sjhb ccb->csio.resid = 192765605Sjhb ccb->csio.dxfer_len - le32toh(clp->bytecount); 192851078Speter } 192965605Sjhb } 193051078Speter if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) { 193151078Speter bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map, 193251078Speter (ccb->ccb_h.flags & CAM_DIR_IN) ? 193351078Speter BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 193451078Speter bus_dmamap_unload(ch->dma.data_tag, slot->dma.data_map); 193551078Speter } 193651078Speter if (et != AHCI_ERR_NONE) 193751078Speter ch->eslots |= (1 << slot->slot); 193851078Speter /* In case of error, freeze device for proper recovery. */ 193951078Speter if ((et != AHCI_ERR_NONE) && (!ch->readlog) && 194051078Speter !(ccb->ccb_h.status & CAM_DEV_QFRZN)) { 194151078Speter xpt_freeze_devq(ccb->ccb_h.path, 1); 194251078Speter ccb->ccb_h.status |= CAM_DEV_QFRZN; 194351078Speter } 194451078Speter /* Set proper result status. */ 194551078Speter ccb->ccb_h.status &= ~CAM_STATUS_MASK; 194651078Speter switch (et) { 194765605Sjhb case AHCI_ERR_NONE: 194872200Sbmilekic ccb->ccb_h.status |= CAM_REQ_CMP; 194951078Speter if (ccb->ccb_h.func_code == XPT_SCSI_IO) 195065605Sjhb ccb->csio.scsi_status = SCSI_STATUS_OK; 195151078Speter break; 195251078Speter case AHCI_ERR_INVALID: 195351078Speter ch->fatalerr = 1; 195451078Speter ccb->ccb_h.status |= CAM_REQ_INVALID; 195551078Speter break; 195651078Speter case AHCI_ERR_INNOCENT: 195765605Sjhb ccb->ccb_h.status |= CAM_REQUEUE_REQ; 195872200Sbmilekic break; 195951078Speter case AHCI_ERR_TFE: 196065605Sjhb case AHCI_ERR_NCQ: 196151078Speter if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 196251078Speter ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR; 196351078Speter ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND; 196451078Speter } else { 196551078Speter ccb->ccb_h.status |= CAM_ATA_STATUS_ERROR; 196651078Speter } 196751078Speter break; 196851078Speter case AHCI_ERR_SATA: 196951078Speter ch->fatalerr = 1; 197051078Speter if (!ch->readlog) { 197151078Speter xpt_freeze_simq(ch->sim, 1); 197251078Speter ccb->ccb_h.status &= ~CAM_STATUS_MASK; 197351078Speter ccb->ccb_h.status |= CAM_RELEASE_SIMQ; 197451078Speter } 197551078Speter ccb->ccb_h.status |= CAM_UNCOR_PARITY; 197672200Sbmilekic break; 197751078Speter case AHCI_ERR_TIMEOUT: 197851078Speter if (!ch->readlog) { 197951078Speter xpt_freeze_simq(ch->sim, 1); 198051078Speter ccb->ccb_h.status &= ~CAM_STATUS_MASK; 198151078Speter ccb->ccb_h.status |= CAM_RELEASE_SIMQ; 198251078Speter } 198351078Speter ccb->ccb_h.status |= CAM_CMD_TIMEOUT; 198451078Speter break; 198551078Speter default: 198651078Speter ch->fatalerr = 1; 198751078Speter ccb->ccb_h.status |= CAM_REQ_CMP_ERR; 198851078Speter } 198951078Speter /* Free slot. */ 199051078Speter ch->oslots &= ~(1 << slot->slot); 199151078Speter ch->rslots &= ~(1 << slot->slot); 199251078Speter ch->aslots &= ~(1 << slot->slot); 199351078Speter slot->state = AHCI_SLOT_EMPTY; 199451078Speter slot->ccb = NULL; 199551078Speter /* Update channel stats. */ 199651078Speter ch->numrslots--; 199751078Speter ch->numrslotspd[ccb->ccb_h.target_id]--; 199851078Speter if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 199951078Speter (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 200051078Speter ch->numtslots--; 200151078Speter ch->numtslotspd[ccb->ccb_h.target_id]--; 200251078Speter } 200351078Speter /* Cancel timeout state if request completed normally. */ 200451078Speter if (et != AHCI_ERR_TIMEOUT) { 2005135329Sphk lastto = (ch->toslots == (1 << slot->slot)); 200657915Simp ch->toslots &= ~(1 << slot->slot); 200757915Simp if (lastto) 200851078Speter xpt_release_simq(ch->sim, TRUE); 200972200Sbmilekic } 201051078Speter /* If it was first request of reset sequence and there is no error, 201151078Speter * proceed to second request. */ 201251078Speter if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 201351078Speter (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) && 201451078Speter (ccb->ataio.cmd.control & ATA_A_RESET) && 201551078Speter et == AHCI_ERR_NONE) { 201651078Speter ccb->ataio.cmd.control &= ~ATA_A_RESET; 201751078Speter ahci_begin_transaction(dev, ccb); 201851078Speter return; 201951078Speter } 202051078Speter /* If it was our READ LOG command - process it. */ 202151078Speter if (ch->readlog) { 202272200Sbmilekic ahci_process_read_log(dev, ccb); 202351078Speter /* If it was NCQ command error, put result on hold. */ 202451078Speter } else if (et == AHCI_ERR_NCQ) { 202551078Speter ch->hold[slot->slot] = ccb; 202651078Speter ch->numhslots++; 202751078Speter } else 202851078Speter xpt_done(ccb); 202951078Speter /* Unfreeze frozen command. */ 203051078Speter if (ch->frozen && !ahci_check_collision(dev, ch->frozen)) { 203151078Speter union ccb *fccb = ch->frozen; 203251078Speter ch->frozen = NULL; 203351078Speter ahci_begin_transaction(dev, fccb); 203451078Speter xpt_release_simq(ch->sim, TRUE); 203551078Speter } 203651078Speter /* If we have no other active commands, ... */ 203751078Speter if (ch->rslots == 0) { 203872200Sbmilekic /* if there was fatal error - reset port. */ 203951078Speter if (ch->toslots != 0 || ch->fatalerr) { 204051078Speter ahci_reset(dev); 204151078Speter } else { 204251078Speter /* if we have slots in error, we can reinit port. */ 204351078Speter if (ch->eslots != 0) { 204451078Speter ahci_stop(dev); 204551078Speter ahci_start(dev, 1); 204651078Speter } 204751078Speter /* if there commands on hold, we can do READ LOG. */ 204851078Speter if (!ch->readlog && ch->numhslots) 204951078Speter ahci_issue_read_log(dev); 205072200Sbmilekic } 205151078Speter /* If all the rest of commands are in timeout - give them chance. */ 205251078Speter } else if ((ch->rslots & ~ch->toslots) == 0 && 205351078Speter et != AHCI_ERR_TIMEOUT) 205451078Speter ahci_rearm_timeout(dev); 205551078Speter /* Start PM timer. */ 205651078Speter if (ch->numrslots == 0 && ch->pm_level > 3 && 205751078Speter (ch->curr[ch->pm_present ? 15 : 0].caps & CTS_SATA_CAPS_D_PMREQ)) { 205872200Sbmilekic callout_schedule(&ch->pm_timer, 205951078Speter (ch->pm_level == 4) ? hz / 1000 : hz / 8); 206051078Speter } 206151078Speter} 206251078Speter 206351078Speterstatic void 206451078Speterahci_issue_read_log(device_t dev) 206551078Speter{ 206651078Speter struct ahci_channel *ch = device_get_softc(dev); 206751078Speter union ccb *ccb; 206851078Speter struct ccb_ataio *ataio; 206951078Speter int i; 207072200Sbmilekic 207151078Speter ch->readlog = 1; 207251078Speter /* Find some holden command. */ 207351078Speter for (i = 0; i < ch->numslots; i++) { 207472200Sbmilekic if (ch->hold[i]) 207551078Speter break; 207651078Speter } 207772200Sbmilekic ccb = xpt_alloc_ccb_nowait(); 207851078Speter if (ccb == NULL) { 207951078Speter device_printf(dev, "Unable allocate READ LOG command"); 208051078Speter return; /* XXX */ 208151078Speter } 208251078Speter ccb->ccb_h = ch->hold[i]->ccb_h; /* Reuse old header. */ 208351654Sphk ccb->ccb_h.func_code = XPT_ATA_IO; 208451078Speter ccb->ccb_h.flags = CAM_DIR_IN; 208551078Speter ccb->ccb_h.timeout = 1000; /* 1s should be enough. */ 208651078Speter ataio = &ccb->ataio; 208751078Speter ataio->data_ptr = malloc(512, M_AHCI, M_NOWAIT); 208851078Speter if (ataio->data_ptr == NULL) { 2089135329Sphk xpt_free_ccb(ccb); 209057915Simp device_printf(dev, "Unable allocate memory for READ LOG command"); 209151078Speter return; /* XXX */ 209272200Sbmilekic } 209351078Speter ataio->dxfer_len = 512; 209451078Speter bzero(&ataio->cmd, sizeof(ataio->cmd)); 209551078Speter ataio->cmd.flags = CAM_ATAIO_48BIT; 209651078Speter ataio->cmd.command = 0x2F; /* READ LOG EXT */ 209751078Speter ataio->cmd.sector_count = 1; 209851078Speter ataio->cmd.sector_count_exp = 0; 209960471Snyan ataio->cmd.lba_low = 0x10; 210060471Snyan ataio->cmd.lba_mid = 0; 210151078Speter ataio->cmd.lba_mid_exp = 0; 210251078Speter /* Freeze SIM while doing READ LOG EXT. */ 210351078Speter xpt_freeze_simq(ch->sim, 1); 210451078Speter ahci_begin_transaction(dev, ccb); 210551078Speter} 210651078Speter 210751078Speterstatic void 210851078Speterahci_process_read_log(device_t dev, union ccb *ccb) 210951078Speter{ 211051078Speter struct ahci_channel *ch = device_get_softc(dev); 211151078Speter uint8_t *data; 211251078Speter struct ata_res *res; 211351078Speter int i; 211460471Snyan 211560471Snyan ch->readlog = 0; 211651078Speter 211751078Speter data = ccb->ataio.data_ptr; 211851078Speter if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP && 211972200Sbmilekic (data[0] & 0x80) == 0) { 212051078Speter for (i = 0; i < ch->numslots; i++) { 212151078Speter if (!ch->hold[i]) 212251078Speter continue; 212351078Speter if ((data[0] & 0x1F) == i) { 2124136478Sphk res = &ch->hold[i]->ataio.res; 2125131094Sphk res->status = data[2]; 212651078Speter res->error = data[3]; 2127131094Sphk res->lba_low = data[4]; 212851078Speter res->lba_mid = data[5]; 2129135329Sphk res->lba_high = data[6]; 2130131094Sphk res->device = data[7]; 2131131094Sphk res->lba_low_exp = data[8]; 2132131094Sphk res->lba_mid_exp = data[9]; 2133131094Sphk res->lba_high_exp = data[10]; 2134131094Sphk res->sector_count = data[12]; 2135131094Sphk res->sector_count_exp = data[13]; 2136131094Sphk } else { 2137131094Sphk ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK; 2138131094Sphk ch->hold[i]->ccb_h.status |= CAM_REQUEUE_REQ; 2139131094Sphk } 2140131094Sphk xpt_done(ch->hold[i]); 2141131094Sphk ch->hold[i] = NULL; 2142131094Sphk ch->numhslots--; 2143131094Sphk } 2144131094Sphk } else { 2145131094Sphk if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) 2146131094Sphk device_printf(dev, "Error while READ LOG EXT\n"); 2147131094Sphk else if ((data[0] & 0x80) == 0) { 2148131094Sphk device_printf(dev, "Non-queued command error in READ LOG EXT\n"); 2149131094Sphk } 2150131094Sphk for (i = 0; i < ch->numslots; i++) { 2151131094Sphk if (!ch->hold[i]) 2152131094Sphk continue; 2153131094Sphk xpt_done(ch->hold[i]); 2154131094Sphk ch->hold[i] = NULL; 215551078Speter ch->numhslots--; 215651078Speter } 2157131094Sphk } 215851078Speter free(ccb->ataio.data_ptr, M_AHCI); 2159131094Sphk xpt_free_ccb(ccb); 216051078Speter xpt_release_simq(ch->sim, TRUE); 2161131094Sphk} 2162131094Sphk 2163131094Sphkstatic void 216451078Speterahci_start(device_t dev, int fbs) 2165131094Sphk{ 2166131094Sphk struct ahci_channel *ch = device_get_softc(dev); 216751078Speter u_int32_t cmd; 216851078Speter 216951078Speter /* Clear SATA error register */ 217051078Speter ATA_OUTL(ch->r_mem, AHCI_P_SERR, 0xFFFFFFFF); 217151078Speter /* Clear any interrupts pending on this channel */ 217251078Speter ATA_OUTL(ch->r_mem, AHCI_P_IS, 0xFFFFFFFF); 217351078Speter /* Configure FIS-based switching if supported. */ 217451078Speter if (ch->chcaps & AHCI_P_CMD_FBSCP) { 217551078Speter ch->fbs_enabled = (fbs && ch->pm_present) ? 1 : 0; 217651078Speter ATA_OUTL(ch->r_mem, AHCI_P_FBS, 217751078Speter ch->fbs_enabled ? AHCI_P_FBS_EN : 0); 217851078Speter } 217951078Speter /* Start operations on this channel */ 218051078Speter cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 218151078Speter cmd &= ~AHCI_P_CMD_PMA; 218251078Speter ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd | AHCI_P_CMD_ST | 218351078Speter (ch->pm_present ? AHCI_P_CMD_PMA : 0)); 218451078Speter} 218553344Speter 218651078Speterstatic void 218751078Speterahci_stop(device_t dev) 218851078Speter{ 218951078Speter struct ahci_channel *ch = device_get_softc(dev); 219051078Speter u_int32_t cmd; 219151078Speter int timeout; 219251078Speter 219351078Speter /* Kill all activity on this channel */ 219451078Speter cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 219551078Speter ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd & ~AHCI_P_CMD_ST); 219651078Speter /* Wait for activity stop. */ 219751078Speter timeout = 0; 219851078Speter do { 219951078Speter DELAY(1000); 220051078Speter if (timeout++ > 1000) { 220151078Speter device_printf(dev, "stopping AHCI engine failed\n"); 220251078Speter break; 220351078Speter } 220451078Speter } while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CR); 220551078Speter ch->eslots = 0; 220651078Speter} 220751078Speter 220851078Speterstatic void 220951078Speterahci_clo(device_t dev) 221051078Speter{ 221151078Speter struct ahci_channel *ch = device_get_softc(dev); 221251078Speter u_int32_t cmd; 221351078Speter int timeout; 221451078Speter 221551078Speter /* Issue Command List Override if supported */ 221651078Speter if (ch->caps & AHCI_CAP_SCLO) { 221751078Speter cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 221851078Speter cmd |= AHCI_P_CMD_CLO; 221951078Speter ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd); 222051078Speter timeout = 0; 222153344Speter do { 222251078Speter DELAY(1000); 222351078Speter if (timeout++ > 1000) { 222451078Speter device_printf(dev, "executing CLO failed\n"); 222572200Sbmilekic break; 222651078Speter } 222772200Sbmilekic } while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CLO); 222851078Speter } 222951078Speter} 223051078Speter 223151078Speterstatic void 223251078Speterahci_stop_fr(device_t dev) 223351078Speter{ 223451078Speter struct ahci_channel *ch = device_get_softc(dev); 223551078Speter u_int32_t cmd; 223651078Speter int timeout; 223753344Speter 223851078Speter /* Kill all FIS reception on this channel */ 223951078Speter cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 224051078Speter ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd & ~AHCI_P_CMD_FRE); 224151078Speter /* Wait for FIS reception stop. */ 224251078Speter timeout = 0; 224351078Speter do { 224451078Speter DELAY(1000); 224551078Speter if (timeout++ > 1000) { 224651078Speter device_printf(dev, "stopping AHCI FR engine failed\n"); 224751078Speter break; 224851078Speter } 224972200Sbmilekic } while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_FR); 225051078Speter} 225151078Speter 225272200Sbmilekicstatic void 225351078Speterahci_start_fr(device_t dev) 225451078Speter{ 225551078Speter struct ahci_channel *ch = device_get_softc(dev); 225651078Speter u_int32_t cmd; 225751078Speter 225851078Speter /* Start FIS reception on this channel */ 225951078Speter cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 226051078Speter ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd | AHCI_P_CMD_FRE); 226151078Speter} 226251078Speter 226351078Speterstatic int 226451078Speterahci_wait_ready(device_t dev, int t) 226551078Speter{ 226651078Speter struct ahci_channel *ch = device_get_softc(dev); 226751078Speter int timeout = 0; 226851078Speter uint32_t val; 226951078Speter 227051078Speter while ((val = ATA_INL(ch->r_mem, AHCI_P_TFD)) & 227151078Speter (ATA_S_BUSY | ATA_S_DRQ)) { 227251078Speter DELAY(1000); 227351078Speter if (timeout++ > t) { 2274120457Sphk device_printf(dev, "device is not ready (timeout %dms) " 2275120457Sphk "tfd = %08x\n", t, val); 2276120457Sphk return (EBUSY); 2277120457Sphk } 2278120457Sphk } 2279120457Sphk if (bootverbose) 2280120457Sphk device_printf(dev, "ready wait time=%dms\n", timeout); 2281120457Sphk return (0); 2282120457Sphk} 2283120457Sphk 2284120457Sphkstatic void 2285120457Sphkahci_reset(device_t dev) 228692739Salfred{ 228793010Sbde struct ahci_channel *ch = device_get_softc(dev); 228893010Sbde struct ahci_controller *ctlr = device_get_softc(device_get_parent(dev)); 228993010Sbde int i; 229051078Speter 2291158943Sphk xpt_freeze_simq(ch->sim, 1); 2292158943Sphk if (bootverbose) 2293158943Sphk device_printf(dev, "AHCI reset...\n"); 2294158943Sphk /* Requeue freezed command. */ 2295158943Sphk if (ch->frozen) { 2296235405Savg union ccb *fccb = ch->frozen; 2297235405Savg ch->frozen = NULL; 229851078Speter fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ; 2299158947Sphk if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) { 230051078Speter xpt_freeze_devq(fccb->ccb_h.path, 1); 230151078Speter fccb->ccb_h.status |= CAM_DEV_QFRZN; 230251078Speter } 230351078Speter xpt_done(fccb); 230451078Speter } 230551078Speter /* Kill the engine and requeue all running commands. */ 230651078Speter ahci_stop(dev); 230751078Speter for (i = 0; i < ch->numslots; i++) { 230851078Speter /* Do we have a running request on slot? */ 230951078Speter if (ch->slot[i].state < AHCI_SLOT_RUNNING) 231051078Speter continue; 231151078Speter /* XXX; Commands in loading state. */ 231251078Speter ahci_end_transaction(&ch->slot[i], AHCI_ERR_INNOCENT); 231351078Speter } 231451078Speter for (i = 0; i < ch->numslots; i++) { 231551078Speter if (!ch->hold[i]) 231651078Speter continue; 231751078Speter xpt_done(ch->hold[i]); 231851078Speter ch->hold[i] = NULL; 231951078Speter ch->numhslots--; 232051078Speter } 232151078Speter if (ch->toslots != 0) 232251078Speter xpt_release_simq(ch->sim, TRUE); 232351078Speter ch->eslots = 0; 232451078Speter ch->toslots = 0; 232551078Speter ch->fatalerr = 0; 232651078Speter /* Tell the XPT about the event */ 232751078Speter xpt_async(AC_BUS_RESET, ch->path, NULL); 232889986Sjhay /* Disable port interrupts */ 232989986Sjhay ATA_OUTL(ch->r_mem, AHCI_P_IE, 0); 233089986Sjhay /* Reset and reconnect PHY, */ 233151078Speter if (!ahci_sata_phy_reset(dev)) { 233289986Sjhay if (bootverbose) 233351078Speter device_printf(dev, 233451078Speter "AHCI reset done: phy reset found no device\n"); 233551078Speter ch->devices = 0; 233651078Speter /* Enable wanted port interrupts */ 233751078Speter ATA_OUTL(ch->r_mem, AHCI_P_IE, 233851078Speter (AHCI_P_IX_CPD | AHCI_P_IX_PRC | AHCI_P_IX_PC)); 233951078Speter xpt_release_simq(ch->sim, TRUE); 234051078Speter return; 234151078Speter } 234251078Speter /* Wait for clearing busy status. */ 234351078Speter if (ahci_wait_ready(dev, 15000)) 234451078Speter ahci_clo(dev); 234589986Sjhay ahci_start(dev, 1); 234651078Speter ch->devices = 1; 234789986Sjhay /* Enable wanted port interrupts */ 234889986Sjhay ATA_OUTL(ch->r_mem, AHCI_P_IE, 234989986Sjhay (AHCI_P_IX_CPD | AHCI_P_IX_TFE | AHCI_P_IX_HBF | 235089986Sjhay AHCI_P_IX_HBD | AHCI_P_IX_IF | AHCI_P_IX_OF | 235151078Speter ((ch->pm_level == 0) ? AHCI_P_IX_PRC | AHCI_P_IX_PC : 0) | 235251078Speter AHCI_P_IX_DP | AHCI_P_IX_UF | (ctlr->ccc ? 0 : AHCI_P_IX_SDB) | 235351078Speter AHCI_P_IX_DS | AHCI_P_IX_PS | (ctlr->ccc ? 0 : AHCI_P_IX_DHR))); 235451078Speter if (bootverbose) 235551078Speter device_printf(dev, "AHCI reset done: device found\n"); 235651078Speter xpt_release_simq(ch->sim, TRUE); 235751078Speter} 235851078Speter 235989986Sjhaystatic int 236051078Speterahci_setup_fis(device_t dev, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag) 236151078Speter{ 236251078Speter struct ahci_channel *ch = device_get_softc(dev); 236351078Speter u_int8_t *fis = &ctp->cfis[0]; 236451078Speter 236551078Speter bzero(ctp->cfis, 64); 236651078Speter fis[0] = 0x27; /* host to device */ 236751078Speter fis[1] = (ccb->ccb_h.target_id & 0x0f); 236851078Speter if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 236951078Speter fis[1] |= 0x80; 237051078Speter fis[2] = ATA_PACKET_CMD; 237151078Speter if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE && 237251078Speter ch->curr[ccb->ccb_h.target_id].mode >= ATA_DMA) 237351078Speter fis[3] = ATA_F_DMA; 237451078Speter else { 237551078Speter fis[5] = ccb->csio.dxfer_len; 237651078Speter fis[6] = ccb->csio.dxfer_len >> 8; 237751078Speter } 237851078Speter fis[7] = ATA_D_LBA; 237951078Speter fis[15] = ATA_A_4BIT; 238051078Speter bzero(ctp->acmd, 32); 238189986Sjhay bcopy((ccb->ccb_h.flags & CAM_CDB_POINTER) ? 238251078Speter ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes, 238351078Speter ctp->acmd, ccb->csio.cdb_len); 238451078Speter } else if ((ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) == 0) { 238589986Sjhay fis[1] |= 0x80; 238651078Speter fis[2] = ccb->ataio.cmd.command; 238751078Speter fis[3] = ccb->ataio.cmd.features; 238851078Speter fis[4] = ccb->ataio.cmd.lba_low; 238951078Speter fis[5] = ccb->ataio.cmd.lba_mid; 239051078Speter fis[6] = ccb->ataio.cmd.lba_high; 239151078Speter fis[7] = ccb->ataio.cmd.device; 239251078Speter fis[8] = ccb->ataio.cmd.lba_low_exp; 239351078Speter fis[9] = ccb->ataio.cmd.lba_mid_exp; 239451078Speter fis[10] = ccb->ataio.cmd.lba_high_exp; 239551078Speter fis[11] = ccb->ataio.cmd.features_exp; 239651078Speter if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) { 239751078Speter fis[12] = tag << 3; 239851078Speter fis[13] = 0; 239951078Speter } else { 240051078Speter fis[12] = ccb->ataio.cmd.sector_count; 240151078Speter fis[13] = ccb->ataio.cmd.sector_count_exp; 240251078Speter } 240351078Speter fis[15] = ATA_A_4BIT; 240451078Speter } else { 240551078Speter fis[15] = ccb->ataio.cmd.control; 240651078Speter } 240751078Speter return (20); 240851078Speter} 240951078Speter 241051078Speterstatic int 241151078Speterahci_sata_connect(struct ahci_channel *ch) 241251078Speter{ 241351078Speter u_int32_t status; 241451078Speter int timeout; 241551078Speter 241651078Speter /* Wait up to 100ms for "connect well" */ 241751078Speter for (timeout = 0; timeout < 100 ; timeout++) { 241851078Speter status = ATA_INL(ch->r_mem, AHCI_P_SSTS); 241951078Speter if (((status & ATA_SS_DET_MASK) == ATA_SS_DET_PHY_ONLINE) && 242051078Speter ((status & ATA_SS_SPD_MASK) != ATA_SS_SPD_NO_SPEED) && 2421158943Sphk ((status & ATA_SS_IPM_MASK) == ATA_SS_IPM_ACTIVE)) 242251078Speter break; 242351078Speter if ((status & ATA_SS_DET_MASK) == ATA_SS_DET_PHY_OFFLINE) { 242451078Speter if (bootverbose) { 242551078Speter device_printf(ch->dev, "SATA offline status=%08x\n", 242689986Sjhay status); 2427128899Sambrisko } 242851078Speter return (0); 242951078Speter } 243051078Speter DELAY(1000); 243151078Speter } 243251078Speter if (timeout >= 100) { 243351078Speter if (bootverbose) { 243451078Speter device_printf(ch->dev, "SATA connect timeout status=%08x\n", 243551078Speter status); 243651078Speter } 243751078Speter return (0); 243851078Speter } 243951078Speter if (bootverbose) { 244051078Speter device_printf(ch->dev, "SATA connect time=%dms status=%08x\n", 244151078Speter timeout, status); 244251078Speter } 244351078Speter /* Clear SATA error register */ 244451078Speter ATA_OUTL(ch->r_mem, AHCI_P_SERR, 0xffffffff); 244551078Speter return (1); 2446128899Sambrisko} 2447128899Sambrisko 2448117167Sjhbstatic int 2449128899Sambriskoahci_sata_phy_reset(device_t dev) 2450128899Sambrisko{ 2451128899Sambrisko struct ahci_channel *ch = device_get_softc(dev); 2452128899Sambrisko int sata_rev; 2453128899Sambrisko uint32_t val; 2454128899Sambrisko 2455128899Sambrisko sata_rev = ch->user[ch->pm_present ? 15 : 0].revision; 2456128899Sambrisko if (sata_rev == 1) 2457128899Sambrisko val = ATA_SC_SPD_SPEED_GEN1; 2458128781Sambrisko else if (sata_rev == 2) 2459128899Sambrisko val = ATA_SC_SPD_SPEED_GEN2; 2460128899Sambrisko else if (sata_rev == 3) 2461174283Sjulian val = ATA_SC_SPD_SPEED_GEN3; 2462128899Sambrisko else 2463128899Sambrisko val = 0; 2464128899Sambrisko ATA_OUTL(ch->r_mem, AHCI_P_SCTL, 2465128899Sambrisko ATA_SC_DET_RESET | val | 246651078Speter ATA_SC_IPM_DIS_PARTIAL | ATA_SC_IPM_DIS_SLUMBER); 246751078Speter DELAY(5000); 246851078Speter ATA_OUTL(ch->r_mem, AHCI_P_SCTL, 246951078Speter ATA_SC_DET_IDLE | val | ((ch->pm_level > 0) ? 0 : 247051078Speter (ATA_SC_IPM_DIS_PARTIAL | ATA_SC_IPM_DIS_SLUMBER))); 247151078Speter DELAY(5000); 247251078Speter if (!ahci_sata_connect(ch)) { 247351078Speter if (ch->pm_level > 0) 247451078Speter ATA_OUTL(ch->r_mem, AHCI_P_SCTL, ATA_SC_DET_DISABLE); 247551078Speter return (0); 247651078Speter } 2477128899Sambrisko return (1); 2478128899Sambrisko} 2479128899Sambrisko 2480128899Sambriskostatic int 2481128899Sambriskoahci_check_ids(device_t dev, union ccb *ccb) 2482128899Sambrisko{ 248351078Speter struct ahci_channel *ch = device_get_softc(dev); 2484128899Sambrisko 248551078Speter if (ccb->ccb_h.target_id > ((ch->caps & AHCI_CAP_SPM) ? 15 : 0)) { 2486128899Sambrisko ccb->ccb_h.status = CAM_TID_INVALID; 2487128899Sambrisko xpt_done(ccb); 2488128899Sambrisko return (-1); 2489128899Sambrisko } 2490128899Sambrisko if (ccb->ccb_h.target_lun != 0) { 2491128899Sambrisko ccb->ccb_h.status = CAM_LUN_INVALID; 2492128899Sambrisko xpt_done(ccb); 2493128899Sambrisko return (-1); 2494128899Sambrisko } 2495131939Smarcel return (0); 2496131939Smarcel} 2497128899Sambrisko 249851078Speterstatic void 249951078Speterahciaction(struct cam_sim *sim, union ccb *ccb) 250051078Speter{ 250151078Speter device_t dev, parent; 250251078Speter struct ahci_channel *ch; 250366230Sjhb 2504158943Sphk CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("ahciaction func_code=%x\n", 250566230Sjhb ccb->ccb_h.func_code)); 250666230Sjhb 2507120457Sphk ch = (struct ahci_channel *)cam_sim_softc(sim); 250866230Sjhb dev = ch->dev; 250966230Sjhb switch (ccb->ccb_h.func_code) { 251085371Sjlemon /* Common cases first */ 2511158943Sphk case XPT_ATA_IO: /* Execute the requested I/O operation */ 251285371Sjlemon case XPT_SCSI_IO: 251385371Sjlemon if (ahci_check_ids(dev, ccb)) 251485371Sjlemon return; 251585371Sjlemon if (ch->devices == 0 || 251685371Sjlemon (ch->pm_present == 0 && 2517235405Savg ccb->ccb_h.target_id > 0 && ccb->ccb_h.target_id < 15)) { 2518235405Savg ccb->ccb_h.status = CAM_SEL_TIMEOUT; 2519235405Savg break; 2520235405Savg } 2521235405Savg /* Check for command collision. */ 2522235405Savg if (ahci_check_collision(dev, ccb)) { 2523235405Savg /* Freeze command. */ 2524235405Savg ch->frozen = ccb; 2525235405Savg /* We have only one frozen slot, so freeze simq also. */ 2526235405Savg xpt_freeze_simq(ch->sim, 1); 252751078Speter return; 2528158947Sphk } 252951078Speter ahci_begin_transaction(dev, ccb); 253051078Speter return; 253151078Speter case XPT_EN_LUN: /* Enable LUN as a target */ 253251078Speter case XPT_TARGET_IO: /* Execute target I/O request */ 253351078Speter case XPT_ACCEPT_TARGET_IO: /* Accept Host Target Mode CDB */ 253498401Sn_hibma case XPT_CONT_TARGET_IO: /* Continue Host Target I/O Connection*/ 2535131939Smarcel case XPT_ABORT: /* Abort the specified CCB */ 2536131939Smarcel /* XXX Implement */ 253798401Sn_hibma ccb->ccb_h.status = CAM_REQ_INVALID; 253898401Sn_hibma break; 253998401Sn_hibma case XPT_SET_TRAN_SETTINGS: 2540131939Smarcel { 254151078Speter struct ccb_trans_settings *cts = &ccb->cts; 254298401Sn_hibma struct ahci_device *d; 2543131939Smarcel 2544131939Smarcel if (ahci_check_ids(dev, ccb)) 2545131939Smarcel return; 254698401Sn_hibma if (cts->type == CTS_TYPE_CURRENT_SETTINGS) 254751078Speter d = &ch->curr[ccb->ccb_h.target_id]; 254898401Sn_hibma else 254951078Speter d = &ch->user[ccb->ccb_h.target_id]; 255051078Speter if (cts->xport_specific.sata.valid & CTS_SATA_VALID_REVISION) 255151078Speter d->revision = cts->xport_specific.sata.revision; 255251078Speter if (cts->xport_specific.sata.valid & CTS_SATA_VALID_MODE) 255351078Speter d->mode = cts->xport_specific.sata.mode; 255451078Speter if (cts->xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT) 255551078Speter d->bytecount = min(8192, cts->xport_specific.sata.bytecount); 255651078Speter if (cts->xport_specific.sata.valid & CTS_SATA_VALID_TAGS) 255751078Speter d->tags = min(ch->numslots, cts->xport_specific.sata.tags); 2558104094Sphk if (cts->xport_specific.sata.valid & CTS_SATA_VALID_PM) 2559158943Sphk ch->pm_present = cts->xport_specific.sata.pm_present; 256051078Speter if (cts->xport_specific.sata.valid & CTS_SATA_VALID_ATAPI) 256188582Sbde d->atapi = cts->xport_specific.sata.atapi; 256251078Speter if (cts->xport_specific.sata.valid & CTS_SATA_VALID_CAPS) 256351078Speter d->caps = cts->xport_specific.sata.caps; 256451078Speter ccb->ccb_h.status = CAM_REQ_CMP; 256598401Sn_hibma break; 256651078Speter } 2567131939Smarcel case XPT_GET_TRAN_SETTINGS: 256898401Sn_hibma /* Get default/user set transfer settings for the target */ 256998401Sn_hibma { 257098401Sn_hibma struct ccb_trans_settings *cts = &ccb->cts; 2571131939Smarcel struct ahci_device *d; 257251078Speter uint32_t status; 257398401Sn_hibma 2574131939Smarcel if (ahci_check_ids(dev, ccb)) 2575131939Smarcel return; 2576131939Smarcel if (cts->type == CTS_TYPE_CURRENT_SETTINGS) 257798401Sn_hibma d = &ch->curr[ccb->ccb_h.target_id]; 257851078Speter else 257988582Sbde d = &ch->user[ccb->ccb_h.target_id]; 2580141530Srwatson cts->protocol = PROTO_ATA; 2581141530Srwatson cts->protocol_version = PROTO_VERSION_UNSPECIFIED; 2582141530Srwatson cts->transport = XPORT_SATA; 258388582Sbde cts->transport_version = XPORT_VERSION_UNSPECIFIED; 258498401Sn_hibma cts->proto_specific.valid = 0; 258551078Speter cts->xport_specific.sata.valid = 0; 258651078Speter if (cts->type == CTS_TYPE_CURRENT_SETTINGS && 258751078Speter (ccb->ccb_h.target_id == 15 || 2588141530Srwatson (ccb->ccb_h.target_id == 0 && !ch->pm_present))) { 2589141530Srwatson status = ATA_INL(ch->r_mem, AHCI_P_SSTS) & ATA_SS_SPD_MASK; 259051078Speter if (status & 0x0f0) { 259151078Speter cts->xport_specific.sata.revision = 259251078Speter (status & 0x0f0) >> 4; 2593131939Smarcel cts->xport_specific.sata.valid |= 2594131939Smarcel CTS_SATA_VALID_REVISION; 2595131939Smarcel } 2596131939Smarcel cts->xport_specific.sata.caps = d->caps & CTS_SATA_CAPS_D; 2597131939Smarcel if (ch->pm_level) { 2598131939Smarcel if (ch->caps & (AHCI_CAP_PSC | AHCI_CAP_SSC)) 2599131939Smarcel cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_PMREQ; 2600131939Smarcel if (ch->caps2 & AHCI_CAP2_APST) 2601131939Smarcel cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_APST; 2602131939Smarcel } 2603131939Smarcel if ((ch->caps & AHCI_CAP_SNCQ) && 2604131939Smarcel (ch->quirks & AHCI_Q_NOAA) == 0) 2605131939Smarcel cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_DMAAA; 2606131939Smarcel cts->xport_specific.sata.caps &= 2607158950Sphk ch->user[ccb->ccb_h.target_id].caps; 2608131939Smarcel cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS; 2609131939Smarcel } else { 2610131939Smarcel cts->xport_specific.sata.revision = d->revision; 261151078Speter cts->xport_specific.sata.valid |= CTS_SATA_VALID_REVISION; 2612131939Smarcel cts->xport_specific.sata.caps = d->caps; 2613131939Smarcel cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS; 261451078Speter } 2615131939Smarcel cts->xport_specific.sata.mode = d->mode; 2616131939Smarcel cts->xport_specific.sata.valid |= CTS_SATA_VALID_MODE; 2617131939Smarcel cts->xport_specific.sata.bytecount = d->bytecount; 2618131939Smarcel cts->xport_specific.sata.valid |= CTS_SATA_VALID_BYTECOUNT; 261998401Sn_hibma cts->xport_specific.sata.pm_present = ch->pm_present; 2620131939Smarcel cts->xport_specific.sata.valid |= CTS_SATA_VALID_PM; 2621131939Smarcel cts->xport_specific.sata.tags = d->tags; 2622131939Smarcel cts->xport_specific.sata.valid |= CTS_SATA_VALID_TAGS; 262351078Speter cts->xport_specific.sata.atapi = d->atapi; 262451078Speter cts->xport_specific.sata.valid |= CTS_SATA_VALID_ATAPI; 2625131939Smarcel ccb->ccb_h.status = CAM_REQ_CMP; 2626131939Smarcel break; 262751078Speter } 2628158943Sphk case XPT_RESET_BUS: /* Reset the specified SCSI bus */ 2629131939Smarcel case XPT_RESET_DEV: /* Bus Device Reset the specified SCSI device */ 263051078Speter ahci_reset(dev); 2631131939Smarcel ccb->ccb_h.status = CAM_REQ_CMP; 2632158947Sphk break; 2633131939Smarcel case XPT_TERM_IO: /* Terminate the I/O process */ 2634158950Sphk /* XXX Implement */ 263551078Speter ccb->ccb_h.status = CAM_REQ_INVALID; 2636131939Smarcel break; 263751078Speter case XPT_PATH_INQ: /* Path routing inquiry */ 2638 { 2639 struct ccb_pathinq *cpi = &ccb->cpi; 2640 2641 parent = device_get_parent(dev); 2642 cpi->version_num = 1; /* XXX??? */ 2643 cpi->hba_inquiry = PI_SDTR_ABLE; 2644 if (ch->caps & AHCI_CAP_SNCQ) 2645 cpi->hba_inquiry |= PI_TAG_ABLE; 2646 if (ch->caps & AHCI_CAP_SPM) 2647 cpi->hba_inquiry |= PI_SATAPM; 2648 cpi->target_sprt = 0; 2649 cpi->hba_misc = PIM_SEQSCAN; 2650 cpi->hba_eng_cnt = 0; 2651 if (ch->caps & AHCI_CAP_SPM) 2652 cpi->max_target = 15; 2653 else 2654 cpi->max_target = 0; 2655 cpi->max_lun = 0; 2656 cpi->initiator_id = 0; 2657 cpi->bus_id = cam_sim_bus(sim); 2658 cpi->base_transfer_speed = 150000; 2659 strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); 2660 strncpy(cpi->hba_vid, "AHCI", HBA_IDLEN); 2661 strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); 2662 cpi->unit_number = cam_sim_unit(sim); 2663 cpi->transport = XPORT_SATA; 2664 cpi->transport_version = XPORT_VERSION_UNSPECIFIED; 2665 cpi->protocol = PROTO_ATA; 2666 cpi->protocol_version = PROTO_VERSION_UNSPECIFIED; 2667 cpi->maxio = MAXPHYS; 2668 /* ATI SB600 can't handle 256 sectors with FPDMA (NCQ). */ 2669 if (pci_get_devid(parent) == 0x43801002) 2670 cpi->maxio = min(cpi->maxio, 128 * 512); 2671 cpi->hba_vendor = pci_get_vendor(parent); 2672 cpi->hba_device = pci_get_device(parent); 2673 cpi->hba_subvendor = pci_get_subvendor(parent); 2674 cpi->hba_subdevice = pci_get_subdevice(parent); 2675 cpi->ccb_h.status = CAM_REQ_CMP; 2676 break; 2677 } 2678 default: 2679 ccb->ccb_h.status = CAM_REQ_INVALID; 2680 break; 2681 } 2682 xpt_done(ccb); 2683} 2684 2685static void 2686ahcipoll(struct cam_sim *sim) 2687{ 2688 struct ahci_channel *ch = (struct ahci_channel *)cam_sim_softc(sim); 2689 2690 ahci_ch_intr(ch->dev); 2691} 2692