11573Srgrimes/*-
21573Srgrimes * Copyright (c) 2009 Yahoo! Inc.
31573Srgrimes * Copyright (c) 2011-2015 LSI Corp.
41573Srgrimes * Copyright (c) 2013-2015 Avago Technologies
51573Srgrimes * All rights reserved.
61573Srgrimes *
71573Srgrimes * Redistribution and use in source and binary forms, with or without
81573Srgrimes * modification, are permitted provided that the following conditions
91573Srgrimes * are met:
101573Srgrimes * 1. Redistributions of source code must retain the above copyright
111573Srgrimes *    notice, this list of conditions and the following disclaimer.
121573Srgrimes * 2. Redistributions in binary form must reproduce the above copyright
131573Srgrimes *    notice, this list of conditions and the following disclaimer in the
141573Srgrimes *    documentation and/or other materials provided with the distribution.
151573Srgrimes *
16251069Semaste * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
171573Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
181573Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
191573Srgrimes * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
201573Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
211573Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
221573Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
231573Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
241573Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
251573Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
261573Srgrimes * SUCH DAMAGE.
271573Srgrimes *
281573Srgrimes * Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD
291573Srgrimes *
301573Srgrimes * $FreeBSD: stable/10/sys/dev/mps/mps.c 322661 2017-08-18 15:38:08Z ken $
311573Srgrimes */
321573Srgrimes
331573Srgrimes#include <sys/cdefs.h>
341573Srgrimes__FBSDID("$FreeBSD: stable/10/sys/dev/mps/mps.c 322661 2017-08-18 15:38:08Z ken $");
351573Srgrimes
3686170Sobrien/* Communications core for Avago Technologies (LSI) MPT2 */
3786170Sobrien
381573Srgrimes/* TODO Move headers to mpsvar */
391573Srgrimes#include <sys/types.h>
401573Srgrimes#include <sys/param.h>
411573Srgrimes#include <sys/systm.h>
421573Srgrimes#include <sys/kernel.h>
431573Srgrimes#include <sys/selinfo.h>
441573Srgrimes#include <sys/lock.h>
451573Srgrimes#include <sys/mutex.h>
46103012Stjr#include <sys/module.h>
471573Srgrimes#include <sys/bus.h>
481573Srgrimes#include <sys/conf.h>
4992889Sobrien#include <sys/bio.h>
5092889Sobrien#include <sys/malloc.h>
511573Srgrimes#include <sys/uio.h>
521573Srgrimes#include <sys/sysctl.h>
53188080Sdanger#include <sys/queue.h>
541573Srgrimes#include <sys/kthread.h>
551573Srgrimes#include <sys/taskqueue.h>
56188080Sdanger#include <sys/endian.h>
571573Srgrimes#include <sys/eventhandler.h>
581573Srgrimes
591573Srgrimes#include <machine/bus.h>
601573Srgrimes#include <machine/resource.h>
611573Srgrimes#include <sys/rman.h>
621573Srgrimes#include <sys/proc.h>
63
64#include <dev/pci/pcivar.h>
65
66#include <cam/cam.h>
67#include <cam/scsi/scsi_all.h>
68
69#include <dev/mps/mpi/mpi2_type.h>
70#include <dev/mps/mpi/mpi2.h>
71#include <dev/mps/mpi/mpi2_ioc.h>
72#include <dev/mps/mpi/mpi2_sas.h>
73#include <dev/mps/mpi/mpi2_cnfg.h>
74#include <dev/mps/mpi/mpi2_init.h>
75#include <dev/mps/mpi/mpi2_tool.h>
76#include <dev/mps/mps_ioctl.h>
77#include <dev/mps/mpsvar.h>
78#include <dev/mps/mps_table.h>
79
80static int mps_diag_reset(struct mps_softc *sc, int sleep_flag);
81static int mps_init_queues(struct mps_softc *sc);
82static int mps_message_unit_reset(struct mps_softc *sc, int sleep_flag);
83static int mps_transition_operational(struct mps_softc *sc);
84static int mps_iocfacts_allocate(struct mps_softc *sc, uint8_t attaching);
85static void mps_iocfacts_free(struct mps_softc *sc);
86static void mps_startup(void *arg);
87static int mps_send_iocinit(struct mps_softc *sc);
88static int mps_alloc_queues(struct mps_softc *sc);
89static int mps_alloc_replies(struct mps_softc *sc);
90static int mps_alloc_requests(struct mps_softc *sc);
91static int mps_attach_log(struct mps_softc *sc);
92static __inline void mps_complete_command(struct mps_softc *sc,
93    struct mps_command *cm);
94static void mps_dispatch_event(struct mps_softc *sc, uintptr_t data,
95    MPI2_EVENT_NOTIFICATION_REPLY *reply);
96static void mps_config_complete(struct mps_softc *sc, struct mps_command *cm);
97static void mps_periodic(void *);
98static int mps_reregister_events(struct mps_softc *sc);
99static void mps_enqueue_request(struct mps_softc *sc, struct mps_command *cm);
100static int mps_get_iocfacts(struct mps_softc *sc, MPI2_IOC_FACTS_REPLY *facts);
101static int mps_wait_db_ack(struct mps_softc *sc, int timeout, int sleep_flag);
102SYSCTL_NODE(_hw, OID_AUTO, mps, CTLFLAG_RD, 0, "MPS Driver Parameters");
103
104MALLOC_DEFINE(M_MPT2, "mps", "mpt2 driver memory");
105
106/*
107 * Do a "Diagnostic Reset" aka a hard reset.  This should get the chip out of
108 * any state and back to its initialization state machine.
109 */
110static char mpt2_reset_magic[] = { 0x00, 0x0f, 0x04, 0x0b, 0x02, 0x07, 0x0d };
111
112/* Added this union to smoothly convert le64toh cm->cm_desc.Words.
113 * Compiler only support unint64_t to be passed as argument.
114 * Otherwise it will through below error
115 * "aggregate value used where an integer was expected"
116 */
117
118typedef union _reply_descriptor {
119        u64 word;
120        struct {
121                u32 low;
122                u32 high;
123        } u;
124}reply_descriptor,address_descriptor;
125
126/* Rate limit chain-fail messages to 1 per minute */
127static struct timeval mps_chainfail_interval = { 60, 0 };
128
129/*
130 * sleep_flag can be either CAN_SLEEP or NO_SLEEP.
131 * If this function is called from process context, it can sleep
132 * and there is no harm to sleep, in case if this fuction is called
133 * from Interrupt handler, we can not sleep and need NO_SLEEP flag set.
134 * based on sleep flags driver will call either msleep, pause or DELAY.
135 * msleep and pause are of same variant, but pause is used when mps_mtx
136 * is not hold by driver.
137 *
138 */
139static int
140mps_diag_reset(struct mps_softc *sc,int sleep_flag)
141{
142	uint32_t reg;
143	int i, error, tries = 0;
144	uint8_t first_wait_done = FALSE;
145
146	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
147
148	/* Clear any pending interrupts */
149	mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
150
151	/*Force NO_SLEEP for threads prohibited to sleep
152 	* e.a Thread from interrupt handler are prohibited to sleep.
153 	*/
154	if (curthread->td_no_sleeping != 0)
155		sleep_flag = NO_SLEEP;
156
157	/* Push the magic sequence */
158	error = ETIMEDOUT;
159	while (tries++ < 20) {
160		for (i = 0; i < sizeof(mpt2_reset_magic); i++)
161			mps_regwrite(sc, MPI2_WRITE_SEQUENCE_OFFSET,
162			    mpt2_reset_magic[i]);
163		/* wait 100 msec */
164		if (mtx_owned(&sc->mps_mtx) && sleep_flag == CAN_SLEEP)
165			msleep(&sc->msleep_fake_chan, &sc->mps_mtx, 0,
166			    "mpsdiag", hz/10);
167		else if (sleep_flag == CAN_SLEEP)
168			pause("mpsdiag", hz/10);
169		else
170			DELAY(100 * 1000);
171
172		reg = mps_regread(sc, MPI2_HOST_DIAGNOSTIC_OFFSET);
173		if (reg & MPI2_DIAG_DIAG_WRITE_ENABLE) {
174			error = 0;
175			break;
176		}
177	}
178	if (error)
179		return (error);
180
181	/* Send the actual reset.  XXX need to refresh the reg? */
182	mps_regwrite(sc, MPI2_HOST_DIAGNOSTIC_OFFSET,
183	    reg | MPI2_DIAG_RESET_ADAPTER);
184
185	/* Wait up to 300 seconds in 50ms intervals */
186	error = ETIMEDOUT;
187	for (i = 0; i < 6000; i++) {
188		/*
189		 * Wait 50 msec. If this is the first time through, wait 256
190		 * msec to satisfy Diag Reset timing requirements.
191		 */
192		if (first_wait_done) {
193			if (mtx_owned(&sc->mps_mtx) && sleep_flag == CAN_SLEEP)
194				msleep(&sc->msleep_fake_chan, &sc->mps_mtx, 0,
195				    "mpsdiag", hz/20);
196			else if (sleep_flag == CAN_SLEEP)
197				pause("mpsdiag", hz/20);
198			else
199				DELAY(50 * 1000);
200		} else {
201			DELAY(256 * 1000);
202			first_wait_done = TRUE;
203		}
204		/*
205		 * Check for the RESET_ADAPTER bit to be cleared first, then
206		 * wait for the RESET state to be cleared, which takes a little
207		 * longer.
208		 */
209		reg = mps_regread(sc, MPI2_HOST_DIAGNOSTIC_OFFSET);
210		if (reg & MPI2_DIAG_RESET_ADAPTER) {
211			continue;
212		}
213		reg = mps_regread(sc, MPI2_DOORBELL_OFFSET);
214		if ((reg & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_RESET) {
215			error = 0;
216			break;
217		}
218	}
219	if (error)
220		return (error);
221
222	mps_regwrite(sc, MPI2_WRITE_SEQUENCE_OFFSET, 0x0);
223
224	return (0);
225}
226
227static int
228mps_message_unit_reset(struct mps_softc *sc, int sleep_flag)
229{
230
231	MPS_FUNCTRACE(sc);
232
233	mps_regwrite(sc, MPI2_DOORBELL_OFFSET,
234	    MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET <<
235	    MPI2_DOORBELL_FUNCTION_SHIFT);
236
237	if (mps_wait_db_ack(sc, 5, sleep_flag) != 0) {
238		mps_dprint(sc, MPS_FAULT, "Doorbell handshake failed : <%s>\n",
239				__func__);
240		return (ETIMEDOUT);
241	}
242
243	return (0);
244}
245
246static int
247mps_transition_ready(struct mps_softc *sc)
248{
249	uint32_t reg, state;
250	int error, tries = 0;
251	int sleep_flags;
252
253	MPS_FUNCTRACE(sc);
254	/* If we are in attach call, do not sleep */
255	sleep_flags = (sc->mps_flags & MPS_FLAGS_ATTACH_DONE)
256					? CAN_SLEEP:NO_SLEEP;
257	error = 0;
258	while (tries++ < 1200) {
259		reg = mps_regread(sc, MPI2_DOORBELL_OFFSET);
260		mps_dprint(sc, MPS_INIT, "Doorbell= 0x%x\n", reg);
261
262		/*
263		 * Ensure the IOC is ready to talk.  If it's not, try
264		 * resetting it.
265		 */
266		if (reg & MPI2_DOORBELL_USED) {
267			mps_diag_reset(sc, sleep_flags);
268			DELAY(50000);
269			continue;
270		}
271
272		/* Is the adapter owned by another peer? */
273		if ((reg & MPI2_DOORBELL_WHO_INIT_MASK) ==
274		    (MPI2_WHOINIT_PCI_PEER << MPI2_DOORBELL_WHO_INIT_SHIFT)) {
275			device_printf(sc->mps_dev, "IOC is under the control "
276			    "of another peer host, aborting initialization.\n");
277			return (ENXIO);
278		}
279
280		state = reg & MPI2_IOC_STATE_MASK;
281		if (state == MPI2_IOC_STATE_READY) {
282			/* Ready to go! */
283			error = 0;
284			break;
285		} else if (state == MPI2_IOC_STATE_FAULT) {
286			mps_dprint(sc, MPS_FAULT, "IOC in fault state 0x%x, resetting\n",
287			    state & MPI2_DOORBELL_FAULT_CODE_MASK);
288			mps_diag_reset(sc, sleep_flags);
289		} else if (state == MPI2_IOC_STATE_OPERATIONAL) {
290			/* Need to take ownership */
291			mps_message_unit_reset(sc, sleep_flags);
292		} else if (state == MPI2_IOC_STATE_RESET) {
293			/* Wait a bit, IOC might be in transition */
294			mps_dprint(sc, MPS_FAULT,
295			    "IOC in unexpected reset state\n");
296		} else {
297			mps_dprint(sc, MPS_FAULT,
298			    "IOC in unknown state 0x%x\n", state);
299			error = EINVAL;
300			break;
301		}
302
303		/* Wait 50ms for things to settle down. */
304		DELAY(50000);
305	}
306
307	if (error)
308		device_printf(sc->mps_dev, "Cannot transition IOC to ready\n");
309
310	return (error);
311}
312
313static int
314mps_transition_operational(struct mps_softc *sc)
315{
316	uint32_t reg, state;
317	int error;
318
319	MPS_FUNCTRACE(sc);
320
321	error = 0;
322	reg = mps_regread(sc, MPI2_DOORBELL_OFFSET);
323	mps_dprint(sc, MPS_INIT, "Doorbell= 0x%x\n", reg);
324
325	state = reg & MPI2_IOC_STATE_MASK;
326	if (state != MPI2_IOC_STATE_READY) {
327		if ((error = mps_transition_ready(sc)) != 0) {
328			mps_dprint(sc, MPS_FAULT,
329			    "%s failed to transition ready\n", __func__);
330			return (error);
331		}
332	}
333
334	error = mps_send_iocinit(sc);
335	return (error);
336}
337
338/*
339 * This is called during attach and when re-initializing due to a Diag Reset.
340 * IOC Facts is used to allocate many of the structures needed by the driver.
341 * If called from attach, de-allocation is not required because the driver has
342 * not allocated any structures yet, but if called from a Diag Reset, previously
343 * allocated structures based on IOC Facts will need to be freed and re-
344 * allocated bases on the latest IOC Facts.
345 */
346static int
347mps_iocfacts_allocate(struct mps_softc *sc, uint8_t attaching)
348{
349	int error;
350	Mpi2IOCFactsReply_t saved_facts;
351	uint8_t saved_mode, reallocating;
352
353	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
354
355	/* Save old IOC Facts and then only reallocate if Facts have changed */
356	if (!attaching) {
357		bcopy(sc->facts, &saved_facts, sizeof(MPI2_IOC_FACTS_REPLY));
358	}
359
360	/*
361	 * Get IOC Facts.  In all cases throughout this function, panic if doing
362	 * a re-initialization and only return the error if attaching so the OS
363	 * can handle it.
364	 */
365	if ((error = mps_get_iocfacts(sc, sc->facts)) != 0) {
366		if (attaching) {
367			mps_dprint(sc, MPS_FAULT, "%s failed to get IOC Facts "
368			    "with error %d\n", __func__, error);
369			return (error);
370		} else {
371			panic("%s failed to get IOC Facts with error %d\n",
372			    __func__, error);
373		}
374	}
375
376	MPS_DPRINT_PAGE(sc, MPS_XINFO, iocfacts, sc->facts);
377
378	snprintf(sc->fw_version, sizeof(sc->fw_version),
379	    "%02d.%02d.%02d.%02d",
380	    sc->facts->FWVersion.Struct.Major,
381	    sc->facts->FWVersion.Struct.Minor,
382	    sc->facts->FWVersion.Struct.Unit,
383	    sc->facts->FWVersion.Struct.Dev);
384
385	mps_printf(sc, "Firmware: %s, Driver: %s\n", sc->fw_version,
386	    MPS_DRIVER_VERSION);
387	mps_printf(sc, "IOCCapabilities: %b\n", sc->facts->IOCCapabilities,
388	    "\20" "\3ScsiTaskFull" "\4DiagTrace" "\5SnapBuf" "\6ExtBuf"
389	    "\7EEDP" "\10BiDirTarg" "\11Multicast" "\14TransRetry" "\15IR"
390	    "\16EventReplay" "\17RaidAccel" "\20MSIXIndex" "\21HostDisc");
391
392	/*
393	 * If the chip doesn't support event replay then a hard reset will be
394	 * required to trigger a full discovery.  Do the reset here then
395	 * retransition to Ready.  A hard reset might have already been done,
396	 * but it doesn't hurt to do it again.  Only do this if attaching, not
397	 * for a Diag Reset.
398	 */
399	if (attaching) {
400		if ((sc->facts->IOCCapabilities &
401		    MPI2_IOCFACTS_CAPABILITY_EVENT_REPLAY) == 0) {
402			mps_diag_reset(sc, NO_SLEEP);
403			if ((error = mps_transition_ready(sc)) != 0) {
404				mps_dprint(sc, MPS_FAULT, "%s failed to "
405				    "transition to ready with error %d\n",
406				    __func__, error);
407				return (error);
408			}
409		}
410	}
411
412	/*
413	 * Set flag if IR Firmware is loaded.  If the RAID Capability has
414	 * changed from the previous IOC Facts, log a warning, but only if
415	 * checking this after a Diag Reset and not during attach.
416	 */
417	saved_mode = sc->ir_firmware;
418	if (sc->facts->IOCCapabilities &
419	    MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID)
420		sc->ir_firmware = 1;
421	if (!attaching) {
422		if (sc->ir_firmware != saved_mode) {
423			mps_dprint(sc, MPS_FAULT, "%s new IR/IT mode in IOC "
424			    "Facts does not match previous mode\n", __func__);
425		}
426	}
427
428	/* Only deallocate and reallocate if relevant IOC Facts have changed */
429	reallocating = FALSE;
430	sc->mps_flags &= ~MPS_FLAGS_REALLOCATED;
431
432	if ((!attaching) &&
433	    ((saved_facts.MsgVersion != sc->facts->MsgVersion) ||
434	    (saved_facts.HeaderVersion != sc->facts->HeaderVersion) ||
435	    (saved_facts.MaxChainDepth != sc->facts->MaxChainDepth) ||
436	    (saved_facts.RequestCredit != sc->facts->RequestCredit) ||
437	    (saved_facts.ProductID != sc->facts->ProductID) ||
438	    (saved_facts.IOCCapabilities != sc->facts->IOCCapabilities) ||
439	    (saved_facts.IOCRequestFrameSize !=
440	    sc->facts->IOCRequestFrameSize) ||
441	    (saved_facts.MaxTargets != sc->facts->MaxTargets) ||
442	    (saved_facts.MaxSasExpanders != sc->facts->MaxSasExpanders) ||
443	    (saved_facts.MaxEnclosures != sc->facts->MaxEnclosures) ||
444	    (saved_facts.HighPriorityCredit != sc->facts->HighPriorityCredit) ||
445	    (saved_facts.MaxReplyDescriptorPostQueueDepth !=
446	    sc->facts->MaxReplyDescriptorPostQueueDepth) ||
447	    (saved_facts.ReplyFrameSize != sc->facts->ReplyFrameSize) ||
448	    (saved_facts.MaxVolumes != sc->facts->MaxVolumes) ||
449	    (saved_facts.MaxPersistentEntries !=
450	    sc->facts->MaxPersistentEntries))) {
451		reallocating = TRUE;
452
453		/* Record that we reallocated everything */
454		sc->mps_flags |= MPS_FLAGS_REALLOCATED;
455	}
456
457	/*
458	 * Some things should be done if attaching or re-allocating after a Diag
459	 * Reset, but are not needed after a Diag Reset if the FW has not
460	 * changed.
461	 */
462	if (attaching || reallocating) {
463		/*
464		 * Check if controller supports FW diag buffers and set flag to
465		 * enable each type.
466		 */
467		if (sc->facts->IOCCapabilities &
468		    MPI2_IOCFACTS_CAPABILITY_DIAG_TRACE_BUFFER)
469			sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_TRACE].
470			    enabled = TRUE;
471		if (sc->facts->IOCCapabilities &
472		    MPI2_IOCFACTS_CAPABILITY_SNAPSHOT_BUFFER)
473			sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_SNAPSHOT].
474			    enabled = TRUE;
475		if (sc->facts->IOCCapabilities &
476		    MPI2_IOCFACTS_CAPABILITY_EXTENDED_BUFFER)
477			sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_EXTENDED].
478			    enabled = TRUE;
479
480		/*
481		 * Set flag if EEDP is supported and if TLR is supported.
482		 */
483		if (sc->facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP)
484			sc->eedp_enabled = TRUE;
485		if (sc->facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR)
486			sc->control_TLR = TRUE;
487
488		/*
489		 * Size the queues. Since the reply queues always need one free
490		 * entry, we'll just deduct one reply message here.
491		 */
492		sc->num_reqs = MIN(MPS_REQ_FRAMES, sc->facts->RequestCredit);
493		sc->num_replies = MIN(MPS_REPLY_FRAMES + MPS_EVT_REPLY_FRAMES,
494		    sc->facts->MaxReplyDescriptorPostQueueDepth) - 1;
495
496		/*
497		 * Initialize all Tail Queues
498		 */
499		TAILQ_INIT(&sc->req_list);
500		TAILQ_INIT(&sc->high_priority_req_list);
501		TAILQ_INIT(&sc->chain_list);
502		TAILQ_INIT(&sc->tm_list);
503	}
504
505	/*
506	 * If doing a Diag Reset and the FW is significantly different
507	 * (reallocating will be set above in IOC Facts comparison), then all
508	 * buffers based on the IOC Facts will need to be freed before they are
509	 * reallocated.
510	 */
511	if (reallocating) {
512		mps_iocfacts_free(sc);
513		mpssas_realloc_targets(sc, saved_facts.MaxTargets +
514		    saved_facts.MaxVolumes);
515	}
516
517	/*
518	 * Any deallocation has been completed.  Now start reallocating
519	 * if needed.  Will only need to reallocate if attaching or if the new
520	 * IOC Facts are different from the previous IOC Facts after a Diag
521	 * Reset. Targets have already been allocated above if needed.
522	 */
523	if (attaching || reallocating) {
524		if (((error = mps_alloc_queues(sc)) != 0) ||
525		    ((error = mps_alloc_replies(sc)) != 0) ||
526		    ((error = mps_alloc_requests(sc)) != 0)) {
527			if (attaching ) {
528				mps_dprint(sc, MPS_FAULT, "%s failed to alloc "
529				    "queues with error %d\n", __func__, error);
530				mps_free(sc);
531				return (error);
532			} else {
533				panic("%s failed to alloc queues with error "
534				    "%d\n", __func__, error);
535			}
536		}
537	}
538
539	/* Always initialize the queues */
540	bzero(sc->free_queue, sc->fqdepth * 4);
541	mps_init_queues(sc);
542
543	/*
544	 * Always get the chip out of the reset state, but only panic if not
545	 * attaching.  If attaching and there is an error, that is handled by
546	 * the OS.
547	 */
548	error = mps_transition_operational(sc);
549	if (error != 0) {
550		if (attaching) {
551			mps_printf(sc, "%s failed to transition to operational "
552			    "with error %d\n", __func__, error);
553			mps_free(sc);
554			return (error);
555		} else {
556			panic("%s failed to transition to operational with "
557			    "error %d\n", __func__, error);
558		}
559	}
560
561	/*
562	 * Finish the queue initialization.
563	 * These are set here instead of in mps_init_queues() because the
564	 * IOC resets these values during the state transition in
565	 * mps_transition_operational().  The free index is set to 1
566	 * because the corresponding index in the IOC is set to 0, and the
567	 * IOC treats the queues as full if both are set to the same value.
568	 * Hence the reason that the queue can't hold all of the possible
569	 * replies.
570	 */
571	sc->replypostindex = 0;
572	mps_regwrite(sc, MPI2_REPLY_FREE_HOST_INDEX_OFFSET, sc->replyfreeindex);
573	mps_regwrite(sc, MPI2_REPLY_POST_HOST_INDEX_OFFSET, 0);
574
575	/*
576	 * Attach the subsystems so they can prepare their event masks.
577	 */
578	/* XXX Should be dynamic so that IM/IR and user modules can attach */
579	if (attaching) {
580		if (((error = mps_attach_log(sc)) != 0) ||
581		    ((error = mps_attach_sas(sc)) != 0) ||
582		    ((error = mps_attach_user(sc)) != 0)) {
583			mps_printf(sc, "%s failed to attach all subsystems: "
584			    "error %d\n", __func__, error);
585			mps_free(sc);
586			return (error);
587		}
588
589		if ((error = mps_pci_setup_interrupts(sc)) != 0) {
590			mps_printf(sc, "%s failed to setup interrupts\n",
591			    __func__);
592			mps_free(sc);
593			return (error);
594		}
595	}
596
597	/*
598	 * Set flag if this is a WD controller.  This shouldn't ever change, but
599	 * reset it after a Diag Reset, just in case.
600	 */
601	sc->WD_available = FALSE;
602	if (pci_get_device(sc->mps_dev) == MPI2_MFGPAGE_DEVID_SSS6200)
603		sc->WD_available = TRUE;
604
605	return (error);
606}
607
608/*
609 * This is called if memory is being free (during detach for example) and when
610 * buffers need to be reallocated due to a Diag Reset.
611 */
612static void
613mps_iocfacts_free(struct mps_softc *sc)
614{
615	struct mps_command *cm;
616	int i;
617
618	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
619
620	if (sc->free_busaddr != 0)
621		bus_dmamap_unload(sc->queues_dmat, sc->queues_map);
622	if (sc->free_queue != NULL)
623		bus_dmamem_free(sc->queues_dmat, sc->free_queue,
624		    sc->queues_map);
625	if (sc->queues_dmat != NULL)
626		bus_dma_tag_destroy(sc->queues_dmat);
627
628	if (sc->chain_busaddr != 0)
629		bus_dmamap_unload(sc->chain_dmat, sc->chain_map);
630	if (sc->chain_frames != NULL)
631		bus_dmamem_free(sc->chain_dmat, sc->chain_frames,
632		    sc->chain_map);
633	if (sc->chain_dmat != NULL)
634		bus_dma_tag_destroy(sc->chain_dmat);
635
636	if (sc->sense_busaddr != 0)
637		bus_dmamap_unload(sc->sense_dmat, sc->sense_map);
638	if (sc->sense_frames != NULL)
639		bus_dmamem_free(sc->sense_dmat, sc->sense_frames,
640		    sc->sense_map);
641	if (sc->sense_dmat != NULL)
642		bus_dma_tag_destroy(sc->sense_dmat);
643
644	if (sc->reply_busaddr != 0)
645		bus_dmamap_unload(sc->reply_dmat, sc->reply_map);
646	if (sc->reply_frames != NULL)
647		bus_dmamem_free(sc->reply_dmat, sc->reply_frames,
648		    sc->reply_map);
649	if (sc->reply_dmat != NULL)
650		bus_dma_tag_destroy(sc->reply_dmat);
651
652	if (sc->req_busaddr != 0)
653		bus_dmamap_unload(sc->req_dmat, sc->req_map);
654	if (sc->req_frames != NULL)
655		bus_dmamem_free(sc->req_dmat, sc->req_frames, sc->req_map);
656	if (sc->req_dmat != NULL)
657		bus_dma_tag_destroy(sc->req_dmat);
658
659	if (sc->chains != NULL)
660		free(sc->chains, M_MPT2);
661	if (sc->commands != NULL) {
662		for (i = 1; i < sc->num_reqs; i++) {
663			cm = &sc->commands[i];
664			bus_dmamap_destroy(sc->buffer_dmat, cm->cm_dmamap);
665		}
666		free(sc->commands, M_MPT2);
667	}
668	if (sc->buffer_dmat != NULL)
669		bus_dma_tag_destroy(sc->buffer_dmat);
670}
671
672/*
673 * The terms diag reset and hard reset are used interchangeably in the MPI
674 * docs to mean resetting the controller chip.  In this code diag reset
675 * cleans everything up, and the hard reset function just sends the reset
676 * sequence to the chip.  This should probably be refactored so that every
677 * subsystem gets a reset notification of some sort, and can clean up
678 * appropriately.
679 */
680int
681mps_reinit(struct mps_softc *sc)
682{
683	int error;
684	struct mpssas_softc *sassc;
685
686	sassc = sc->sassc;
687
688	MPS_FUNCTRACE(sc);
689
690	mtx_assert(&sc->mps_mtx, MA_OWNED);
691
692	if (sc->mps_flags & MPS_FLAGS_DIAGRESET) {
693		mps_dprint(sc, MPS_INIT, "%s reset already in progress\n",
694			   __func__);
695		return 0;
696	}
697
698	mps_dprint(sc, MPS_INFO, "Reinitializing controller,\n");
699	/* make sure the completion callbacks can recognize they're getting
700	 * a NULL cm_reply due to a reset.
701	 */
702	sc->mps_flags |= MPS_FLAGS_DIAGRESET;
703
704	/*
705	 * Mask interrupts here.
706	 */
707	mps_dprint(sc, MPS_INIT, "%s mask interrupts\n", __func__);
708	mps_mask_intr(sc);
709
710	error = mps_diag_reset(sc, CAN_SLEEP);
711	if (error != 0) {
712		/* XXXSL No need to panic here */
713		panic("%s hard reset failed with error %d\n",
714		    __func__, error);
715	}
716
717	/* Restore the PCI state, including the MSI-X registers */
718	mps_pci_restore(sc);
719
720	/* Give the I/O subsystem special priority to get itself prepared */
721	mpssas_handle_reinit(sc);
722
723	/*
724	 * Get IOC Facts and allocate all structures based on this information.
725	 * The attach function will also call mps_iocfacts_allocate at startup.
726	 * If relevant values have changed in IOC Facts, this function will free
727	 * all of the memory based on IOC Facts and reallocate that memory.
728	 */
729	if ((error = mps_iocfacts_allocate(sc, FALSE)) != 0) {
730		panic("%s IOC Facts based allocation failed with error %d\n",
731		    __func__, error);
732	}
733
734	/*
735	 * Mapping structures will be re-allocated after getting IOC Page8, so
736	 * free these structures here.
737	 */
738	mps_mapping_exit(sc);
739
740	/*
741	 * The static page function currently read is IOC Page8.  Others can be
742	 * added in future.  It's possible that the values in IOC Page8 have
743	 * changed after a Diag Reset due to user modification, so always read
744	 * these.  Interrupts are masked, so unmask them before getting config
745	 * pages.
746	 */
747	mps_unmask_intr(sc);
748	sc->mps_flags &= ~MPS_FLAGS_DIAGRESET;
749	mps_base_static_config_pages(sc);
750
751	/*
752	 * Some mapping info is based in IOC Page8 data, so re-initialize the
753	 * mapping tables.
754	 */
755	mps_mapping_initialize(sc);
756
757	/*
758	 * Restart will reload the event masks clobbered by the reset, and
759	 * then enable the port.
760	 */
761	mps_reregister_events(sc);
762
763	/* the end of discovery will release the simq, so we're done. */
764	mps_dprint(sc, MPS_INFO, "%s finished sc %p post %u free %u\n",
765	    __func__, sc, sc->replypostindex, sc->replyfreeindex);
766
767	mpssas_release_simq_reinit(sassc);
768
769	return 0;
770}
771
772/* Wait for the chip to ACK a word that we've put into its FIFO
773 * Wait for <timeout> seconds. In single loop wait for busy loop
774 * for 500 microseconds.
775 * Total is [ 0.5 * (2000 * <timeout>) ] in miliseconds.
776 * */
777static int
778mps_wait_db_ack(struct mps_softc *sc, int timeout, int sleep_flag)
779{
780
781	u32 cntdn, count;
782	u32 int_status;
783	u32 doorbell;
784
785	count = 0;
786	cntdn = (sleep_flag == CAN_SLEEP) ? 1000*timeout : 2000*timeout;
787	do {
788		int_status = mps_regread(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET);
789		if (!(int_status & MPI2_HIS_SYS2IOC_DB_STATUS)) {
790			mps_dprint(sc, MPS_INIT,
791			"%s: successfull count(%d), timeout(%d)\n",
792			__func__, count, timeout);
793		return 0;
794		} else if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
795			doorbell = mps_regread(sc, MPI2_DOORBELL_OFFSET);
796			if ((doorbell & MPI2_IOC_STATE_MASK) ==
797				MPI2_IOC_STATE_FAULT) {
798				mps_dprint(sc, MPS_FAULT,
799					"fault_state(0x%04x)!\n", doorbell);
800				return (EFAULT);
801			}
802		} else if (int_status == 0xFFFFFFFF)
803			goto out;
804
805		/* If it can sleep, sleep for 1 milisecond, else busy loop for
806		* 0.5 milisecond */
807		if (mtx_owned(&sc->mps_mtx) && sleep_flag == CAN_SLEEP)
808			msleep(&sc->msleep_fake_chan, &sc->mps_mtx, 0,
809			"mpsdba", hz/1000);
810		else if (sleep_flag == CAN_SLEEP)
811			pause("mpsdba", hz/1000);
812		else
813			DELAY(500);
814		count++;
815	} while (--cntdn);
816
817	out:
818	mps_dprint(sc, MPS_FAULT, "%s: failed due to timeout count(%d), "
819		"int_status(%x)!\n", __func__, count, int_status);
820	return (ETIMEDOUT);
821
822}
823
824/* Wait for the chip to signal that the next word in its FIFO can be fetched */
825static int
826mps_wait_db_int(struct mps_softc *sc)
827{
828	int retry;
829
830	for (retry = 0; retry < MPS_DB_MAX_WAIT; retry++) {
831		if ((mps_regread(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET) &
832		    MPI2_HIS_IOC2SYS_DB_STATUS) != 0)
833			return (0);
834		DELAY(2000);
835	}
836	return (ETIMEDOUT);
837}
838
839/* Step through the synchronous command state machine, i.e. "Doorbell mode" */
840static int
841mps_request_sync(struct mps_softc *sc, void *req, MPI2_DEFAULT_REPLY *reply,
842    int req_sz, int reply_sz, int timeout)
843{
844	uint32_t *data32;
845	uint16_t *data16;
846	int i, count, ioc_sz, residual;
847	int sleep_flags = CAN_SLEEP;
848
849	if (curthread->td_no_sleeping != 0)
850		sleep_flags = NO_SLEEP;
851
852	/* Step 1 */
853	mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
854
855	/* Step 2 */
856	if (mps_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_USED)
857		return (EBUSY);
858
859	/* Step 3
860	 * Announce that a message is coming through the doorbell.  Messages
861	 * are pushed at 32bit words, so round up if needed.
862	 */
863	count = (req_sz + 3) / 4;
864	mps_regwrite(sc, MPI2_DOORBELL_OFFSET,
865	    (MPI2_FUNCTION_HANDSHAKE << MPI2_DOORBELL_FUNCTION_SHIFT) |
866	    (count << MPI2_DOORBELL_ADD_DWORDS_SHIFT));
867
868	/* Step 4 */
869	if (mps_wait_db_int(sc) ||
870	    (mps_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_USED) == 0) {
871		mps_dprint(sc, MPS_FAULT, "Doorbell failed to activate\n");
872		return (ENXIO);
873	}
874	mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
875	if (mps_wait_db_ack(sc, 5, sleep_flags) != 0) {
876		mps_dprint(sc, MPS_FAULT, "Doorbell handshake failed\n");
877		return (ENXIO);
878	}
879
880	/* Step 5 */
881	/* Clock out the message data synchronously in 32-bit dwords*/
882	data32 = (uint32_t *)req;
883	for (i = 0; i < count; i++) {
884		mps_regwrite(sc, MPI2_DOORBELL_OFFSET, htole32(data32[i]));
885		if (mps_wait_db_ack(sc, 5, sleep_flags) != 0) {
886			mps_dprint(sc, MPS_FAULT,
887			    "Timeout while writing doorbell\n");
888			return (ENXIO);
889		}
890	}
891
892	/* Step 6 */
893	/* Clock in the reply in 16-bit words.  The total length of the
894	 * message is always in the 4th byte, so clock out the first 2 words
895	 * manually, then loop the rest.
896	 */
897	data16 = (uint16_t *)reply;
898	if (mps_wait_db_int(sc) != 0) {
899		mps_dprint(sc, MPS_FAULT, "Timeout reading doorbell 0\n");
900		return (ENXIO);
901	}
902	data16[0] =
903	    mps_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_DATA_MASK;
904	mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
905	if (mps_wait_db_int(sc) != 0) {
906		mps_dprint(sc, MPS_FAULT, "Timeout reading doorbell 1\n");
907		return (ENXIO);
908	}
909	data16[1] =
910	    mps_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_DATA_MASK;
911	mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
912
913	/* Number of 32bit words in the message */
914	ioc_sz = reply->MsgLength;
915
916	/*
917	 * Figure out how many 16bit words to clock in without overrunning.
918	 * The precision loss with dividing reply_sz can safely be
919	 * ignored because the messages can only be multiples of 32bits.
920	 */
921	residual = 0;
922	count = MIN((reply_sz / 4), ioc_sz) * 2;
923	if (count < ioc_sz * 2) {
924		residual = ioc_sz * 2 - count;
925		mps_dprint(sc, MPS_ERROR, "Driver error, throwing away %d "
926		    "residual message words\n", residual);
927	}
928
929	for (i = 2; i < count; i++) {
930		if (mps_wait_db_int(sc) != 0) {
931			mps_dprint(sc, MPS_FAULT,
932			    "Timeout reading doorbell %d\n", i);
933			return (ENXIO);
934		}
935		data16[i] = mps_regread(sc, MPI2_DOORBELL_OFFSET) &
936		    MPI2_DOORBELL_DATA_MASK;
937		mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
938	}
939
940	/*
941	 * Pull out residual words that won't fit into the provided buffer.
942	 * This keeps the chip from hanging due to a driver programming
943	 * error.
944	 */
945	while (residual--) {
946		if (mps_wait_db_int(sc) != 0) {
947			mps_dprint(sc, MPS_FAULT,
948			    "Timeout reading doorbell\n");
949			return (ENXIO);
950		}
951		(void)mps_regread(sc, MPI2_DOORBELL_OFFSET);
952		mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
953	}
954
955	/* Step 7 */
956	if (mps_wait_db_int(sc) != 0) {
957		mps_dprint(sc, MPS_FAULT, "Timeout waiting to exit doorbell\n");
958		return (ENXIO);
959	}
960	if (mps_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_USED)
961		mps_dprint(sc, MPS_FAULT, "Warning, doorbell still active\n");
962	mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
963
964	return (0);
965}
966
967static void
968mps_enqueue_request(struct mps_softc *sc, struct mps_command *cm)
969{
970	reply_descriptor rd;
971	MPS_FUNCTRACE(sc);
972	mps_dprint(sc, MPS_TRACE, "SMID %u cm %p ccb %p\n",
973	    cm->cm_desc.Default.SMID, cm, cm->cm_ccb);
974
975	if (sc->mps_flags & MPS_FLAGS_ATTACH_DONE && !(sc->mps_flags & MPS_FLAGS_SHUTDOWN))
976		mtx_assert(&sc->mps_mtx, MA_OWNED);
977
978	if (++sc->io_cmds_active > sc->io_cmds_highwater)
979		sc->io_cmds_highwater++;
980	rd.u.low = cm->cm_desc.Words.Low;
981	rd.u.high = cm->cm_desc.Words.High;
982	rd.word = htole64(rd.word);
983	/* TODO-We may need to make below regwrite atomic */
984	mps_regwrite(sc, MPI2_REQUEST_DESCRIPTOR_POST_LOW_OFFSET,
985	    rd.u.low);
986	mps_regwrite(sc, MPI2_REQUEST_DESCRIPTOR_POST_HIGH_OFFSET,
987	    rd.u.high);
988}
989
990/*
991 * Just the FACTS, ma'am.
992 */
993static int
994mps_get_iocfacts(struct mps_softc *sc, MPI2_IOC_FACTS_REPLY *facts)
995{
996	MPI2_DEFAULT_REPLY *reply;
997	MPI2_IOC_FACTS_REQUEST request;
998	int error, req_sz, reply_sz;
999
1000	MPS_FUNCTRACE(sc);
1001
1002	req_sz = sizeof(MPI2_IOC_FACTS_REQUEST);
1003	reply_sz = sizeof(MPI2_IOC_FACTS_REPLY);
1004	reply = (MPI2_DEFAULT_REPLY *)facts;
1005
1006	bzero(&request, req_sz);
1007	request.Function = MPI2_FUNCTION_IOC_FACTS;
1008	error = mps_request_sync(sc, &request, reply, req_sz, reply_sz, 5);
1009
1010	return (error);
1011}
1012
1013static int
1014mps_send_iocinit(struct mps_softc *sc)
1015{
1016	MPI2_IOC_INIT_REQUEST	init;
1017	MPI2_DEFAULT_REPLY	reply;
1018	int req_sz, reply_sz, error;
1019	struct timeval now;
1020	uint64_t time_in_msec;
1021
1022	MPS_FUNCTRACE(sc);
1023
1024	req_sz = sizeof(MPI2_IOC_INIT_REQUEST);
1025	reply_sz = sizeof(MPI2_IOC_INIT_REPLY);
1026	bzero(&init, req_sz);
1027	bzero(&reply, reply_sz);
1028
1029	/*
1030	 * Fill in the init block.  Note that most addresses are
1031	 * deliberately in the lower 32bits of memory.  This is a micro-
1032	 * optimzation for PCI/PCIX, though it's not clear if it helps PCIe.
1033	 */
1034	init.Function = MPI2_FUNCTION_IOC_INIT;
1035	init.WhoInit = MPI2_WHOINIT_HOST_DRIVER;
1036	init.MsgVersion = htole16(MPI2_VERSION);
1037	init.HeaderVersion = htole16(MPI2_HEADER_VERSION);
1038	init.SystemRequestFrameSize = htole16(sc->facts->IOCRequestFrameSize);
1039	init.ReplyDescriptorPostQueueDepth = htole16(sc->pqdepth);
1040	init.ReplyFreeQueueDepth = htole16(sc->fqdepth);
1041	init.SenseBufferAddressHigh = 0;
1042	init.SystemReplyAddressHigh = 0;
1043	init.SystemRequestFrameBaseAddress.High = 0;
1044	init.SystemRequestFrameBaseAddress.Low = htole32((uint32_t)sc->req_busaddr);
1045	init.ReplyDescriptorPostQueueAddress.High = 0;
1046	init.ReplyDescriptorPostQueueAddress.Low = htole32((uint32_t)sc->post_busaddr);
1047	init.ReplyFreeQueueAddress.High = 0;
1048	init.ReplyFreeQueueAddress.Low = htole32((uint32_t)sc->free_busaddr);
1049	getmicrotime(&now);
1050	time_in_msec = (now.tv_sec * 1000 + now.tv_usec/1000);
1051	init.TimeStamp.High = htole32((time_in_msec >> 32) & 0xFFFFFFFF);
1052	init.TimeStamp.Low = htole32(time_in_msec & 0xFFFFFFFF);
1053
1054	error = mps_request_sync(sc, &init, &reply, req_sz, reply_sz, 5);
1055	if ((reply.IOCStatus & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS)
1056		error = ENXIO;
1057
1058	mps_dprint(sc, MPS_INIT, "IOCInit status= 0x%x\n", reply.IOCStatus);
1059	return (error);
1060}
1061
1062void
1063mps_memaddr_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
1064{
1065	bus_addr_t *addr;
1066
1067	addr = arg;
1068	*addr = segs[0].ds_addr;
1069}
1070
1071static int
1072mps_alloc_queues(struct mps_softc *sc)
1073{
1074	bus_addr_t queues_busaddr;
1075	uint8_t *queues;
1076	int qsize, fqsize, pqsize;
1077
1078	/*
1079	 * The reply free queue contains 4 byte entries in multiples of 16 and
1080	 * aligned on a 16 byte boundary. There must always be an unused entry.
1081	 * This queue supplies fresh reply frames for the firmware to use.
1082	 *
1083	 * The reply descriptor post queue contains 8 byte entries in
1084	 * multiples of 16 and aligned on a 16 byte boundary.  This queue
1085	 * contains filled-in reply frames sent from the firmware to the host.
1086	 *
1087	 * These two queues are allocated together for simplicity.
1088	 */
1089	sc->fqdepth = roundup2((sc->num_replies + 1), 16);
1090	sc->pqdepth = roundup2((sc->num_replies + 1), 16);
1091	fqsize= sc->fqdepth * 4;
1092	pqsize = sc->pqdepth * 8;
1093	qsize = fqsize + pqsize;
1094
1095        if (bus_dma_tag_create( sc->mps_parent_dmat,    /* parent */
1096				16, 0,			/* algnmnt, boundary */
1097				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
1098				BUS_SPACE_MAXADDR,	/* highaddr */
1099				NULL, NULL,		/* filter, filterarg */
1100                                qsize,			/* maxsize */
1101                                1,			/* nsegments */
1102                                qsize,			/* maxsegsize */
1103                                0,			/* flags */
1104                                NULL, NULL,		/* lockfunc, lockarg */
1105                                &sc->queues_dmat)) {
1106		device_printf(sc->mps_dev, "Cannot allocate queues DMA tag\n");
1107		return (ENOMEM);
1108        }
1109        if (bus_dmamem_alloc(sc->queues_dmat, (void **)&queues, BUS_DMA_NOWAIT,
1110	    &sc->queues_map)) {
1111		device_printf(sc->mps_dev, "Cannot allocate queues memory\n");
1112		return (ENOMEM);
1113        }
1114        bzero(queues, qsize);
1115        bus_dmamap_load(sc->queues_dmat, sc->queues_map, queues, qsize,
1116	    mps_memaddr_cb, &queues_busaddr, 0);
1117
1118	sc->free_queue = (uint32_t *)queues;
1119	sc->free_busaddr = queues_busaddr;
1120	sc->post_queue = (MPI2_REPLY_DESCRIPTORS_UNION *)(queues + fqsize);
1121	sc->post_busaddr = queues_busaddr + fqsize;
1122
1123	return (0);
1124}
1125
1126static int
1127mps_alloc_replies(struct mps_softc *sc)
1128{
1129	int rsize, num_replies;
1130
1131	/*
1132	 * sc->num_replies should be one less than sc->fqdepth.  We need to
1133	 * allocate space for sc->fqdepth replies, but only sc->num_replies
1134	 * replies can be used at once.
1135	 */
1136	num_replies = max(sc->fqdepth, sc->num_replies);
1137
1138	rsize = sc->facts->ReplyFrameSize * num_replies * 4;
1139        if (bus_dma_tag_create( sc->mps_parent_dmat,    /* parent */
1140				4, 0,			/* algnmnt, boundary */
1141				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
1142				BUS_SPACE_MAXADDR,	/* highaddr */
1143				NULL, NULL,		/* filter, filterarg */
1144                                rsize,			/* maxsize */
1145                                1,			/* nsegments */
1146                                rsize,			/* maxsegsize */
1147                                0,			/* flags */
1148                                NULL, NULL,		/* lockfunc, lockarg */
1149                                &sc->reply_dmat)) {
1150		device_printf(sc->mps_dev, "Cannot allocate replies DMA tag\n");
1151		return (ENOMEM);
1152        }
1153        if (bus_dmamem_alloc(sc->reply_dmat, (void **)&sc->reply_frames,
1154	    BUS_DMA_NOWAIT, &sc->reply_map)) {
1155		device_printf(sc->mps_dev, "Cannot allocate replies memory\n");
1156		return (ENOMEM);
1157        }
1158        bzero(sc->reply_frames, rsize);
1159        bus_dmamap_load(sc->reply_dmat, sc->reply_map, sc->reply_frames, rsize,
1160	    mps_memaddr_cb, &sc->reply_busaddr, 0);
1161
1162	return (0);
1163}
1164
1165static int
1166mps_alloc_requests(struct mps_softc *sc)
1167{
1168	struct mps_command *cm;
1169	struct mps_chain *chain;
1170	int i, rsize, nsegs;
1171
1172	rsize = sc->facts->IOCRequestFrameSize * sc->num_reqs * 4;
1173        if (bus_dma_tag_create( sc->mps_parent_dmat,    /* parent */
1174				16, 0,			/* algnmnt, boundary */
1175				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
1176				BUS_SPACE_MAXADDR,	/* highaddr */
1177				NULL, NULL,		/* filter, filterarg */
1178                                rsize,			/* maxsize */
1179                                1,			/* nsegments */
1180                                rsize,			/* maxsegsize */
1181                                0,			/* flags */
1182                                NULL, NULL,		/* lockfunc, lockarg */
1183                                &sc->req_dmat)) {
1184		device_printf(sc->mps_dev, "Cannot allocate request DMA tag\n");
1185		return (ENOMEM);
1186        }
1187        if (bus_dmamem_alloc(sc->req_dmat, (void **)&sc->req_frames,
1188	    BUS_DMA_NOWAIT, &sc->req_map)) {
1189		device_printf(sc->mps_dev, "Cannot allocate request memory\n");
1190		return (ENOMEM);
1191        }
1192        bzero(sc->req_frames, rsize);
1193        bus_dmamap_load(sc->req_dmat, sc->req_map, sc->req_frames, rsize,
1194	    mps_memaddr_cb, &sc->req_busaddr, 0);
1195
1196	rsize = sc->facts->IOCRequestFrameSize * sc->max_chains * 4;
1197        if (bus_dma_tag_create( sc->mps_parent_dmat,    /* parent */
1198				16, 0,			/* algnmnt, boundary */
1199				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
1200				BUS_SPACE_MAXADDR,	/* highaddr */
1201				NULL, NULL,		/* filter, filterarg */
1202                                rsize,			/* maxsize */
1203                                1,			/* nsegments */
1204                                rsize,			/* maxsegsize */
1205                                0,			/* flags */
1206                                NULL, NULL,		/* lockfunc, lockarg */
1207                                &sc->chain_dmat)) {
1208		device_printf(sc->mps_dev, "Cannot allocate chain DMA tag\n");
1209		return (ENOMEM);
1210        }
1211        if (bus_dmamem_alloc(sc->chain_dmat, (void **)&sc->chain_frames,
1212	    BUS_DMA_NOWAIT, &sc->chain_map)) {
1213		device_printf(sc->mps_dev, "Cannot allocate chain memory\n");
1214		return (ENOMEM);
1215        }
1216        bzero(sc->chain_frames, rsize);
1217        bus_dmamap_load(sc->chain_dmat, sc->chain_map, sc->chain_frames, rsize,
1218	    mps_memaddr_cb, &sc->chain_busaddr, 0);
1219
1220	rsize = MPS_SENSE_LEN * sc->num_reqs;
1221        if (bus_dma_tag_create( sc->mps_parent_dmat,    /* parent */
1222				1, 0,			/* algnmnt, boundary */
1223				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
1224				BUS_SPACE_MAXADDR,	/* highaddr */
1225				NULL, NULL,		/* filter, filterarg */
1226                                rsize,			/* maxsize */
1227                                1,			/* nsegments */
1228                                rsize,			/* maxsegsize */
1229                                0,			/* flags */
1230                                NULL, NULL,		/* lockfunc, lockarg */
1231                                &sc->sense_dmat)) {
1232		device_printf(sc->mps_dev, "Cannot allocate sense DMA tag\n");
1233		return (ENOMEM);
1234        }
1235        if (bus_dmamem_alloc(sc->sense_dmat, (void **)&sc->sense_frames,
1236	    BUS_DMA_NOWAIT, &sc->sense_map)) {
1237		device_printf(sc->mps_dev, "Cannot allocate sense memory\n");
1238		return (ENOMEM);
1239        }
1240        bzero(sc->sense_frames, rsize);
1241        bus_dmamap_load(sc->sense_dmat, sc->sense_map, sc->sense_frames, rsize,
1242	    mps_memaddr_cb, &sc->sense_busaddr, 0);
1243
1244	sc->chains = malloc(sizeof(struct mps_chain) * sc->max_chains, M_MPT2,
1245	    M_WAITOK | M_ZERO);
1246	if(!sc->chains) {
1247		device_printf(sc->mps_dev,
1248		"Cannot allocate chains memory %s %d\n",
1249		 __func__, __LINE__);
1250		return (ENOMEM);
1251	}
1252	for (i = 0; i < sc->max_chains; i++) {
1253		chain = &sc->chains[i];
1254		chain->chain = (MPI2_SGE_IO_UNION *)(sc->chain_frames +
1255		    i * sc->facts->IOCRequestFrameSize * 4);
1256		chain->chain_busaddr = sc->chain_busaddr +
1257		    i * sc->facts->IOCRequestFrameSize * 4;
1258		mps_free_chain(sc, chain);
1259		sc->chain_free_lowwater++;
1260	}
1261
1262	/* XXX Need to pick a more precise value */
1263	nsegs = (MAXPHYS / PAGE_SIZE) + 1;
1264        if (bus_dma_tag_create( sc->mps_parent_dmat,    /* parent */
1265				1, 0,			/* algnmnt, boundary */
1266				BUS_SPACE_MAXADDR,	/* lowaddr */
1267				BUS_SPACE_MAXADDR,	/* highaddr */
1268				NULL, NULL,		/* filter, filterarg */
1269                                BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
1270                                nsegs,			/* nsegments */
1271                                BUS_SPACE_MAXSIZE_24BIT,/* maxsegsize */
1272                                BUS_DMA_ALLOCNOW,	/* flags */
1273                                busdma_lock_mutex,	/* lockfunc */
1274				&sc->mps_mtx,		/* lockarg */
1275                                &sc->buffer_dmat)) {
1276		device_printf(sc->mps_dev, "Cannot allocate buffer DMA tag\n");
1277		return (ENOMEM);
1278        }
1279
1280	/*
1281	 * SMID 0 cannot be used as a free command per the firmware spec.
1282	 * Just drop that command instead of risking accounting bugs.
1283	 */
1284	sc->commands = malloc(sizeof(struct mps_command) * sc->num_reqs,
1285	    M_MPT2, M_WAITOK | M_ZERO);
1286	if(!sc->commands) {
1287		device_printf(sc->mps_dev, "Cannot allocate memory %s %d\n",
1288		 __func__, __LINE__);
1289		return (ENOMEM);
1290	}
1291	for (i = 1; i < sc->num_reqs; i++) {
1292		cm = &sc->commands[i];
1293		cm->cm_req = sc->req_frames +
1294		    i * sc->facts->IOCRequestFrameSize * 4;
1295		cm->cm_req_busaddr = sc->req_busaddr +
1296		    i * sc->facts->IOCRequestFrameSize * 4;
1297		cm->cm_sense = &sc->sense_frames[i];
1298		cm->cm_sense_busaddr = sc->sense_busaddr + i * MPS_SENSE_LEN;
1299		cm->cm_desc.Default.SMID = i;
1300		cm->cm_sc = sc;
1301		TAILQ_INIT(&cm->cm_chain_list);
1302		callout_init_mtx(&cm->cm_callout, &sc->mps_mtx, 0);
1303
1304		/* XXX Is a failure here a critical problem? */
1305		if (bus_dmamap_create(sc->buffer_dmat, 0, &cm->cm_dmamap) == 0)
1306			if (i <= sc->facts->HighPriorityCredit)
1307				mps_free_high_priority_command(sc, cm);
1308			else
1309				mps_free_command(sc, cm);
1310		else {
1311			panic("failed to allocate command %d\n", i);
1312			sc->num_reqs = i;
1313			break;
1314		}
1315	}
1316
1317	return (0);
1318}
1319
1320static int
1321mps_init_queues(struct mps_softc *sc)
1322{
1323	int i;
1324
1325	memset((uint8_t *)sc->post_queue, 0xff, sc->pqdepth * 8);
1326
1327	/*
1328	 * According to the spec, we need to use one less reply than we
1329	 * have space for on the queue.  So sc->num_replies (the number we
1330	 * use) should be less than sc->fqdepth (allocated size).
1331	 */
1332	if (sc->num_replies >= sc->fqdepth)
1333		return (EINVAL);
1334
1335	/*
1336	 * Initialize all of the free queue entries.
1337	 */
1338	for (i = 0; i < sc->fqdepth; i++)
1339		sc->free_queue[i] = sc->reply_busaddr + (i * sc->facts->ReplyFrameSize * 4);
1340	sc->replyfreeindex = sc->num_replies;
1341
1342	return (0);
1343}
1344
1345/* Get the driver parameter tunables.  Lowest priority are the driver defaults.
1346 * Next are the global settings, if they exist.  Highest are the per-unit
1347 * settings, if they exist.
1348 */
1349void
1350mps_get_tunables(struct mps_softc *sc)
1351{
1352	char tmpstr[80];
1353
1354	/* XXX default to some debugging for now */
1355	sc->mps_debug = MPS_INFO|MPS_FAULT;
1356	sc->disable_msix = 0;
1357	sc->disable_msi = 0;
1358	sc->max_chains = MPS_CHAIN_FRAMES;
1359	sc->max_io_pages = MPS_MAXIO_PAGES;
1360	sc->enable_ssu = MPS_SSU_ENABLE_SSD_DISABLE_HDD;
1361	sc->spinup_wait_time = DEFAULT_SPINUP_WAIT;
1362	sc->use_phynum = 1;
1363
1364	/*
1365	 * Grab the global variables.
1366	 */
1367	TUNABLE_INT_FETCH("hw.mps.debug_level", &sc->mps_debug);
1368	TUNABLE_INT_FETCH("hw.mps.disable_msix", &sc->disable_msix);
1369	TUNABLE_INT_FETCH("hw.mps.disable_msi", &sc->disable_msi);
1370	TUNABLE_INT_FETCH("hw.mps.max_chains", &sc->max_chains);
1371	TUNABLE_INT_FETCH("hw.mps.max_io_pages", &sc->max_io_pages);
1372	TUNABLE_INT_FETCH("hw.mps.enable_ssu", &sc->enable_ssu);
1373	TUNABLE_INT_FETCH("hw.mps.spinup_wait_time", &sc->spinup_wait_time);
1374	TUNABLE_INT_FETCH("hw.mps.use_phy_num", &sc->use_phynum);
1375
1376	/* Grab the unit-instance variables */
1377	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.debug_level",
1378	    device_get_unit(sc->mps_dev));
1379	TUNABLE_INT_FETCH(tmpstr, &sc->mps_debug);
1380
1381	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.disable_msix",
1382	    device_get_unit(sc->mps_dev));
1383	TUNABLE_INT_FETCH(tmpstr, &sc->disable_msix);
1384
1385	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.disable_msi",
1386	    device_get_unit(sc->mps_dev));
1387	TUNABLE_INT_FETCH(tmpstr, &sc->disable_msi);
1388
1389	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.max_chains",
1390	    device_get_unit(sc->mps_dev));
1391	TUNABLE_INT_FETCH(tmpstr, &sc->max_chains);
1392
1393	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.max_io_pages",
1394	    device_get_unit(sc->mps_dev));
1395	TUNABLE_INT_FETCH(tmpstr, &sc->max_io_pages);
1396
1397	bzero(sc->exclude_ids, sizeof(sc->exclude_ids));
1398	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.exclude_ids",
1399	    device_get_unit(sc->mps_dev));
1400	TUNABLE_STR_FETCH(tmpstr, sc->exclude_ids, sizeof(sc->exclude_ids));
1401
1402	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.enable_ssu",
1403	    device_get_unit(sc->mps_dev));
1404	TUNABLE_INT_FETCH(tmpstr, &sc->enable_ssu);
1405
1406	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.spinup_wait_time",
1407	    device_get_unit(sc->mps_dev));
1408	TUNABLE_INT_FETCH(tmpstr, &sc->spinup_wait_time);
1409
1410	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.use_phy_num",
1411	    device_get_unit(sc->mps_dev));
1412	TUNABLE_INT_FETCH(tmpstr, &sc->use_phynum);
1413}
1414
1415static void
1416mps_setup_sysctl(struct mps_softc *sc)
1417{
1418	struct sysctl_ctx_list	*sysctl_ctx = NULL;
1419	struct sysctl_oid	*sysctl_tree = NULL;
1420	char tmpstr[80], tmpstr2[80];
1421
1422	/*
1423	 * Setup the sysctl variable so the user can change the debug level
1424	 * on the fly.
1425	 */
1426	snprintf(tmpstr, sizeof(tmpstr), "MPS controller %d",
1427	    device_get_unit(sc->mps_dev));
1428	snprintf(tmpstr2, sizeof(tmpstr2), "%d", device_get_unit(sc->mps_dev));
1429
1430	sysctl_ctx = device_get_sysctl_ctx(sc->mps_dev);
1431	if (sysctl_ctx != NULL)
1432		sysctl_tree = device_get_sysctl_tree(sc->mps_dev);
1433
1434	if (sysctl_tree == NULL) {
1435		sysctl_ctx_init(&sc->sysctl_ctx);
1436		sc->sysctl_tree = SYSCTL_ADD_NODE(&sc->sysctl_ctx,
1437		    SYSCTL_STATIC_CHILDREN(_hw_mps), OID_AUTO, tmpstr2,
1438		    CTLFLAG_RD, 0, tmpstr);
1439		if (sc->sysctl_tree == NULL)
1440			return;
1441		sysctl_ctx = &sc->sysctl_ctx;
1442		sysctl_tree = sc->sysctl_tree;
1443	}
1444
1445	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1446	    OID_AUTO, "debug_level", CTLFLAG_RW, &sc->mps_debug, 0,
1447	    "mps debug level");
1448
1449	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1450	    OID_AUTO, "disable_msix", CTLFLAG_RD, &sc->disable_msix, 0,
1451	    "Disable the use of MSI-X interrupts");
1452
1453	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1454	    OID_AUTO, "disable_msi", CTLFLAG_RD, &sc->disable_msi, 0,
1455	    "Disable the use of MSI interrupts");
1456
1457	SYSCTL_ADD_STRING(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1458	    OID_AUTO, "firmware_version", CTLFLAG_RW, sc->fw_version,
1459	    strlen(sc->fw_version), "firmware version");
1460
1461	SYSCTL_ADD_STRING(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1462	    OID_AUTO, "driver_version", CTLFLAG_RW, MPS_DRIVER_VERSION,
1463	    strlen(MPS_DRIVER_VERSION), "driver version");
1464
1465	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1466	    OID_AUTO, "io_cmds_active", CTLFLAG_RD,
1467	    &sc->io_cmds_active, 0, "number of currently active commands");
1468
1469	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1470	    OID_AUTO, "io_cmds_highwater", CTLFLAG_RD,
1471	    &sc->io_cmds_highwater, 0, "maximum active commands seen");
1472
1473	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1474	    OID_AUTO, "chain_free", CTLFLAG_RD,
1475	    &sc->chain_free, 0, "number of free chain elements");
1476
1477	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1478	    OID_AUTO, "chain_free_lowwater", CTLFLAG_RD,
1479	    &sc->chain_free_lowwater, 0,"lowest number of free chain elements");
1480
1481	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1482	    OID_AUTO, "max_chains", CTLFLAG_RD,
1483	    &sc->max_chains, 0,"maximum chain frames that will be allocated");
1484
1485	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1486	    OID_AUTO, "max_io_pages", CTLFLAG_RD,
1487	    &sc->max_io_pages, 0,"maximum pages to allow per I/O (if <1 use "
1488	    "IOCFacts)");
1489
1490	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1491	    OID_AUTO, "enable_ssu", CTLFLAG_RW, &sc->enable_ssu, 0,
1492	    "enable SSU to SATA SSD/HDD at shutdown");
1493
1494	SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1495	    OID_AUTO, "chain_alloc_fail", CTLFLAG_RD,
1496	    &sc->chain_alloc_fail, "chain allocation failures");
1497
1498	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1499	    OID_AUTO, "spinup_wait_time", CTLFLAG_RD,
1500	    &sc->spinup_wait_time, DEFAULT_SPINUP_WAIT, "seconds to wait for "
1501	    "spinup after SATA ID error");
1502
1503	SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1504	    OID_AUTO, "mapping_table_dump", CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
1505	    mps_mapping_dump, "A", "Mapping Table Dump");
1506
1507	SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1508	    OID_AUTO, "encl_table_dump", CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
1509	    mps_mapping_encl_dump, "A", "Enclosure Table Dump");
1510
1511	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1512	    OID_AUTO, "use_phy_num", CTLFLAG_RD, &sc->use_phynum, 0,
1513	    "Use the phy number for enumeration");
1514}
1515
1516int
1517mps_attach(struct mps_softc *sc)
1518{
1519	int error;
1520
1521	MPS_FUNCTRACE(sc);
1522
1523	mtx_init(&sc->mps_mtx, "MPT2SAS lock", NULL, MTX_DEF);
1524	callout_init_mtx(&sc->periodic, &sc->mps_mtx, 0);
1525	callout_init_mtx(&sc->device_check_callout, &sc->mps_mtx, 0);
1526	TAILQ_INIT(&sc->event_list);
1527	timevalclear(&sc->lastfail);
1528
1529	if ((error = mps_transition_ready(sc)) != 0) {
1530		mps_printf(sc, "%s failed to transition ready\n", __func__);
1531		return (error);
1532	}
1533
1534	sc->facts = malloc(sizeof(MPI2_IOC_FACTS_REPLY), M_MPT2,
1535	    M_ZERO|M_NOWAIT);
1536	if(!sc->facts) {
1537		device_printf(sc->mps_dev, "Cannot allocate memory %s %d\n",
1538		 __func__, __LINE__);
1539		return (ENOMEM);
1540	}
1541
1542	/*
1543	 * Get IOC Facts and allocate all structures based on this information.
1544	 * A Diag Reset will also call mps_iocfacts_allocate and re-read the IOC
1545	 * Facts. If relevant values have changed in IOC Facts, this function
1546	 * will free all of the memory based on IOC Facts and reallocate that
1547	 * memory.  If this fails, any allocated memory should already be freed.
1548	 */
1549	if ((error = mps_iocfacts_allocate(sc, TRUE)) != 0) {
1550		mps_dprint(sc, MPS_FAULT, "%s IOC Facts based allocation "
1551		    "failed with error %d\n", __func__, error);
1552		return (error);
1553	}
1554
1555	/* Start the periodic watchdog check on the IOC Doorbell */
1556	mps_periodic(sc);
1557
1558	/*
1559	 * The portenable will kick off discovery events that will drive the
1560	 * rest of the initialization process.  The CAM/SAS module will
1561	 * hold up the boot sequence until discovery is complete.
1562	 */
1563	sc->mps_ich.ich_func = mps_startup;
1564	sc->mps_ich.ich_arg = sc;
1565	if (config_intrhook_establish(&sc->mps_ich) != 0) {
1566		mps_dprint(sc, MPS_ERROR, "Cannot establish MPS config hook\n");
1567		error = EINVAL;
1568	}
1569
1570	/*
1571	 * Allow IR to shutdown gracefully when shutdown occurs.
1572	 */
1573	sc->shutdown_eh = EVENTHANDLER_REGISTER(shutdown_final,
1574	    mpssas_ir_shutdown, sc, SHUTDOWN_PRI_DEFAULT);
1575
1576	if (sc->shutdown_eh == NULL)
1577		mps_dprint(sc, MPS_ERROR, "shutdown event registration "
1578		    "failed\n");
1579
1580	mps_setup_sysctl(sc);
1581
1582	sc->mps_flags |= MPS_FLAGS_ATTACH_DONE;
1583
1584	return (error);
1585}
1586
1587/* Run through any late-start handlers. */
1588static void
1589mps_startup(void *arg)
1590{
1591	struct mps_softc *sc;
1592
1593	sc = (struct mps_softc *)arg;
1594
1595	mps_lock(sc);
1596	mps_unmask_intr(sc);
1597
1598	/* initialize device mapping tables */
1599	mps_base_static_config_pages(sc);
1600	mps_mapping_initialize(sc);
1601	mpssas_startup(sc);
1602	mps_unlock(sc);
1603}
1604
1605/* Periodic watchdog.  Is called with the driver lock already held. */
1606static void
1607mps_periodic(void *arg)
1608{
1609	struct mps_softc *sc;
1610	uint32_t db;
1611
1612	sc = (struct mps_softc *)arg;
1613	if (sc->mps_flags & MPS_FLAGS_SHUTDOWN)
1614		return;
1615
1616	db = mps_regread(sc, MPI2_DOORBELL_OFFSET);
1617	if ((db & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
1618		mps_dprint(sc, MPS_FAULT, "IOC Fault 0x%08x, Resetting\n", db);
1619		mps_reinit(sc);
1620	}
1621
1622	callout_reset(&sc->periodic, MPS_PERIODIC_DELAY * hz, mps_periodic, sc);
1623}
1624
1625static void
1626mps_log_evt_handler(struct mps_softc *sc, uintptr_t data,
1627    MPI2_EVENT_NOTIFICATION_REPLY *event)
1628{
1629	MPI2_EVENT_DATA_LOG_ENTRY_ADDED *entry;
1630
1631	MPS_DPRINT_EVENT(sc, generic, event);
1632
1633	switch (event->Event) {
1634	case MPI2_EVENT_LOG_DATA:
1635		mps_dprint(sc, MPS_EVENT, "MPI2_EVENT_LOG_DATA:\n");
1636		if (sc->mps_debug & MPS_EVENT)
1637			hexdump(event->EventData, event->EventDataLength, NULL, 0);
1638		break;
1639	case MPI2_EVENT_LOG_ENTRY_ADDED:
1640		entry = (MPI2_EVENT_DATA_LOG_ENTRY_ADDED *)event->EventData;
1641		mps_dprint(sc, MPS_EVENT, "MPI2_EVENT_LOG_ENTRY_ADDED event "
1642		    "0x%x Sequence %d:\n", entry->LogEntryQualifier,
1643		     entry->LogSequence);
1644		break;
1645	default:
1646		break;
1647	}
1648	return;
1649}
1650
1651static int
1652mps_attach_log(struct mps_softc *sc)
1653{
1654	u32 events[MPI2_EVENT_NOTIFY_EVENTMASK_WORDS];
1655
1656	bzero(events, 16);
1657	setbit(events, MPI2_EVENT_LOG_DATA);
1658	setbit(events, MPI2_EVENT_LOG_ENTRY_ADDED);
1659
1660	mps_register_events(sc, events, mps_log_evt_handler, NULL,
1661	    &sc->mps_log_eh);
1662
1663	return (0);
1664}
1665
1666static int
1667mps_detach_log(struct mps_softc *sc)
1668{
1669
1670	if (sc->mps_log_eh != NULL)
1671		mps_deregister_events(sc, sc->mps_log_eh);
1672	return (0);
1673}
1674
1675/*
1676 * Free all of the driver resources and detach submodules.  Should be called
1677 * without the lock held.
1678 */
1679int
1680mps_free(struct mps_softc *sc)
1681{
1682	int error;
1683
1684	/* Turn off the watchdog */
1685	mps_lock(sc);
1686	sc->mps_flags |= MPS_FLAGS_SHUTDOWN;
1687	mps_unlock(sc);
1688	/* Lock must not be held for this */
1689	callout_drain(&sc->periodic);
1690	callout_drain(&sc->device_check_callout);
1691
1692	if (((error = mps_detach_log(sc)) != 0) ||
1693	    ((error = mps_detach_sas(sc)) != 0))
1694		return (error);
1695
1696	mps_detach_user(sc);
1697
1698	/* Put the IOC back in the READY state. */
1699	mps_lock(sc);
1700	if ((error = mps_transition_ready(sc)) != 0) {
1701		mps_unlock(sc);
1702		return (error);
1703	}
1704	mps_unlock(sc);
1705
1706	if (sc->facts != NULL)
1707		free(sc->facts, M_MPT2);
1708
1709	/*
1710	 * Free all buffers that are based on IOC Facts.  A Diag Reset may need
1711	 * to free these buffers too.
1712	 */
1713	mps_iocfacts_free(sc);
1714
1715	if (sc->sysctl_tree != NULL)
1716		sysctl_ctx_free(&sc->sysctl_ctx);
1717
1718	/* Deregister the shutdown function */
1719	if (sc->shutdown_eh != NULL)
1720		EVENTHANDLER_DEREGISTER(shutdown_final, sc->shutdown_eh);
1721
1722	mtx_destroy(&sc->mps_mtx);
1723
1724	return (0);
1725}
1726
1727static __inline void
1728mps_complete_command(struct mps_softc *sc, struct mps_command *cm)
1729{
1730	MPS_FUNCTRACE(sc);
1731
1732	if (cm == NULL) {
1733		mps_dprint(sc, MPS_ERROR, "Completing NULL command\n");
1734		return;
1735	}
1736
1737	if (cm->cm_flags & MPS_CM_FLAGS_POLLED)
1738		cm->cm_flags |= MPS_CM_FLAGS_COMPLETE;
1739
1740	if (cm->cm_complete != NULL) {
1741		mps_dprint(sc, MPS_TRACE,
1742			   "%s cm %p calling cm_complete %p data %p reply %p\n",
1743			   __func__, cm, cm->cm_complete, cm->cm_complete_data,
1744			   cm->cm_reply);
1745		cm->cm_complete(sc, cm);
1746	}
1747
1748	if (cm->cm_flags & MPS_CM_FLAGS_WAKEUP) {
1749		mps_dprint(sc, MPS_TRACE, "waking up %p\n", cm);
1750		wakeup(cm);
1751	}
1752
1753	if (cm->cm_sc->io_cmds_active != 0) {
1754		cm->cm_sc->io_cmds_active--;
1755	} else {
1756		mps_dprint(sc, MPS_ERROR, "Warning: io_cmds_active is "
1757		    "out of sync - resynching to 0\n");
1758	}
1759}
1760
1761
1762static void
1763mps_sas_log_info(struct mps_softc *sc , u32 log_info)
1764{
1765	union loginfo_type {
1766		u32     loginfo;
1767		struct {
1768			u32     subcode:16;
1769			u32     code:8;
1770			u32     originator:4;
1771			u32     bus_type:4;
1772		} dw;
1773	};
1774	union loginfo_type sas_loginfo;
1775	char *originator_str = NULL;
1776
1777	sas_loginfo.loginfo = log_info;
1778	if (sas_loginfo.dw.bus_type != 3 /*SAS*/)
1779		return;
1780
1781	/* each nexus loss loginfo */
1782	if (log_info == 0x31170000)
1783		return;
1784
1785	/* eat the loginfos associated with task aborts */
1786	if ((log_info == 30050000 || log_info ==
1787	    0x31140000 || log_info == 0x31130000))
1788		return;
1789
1790	switch (sas_loginfo.dw.originator) {
1791	case 0:
1792		originator_str = "IOP";
1793		break;
1794	case 1:
1795		originator_str = "PL";
1796		break;
1797	case 2:
1798		originator_str = "IR";
1799		break;
1800}
1801
1802	mps_dprint(sc, MPS_LOG, "log_info(0x%08x): originator(%s), "
1803	"code(0x%02x), sub_code(0x%04x)\n", log_info,
1804	originator_str, sas_loginfo.dw.code,
1805	sas_loginfo.dw.subcode);
1806}
1807
1808static void
1809mps_display_reply_info(struct mps_softc *sc, uint8_t *reply)
1810{
1811	MPI2DefaultReply_t *mpi_reply;
1812	u16 sc_status;
1813
1814	mpi_reply = (MPI2DefaultReply_t*)reply;
1815	sc_status = le16toh(mpi_reply->IOCStatus);
1816	if (sc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE)
1817		mps_sas_log_info(sc, le32toh(mpi_reply->IOCLogInfo));
1818}
1819void
1820mps_intr(void *data)
1821{
1822	struct mps_softc *sc;
1823	uint32_t status;
1824
1825	sc = (struct mps_softc *)data;
1826	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
1827
1828	/*
1829	 * Check interrupt status register to flush the bus.  This is
1830	 * needed for both INTx interrupts and driver-driven polling
1831	 */
1832	status = mps_regread(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET);
1833	if ((status & MPI2_HIS_REPLY_DESCRIPTOR_INTERRUPT) == 0)
1834		return;
1835
1836	mps_lock(sc);
1837	mps_intr_locked(data);
1838	mps_unlock(sc);
1839	return;
1840}
1841
1842/*
1843 * In theory, MSI/MSIX interrupts shouldn't need to read any registers on the
1844 * chip.  Hopefully this theory is correct.
1845 */
1846void
1847mps_intr_msi(void *data)
1848{
1849	struct mps_softc *sc;
1850
1851	sc = (struct mps_softc *)data;
1852	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
1853	mps_lock(sc);
1854	mps_intr_locked(data);
1855	mps_unlock(sc);
1856	return;
1857}
1858
1859/*
1860 * The locking is overly broad and simplistic, but easy to deal with for now.
1861 */
1862void
1863mps_intr_locked(void *data)
1864{
1865	MPI2_REPLY_DESCRIPTORS_UNION *desc;
1866	struct mps_softc *sc;
1867	struct mps_command *cm = NULL;
1868	uint8_t flags;
1869	u_int pq;
1870	MPI2_DIAG_RELEASE_REPLY *rel_rep;
1871	mps_fw_diagnostic_buffer_t *pBuffer;
1872
1873	sc = (struct mps_softc *)data;
1874
1875	pq = sc->replypostindex;
1876	mps_dprint(sc, MPS_TRACE,
1877	    "%s sc %p starting with replypostindex %u\n",
1878	    __func__, sc, sc->replypostindex);
1879
1880	for ( ;; ) {
1881		cm = NULL;
1882		desc = &sc->post_queue[sc->replypostindex];
1883		flags = desc->Default.ReplyFlags &
1884		    MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
1885		if ((flags == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
1886		 || (le32toh(desc->Words.High) == 0xffffffff))
1887			break;
1888
1889		/* increment the replypostindex now, so that event handlers
1890		 * and cm completion handlers which decide to do a diag
1891		 * reset can zero it without it getting incremented again
1892		 * afterwards, and we break out of this loop on the next
1893		 * iteration since the reply post queue has been cleared to
1894		 * 0xFF and all descriptors look unused (which they are).
1895		 */
1896		if (++sc->replypostindex >= sc->pqdepth)
1897			sc->replypostindex = 0;
1898
1899		switch (flags) {
1900		case MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS:
1901			cm = &sc->commands[le16toh(desc->SCSIIOSuccess.SMID)];
1902			cm->cm_reply = NULL;
1903			break;
1904		case MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY:
1905		{
1906			uint32_t baddr;
1907			uint8_t *reply;
1908
1909			/*
1910			 * Re-compose the reply address from the address
1911			 * sent back from the chip.  The ReplyFrameAddress
1912			 * is the lower 32 bits of the physical address of
1913			 * particular reply frame.  Convert that address to
1914			 * host format, and then use that to provide the
1915			 * offset against the virtual address base
1916			 * (sc->reply_frames).
1917			 */
1918			baddr = le32toh(desc->AddressReply.ReplyFrameAddress);
1919			reply = sc->reply_frames +
1920				(baddr - ((uint32_t)sc->reply_busaddr));
1921			/*
1922			 * Make sure the reply we got back is in a valid
1923			 * range.  If not, go ahead and panic here, since
1924			 * we'll probably panic as soon as we deference the
1925			 * reply pointer anyway.
1926			 */
1927			if ((reply < sc->reply_frames)
1928			 || (reply > (sc->reply_frames +
1929			     (sc->fqdepth * sc->facts->ReplyFrameSize * 4)))) {
1930				printf("%s: WARNING: reply %p out of range!\n",
1931				       __func__, reply);
1932				printf("%s: reply_frames %p, fqdepth %d, "
1933				       "frame size %d\n", __func__,
1934				       sc->reply_frames, sc->fqdepth,
1935				       sc->facts->ReplyFrameSize * 4);
1936				printf("%s: baddr %#x,\n", __func__, baddr);
1937				/* LSI-TODO. See Linux Code. Need Gracefull exit*/
1938				panic("Reply address out of range");
1939			}
1940			if (le16toh(desc->AddressReply.SMID) == 0) {
1941				if (((MPI2_DEFAULT_REPLY *)reply)->Function ==
1942				    MPI2_FUNCTION_DIAG_BUFFER_POST) {
1943					/*
1944					 * If SMID is 0 for Diag Buffer Post,
1945					 * this implies that the reply is due to
1946					 * a release function with a status that
1947					 * the buffer has been released.  Set
1948					 * the buffer flags accordingly.
1949					 */
1950					rel_rep =
1951					    (MPI2_DIAG_RELEASE_REPLY *)reply;
1952					if ((le16toh(rel_rep->IOCStatus) &
1953					    MPI2_IOCSTATUS_MASK) ==
1954					    MPI2_IOCSTATUS_DIAGNOSTIC_RELEASED)
1955					{
1956						pBuffer =
1957						    &sc->fw_diag_buffer_list[
1958						    rel_rep->BufferType];
1959						pBuffer->valid_data = TRUE;
1960						pBuffer->owned_by_firmware =
1961						    FALSE;
1962						pBuffer->immediate = FALSE;
1963					}
1964				} else
1965					mps_dispatch_event(sc, baddr,
1966					    (MPI2_EVENT_NOTIFICATION_REPLY *)
1967					    reply);
1968			} else {
1969				cm = &sc->commands[le16toh(desc->AddressReply.SMID)];
1970				cm->cm_reply = reply;
1971				cm->cm_reply_data =
1972				    le32toh(desc->AddressReply.ReplyFrameAddress);
1973			}
1974			break;
1975		}
1976		case MPI2_RPY_DESCRIPT_FLAGS_TARGETASSIST_SUCCESS:
1977		case MPI2_RPY_DESCRIPT_FLAGS_TARGET_COMMAND_BUFFER:
1978		case MPI2_RPY_DESCRIPT_FLAGS_RAID_ACCELERATOR_SUCCESS:
1979		default:
1980			/* Unhandled */
1981			mps_dprint(sc, MPS_ERROR, "Unhandled reply 0x%x\n",
1982			    desc->Default.ReplyFlags);
1983			cm = NULL;
1984			break;
1985		}
1986
1987
1988		if (cm != NULL) {
1989			// Print Error reply frame
1990			if (cm->cm_reply)
1991				mps_display_reply_info(sc,cm->cm_reply);
1992			mps_complete_command(sc, cm);
1993		}
1994
1995		desc->Words.Low = 0xffffffff;
1996		desc->Words.High = 0xffffffff;
1997	}
1998
1999	if (pq != sc->replypostindex) {
2000		mps_dprint(sc, MPS_TRACE,
2001		    "%s sc %p writing postindex %d\n",
2002		    __func__, sc, sc->replypostindex);
2003		mps_regwrite(sc, MPI2_REPLY_POST_HOST_INDEX_OFFSET, sc->replypostindex);
2004	}
2005
2006	return;
2007}
2008
2009static void
2010mps_dispatch_event(struct mps_softc *sc, uintptr_t data,
2011    MPI2_EVENT_NOTIFICATION_REPLY *reply)
2012{
2013	struct mps_event_handle *eh;
2014	int event, handled = 0;
2015
2016	event = le16toh(reply->Event);
2017	TAILQ_FOREACH(eh, &sc->event_list, eh_list) {
2018		if (isset(eh->mask, event)) {
2019			eh->callback(sc, data, reply);
2020			handled++;
2021		}
2022	}
2023
2024	if (handled == 0)
2025		mps_dprint(sc, MPS_EVENT, "Unhandled event 0x%x\n", le16toh(event));
2026
2027	/*
2028	 * This is the only place that the event/reply should be freed.
2029	 * Anything wanting to hold onto the event data should have
2030	 * already copied it into their own storage.
2031	 */
2032	mps_free_reply(sc, data);
2033}
2034
2035static void
2036mps_reregister_events_complete(struct mps_softc *sc, struct mps_command *cm)
2037{
2038	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
2039
2040	if (cm->cm_reply)
2041		MPS_DPRINT_EVENT(sc, generic,
2042			(MPI2_EVENT_NOTIFICATION_REPLY *)cm->cm_reply);
2043
2044	mps_free_command(sc, cm);
2045
2046	/* next, send a port enable */
2047	mpssas_startup(sc);
2048}
2049
2050/*
2051 * For both register_events and update_events, the caller supplies a bitmap
2052 * of events that it _wants_.  These functions then turn that into a bitmask
2053 * suitable for the controller.
2054 */
2055int
2056mps_register_events(struct mps_softc *sc, u32 *mask,
2057    mps_evt_callback_t *cb, void *data, struct mps_event_handle **handle)
2058{
2059	struct mps_event_handle *eh;
2060	int error = 0;
2061
2062	eh = malloc(sizeof(struct mps_event_handle), M_MPT2, M_WAITOK|M_ZERO);
2063	if(!eh) {
2064		device_printf(sc->mps_dev, "Cannot allocate memory %s %d\n",
2065		 __func__, __LINE__);
2066		return (ENOMEM);
2067	}
2068	eh->callback = cb;
2069	eh->data = data;
2070	TAILQ_INSERT_TAIL(&sc->event_list, eh, eh_list);
2071	if (mask != NULL)
2072		error = mps_update_events(sc, eh, mask);
2073	*handle = eh;
2074
2075	return (error);
2076}
2077
2078int
2079mps_update_events(struct mps_softc *sc, struct mps_event_handle *handle,
2080    u32 *mask)
2081{
2082	MPI2_EVENT_NOTIFICATION_REQUEST *evtreq;
2083	MPI2_EVENT_NOTIFICATION_REPLY *reply = NULL;
2084	struct mps_command *cm;
2085	int error, i;
2086
2087	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
2088
2089	if ((mask != NULL) && (handle != NULL))
2090		bcopy(mask, &handle->mask[0], sizeof(u32) *
2091				MPI2_EVENT_NOTIFY_EVENTMASK_WORDS);
2092
2093	for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
2094		sc->event_mask[i] = -1;
2095
2096	for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
2097		sc->event_mask[i] &= ~handle->mask[i];
2098
2099
2100	if ((cm = mps_alloc_command(sc)) == NULL)
2101		return (EBUSY);
2102	evtreq = (MPI2_EVENT_NOTIFICATION_REQUEST *)cm->cm_req;
2103	evtreq->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
2104	evtreq->MsgFlags = 0;
2105	evtreq->SASBroadcastPrimitiveMasks = 0;
2106#ifdef MPS_DEBUG_ALL_EVENTS
2107	{
2108		u_char fullmask[16];
2109		memset(fullmask, 0x00, 16);
2110		bcopy(fullmask, &evtreq->EventMasks[0], sizeof(u32) *
2111				MPI2_EVENT_NOTIFY_EVENTMASK_WORDS);
2112	}
2113#else
2114        for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
2115                evtreq->EventMasks[i] =
2116                    htole32(sc->event_mask[i]);
2117#endif
2118	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
2119	cm->cm_data = NULL;
2120
2121	error = mps_wait_command(sc, &cm, 60, 0);
2122	if (cm != NULL)
2123		reply = (MPI2_EVENT_NOTIFICATION_REPLY *)cm->cm_reply;
2124	if ((reply == NULL) ||
2125	    (reply->IOCStatus & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS)
2126		error = ENXIO;
2127
2128	if (reply)
2129		MPS_DPRINT_EVENT(sc, generic, reply);
2130
2131	mps_dprint(sc, MPS_TRACE, "%s finished error %d\n", __func__, error);
2132
2133	if (cm != NULL)
2134		mps_free_command(sc, cm);
2135	return (error);
2136}
2137
2138static int
2139mps_reregister_events(struct mps_softc *sc)
2140{
2141	MPI2_EVENT_NOTIFICATION_REQUEST *evtreq;
2142	struct mps_command *cm;
2143	struct mps_event_handle *eh;
2144	int error, i;
2145
2146	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
2147
2148	/* first, reregister events */
2149
2150	for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
2151		sc->event_mask[i] = -1;
2152
2153	TAILQ_FOREACH(eh, &sc->event_list, eh_list) {
2154		for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
2155			sc->event_mask[i] &= ~eh->mask[i];
2156	}
2157
2158	if ((cm = mps_alloc_command(sc)) == NULL)
2159		return (EBUSY);
2160	evtreq = (MPI2_EVENT_NOTIFICATION_REQUEST *)cm->cm_req;
2161	evtreq->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
2162	evtreq->MsgFlags = 0;
2163	evtreq->SASBroadcastPrimitiveMasks = 0;
2164#ifdef MPS_DEBUG_ALL_EVENTS
2165	{
2166		u_char fullmask[16];
2167		memset(fullmask, 0x00, 16);
2168		bcopy(fullmask, &evtreq->EventMasks[0], sizeof(u32) *
2169			MPI2_EVENT_NOTIFY_EVENTMASK_WORDS);
2170	}
2171#else
2172        for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
2173                evtreq->EventMasks[i] =
2174                    htole32(sc->event_mask[i]);
2175#endif
2176	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
2177	cm->cm_data = NULL;
2178	cm->cm_complete = mps_reregister_events_complete;
2179
2180	error = mps_map_command(sc, cm);
2181
2182	mps_dprint(sc, MPS_TRACE, "%s finished with error %d\n", __func__,
2183	    error);
2184	return (error);
2185}
2186
2187void
2188mps_deregister_events(struct mps_softc *sc, struct mps_event_handle *handle)
2189{
2190
2191	TAILQ_REMOVE(&sc->event_list, handle, eh_list);
2192	free(handle, M_MPT2);
2193}
2194
2195/*
2196 * Add a chain element as the next SGE for the specified command.
2197 * Reset cm_sge and cm_sgesize to indicate all the available space.
2198 */
2199static int
2200mps_add_chain(struct mps_command *cm)
2201{
2202	MPI2_SGE_CHAIN32 *sgc;
2203	struct mps_chain *chain;
2204	int space;
2205
2206	if (cm->cm_sglsize < MPS_SGC_SIZE)
2207		panic("MPS: Need SGE Error Code\n");
2208
2209	chain = mps_alloc_chain(cm->cm_sc);
2210	if (chain == NULL)
2211		return (ENOBUFS);
2212
2213	space = (int)cm->cm_sc->facts->IOCRequestFrameSize * 4;
2214
2215	/*
2216	 * Note: a double-linked list is used to make it easier to
2217	 * walk for debugging.
2218	 */
2219	TAILQ_INSERT_TAIL(&cm->cm_chain_list, chain, chain_link);
2220
2221	sgc = (MPI2_SGE_CHAIN32 *)&cm->cm_sge->MpiChain;
2222	sgc->Length = htole16(space);
2223	sgc->NextChainOffset = 0;
2224	/* TODO Looks like bug in Setting sgc->Flags.
2225	 *	sgc->Flags = ( MPI2_SGE_FLAGS_CHAIN_ELEMENT | MPI2_SGE_FLAGS_64_BIT_ADDRESSING |
2226	 *	            MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT
2227	 *	This is fine.. because we are not using simple element. In case of
2228	 *	MPI2_SGE_CHAIN32, we have seperate Length and Flags feild.
2229 	 */
2230	sgc->Flags = MPI2_SGE_FLAGS_CHAIN_ELEMENT;
2231	sgc->Address = htole32(chain->chain_busaddr);
2232
2233	cm->cm_sge = (MPI2_SGE_IO_UNION *)&chain->chain->MpiSimple;
2234	cm->cm_sglsize = space;
2235	return (0);
2236}
2237
2238/*
2239 * Add one scatter-gather element (chain, simple, transaction context)
2240 * to the scatter-gather list for a command.  Maintain cm_sglsize and
2241 * cm_sge as the remaining size and pointer to the next SGE to fill
2242 * in, respectively.
2243 */
2244int
2245mps_push_sge(struct mps_command *cm, void *sgep, size_t len, int segsleft)
2246{
2247	MPI2_SGE_TRANSACTION_UNION *tc = sgep;
2248	MPI2_SGE_SIMPLE64 *sge = sgep;
2249	int error, type;
2250	uint32_t saved_buf_len, saved_address_low, saved_address_high;
2251
2252	type = (tc->Flags & MPI2_SGE_FLAGS_ELEMENT_MASK);
2253
2254#ifdef INVARIANTS
2255	switch (type) {
2256	case MPI2_SGE_FLAGS_TRANSACTION_ELEMENT: {
2257		if (len != tc->DetailsLength + 4)
2258			panic("TC %p length %u or %zu?", tc,
2259			    tc->DetailsLength + 4, len);
2260		}
2261		break;
2262	case MPI2_SGE_FLAGS_CHAIN_ELEMENT:
2263		/* Driver only uses 32-bit chain elements */
2264		if (len != MPS_SGC_SIZE)
2265			panic("CHAIN %p length %u or %zu?", sgep,
2266			    MPS_SGC_SIZE, len);
2267		break;
2268	case MPI2_SGE_FLAGS_SIMPLE_ELEMENT:
2269		/* Driver only uses 64-bit SGE simple elements */
2270		if (len != MPS_SGE64_SIZE)
2271			panic("SGE simple %p length %u or %zu?", sge,
2272			    MPS_SGE64_SIZE, len);
2273		if (((le32toh(sge->FlagsLength) >> MPI2_SGE_FLAGS_SHIFT) &
2274		    MPI2_SGE_FLAGS_ADDRESS_SIZE) == 0)
2275			panic("SGE simple %p not marked 64-bit?", sge);
2276
2277		break;
2278	default:
2279		panic("Unexpected SGE %p, flags %02x", tc, tc->Flags);
2280	}
2281#endif
2282
2283	/*
2284	 * case 1: 1 more segment, enough room for it
2285	 * case 2: 2 more segments, enough room for both
2286	 * case 3: >=2 more segments, only enough room for 1 and a chain
2287	 * case 4: >=1 more segment, enough room for only a chain
2288	 * case 5: >=1 more segment, no room for anything (error)
2289         */
2290
2291	/*
2292	 * There should be room for at least a chain element, or this
2293	 * code is buggy.  Case (5).
2294	 */
2295	if (cm->cm_sglsize < MPS_SGC_SIZE)
2296		panic("MPS: Need SGE Error Code\n");
2297
2298	if (segsleft >= 2 &&
2299	    cm->cm_sglsize < len + MPS_SGC_SIZE + MPS_SGE64_SIZE) {
2300		/*
2301		 * There are 2 or more segments left to add, and only
2302		 * enough room for 1 and a chain.  Case (3).
2303		 *
2304		 * Mark as last element in this chain if necessary.
2305		 */
2306		if (type == MPI2_SGE_FLAGS_SIMPLE_ELEMENT) {
2307			sge->FlagsLength |= htole32(
2308			    MPI2_SGE_FLAGS_LAST_ELEMENT << MPI2_SGE_FLAGS_SHIFT);
2309		}
2310
2311		/*
2312		 * Add the item then a chain.  Do the chain now,
2313		 * rather than on the next iteration, to simplify
2314		 * understanding the code.
2315		 */
2316		cm->cm_sglsize -= len;
2317		bcopy(sgep, cm->cm_sge, len);
2318		cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + len);
2319		return (mps_add_chain(cm));
2320	}
2321
2322	if (segsleft >= 1 && cm->cm_sglsize < len + MPS_SGC_SIZE) {
2323		/*
2324		 * 1 or more segment, enough room for only a chain.
2325		 * Hope the previous element wasn't a Simple entry
2326		 * that needed to be marked with
2327		 * MPI2_SGE_FLAGS_LAST_ELEMENT.  Case (4).
2328		 */
2329		if ((error = mps_add_chain(cm)) != 0)
2330			return (error);
2331	}
2332
2333#ifdef INVARIANTS
2334	/* Case 1: 1 more segment, enough room for it. */
2335	if (segsleft == 1 && cm->cm_sglsize < len)
2336		panic("1 seg left and no room? %u versus %zu",
2337		    cm->cm_sglsize, len);
2338
2339	/* Case 2: 2 more segments, enough room for both */
2340	if (segsleft == 2 && cm->cm_sglsize < len + MPS_SGE64_SIZE)
2341		panic("2 segs left and no room? %u versus %zu",
2342		    cm->cm_sglsize, len);
2343#endif
2344
2345	if (segsleft == 1 && type == MPI2_SGE_FLAGS_SIMPLE_ELEMENT) {
2346		/*
2347		 * If this is a bi-directional request, need to account for that
2348		 * here.  Save the pre-filled sge values.  These will be used
2349		 * either for the 2nd SGL or for a single direction SGL.  If
2350		 * cm_out_len is non-zero, this is a bi-directional request, so
2351		 * fill in the OUT SGL first, then the IN SGL, otherwise just
2352		 * fill in the IN SGL.  Note that at this time, when filling in
2353		 * 2 SGL's for a bi-directional request, they both use the same
2354		 * DMA buffer (same cm command).
2355		 */
2356		saved_buf_len = le32toh(sge->FlagsLength) & 0x00FFFFFF;
2357		saved_address_low = sge->Address.Low;
2358		saved_address_high = sge->Address.High;
2359		if (cm->cm_out_len) {
2360			sge->FlagsLength = htole32(cm->cm_out_len |
2361			    ((uint32_t)(MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
2362			    MPI2_SGE_FLAGS_END_OF_BUFFER |
2363			    MPI2_SGE_FLAGS_HOST_TO_IOC |
2364			    MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
2365			    MPI2_SGE_FLAGS_SHIFT));
2366			cm->cm_sglsize -= len;
2367			bcopy(sgep, cm->cm_sge, len);
2368			cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge
2369			    + len);
2370		}
2371		saved_buf_len |=
2372		    ((uint32_t)(MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
2373		    MPI2_SGE_FLAGS_END_OF_BUFFER |
2374		    MPI2_SGE_FLAGS_LAST_ELEMENT |
2375		    MPI2_SGE_FLAGS_END_OF_LIST |
2376		    MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
2377		    MPI2_SGE_FLAGS_SHIFT);
2378		if (cm->cm_flags & MPS_CM_FLAGS_DATAIN) {
2379			saved_buf_len |=
2380			    ((uint32_t)(MPI2_SGE_FLAGS_IOC_TO_HOST) <<
2381			    MPI2_SGE_FLAGS_SHIFT);
2382		} else {
2383			saved_buf_len |=
2384			    ((uint32_t)(MPI2_SGE_FLAGS_HOST_TO_IOC) <<
2385			    MPI2_SGE_FLAGS_SHIFT);
2386		}
2387		sge->FlagsLength = htole32(saved_buf_len);
2388		sge->Address.Low = saved_address_low;
2389		sge->Address.High = saved_address_high;
2390	}
2391
2392	cm->cm_sglsize -= len;
2393	bcopy(sgep, cm->cm_sge, len);
2394	cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + len);
2395	return (0);
2396}
2397
2398/*
2399 * Add one dma segment to the scatter-gather list for a command.
2400 */
2401int
2402mps_add_dmaseg(struct mps_command *cm, vm_paddr_t pa, size_t len, u_int flags,
2403    int segsleft)
2404{
2405	MPI2_SGE_SIMPLE64 sge;
2406
2407	/*
2408	 * This driver always uses 64-bit address elements for simplicity.
2409	 */
2410	bzero(&sge, sizeof(sge));
2411	flags |= MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
2412	    MPI2_SGE_FLAGS_64_BIT_ADDRESSING;
2413	sge.FlagsLength = htole32(len | (flags << MPI2_SGE_FLAGS_SHIFT));
2414	mps_from_u64(pa, &sge.Address);
2415
2416	return (mps_push_sge(cm, &sge, sizeof sge, segsleft));
2417}
2418
2419static void
2420mps_data_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
2421{
2422	struct mps_softc *sc;
2423	struct mps_command *cm;
2424	u_int i, dir, sflags;
2425
2426	cm = (struct mps_command *)arg;
2427	sc = cm->cm_sc;
2428
2429	/*
2430	 * In this case, just print out a warning and let the chip tell the
2431	 * user they did the wrong thing.
2432	 */
2433	if ((cm->cm_max_segs != 0) && (nsegs > cm->cm_max_segs)) {
2434		mps_dprint(sc, MPS_ERROR,
2435			   "%s: warning: busdma returned %d segments, "
2436			   "more than the %d allowed\n", __func__, nsegs,
2437			   cm->cm_max_segs);
2438	}
2439
2440	/*
2441	 * Set up DMA direction flags.  Bi-directional requests are also handled
2442	 * here.  In that case, both direction flags will be set.
2443	 */
2444	sflags = 0;
2445	if (cm->cm_flags & MPS_CM_FLAGS_SMP_PASS) {
2446		/*
2447		 * We have to add a special case for SMP passthrough, there
2448		 * is no easy way to generically handle it.  The first
2449		 * S/G element is used for the command (therefore the
2450		 * direction bit needs to be set).  The second one is used
2451		 * for the reply.  We'll leave it to the caller to make
2452		 * sure we only have two buffers.
2453		 */
2454		/*
2455		 * Even though the busdma man page says it doesn't make
2456		 * sense to have both direction flags, it does in this case.
2457		 * We have one s/g element being accessed in each direction.
2458		 */
2459		dir = BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD;
2460
2461		/*
2462		 * Set the direction flag on the first buffer in the SMP
2463		 * passthrough request.  We'll clear it for the second one.
2464		 */
2465		sflags |= MPI2_SGE_FLAGS_DIRECTION |
2466			  MPI2_SGE_FLAGS_END_OF_BUFFER;
2467	} else if (cm->cm_flags & MPS_CM_FLAGS_DATAOUT) {
2468		sflags |= MPI2_SGE_FLAGS_HOST_TO_IOC;
2469		dir = BUS_DMASYNC_PREWRITE;
2470	} else
2471		dir = BUS_DMASYNC_PREREAD;
2472
2473	for (i = 0; i < nsegs; i++) {
2474		if ((cm->cm_flags & MPS_CM_FLAGS_SMP_PASS) && (i != 0)) {
2475			sflags &= ~MPI2_SGE_FLAGS_DIRECTION;
2476		}
2477		error = mps_add_dmaseg(cm, segs[i].ds_addr, segs[i].ds_len,
2478		    sflags, nsegs - i);
2479		if (error != 0) {
2480			/* Resource shortage, roll back! */
2481			if (ratecheck(&sc->lastfail, &mps_chainfail_interval))
2482				mps_dprint(sc, MPS_INFO, "Out of chain frames, "
2483				    "consider increasing hw.mps.max_chains.\n");
2484			cm->cm_flags |= MPS_CM_FLAGS_CHAIN_FAILED;
2485			mps_complete_command(sc, cm);
2486			return;
2487		}
2488	}
2489
2490	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
2491	mps_enqueue_request(sc, cm);
2492
2493	return;
2494}
2495
2496static void
2497mps_data_cb2(void *arg, bus_dma_segment_t *segs, int nsegs, bus_size_t mapsize,
2498	     int error)
2499{
2500	mps_data_cb(arg, segs, nsegs, error);
2501}
2502
2503/*
2504 * This is the routine to enqueue commands ansynchronously.
2505 * Note that the only error path here is from bus_dmamap_load(), which can
2506 * return EINPROGRESS if it is waiting for resources.  Other than this, it's
2507 * assumed that if you have a command in-hand, then you have enough credits
2508 * to use it.
2509 */
2510int
2511mps_map_command(struct mps_softc *sc, struct mps_command *cm)
2512{
2513	int error = 0;
2514
2515	if (cm->cm_flags & MPS_CM_FLAGS_USE_UIO) {
2516		error = bus_dmamap_load_uio(sc->buffer_dmat, cm->cm_dmamap,
2517		    &cm->cm_uio, mps_data_cb2, cm, 0);
2518	} else if (cm->cm_flags & MPS_CM_FLAGS_USE_CCB) {
2519		error = bus_dmamap_load_ccb(sc->buffer_dmat, cm->cm_dmamap,
2520		    cm->cm_data, mps_data_cb, cm, 0);
2521	} else if ((cm->cm_data != NULL) && (cm->cm_length != 0)) {
2522		error = bus_dmamap_load(sc->buffer_dmat, cm->cm_dmamap,
2523		    cm->cm_data, cm->cm_length, mps_data_cb, cm, 0);
2524	} else {
2525		/* Add a zero-length element as needed */
2526		if (cm->cm_sge != NULL)
2527			mps_add_dmaseg(cm, 0, 0, 0, 1);
2528		mps_enqueue_request(sc, cm);
2529	}
2530
2531	return (error);
2532}
2533
2534/*
2535 * This is the routine to enqueue commands synchronously.  An error of
2536 * EINPROGRESS from mps_map_command() is ignored since the command will
2537 * be executed and enqueued automatically.  Other errors come from msleep().
2538 */
2539int
2540mps_wait_command(struct mps_softc *sc, struct mps_command **cmp, int timeout,
2541    int sleep_flag)
2542{
2543	int error, rc;
2544	struct timeval cur_time, start_time;
2545	struct mps_command *cm = *cmp;
2546
2547	if (sc->mps_flags & MPS_FLAGS_DIAGRESET)
2548		return  EBUSY;
2549
2550	cm->cm_complete = NULL;
2551	cm->cm_flags |= MPS_CM_FLAGS_POLLED;
2552	error = mps_map_command(sc, cm);
2553	if ((error != 0) && (error != EINPROGRESS))
2554		return (error);
2555
2556	/*
2557	 * Check for context and wait for 50 mSec at a time until time has
2558	 * expired or the command has finished.  If msleep can't be used, need
2559	 * to poll.
2560	 */
2561	if (curthread->td_no_sleeping != 0)
2562		sleep_flag = NO_SLEEP;
2563	getmicrouptime(&start_time);
2564	if (mtx_owned(&sc->mps_mtx) && sleep_flag == CAN_SLEEP) {
2565		cm->cm_flags |= MPS_CM_FLAGS_WAKEUP;
2566		error = msleep(cm, &sc->mps_mtx, 0, "mpswait", timeout*hz);
2567		if (error == EWOULDBLOCK) {
2568			/*
2569			 * Record the actual elapsed time in the case of a
2570			 * timeout for the message below.
2571			 */
2572			getmicrouptime(&cur_time);
2573			timevalsub(&cur_time, &start_time);
2574		}
2575	} else {
2576		while ((cm->cm_flags & MPS_CM_FLAGS_COMPLETE) == 0) {
2577			mps_intr_locked(sc);
2578			if (sleep_flag == CAN_SLEEP)
2579				pause("mpswait", hz/20);
2580			else
2581				DELAY(50000);
2582
2583			getmicrouptime(&cur_time);
2584			timevalsub(&cur_time, &start_time);
2585			if (cur_time.tv_sec > timeout) {
2586				error = EWOULDBLOCK;
2587				break;
2588			}
2589		}
2590	}
2591
2592	if (error == EWOULDBLOCK) {
2593		mps_dprint(sc, MPS_FAULT, "Calling Reinit from %s, timeout=%d,"
2594		    " elapsed=%jd\n", __func__, timeout,
2595		    (intmax_t)cur_time.tv_sec);
2596		rc = mps_reinit(sc);
2597		mps_dprint(sc, MPS_FAULT, "Reinit %s\n", (rc == 0) ? "success" :
2598		    "failed");
2599		if (sc->mps_flags & MPS_FLAGS_REALLOCATED) {
2600			/*
2601			 * Tell the caller that we freed the command in a
2602			 * reinit.
2603			 */
2604			*cmp = NULL;
2605		}
2606		error = ETIMEDOUT;
2607	}
2608	return (error);
2609}
2610
2611/*
2612 * The MPT driver had a verbose interface for config pages.  In this driver,
2613 * reduce it to much simplier terms, similar to the Linux driver.
2614 */
2615int
2616mps_read_config_page(struct mps_softc *sc, struct mps_config_params *params)
2617{
2618	MPI2_CONFIG_REQUEST *req;
2619	struct mps_command *cm;
2620	int error;
2621
2622	if (sc->mps_flags & MPS_FLAGS_BUSY) {
2623		return (EBUSY);
2624	}
2625
2626	cm = mps_alloc_command(sc);
2627	if (cm == NULL) {
2628		return (EBUSY);
2629	}
2630
2631	req = (MPI2_CONFIG_REQUEST *)cm->cm_req;
2632	req->Function = MPI2_FUNCTION_CONFIG;
2633	req->Action = params->action;
2634	req->SGLFlags = 0;
2635	req->ChainOffset = 0;
2636	req->PageAddress = params->page_address;
2637	if (params->hdr.Struct.PageType == MPI2_CONFIG_PAGETYPE_EXTENDED) {
2638		MPI2_CONFIG_EXTENDED_PAGE_HEADER *hdr;
2639
2640		hdr = &params->hdr.Ext;
2641		req->ExtPageType = hdr->ExtPageType;
2642		req->ExtPageLength = hdr->ExtPageLength;
2643		req->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
2644		req->Header.PageLength = 0; /* Must be set to zero */
2645		req->Header.PageNumber = hdr->PageNumber;
2646		req->Header.PageVersion = hdr->PageVersion;
2647	} else {
2648		MPI2_CONFIG_PAGE_HEADER *hdr;
2649
2650		hdr = &params->hdr.Struct;
2651		req->Header.PageType = hdr->PageType;
2652		req->Header.PageNumber = hdr->PageNumber;
2653		req->Header.PageLength = hdr->PageLength;
2654		req->Header.PageVersion = hdr->PageVersion;
2655	}
2656
2657	cm->cm_data = params->buffer;
2658	cm->cm_length = params->length;
2659	if (cm->cm_data != NULL) {
2660		cm->cm_sge = &req->PageBufferSGE;
2661		cm->cm_sglsize = sizeof(MPI2_SGE_IO_UNION);
2662		cm->cm_flags = MPS_CM_FLAGS_SGE_SIMPLE | MPS_CM_FLAGS_DATAIN;
2663	} else
2664		cm->cm_sge = NULL;
2665	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
2666
2667	cm->cm_complete_data = params;
2668	if (params->callback != NULL) {
2669		cm->cm_complete = mps_config_complete;
2670		return (mps_map_command(sc, cm));
2671	} else {
2672		error = mps_wait_command(sc, &cm, 0, CAN_SLEEP);
2673		if (error) {
2674			mps_dprint(sc, MPS_FAULT,
2675			    "Error %d reading config page\n", error);
2676			if (cm != NULL)
2677				mps_free_command(sc, cm);
2678			return (error);
2679		}
2680		mps_config_complete(sc, cm);
2681	}
2682
2683	return (0);
2684}
2685
2686int
2687mps_write_config_page(struct mps_softc *sc, struct mps_config_params *params)
2688{
2689	return (EINVAL);
2690}
2691
2692static void
2693mps_config_complete(struct mps_softc *sc, struct mps_command *cm)
2694{
2695	MPI2_CONFIG_REPLY *reply;
2696	struct mps_config_params *params;
2697
2698	MPS_FUNCTRACE(sc);
2699	params = cm->cm_complete_data;
2700
2701	if (cm->cm_data != NULL) {
2702		bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,
2703		    BUS_DMASYNC_POSTREAD);
2704		bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2705	}
2706
2707	/*
2708	 * XXX KDM need to do more error recovery?  This results in the
2709	 * device in question not getting probed.
2710	 */
2711	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2712		params->status = MPI2_IOCSTATUS_BUSY;
2713		goto done;
2714	}
2715
2716	reply = (MPI2_CONFIG_REPLY *)cm->cm_reply;
2717	if (reply == NULL) {
2718		params->status = MPI2_IOCSTATUS_BUSY;
2719		goto done;
2720	}
2721	params->status = reply->IOCStatus;
2722	if (params->hdr.Struct.PageType == MPI2_CONFIG_PAGETYPE_EXTENDED) {
2723		params->hdr.Ext.ExtPageType = reply->ExtPageType;
2724		params->hdr.Ext.ExtPageLength = reply->ExtPageLength;
2725		params->hdr.Ext.PageType = reply->Header.PageType;
2726		params->hdr.Ext.PageNumber = reply->Header.PageNumber;
2727		params->hdr.Ext.PageVersion = reply->Header.PageVersion;
2728	} else {
2729		params->hdr.Struct.PageType = reply->Header.PageType;
2730		params->hdr.Struct.PageNumber = reply->Header.PageNumber;
2731		params->hdr.Struct.PageLength = reply->Header.PageLength;
2732		params->hdr.Struct.PageVersion = reply->Header.PageVersion;
2733	}
2734
2735done:
2736	mps_free_command(sc, cm);
2737	if (params->callback != NULL)
2738		params->callback(sc, params);
2739
2740	return;
2741}
2742