1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26#include <sys/types.h>
27#include <sys/param.h>
28#include <sys/t_lock.h>
29#include <sys/systm.h>
30#include <sys/spl.h>
31#include <sys/cmn_err.h>
32#include <sys/debug.h>
33#include <sys/kdi_impl.h>
34#include <sys/cpuvar.h>
35#include <sys/cpuvar.h>
36#include <sys/archsystm.h>
37
38/*
39 * Handle software interrupts through 'softcall' mechanism
40 *
41 * At present softcall mechanism uses a global list headed by softhead.
42 * Entries are added to tail and removed from head so as to preserve FIFO
43 * nature of entries in the softcall list. softcall() takes care of adding
44 * entries to the softtail.
45 *
46 * softint must take care of executing the entries in the FIFO
47 * order. It could be called simultaneously from multiple cpus, however only
48 * one instance of softint should process the softcall list with the exception
49 * when CPU is stuck due to high interrupt load and can't execute callbacks.
50 * State diagram is as follows :-
51 *
52 *	- Upper half which is same as old state machine
53 *	  (IDLE->PEND->DRAIN->IDLE)
54 *
55 *	- Lower half which steals the entries from softcall queue and execute
56 *        in the context of softint interrupt handler. The interrupt handler
57 *        is fired on a different CPU by sending a cross-call.
58 *
59 * Starting state is IDLE.
60 *
61 * 				softint()
62 *
63 *
64 *				(c)
65 * 	____________________________________________________
66 * 	|                          ^                         ^
67 * 	v            (a)           |           (b)           |
68 * 	IDLE--------------------->PEND--------------------->DRAIN
69 *	^                         |                         |
70 * 	|                         |                         |
71 * 	|                         |                         |
72 * 	|                         |                         |
73 * 	|                         |                         |
74 * 	|                         d                         d
75 * 	|                         |                         |
76 * 	|                         v                         v
77 * 	|                         PEND                      DRAIN
78 * 	|            (e)           &                          &
79 * 	|<-----------------------STEAL                      STEAL
80 * 	^                                                    |
81 * 	|                                                    |
82 * 	|                         (e)                        v
83 * 	|_________________________<__________________________|
84 *
85 *
86 *
87 * Edge (a)->(b)->(c) are same as old state machine and these
88 * are mutually exclusive state.
89 *
90 * a - When an entry is being enqueued to softcall queue then the state
91 *     moves from IDLE to PEND.
92 *
93 * b - When interrupt handler has started processing softcall queue.
94 *
95 * c - When interrupt handler finished processing softcall queue, the
96 *     state of machines goes back to IDLE.
97 *
98 * d - softcall() generates another softlevel1 iff interrupt handler
99 *     hasn't run recently.
100 *
101 * e - Either PEND|STEAL or DRAIN|STEAL is set. We let softlevel1
102 *     handler exit because we have processed all the entries.
103 *
104 * When CPU is being pinned by higher level interrupts for more than
105 * softcall_delay clock ticks, SOFT_STEAL is OR'ed so that softlevel1
106 * handler on the other CPU can drain the queue.
107 *
108 * These states are needed for softcall mechanism since Solaris has only
109 * one interface (ie. siron ) as of now for :
110 *
111 * - raising a soft interrupt architecture independently (ie not through
112 *   setsoftint(..) )
113 * - to process the softcall queue.
114 */
115
116#define	NSOFTCALLS	200
117
118/*
119 * Defined states for softcall processing.
120 */
121#define	SOFT_IDLE		0x01	/* no processing is needed */
122#define	SOFT_PEND		0x02	/* softcall list needs processing */
123#define	SOFT_DRAIN		0x04	/* list is being processed */
124#define	SOFT_STEAL		0x08	/* list is being stolen for draining */
125
126typedef struct softcall {
127	void (*sc_func)(void *);	/* function to call */
128	void *sc_arg;			/* arg to pass to func */
129	struct softcall *sc_next;	/* next in list */
130} softcall_t;
131
132/*
133 * softcall list and state variables.
134 */
135static softcall_t *softcalls;
136static softcall_t *softhead, *softtail, *softfree;
137static uint_t	softcall_state;
138static clock_t softcall_tick;
139static clock_t softcall_countstart, softcall_lastpoke;
140static uint_t softcall_pokecount;
141
142/*
143 * Max number of pokes per second before increasing softcall_delay
144 */
145uint_t softcall_pokemax = 10;
146
147/*
148 * This ensures that softcall entries don't get stuck for long. It's expressed
149 * in 10 milliseconds as 1 unit. When hires_tick is set or other clock frequency
150 * is used, softcall_init() ensures that it's still expressed as 1 =  10 milli
151 * seconds.
152 */
153unsigned int softcall_delay = 1;
154
155/*
156 * The last CPU which will drain softcall queue.
157 */
158static int softcall_latest_cpuid = -1;
159
160/*
161 * CPUSET to hold the CPU which is processing softcall queue
162 * currently. There can be more than one CPU having bit set
163 * but it will happen only when they are stuck.
164 */
165static cpuset_t *softcall_cpuset = NULL;
166
167/*
168 * protects softcall lists and control variable softcall_state.
169 */
170static kmutex_t	softcall_lock;
171
172static void (*kdi_softcall_func)(void);
173extern void siron_poke_cpu(cpuset_t);
174
175extern void siron(void);
176extern void kdi_siron(void);
177
178
179void
180softcall_init(void)
181{
182	softcall_t *sc;
183
184	softcalls = kmem_zalloc(sizeof (softcall_t) * NSOFTCALLS, KM_SLEEP);
185	softcall_cpuset = kmem_zalloc(sizeof (cpuset_t), KM_SLEEP);
186	for (sc = softcalls; sc < &softcalls[NSOFTCALLS]; sc++) {
187		sc->sc_next = softfree;
188		softfree = sc;
189	}
190	mutex_init(&softcall_lock, NULL, MUTEX_SPIN,
191	    (void *)ipltospl(SPL8));
192	softcall_state = SOFT_IDLE;
193	softcall_tick = ddi_get_lbolt();
194
195	/*
196	 * Since softcall_delay is expressed as 1 = 10 milliseconds.
197	 */
198	softcall_delay = softcall_delay * (hz/100);
199	CPUSET_ZERO(*softcall_cpuset);
200}
201
202/*
203 * Gets called when softcall queue is not moving forward. We choose
204 * a CPU and poke except the ones which are already poked.
205 */
206static int
207softcall_choose_cpu()
208{
209	cpu_t *cplist = CPU;
210	cpu_t *cp;
211	int intr_load = INT_MAX;
212	int cpuid = -1;
213	cpuset_t poke;
214	int s;
215
216	ASSERT(getpil() >= DISP_LEVEL);
217	ASSERT(ncpus > 1);
218	ASSERT(MUTEX_HELD(&softcall_lock));
219
220	CPUSET_ZERO(poke);
221
222	/*
223	 * The hint is to start from current CPU.
224	 */
225	cp = cplist;
226	do {
227		/*
228		 * Don't select this CPU if :
229		 *   - in cpuset already
230		 *   - CPU is not accepting interrupts
231		 *   - CPU is being offlined
232		 */
233		if (CPU_IN_SET(*softcall_cpuset, cp->cpu_id) ||
234		    (cp->cpu_flags & CPU_ENABLE) == 0 ||
235		    (cp == cpu_inmotion))
236			continue;
237#if defined(__x86)
238		/*
239		 * Don't select this CPU if a hypervisor indicates it
240		 * isn't currently scheduled onto a physical cpu.  We are
241		 * looking for a cpu that can respond quickly and the time
242		 * to get the virtual cpu scheduled and switched to running
243		 * state is likely to be relatively lengthy.
244		 */
245		if (vcpu_on_pcpu(cp->cpu_id) == VCPU_NOT_ON_PCPU)
246			continue;
247#endif	/* __x86 */
248
249		/* if CPU is not busy */
250		if (cp->cpu_intrload == 0) {
251			cpuid = cp->cpu_id;
252			break;
253		}
254
255		if (cp->cpu_intrload < intr_load) {
256			cpuid = cp->cpu_id;
257			intr_load = cp->cpu_intrload;
258		} else if (cp->cpu_intrload == intr_load) {
259			/*
260			 * We want to poke CPUs having similar
261			 * load because we don't know which CPU is
262			 * can acknowledge level1 interrupt. The
263			 * list of such CPUs should not be large.
264			 */
265			if (cpuid != -1) {
266				/*
267				 * Put the last CPU chosen because
268				 * it also has same interrupt load.
269				 */
270				CPUSET_ADD(poke, cpuid);
271				cpuid = -1;
272			}
273
274			CPUSET_ADD(poke, cp->cpu_id);
275		}
276	} while ((cp = cp->cpu_next_onln) != cplist);
277
278	/* if we found a CPU which suits best to poke */
279	if (cpuid != -1) {
280		CPUSET_ZERO(poke);
281		CPUSET_ADD(poke, cpuid);
282	}
283
284	if (CPUSET_ISNULL(poke)) {
285		mutex_exit(&softcall_lock);
286		return (0);
287	}
288
289	/*
290	 * We first set the bit in cpuset and then poke.
291	 */
292	CPUSET_XOR(*softcall_cpuset, poke);
293	mutex_exit(&softcall_lock);
294
295	/*
296	 * If softcall() was called at low pil then we may
297	 * get preempted before we raise PIL. It should be okay
298	 * because we are just going to poke CPUs now or at most
299	 * another thread may start choosing CPUs in this routine.
300	 */
301	s = splhigh();
302	siron_poke_cpu(poke);
303	splx(s);
304	return (1);
305}
306
307
308/*
309 * Call function func with argument arg
310 * at some later time at software interrupt priority
311 */
312void
313softcall(void (*func)(void *), void *arg)
314{
315	softcall_t *sc;
316	clock_t w, now;
317
318	/*
319	 * protect against cross-calls
320	 */
321	mutex_enter(&softcall_lock);
322	/* coalesce identical softcalls */
323	for (sc = softhead; sc != 0; sc = sc->sc_next) {
324		if (sc->sc_func == func && sc->sc_arg == arg) {
325			goto intr;
326		}
327	}
328
329	if ((sc = softfree) == 0)
330		panic("too many softcalls");
331
332	softfree = sc->sc_next;
333	sc->sc_func = func;
334	sc->sc_arg = arg;
335	sc->sc_next = 0;
336
337	if (softhead) {
338		softtail->sc_next = sc;
339		softtail = sc;
340	} else
341		softhead = softtail = sc;
342
343intr:
344	if (softcall_state & SOFT_IDLE) {
345		softcall_state = SOFT_PEND;
346		softcall_tick = ddi_get_lbolt();
347		mutex_exit(&softcall_lock);
348		siron();
349	} else if (softcall_state & (SOFT_DRAIN|SOFT_PEND)) {
350		now = ddi_get_lbolt();
351		w = now - softcall_tick;
352		if (w <= softcall_delay || ncpus == 1) {
353			mutex_exit(&softcall_lock);
354			return;
355		}
356		/*
357		 * Did we poke less than a second ago?
358		 */
359		if (now - softcall_lastpoke < hz) {
360			/*
361			 * We did, increment the poke count and
362			 * see if we are poking too often
363			 */
364			if (softcall_pokecount++ == 0)
365				softcall_countstart = now;
366			if (softcall_pokecount > softcall_pokemax) {
367				/*
368				 * If poking too much increase the delay
369				 */
370				if (now - softcall_countstart <= hz)
371					softcall_delay++;
372				softcall_pokecount = 0;
373			}
374		} else {
375			/*
376			 * poke rate has dropped off, reset the poke monitor
377			 */
378			softcall_pokecount = 0;
379		}
380		softcall_lastpoke = now;
381		if (!(softcall_state & SOFT_STEAL)) {
382			softcall_state |= SOFT_STEAL;
383
384			/*
385			 * We want to give some more chance before
386			 * fishing around again.
387			 */
388			softcall_tick = now;
389		}
390
391		/* softcall_lock will be released by this routine */
392		(void) softcall_choose_cpu();
393	}
394}
395
396void
397kdi_softcall(void (*func)(void))
398{
399	kdi_softcall_func = func;
400
401	if (softhead == NULL)
402		kdi_siron();
403}
404
405/*
406 * Called to process software interrupts take one off queue, call it,
407 * repeat.
408 *
409 * Note queue may change during call; softcall_lock, state variables
410 * softcall_state and softcall_latest_cpuid ensures that -
411 * - we don't have multiple cpus pulling from the list (thus causing
412 *   a violation of FIFO order with an exception when we are stuck).
413 * - we don't miss a new entry having been added to the head.
414 * - we don't miss a wakeup.
415 */
416
417void
418softint(void)
419{
420	softcall_t *sc = NULL;
421	void (*func)();
422	caddr_t arg;
423	int cpu_id = CPU->cpu_id;
424
425	/*
426	 * Don't process softcall queue if current CPU is quiesced or
427	 * offlined. This can happen when a CPU is running pause
428	 * thread but softcall already sent a xcall.
429	 */
430	if (CPU->cpu_flags & (CPU_QUIESCED|CPU_OFFLINE)) {
431		if (softcall_cpuset != NULL &&
432		    CPU_IN_SET(*softcall_cpuset, cpu_id)) {
433			CPUSET_DEL(*softcall_cpuset, cpu_id);
434			goto out;
435		}
436	}
437
438	mutex_enter(&softcall_lock);
439
440	if (softcall_state & (SOFT_STEAL|SOFT_PEND)) {
441		softcall_state = SOFT_DRAIN;
442	} else  {
443		/*
444		 * The check for softcall_cpuset being
445		 * NULL is required because it may get
446		 * called very early during boot.
447		 */
448		if (softcall_cpuset != NULL &&
449		    CPU_IN_SET(*softcall_cpuset, cpu_id))
450			CPUSET_DEL(*softcall_cpuset, cpu_id);
451		mutex_exit(&softcall_lock);
452		goto out;
453	}
454
455	/*
456	 * Setting softcall_latest_cpuid to current CPU ensures
457	 * that there is only one active softlevel1 handler to
458	 * process softcall queues.
459	 *
460	 * Since softcall_lock lock is dropped before calling
461	 * func (callback), we need softcall_latest_cpuid
462	 * to prevent two softlevel1 hanlders working on the
463	 * queue when the first softlevel1 handler gets
464	 * stuck due to high interrupt load.
465	 */
466	softcall_latest_cpuid = cpu_id;
467
468	/* add ourself to the cpuset */
469	if (!CPU_IN_SET(*softcall_cpuset, cpu_id))
470		CPUSET_ADD(*softcall_cpuset, cpu_id);
471
472	for (;;) {
473		softcall_tick = ddi_get_lbolt();
474		if ((sc = softhead) != NULL) {
475			func = sc->sc_func;
476			arg = sc->sc_arg;
477			softhead = sc->sc_next;
478			sc->sc_next = softfree;
479			softfree = sc;
480		}
481
482		if (sc == NULL) {
483			if (CPU_IN_SET(*softcall_cpuset, cpu_id))
484				CPUSET_DEL(*softcall_cpuset, cpu_id);
485
486			softcall_state = SOFT_IDLE;
487			ASSERT(softcall_latest_cpuid == cpu_id);
488			softcall_latest_cpuid = -1;
489
490			mutex_exit(&softcall_lock);
491			break;
492		}
493
494		mutex_exit(&softcall_lock);
495		func(arg);
496		mutex_enter(&softcall_lock);
497
498		/*
499		 * No longer need softcall processing from current
500		 * interrupt handler because either
501		 *  (a) softcall is in SOFT_IDLE state or
502		 *  (b) There is a CPU already draining softcall
503		 *	queue and the current softlevel1 is no
504		 *	longer required.
505		 */
506		if (softcall_latest_cpuid != cpu_id) {
507			if (CPU_IN_SET(*softcall_cpuset, cpu_id))
508				CPUSET_DEL(*softcall_cpuset, cpu_id);
509
510			mutex_exit(&softcall_lock);
511			break;
512		}
513	}
514
515out:
516	if ((func = kdi_softcall_func) != NULL) {
517		kdi_softcall_func = NULL;
518		func();
519	}
520}
521