1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright 2003-2011 Netlogic Microsystems (Netlogic). All rights
5 * reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are
9 * met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in
15 *    the documentation and/or other materials provided with the
16 *    distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY Netlogic Microsystems ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NETLOGIC OR CONTRIBUTORS BE
22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
28 * THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 * NETLOGIC_BSD */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD$");
34#include <sys/types.h>
35#include <sys/systm.h>
36#include <sys/param.h>
37#include <sys/lock.h>
38#include <sys/mutex.h>
39#include <sys/proc.h>
40#include <sys/limits.h>
41#include <sys/bus.h>
42#include <sys/sbuf.h>
43
44#include <sys/ktr.h>
45#include <sys/kernel.h>
46#include <sys/kthread.h>
47#include <sys/proc.h>
48#include <sys/resourcevar.h>
49#include <sys/sched.h>
50#include <sys/unistd.h>
51#include <sys/sysctl.h>
52#include <sys/malloc.h>
53
54#include <machine/reg.h>
55#include <machine/cpu.h>
56#include <machine/hwfunc.h>
57#include <machine/mips_opcode.h>
58#include <machine/intr_machdep.h>
59
60#include <mips/nlm/hal/mips-extns.h>
61#include <mips/nlm/hal/haldefs.h>
62#include <mips/nlm/hal/iomap.h>
63#include <mips/nlm/hal/cop2.h>
64#include <mips/nlm/hal/fmn.h>
65#include <mips/nlm/hal/pic.h>
66
67#include <mips/nlm/msgring.h>
68#include <mips/nlm/interrupt.h>
69#include <mips/nlm/xlp.h>
70
71#define	MSGRNG_NSTATIONS	1024
72/*
73 * Keep track of our message ring handler threads, each core has a
74 * different message station. Ideally we will need to start a few
75 * message handling threads every core, and wake them up depending on
76 * load
77 */
78struct msgring_thread {
79	struct thread	*thread;	/* msgring handler threads */
80	int	needed;			/* thread needs to wake up */
81};
82static struct msgring_thread msgring_threads[XLP_MAX_CORES * XLP_MAX_THREADS];
83static struct proc *msgring_proc;	/* all threads are under a proc */
84
85/*
86 * The device drivers can register a handler for the messages sent
87 * from a station (corresponding to the device).
88 */
89struct tx_stn_handler {
90	msgring_handler action;
91	void *arg;
92};
93static struct tx_stn_handler msgmap[MSGRNG_NSTATIONS];
94static struct mtx	msgmap_lock;
95uint32_t xlp_msg_thread_mask;
96static int xlp_msg_threads_per_core = XLP_MAX_THREADS;
97
98static void create_msgring_thread(int hwtid);
99static int msgring_process_fast_intr(void *arg);
100
101/* Debug counters */
102static int msgring_nintr[XLP_MAX_CORES * XLP_MAX_THREADS];
103static int msgring_wakeup_sleep[XLP_MAX_CORES * XLP_MAX_THREADS];
104static int msgring_wakeup_nosleep[XLP_MAX_CORES * XLP_MAX_THREADS];
105static int fmn_msgcount[XLP_MAX_CORES * XLP_MAX_THREADS][4];
106static int fmn_loops[XLP_MAX_CORES * XLP_MAX_THREADS];
107
108/* Whether polled driver implementation */
109static int polled = 0;
110
111/* We do only i/o device credit setup here. CPU credit setup is now
112 * moved to xlp_msgring_cpu_init() so that the credits get setup
113 * only if the CPU exists. xlp_msgring_cpu_init() gets called from
114 * platform_init_ap; and this makes it easy for us to setup CMS
115 * credits for various types of XLP chips, with varying number of
116 * cpu's and cores.
117 */
118static void
119xlp_cms_credit_setup(int credit)
120{
121	uint64_t cmspcibase, cmsbase, pcibase;
122	uint32_t devoffset;
123	int dev, fn, maxqid;
124	int src, qid, i;
125
126	for (i = 0; i < XLP_MAX_NODES; i++) {
127		cmspcibase = nlm_get_cms_pcibase(i);
128		if (!nlm_dev_exists(XLP_IO_CMS_OFFSET(i)))
129			continue;
130		cmsbase = nlm_get_cms_regbase(i);
131		maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0);
132		for (dev = 0; dev < 8; dev++) {
133			for (fn = 0; fn < 8; fn++) {
134				devoffset = XLP_HDR_OFFSET(i, 0, dev, fn);
135				if (nlm_dev_exists(devoffset) == 0)
136					continue;
137				pcibase = nlm_pcicfg_base(devoffset);
138				src = nlm_qidstart(pcibase);
139				if (src == 0)
140					continue;
141#if 0 /* Debug */
142				printf("Setup CMS credits for queues ");
143				printf("[%d to %d] from src %d\n", 0,
144				    maxqid, src);
145#endif
146				for (qid = 0; qid < maxqid; qid++)
147					nlm_cms_setup_credits(cmsbase, qid,
148					    src, credit);
149			}
150		}
151	}
152}
153
154void
155xlp_msgring_cpu_init(int node, int cpu, int credit)
156{
157	uint64_t cmspcibase = nlm_get_cms_pcibase(node);
158	uint64_t cmsbase = nlm_get_cms_regbase(node);
159	int qid, maxqid, src;
160
161	maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0);
162
163	/* cpu credit setup is done only from thread-0 of each core */
164	if((cpu % 4) == 0) {
165		src = cpu << 2; /* each thread has 4 vc's */
166		for (qid = 0; qid < maxqid; qid++)
167			nlm_cms_setup_credits(cmsbase, qid, src, credit);
168	}
169}
170
171/*
172 * Drain out max_messages for the buckets set in the bucket mask.
173 * Use max_msgs = 0 to drain out all messages.
174 */
175int
176xlp_handle_msg_vc(u_int vcmask, int max_msgs)
177{
178	struct nlm_fmn_msg msg;
179	int srcid = 0, size = 0, code = 0;
180	struct tx_stn_handler *he;
181	uint32_t mflags, status;
182	int n_msgs = 0, vc, m, hwtid;
183	u_int msgmask;
184
185	hwtid = nlm_cpuid();
186	for (;;) {
187		/* check if VC empty */
188		mflags = nlm_save_flags_cop2();
189		status = nlm_read_c2_msgstatus1();
190		nlm_restore_flags(mflags);
191
192		msgmask = ((status >> 24) & 0xf) ^ 0xf;
193		msgmask &= vcmask;
194		if (msgmask == 0)
195			    break;
196		m = 0;
197		for (vc = 0; vc < 4; vc++) {
198			if ((msgmask & (1 << vc)) == 0)
199				continue;
200
201			mflags = nlm_save_flags_cop2();
202			status = nlm_fmn_msgrcv(vc, &srcid, &size, &code,
203			    &msg);
204			nlm_restore_flags(mflags);
205			if (status != 0)	/*  no msg or error */
206				continue;
207			if (srcid < 0 || srcid >= 1024) {
208				printf("[%s]: bad src id %d\n", __func__,
209				    srcid);
210				continue;
211			}
212			he = &msgmap[srcid];
213			if(he->action != NULL)
214				(he->action)(vc, size, code, srcid, &msg,
215				he->arg);
216#if 0
217			else
218				printf("[%s]: No Handler for msg from stn %d,"
219				    " vc=%d, size=%d, msg0=%jx, droppinge\n",
220				    __func__, srcid, vc, size,
221				    (uintmax_t)msg.msg[0]);
222#endif
223			fmn_msgcount[hwtid][vc] += 1;
224			m++;	/* msgs handled in this iter */
225		}
226		if (m == 0)
227			break;	/* nothing done in this iter */
228		n_msgs += m;
229		if (max_msgs > 0 && n_msgs >= max_msgs)
230			break;
231	}
232
233	return (n_msgs);
234}
235
236static void
237xlp_discard_msg_vc(u_int vcmask)
238{
239	struct nlm_fmn_msg msg;
240	int srcid = 0, size = 0, code = 0, vc;
241	uint32_t mflags, status;
242
243	for (vc = 0; vc < 4; vc++) {
244		for (;;) {
245			mflags = nlm_save_flags_cop2();
246			status = nlm_fmn_msgrcv(vc, &srcid,
247			    &size, &code, &msg);
248			nlm_restore_flags(mflags);
249
250			/* break if there is no msg or error */
251			if (status != 0)
252				break;
253		}
254	}
255}
256
257void
258xlp_cms_enable_intr(int node, int cpu, int type, int watermark)
259{
260	uint64_t cmsbase;
261	int i, qid;
262
263	cmsbase = nlm_get_cms_regbase(node);
264
265	for (i = 0; i < 4; i++) {
266		qid = (i + (cpu * 4)) & 0x7f;
267		nlm_cms_per_queue_level_intr(cmsbase, qid, type, watermark);
268		nlm_cms_per_queue_timer_intr(cmsbase, qid, 0x1, 0);
269	}
270}
271
272static int
273msgring_process_fast_intr(void *arg)
274{
275	struct msgring_thread *mthd;
276	struct thread *td;
277	int	cpu;
278
279	cpu = nlm_cpuid();
280	mthd = &msgring_threads[cpu];
281	msgring_nintr[cpu]++;
282	td = mthd->thread;
283
284	/* clear pending interrupts */
285	nlm_write_c0_eirr(1ULL << IRQ_MSGRING);
286
287	/* wake up the target thread */
288	mthd->needed = 1;
289	thread_lock(td);
290	if (TD_AWAITING_INTR(td)) {
291		msgring_wakeup_sleep[cpu]++;
292		TD_CLR_IWAIT(td);
293		sched_add(td, SRQ_INTR);
294	} else
295		msgring_wakeup_nosleep[cpu]++;
296
297	thread_unlock(td);
298
299	return (FILTER_HANDLED);
300}
301
302static void
303msgring_process(void * arg)
304{
305	volatile struct msgring_thread *mthd;
306	struct thread *td;
307	uint32_t mflags, msgstatus1;
308	int hwtid, nmsgs;
309
310	hwtid = (intptr_t)arg;
311	mthd = &msgring_threads[hwtid];
312	td = mthd->thread;
313	KASSERT(curthread == td,
314	    ("%s:msg_ithread and proc linkage out of sync", __func__));
315
316	/* First bind this thread to the right CPU */
317	thread_lock(td);
318	sched_bind(td, xlp_hwtid_to_cpuid[hwtid]);
319	thread_unlock(td);
320
321	if (hwtid != nlm_cpuid())
322		printf("Misscheduled hwtid %d != cpuid %d\n", hwtid,
323		    nlm_cpuid());
324
325	xlp_discard_msg_vc(0xf);
326	xlp_msgring_cpu_init(nlm_nodeid(), nlm_cpuid(), CMS_DEFAULT_CREDIT);
327	if (polled == 0) {
328		mflags = nlm_save_flags_cop2();
329		nlm_fmn_cpu_init(IRQ_MSGRING, 0, 0, 0, 0, 0);
330		nlm_restore_flags(mflags);
331		xlp_cms_enable_intr(nlm_nodeid(), nlm_cpuid(), 0x2, 0);
332		/* clear pending interrupts.
333		 *  they will get re-raised if still valid */
334		nlm_write_c0_eirr(1ULL << IRQ_MSGRING);
335	}
336
337	/* start processing messages */
338	for (;;) {
339		atomic_store_rel_int(&mthd->needed, 0);
340		nmsgs = xlp_handle_msg_vc(0xf, 0);
341
342		/* sleep */
343		if (polled == 0) {
344			/* clear VC-pend bits */
345			mflags = nlm_save_flags_cop2();
346			msgstatus1 = nlm_read_c2_msgstatus1();
347			msgstatus1 |= (0xf << 16);
348			nlm_write_c2_msgstatus1(msgstatus1);
349			nlm_restore_flags(mflags);
350
351			thread_lock(td);
352			if (mthd->needed) {
353				thread_unlock(td);
354				continue;
355			}
356			sched_class(td, PRI_ITHD);
357			TD_SET_IWAIT(td);
358			mi_switch(SW_VOL, NULL);
359			thread_unlock(td);
360		} else
361			pause("wmsg", 1);
362
363		fmn_loops[hwtid]++;
364	}
365}
366
367static void
368create_msgring_thread(int hwtid)
369{
370	struct msgring_thread *mthd;
371	struct thread *td;
372	int	error;
373
374	mthd = &msgring_threads[hwtid];
375	error = kproc_kthread_add(msgring_process, (void *)(uintptr_t)hwtid,
376	    &msgring_proc, &td, RFSTOPPED, 2, "msgrngproc",
377	    "msgthr%d", hwtid);
378	if (error)
379		panic("kproc_kthread_add() failed with %d", error);
380	mthd->thread = td;
381
382	thread_lock(td);
383	sched_class(td, PRI_ITHD);
384	sched_add(td, SRQ_INTR);
385	thread_unlock(td);
386}
387
388int
389register_msgring_handler(int startb, int endb, msgring_handler action,
390    void *arg)
391{
392	int	i;
393
394	if (bootverbose)
395		printf("Register handler %d-%d %p(%p)\n",
396		    startb, endb, action, arg);
397	KASSERT(startb >= 0 && startb <= endb && endb < MSGRNG_NSTATIONS,
398	    ("Invalid value for bucket range %d,%d", startb, endb));
399
400	mtx_lock_spin(&msgmap_lock);
401	for (i = startb; i <= endb; i++) {
402		KASSERT(msgmap[i].action == NULL,
403		   ("Bucket %d already used [action %p]", i, msgmap[i].action));
404		msgmap[i].action = action;
405		msgmap[i].arg = arg;
406	}
407	mtx_unlock_spin(&msgmap_lock);
408	return (0);
409}
410
411/*
412 * Initialize the messaging subsystem.
413 *
414 * Message Stations are shared among all threads in a cpu core, this
415 * has to be called once from every core which is online.
416 */
417static void
418xlp_msgring_config(void *arg)
419{
420	void *cookie;
421	unsigned int thrmask, mask;
422	int i;
423
424	/* used polled handler for Ax silion */
425	if (nlm_is_xlp8xx_ax())
426		polled = 1;
427
428	/* Don't poll on all threads, if polled */
429	if (polled)
430		xlp_msg_threads_per_core -= 1;
431
432	mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN);
433	if (xlp_threads_per_core < xlp_msg_threads_per_core)
434		xlp_msg_threads_per_core = xlp_threads_per_core;
435	thrmask = ((1 << xlp_msg_threads_per_core) - 1);
436	mask = 0;
437	for (i = 0; i < XLP_MAX_CORES; i++) {
438		mask <<= XLP_MAX_THREADS;
439		mask |= thrmask;
440	}
441	xlp_msg_thread_mask = xlp_hw_thread_mask & mask;
442#if 0
443	printf("CMS Message handler thread mask %#jx\n",
444	    (uintmax_t)xlp_msg_thread_mask);
445#endif
446	xlp_cms_credit_setup(CMS_DEFAULT_CREDIT);
447	create_msgring_thread(0);
448	cpu_establish_hardintr("msgring", msgring_process_fast_intr, NULL,
449	    NULL, IRQ_MSGRING, INTR_TYPE_NET, &cookie);
450}
451
452/*
453 * Start message ring processing threads on other CPUs, after SMP start
454 */
455static void
456start_msgring_threads(void *arg)
457{
458	int	hwt;
459
460	for (hwt = 1; hwt < XLP_MAX_CORES * XLP_MAX_THREADS; hwt++) {
461		if ((xlp_msg_thread_mask & (1 << hwt)) == 0)
462			continue;
463		create_msgring_thread(hwt);
464	}
465}
466
467SYSINIT(xlp_msgring_config, SI_SUB_DRIVERS, SI_ORDER_FIRST,
468    xlp_msgring_config, NULL);
469SYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE,
470    start_msgring_threads, NULL);
471
472/*
473 * DEBUG support, XXX: static buffer, not locked
474 */
475static int
476sys_print_debug(SYSCTL_HANDLER_ARGS)
477{
478	struct sbuf sb;
479	int error, i;
480
481	sbuf_new_for_sysctl(&sb, NULL, 64, req);
482	sbuf_printf(&sb,
483	    "\nID     vc0       vc1       vc2     vc3     loops\n");
484	for (i = 0; i < 32; i++) {
485		if ((xlp_hw_thread_mask & (1 << i)) == 0)
486			continue;
487		sbuf_printf(&sb, "%2d: %8d %8d %8d %8d %8d\n", i,
488		    fmn_msgcount[i][0], fmn_msgcount[i][1],
489		    fmn_msgcount[i][2], fmn_msgcount[i][3],
490		    fmn_loops[i]);
491	}
492	error = sbuf_finish(&sb);
493	sbuf_delete(&sb);
494	return (error);
495}
496
497SYSCTL_PROC(_debug, OID_AUTO, msgring, CTLTYPE_STRING | CTLFLAG_RD, 0, 0,
498    sys_print_debug, "A", "msgring debug info");
499