1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright 2003-2011 Netlogic Microsystems (Netlogic). All rights 5 * reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are 9 * met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY Netlogic Microsystems ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NETLOGIC OR CONTRIBUTORS BE 22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 28 * THE POSSIBILITY OF SUCH DAMAGE. 29 * 30 * NETLOGIC_BSD */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD$"); 34#include <sys/types.h> 35#include <sys/systm.h> 36#include <sys/param.h> 37#include <sys/lock.h> 38#include <sys/mutex.h> 39#include <sys/proc.h> 40#include <sys/limits.h> 41#include <sys/bus.h> 42#include <sys/sbuf.h> 43 44#include <sys/ktr.h> 45#include <sys/kernel.h> 46#include <sys/kthread.h> 47#include <sys/proc.h> 48#include <sys/resourcevar.h> 49#include <sys/sched.h> 50#include <sys/unistd.h> 51#include <sys/sysctl.h> 52#include <sys/malloc.h> 53 54#include <machine/reg.h> 55#include <machine/cpu.h> 56#include <machine/hwfunc.h> 57#include <machine/mips_opcode.h> 58#include <machine/intr_machdep.h> 59 60#include <mips/nlm/hal/mips-extns.h> 61#include <mips/nlm/hal/haldefs.h> 62#include <mips/nlm/hal/iomap.h> 63#include <mips/nlm/hal/cop2.h> 64#include <mips/nlm/hal/fmn.h> 65#include <mips/nlm/hal/pic.h> 66 67#include <mips/nlm/msgring.h> 68#include <mips/nlm/interrupt.h> 69#include <mips/nlm/xlp.h> 70 71#define MSGRNG_NSTATIONS 1024 72/* 73 * Keep track of our message ring handler threads, each core has a 74 * different message station. Ideally we will need to start a few 75 * message handling threads every core, and wake them up depending on 76 * load 77 */ 78struct msgring_thread { 79 struct thread *thread; /* msgring handler threads */ 80 int needed; /* thread needs to wake up */ 81}; 82static struct msgring_thread msgring_threads[XLP_MAX_CORES * XLP_MAX_THREADS]; 83static struct proc *msgring_proc; /* all threads are under a proc */ 84 85/* 86 * The device drivers can register a handler for the messages sent 87 * from a station (corresponding to the device). 88 */ 89struct tx_stn_handler { 90 msgring_handler action; 91 void *arg; 92}; 93static struct tx_stn_handler msgmap[MSGRNG_NSTATIONS]; 94static struct mtx msgmap_lock; 95uint32_t xlp_msg_thread_mask; 96static int xlp_msg_threads_per_core = XLP_MAX_THREADS; 97 98static void create_msgring_thread(int hwtid); 99static int msgring_process_fast_intr(void *arg); 100 101/* Debug counters */ 102static int msgring_nintr[XLP_MAX_CORES * XLP_MAX_THREADS]; 103static int msgring_wakeup_sleep[XLP_MAX_CORES * XLP_MAX_THREADS]; 104static int msgring_wakeup_nosleep[XLP_MAX_CORES * XLP_MAX_THREADS]; 105static int fmn_msgcount[XLP_MAX_CORES * XLP_MAX_THREADS][4]; 106static int fmn_loops[XLP_MAX_CORES * XLP_MAX_THREADS]; 107 108/* Whether polled driver implementation */ 109static int polled = 0; 110 111/* We do only i/o device credit setup here. CPU credit setup is now 112 * moved to xlp_msgring_cpu_init() so that the credits get setup 113 * only if the CPU exists. xlp_msgring_cpu_init() gets called from 114 * platform_init_ap; and this makes it easy for us to setup CMS 115 * credits for various types of XLP chips, with varying number of 116 * cpu's and cores. 117 */ 118static void 119xlp_cms_credit_setup(int credit) 120{ 121 uint64_t cmspcibase, cmsbase, pcibase; 122 uint32_t devoffset; 123 int dev, fn, maxqid; 124 int src, qid, i; 125 126 for (i = 0; i < XLP_MAX_NODES; i++) { 127 cmspcibase = nlm_get_cms_pcibase(i); 128 if (!nlm_dev_exists(XLP_IO_CMS_OFFSET(i))) 129 continue; 130 cmsbase = nlm_get_cms_regbase(i); 131 maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0); 132 for (dev = 0; dev < 8; dev++) { 133 for (fn = 0; fn < 8; fn++) { 134 devoffset = XLP_HDR_OFFSET(i, 0, dev, fn); 135 if (nlm_dev_exists(devoffset) == 0) 136 continue; 137 pcibase = nlm_pcicfg_base(devoffset); 138 src = nlm_qidstart(pcibase); 139 if (src == 0) 140 continue; 141#if 0 /* Debug */ 142 printf("Setup CMS credits for queues "); 143 printf("[%d to %d] from src %d\n", 0, 144 maxqid, src); 145#endif 146 for (qid = 0; qid < maxqid; qid++) 147 nlm_cms_setup_credits(cmsbase, qid, 148 src, credit); 149 } 150 } 151 } 152} 153 154void 155xlp_msgring_cpu_init(int node, int cpu, int credit) 156{ 157 uint64_t cmspcibase = nlm_get_cms_pcibase(node); 158 uint64_t cmsbase = nlm_get_cms_regbase(node); 159 int qid, maxqid, src; 160 161 maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0); 162 163 /* cpu credit setup is done only from thread-0 of each core */ 164 if((cpu % 4) == 0) { 165 src = cpu << 2; /* each thread has 4 vc's */ 166 for (qid = 0; qid < maxqid; qid++) 167 nlm_cms_setup_credits(cmsbase, qid, src, credit); 168 } 169} 170 171/* 172 * Drain out max_messages for the buckets set in the bucket mask. 173 * Use max_msgs = 0 to drain out all messages. 174 */ 175int 176xlp_handle_msg_vc(u_int vcmask, int max_msgs) 177{ 178 struct nlm_fmn_msg msg; 179 int srcid = 0, size = 0, code = 0; 180 struct tx_stn_handler *he; 181 uint32_t mflags, status; 182 int n_msgs = 0, vc, m, hwtid; 183 u_int msgmask; 184 185 hwtid = nlm_cpuid(); 186 for (;;) { 187 /* check if VC empty */ 188 mflags = nlm_save_flags_cop2(); 189 status = nlm_read_c2_msgstatus1(); 190 nlm_restore_flags(mflags); 191 192 msgmask = ((status >> 24) & 0xf) ^ 0xf; 193 msgmask &= vcmask; 194 if (msgmask == 0) 195 break; 196 m = 0; 197 for (vc = 0; vc < 4; vc++) { 198 if ((msgmask & (1 << vc)) == 0) 199 continue; 200 201 mflags = nlm_save_flags_cop2(); 202 status = nlm_fmn_msgrcv(vc, &srcid, &size, &code, 203 &msg); 204 nlm_restore_flags(mflags); 205 if (status != 0) /* no msg or error */ 206 continue; 207 if (srcid < 0 || srcid >= 1024) { 208 printf("[%s]: bad src id %d\n", __func__, 209 srcid); 210 continue; 211 } 212 he = &msgmap[srcid]; 213 if(he->action != NULL) 214 (he->action)(vc, size, code, srcid, &msg, 215 he->arg); 216#if 0 217 else 218 printf("[%s]: No Handler for msg from stn %d," 219 " vc=%d, size=%d, msg0=%jx, droppinge\n", 220 __func__, srcid, vc, size, 221 (uintmax_t)msg.msg[0]); 222#endif 223 fmn_msgcount[hwtid][vc] += 1; 224 m++; /* msgs handled in this iter */ 225 } 226 if (m == 0) 227 break; /* nothing done in this iter */ 228 n_msgs += m; 229 if (max_msgs > 0 && n_msgs >= max_msgs) 230 break; 231 } 232 233 return (n_msgs); 234} 235 236static void 237xlp_discard_msg_vc(u_int vcmask) 238{ 239 struct nlm_fmn_msg msg; 240 int srcid = 0, size = 0, code = 0, vc; 241 uint32_t mflags, status; 242 243 for (vc = 0; vc < 4; vc++) { 244 for (;;) { 245 mflags = nlm_save_flags_cop2(); 246 status = nlm_fmn_msgrcv(vc, &srcid, 247 &size, &code, &msg); 248 nlm_restore_flags(mflags); 249 250 /* break if there is no msg or error */ 251 if (status != 0) 252 break; 253 } 254 } 255} 256 257void 258xlp_cms_enable_intr(int node, int cpu, int type, int watermark) 259{ 260 uint64_t cmsbase; 261 int i, qid; 262 263 cmsbase = nlm_get_cms_regbase(node); 264 265 for (i = 0; i < 4; i++) { 266 qid = (i + (cpu * 4)) & 0x7f; 267 nlm_cms_per_queue_level_intr(cmsbase, qid, type, watermark); 268 nlm_cms_per_queue_timer_intr(cmsbase, qid, 0x1, 0); 269 } 270} 271 272static int 273msgring_process_fast_intr(void *arg) 274{ 275 struct msgring_thread *mthd; 276 struct thread *td; 277 int cpu; 278 279 cpu = nlm_cpuid(); 280 mthd = &msgring_threads[cpu]; 281 msgring_nintr[cpu]++; 282 td = mthd->thread; 283 284 /* clear pending interrupts */ 285 nlm_write_c0_eirr(1ULL << IRQ_MSGRING); 286 287 /* wake up the target thread */ 288 mthd->needed = 1; 289 thread_lock(td); 290 if (TD_AWAITING_INTR(td)) { 291 msgring_wakeup_sleep[cpu]++; 292 TD_CLR_IWAIT(td); 293 sched_add(td, SRQ_INTR); 294 } else { 295 thread_unlock(td); 296 msgring_wakeup_nosleep[cpu]++; 297 } 298 299 return (FILTER_HANDLED); 300} 301 302static void 303msgring_process(void * arg) 304{ 305 volatile struct msgring_thread *mthd; 306 struct thread *td; 307 uint32_t mflags, msgstatus1; 308 int hwtid, nmsgs; 309 310 hwtid = (intptr_t)arg; 311 mthd = &msgring_threads[hwtid]; 312 td = mthd->thread; 313 KASSERT(curthread == td, 314 ("%s:msg_ithread and proc linkage out of sync", __func__)); 315 316 /* First bind this thread to the right CPU */ 317 thread_lock(td); 318 sched_bind(td, xlp_hwtid_to_cpuid[hwtid]); 319 thread_unlock(td); 320 321 if (hwtid != nlm_cpuid()) 322 printf("Misscheduled hwtid %d != cpuid %d\n", hwtid, 323 nlm_cpuid()); 324 325 xlp_discard_msg_vc(0xf); 326 xlp_msgring_cpu_init(nlm_nodeid(), nlm_cpuid(), CMS_DEFAULT_CREDIT); 327 if (polled == 0) { 328 mflags = nlm_save_flags_cop2(); 329 nlm_fmn_cpu_init(IRQ_MSGRING, 0, 0, 0, 0, 0); 330 nlm_restore_flags(mflags); 331 xlp_cms_enable_intr(nlm_nodeid(), nlm_cpuid(), 0x2, 0); 332 /* clear pending interrupts. 333 * they will get re-raised if still valid */ 334 nlm_write_c0_eirr(1ULL << IRQ_MSGRING); 335 } 336 337 /* start processing messages */ 338 for (;;) { 339 atomic_store_rel_int(&mthd->needed, 0); 340 nmsgs = xlp_handle_msg_vc(0xf, 0); 341 342 /* sleep */ 343 if (polled == 0) { 344 /* clear VC-pend bits */ 345 mflags = nlm_save_flags_cop2(); 346 msgstatus1 = nlm_read_c2_msgstatus1(); 347 msgstatus1 |= (0xf << 16); 348 nlm_write_c2_msgstatus1(msgstatus1); 349 nlm_restore_flags(mflags); 350 351 thread_lock(td); 352 if (mthd->needed) { 353 thread_unlock(td); 354 continue; 355 } 356 sched_class(td, PRI_ITHD); 357 TD_SET_IWAIT(td); 358 mi_switch(SW_VOL); 359 } else 360 pause("wmsg", 1); 361 362 fmn_loops[hwtid]++; 363 } 364} 365 366static void 367create_msgring_thread(int hwtid) 368{ 369 struct msgring_thread *mthd; 370 struct thread *td; 371 int error; 372 373 mthd = &msgring_threads[hwtid]; 374 error = kproc_kthread_add(msgring_process, (void *)(uintptr_t)hwtid, 375 &msgring_proc, &td, RFSTOPPED, 2, "msgrngproc", 376 "msgthr%d", hwtid); 377 if (error) 378 panic("kproc_kthread_add() failed with %d", error); 379 mthd->thread = td; 380 381 thread_lock(td); 382 sched_class(td, PRI_ITHD); 383 sched_add(td, SRQ_INTR); 384} 385 386int 387register_msgring_handler(int startb, int endb, msgring_handler action, 388 void *arg) 389{ 390 int i; 391 392 if (bootverbose) 393 printf("Register handler %d-%d %p(%p)\n", 394 startb, endb, action, arg); 395 KASSERT(startb >= 0 && startb <= endb && endb < MSGRNG_NSTATIONS, 396 ("Invalid value for bucket range %d,%d", startb, endb)); 397 398 mtx_lock_spin(&msgmap_lock); 399 for (i = startb; i <= endb; i++) { 400 KASSERT(msgmap[i].action == NULL, 401 ("Bucket %d already used [action %p]", i, msgmap[i].action)); 402 msgmap[i].action = action; 403 msgmap[i].arg = arg; 404 } 405 mtx_unlock_spin(&msgmap_lock); 406 return (0); 407} 408 409/* 410 * Initialize the messaging subsystem. 411 * 412 * Message Stations are shared among all threads in a cpu core, this 413 * has to be called once from every core which is online. 414 */ 415static void 416xlp_msgring_config(void *arg) 417{ 418 void *cookie; 419 unsigned int thrmask, mask; 420 int i; 421 422 /* used polled handler for Ax silion */ 423 if (nlm_is_xlp8xx_ax()) 424 polled = 1; 425 426 /* Don't poll on all threads, if polled */ 427 if (polled) 428 xlp_msg_threads_per_core -= 1; 429 430 mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN); 431 if (xlp_threads_per_core < xlp_msg_threads_per_core) 432 xlp_msg_threads_per_core = xlp_threads_per_core; 433 thrmask = ((1 << xlp_msg_threads_per_core) - 1); 434 mask = 0; 435 for (i = 0; i < XLP_MAX_CORES; i++) { 436 mask <<= XLP_MAX_THREADS; 437 mask |= thrmask; 438 } 439 xlp_msg_thread_mask = xlp_hw_thread_mask & mask; 440#if 0 441 printf("CMS Message handler thread mask %#jx\n", 442 (uintmax_t)xlp_msg_thread_mask); 443#endif 444 xlp_cms_credit_setup(CMS_DEFAULT_CREDIT); 445 create_msgring_thread(0); 446 cpu_establish_hardintr("msgring", msgring_process_fast_intr, NULL, 447 NULL, IRQ_MSGRING, INTR_TYPE_NET, &cookie); 448} 449 450/* 451 * Start message ring processing threads on other CPUs, after SMP start 452 */ 453static void 454start_msgring_threads(void *arg) 455{ 456 int hwt; 457 458 for (hwt = 1; hwt < XLP_MAX_CORES * XLP_MAX_THREADS; hwt++) { 459 if ((xlp_msg_thread_mask & (1 << hwt)) == 0) 460 continue; 461 create_msgring_thread(hwt); 462 } 463} 464 465SYSINIT(xlp_msgring_config, SI_SUB_DRIVERS, SI_ORDER_FIRST, 466 xlp_msgring_config, NULL); 467SYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE, 468 start_msgring_threads, NULL); 469 470/* 471 * DEBUG support, XXX: static buffer, not locked 472 */ 473static int 474sys_print_debug(SYSCTL_HANDLER_ARGS) 475{ 476 struct sbuf sb; 477 int error, i; 478 479 sbuf_new_for_sysctl(&sb, NULL, 64, req); 480 sbuf_printf(&sb, 481 "\nID vc0 vc1 vc2 vc3 loops\n"); 482 for (i = 0; i < 32; i++) { 483 if ((xlp_hw_thread_mask & (1 << i)) == 0) 484 continue; 485 sbuf_printf(&sb, "%2d: %8d %8d %8d %8d %8d\n", i, 486 fmn_msgcount[i][0], fmn_msgcount[i][1], 487 fmn_msgcount[i][2], fmn_msgcount[i][3], 488 fmn_loops[i]); 489 } 490 error = sbuf_finish(&sb); 491 sbuf_delete(&sb); 492 return (error); 493} 494 495SYSCTL_PROC(_debug, OID_AUTO, msgring, 496 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, 0, 0, 497 sys_print_debug, "A", 498 "msgring debug info"); 499