1/*- 2 * Copyright 2003-2011 Netlogic Microsystems (Netlogic). All rights 3 * reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are 7 * met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in 13 * the documentation and/or other materials provided with the 14 * distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY Netlogic Microsystems ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NETLOGIC OR CONTRIBUTORS BE 20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 26 * THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * NETLOGIC_BSD */ 29 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD: releng/10.3/sys/mips/nlm/cms.c 233534 2012-03-27 07:47:13Z jchandra $"); 32#include <sys/types.h> 33#include <sys/systm.h> 34#include <sys/param.h> 35#include <sys/lock.h> 36#include <sys/mutex.h> 37#include <sys/proc.h> 38#include <sys/limits.h> 39#include <sys/bus.h> 40 41#include <sys/ktr.h> 42#include <sys/kernel.h> 43#include <sys/kthread.h> 44#include <sys/proc.h> 45#include <sys/resourcevar.h> 46#include <sys/sched.h> 47#include <sys/unistd.h> 48#include <sys/sysctl.h> 49#include <sys/malloc.h> 50 51#include <machine/reg.h> 52#include <machine/cpu.h> 53#include <machine/hwfunc.h> 54#include <machine/mips_opcode.h> 55#include <machine/param.h> 56#include <machine/intr_machdep.h> 57 58#include <mips/nlm/hal/mips-extns.h> 59#include <mips/nlm/hal/haldefs.h> 60#include <mips/nlm/hal/iomap.h> 61#include <mips/nlm/hal/cop2.h> 62#include <mips/nlm/hal/fmn.h> 63#include <mips/nlm/hal/pic.h> 64 65#include <mips/nlm/msgring.h> 66#include <mips/nlm/interrupt.h> 67#include <mips/nlm/xlp.h> 68 69#define MSGRNG_NSTATIONS 1024 70/* 71 * Keep track of our message ring handler threads, each core has a 72 * different message station. Ideally we will need to start a few 73 * message handling threads every core, and wake them up depending on 74 * load 75 */ 76struct msgring_thread { 77 struct thread *thread; /* msgring handler threads */ 78 int needed; /* thread needs to wake up */ 79}; 80static struct msgring_thread msgring_threads[XLP_MAX_CORES * XLP_MAX_THREADS]; 81static struct proc *msgring_proc; /* all threads are under a proc */ 82 83/* 84 * The device drivers can register a handler for the messages sent 85 * from a station (corresponding to the device). 86 */ 87struct tx_stn_handler { 88 msgring_handler action; 89 void *arg; 90}; 91static struct tx_stn_handler msgmap[MSGRNG_NSTATIONS]; 92static struct mtx msgmap_lock; 93uint32_t xlp_msg_thread_mask; 94static int xlp_msg_threads_per_core = XLP_MAX_THREADS; 95 96static void create_msgring_thread(int hwtid); 97static int msgring_process_fast_intr(void *arg); 98 99/* Debug counters */ 100static int msgring_nintr[XLP_MAX_CORES * XLP_MAX_THREADS]; 101static int msgring_wakeup_sleep[XLP_MAX_CORES * XLP_MAX_THREADS]; 102static int msgring_wakeup_nosleep[XLP_MAX_CORES * XLP_MAX_THREADS]; 103static int fmn_msgcount[XLP_MAX_CORES * XLP_MAX_THREADS][4]; 104static int fmn_loops[XLP_MAX_CORES * XLP_MAX_THREADS]; 105 106/* Whether polled driver implementation */ 107static int polled = 0; 108 109/* We do only i/o device credit setup here. CPU credit setup is now 110 * moved to xlp_msgring_cpu_init() so that the credits get setup 111 * only if the CPU exists. xlp_msgring_cpu_init() gets called from 112 * platform_init_ap; and this makes it easy for us to setup CMS 113 * credits for various types of XLP chips, with varying number of 114 * cpu's and cores. 115 */ 116static void 117xlp_cms_credit_setup(int credit) 118{ 119 uint64_t cmspcibase, cmsbase, pcibase; 120 uint32_t devoffset; 121 int dev, fn, maxqid; 122 int src, qid, i; 123 124 for (i = 0; i < XLP_MAX_NODES; i++) { 125 cmspcibase = nlm_get_cms_pcibase(i); 126 if (!nlm_dev_exists(XLP_IO_CMS_OFFSET(i))) 127 continue; 128 cmsbase = nlm_get_cms_regbase(i); 129 maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0); 130 for (dev = 0; dev < 8; dev++) { 131 for (fn = 0; fn < 8; fn++) { 132 devoffset = XLP_HDR_OFFSET(i, 0, dev, fn); 133 if (nlm_dev_exists(devoffset) == 0) 134 continue; 135 pcibase = nlm_pcicfg_base(devoffset); 136 src = nlm_qidstart(pcibase); 137 if (src == 0) 138 continue; 139#if 0 /* Debug */ 140 printf("Setup CMS credits for queues "); 141 printf("[%d to %d] from src %d\n", 0, 142 maxqid, src); 143#endif 144 for (qid = 0; qid < maxqid; qid++) 145 nlm_cms_setup_credits(cmsbase, qid, 146 src, credit); 147 } 148 } 149 } 150} 151 152void 153xlp_msgring_cpu_init(int node, int cpu, int credit) 154{ 155 uint64_t cmspcibase = nlm_get_cms_pcibase(node); 156 uint64_t cmsbase = nlm_get_cms_regbase(node); 157 int qid, maxqid, src; 158 159 maxqid = nlm_read_reg(cmspcibase, XLP_PCI_DEVINFO_REG0); 160 161 /* cpu credit setup is done only from thread-0 of each core */ 162 if((cpu % 4) == 0) { 163 src = cpu << 2; /* each thread has 4 vc's */ 164 for (qid = 0; qid < maxqid; qid++) 165 nlm_cms_setup_credits(cmsbase, qid, src, credit); 166 } 167} 168 169/* 170 * Drain out max_messages for the buckets set in the bucket mask. 171 * Use max_msgs = 0 to drain out all messages. 172 */ 173int 174xlp_handle_msg_vc(u_int vcmask, int max_msgs) 175{ 176 struct nlm_fmn_msg msg; 177 int srcid = 0, size = 0, code = 0; 178 struct tx_stn_handler *he; 179 uint32_t mflags, status; 180 int n_msgs = 0, vc, m, hwtid; 181 u_int msgmask; 182 183 hwtid = nlm_cpuid(); 184 for (;;) { 185 /* check if VC empty */ 186 mflags = nlm_save_flags_cop2(); 187 status = nlm_read_c2_msgstatus1(); 188 nlm_restore_flags(mflags); 189 190 msgmask = ((status >> 24) & 0xf) ^ 0xf; 191 msgmask &= vcmask; 192 if (msgmask == 0) 193 break; 194 m = 0; 195 for (vc = 0; vc < 4; vc++) { 196 if ((msgmask & (1 << vc)) == 0) 197 continue; 198 199 mflags = nlm_save_flags_cop2(); 200 status = nlm_fmn_msgrcv(vc, &srcid, &size, &code, 201 &msg); 202 nlm_restore_flags(mflags); 203 if (status != 0) /* no msg or error */ 204 continue; 205 if (srcid < 0 && srcid >= 1024) { 206 printf("[%s]: bad src id %d\n", __func__, 207 srcid); 208 continue; 209 } 210 he = &msgmap[srcid]; 211 if(he->action != NULL) 212 (he->action)(vc, size, code, srcid, &msg, 213 he->arg); 214#if 0 215 else 216 printf("[%s]: No Handler for msg from stn %d," 217 " vc=%d, size=%d, msg0=%jx, droppinge\n", 218 __func__, srcid, vc, size, 219 (uintmax_t)msg.msg[0]); 220#endif 221 fmn_msgcount[hwtid][vc] += 1; 222 m++; /* msgs handled in this iter */ 223 } 224 if (m == 0) 225 break; /* nothing done in this iter */ 226 n_msgs += m; 227 if (max_msgs > 0 && n_msgs >= max_msgs) 228 break; 229 } 230 231 return (n_msgs); 232} 233 234static void 235xlp_discard_msg_vc(u_int vcmask) 236{ 237 struct nlm_fmn_msg msg; 238 int srcid = 0, size = 0, code = 0, vc; 239 uint32_t mflags, status; 240 241 for (vc = 0; vc < 4; vc++) { 242 for (;;) { 243 mflags = nlm_save_flags_cop2(); 244 status = nlm_fmn_msgrcv(vc, &srcid, 245 &size, &code, &msg); 246 nlm_restore_flags(mflags); 247 248 /* break if there is no msg or error */ 249 if (status != 0) 250 break; 251 } 252 } 253} 254 255void 256xlp_cms_enable_intr(int node, int cpu, int type, int watermark) 257{ 258 uint64_t cmsbase; 259 int i, qid; 260 261 cmsbase = nlm_get_cms_regbase(node); 262 263 for (i = 0; i < 4; i++) { 264 qid = (i + (cpu * 4)) & 0x7f; 265 nlm_cms_per_queue_level_intr(cmsbase, qid, type, watermark); 266 nlm_cms_per_queue_timer_intr(cmsbase, qid, 0x1, 0); 267 } 268} 269 270static int 271msgring_process_fast_intr(void *arg) 272{ 273 struct msgring_thread *mthd; 274 struct thread *td; 275 int cpu; 276 277 cpu = nlm_cpuid(); 278 mthd = &msgring_threads[cpu]; 279 msgring_nintr[cpu]++; 280 td = mthd->thread; 281 282 /* clear pending interrupts */ 283 nlm_write_c0_eirr(1ULL << IRQ_MSGRING); 284 285 /* wake up the target thread */ 286 mthd->needed = 1; 287 thread_lock(td); 288 if (TD_AWAITING_INTR(td)) { 289 msgring_wakeup_sleep[cpu]++; 290 TD_CLR_IWAIT(td); 291 sched_add(td, SRQ_INTR); 292 } else 293 msgring_wakeup_nosleep[cpu]++; 294 295 thread_unlock(td); 296 297 return (FILTER_HANDLED); 298} 299 300static void 301msgring_process(void * arg) 302{ 303 volatile struct msgring_thread *mthd; 304 struct thread *td; 305 uint32_t mflags, msgstatus1; 306 int hwtid, nmsgs; 307 308 hwtid = (intptr_t)arg; 309 mthd = &msgring_threads[hwtid]; 310 td = mthd->thread; 311 KASSERT(curthread == td, 312 ("%s:msg_ithread and proc linkage out of sync", __func__)); 313 314 /* First bind this thread to the right CPU */ 315 thread_lock(td); 316 sched_bind(td, xlp_hwtid_to_cpuid[hwtid]); 317 thread_unlock(td); 318 319 if (hwtid != nlm_cpuid()) 320 printf("Misscheduled hwtid %d != cpuid %d\n", hwtid, 321 nlm_cpuid()); 322 323 xlp_discard_msg_vc(0xf); 324 xlp_msgring_cpu_init(nlm_nodeid(), nlm_cpuid(), CMS_DEFAULT_CREDIT); 325 if (polled == 0) { 326 mflags = nlm_save_flags_cop2(); 327 nlm_fmn_cpu_init(IRQ_MSGRING, 0, 0, 0, 0, 0); 328 nlm_restore_flags(mflags); 329 xlp_cms_enable_intr(nlm_nodeid(), nlm_cpuid(), 0x2, 0); 330 /* clear pending interrupts. 331 * they will get re-raised if still valid */ 332 nlm_write_c0_eirr(1ULL << IRQ_MSGRING); 333 } 334 335 /* start processing messages */ 336 for (;;) { 337 atomic_store_rel_int(&mthd->needed, 0); 338 nmsgs = xlp_handle_msg_vc(0xf, 0); 339 340 /* sleep */ 341 if (polled == 0) { 342 /* clear VC-pend bits */ 343 mflags = nlm_save_flags_cop2(); 344 msgstatus1 = nlm_read_c2_msgstatus1(); 345 msgstatus1 |= (0xf << 16); 346 nlm_write_c2_msgstatus1(msgstatus1); 347 nlm_restore_flags(mflags); 348 349 thread_lock(td); 350 if (mthd->needed) { 351 thread_unlock(td); 352 continue; 353 } 354 sched_class(td, PRI_ITHD); 355 TD_SET_IWAIT(td); 356 mi_switch(SW_VOL, NULL); 357 thread_unlock(td); 358 } else 359 pause("wmsg", 1); 360 361 fmn_loops[hwtid]++; 362 } 363} 364 365static void 366create_msgring_thread(int hwtid) 367{ 368 struct msgring_thread *mthd; 369 struct thread *td; 370 int error; 371 372 mthd = &msgring_threads[hwtid]; 373 error = kproc_kthread_add(msgring_process, (void *)(uintptr_t)hwtid, 374 &msgring_proc, &td, RFSTOPPED, 2, "msgrngproc", 375 "msgthr%d", hwtid); 376 if (error) 377 panic("kproc_kthread_add() failed with %d", error); 378 mthd->thread = td; 379 380 thread_lock(td); 381 sched_class(td, PRI_ITHD); 382 sched_add(td, SRQ_INTR); 383 thread_unlock(td); 384} 385 386int 387register_msgring_handler(int startb, int endb, msgring_handler action, 388 void *arg) 389{ 390 int i; 391 392 if (bootverbose) 393 printf("Register handler %d-%d %p(%p)\n", 394 startb, endb, action, arg); 395 KASSERT(startb >= 0 && startb <= endb && endb < MSGRNG_NSTATIONS, 396 ("Invalid value for bucket range %d,%d", startb, endb)); 397 398 mtx_lock_spin(&msgmap_lock); 399 for (i = startb; i <= endb; i++) { 400 KASSERT(msgmap[i].action == NULL, 401 ("Bucket %d already used [action %p]", i, msgmap[i].action)); 402 msgmap[i].action = action; 403 msgmap[i].arg = arg; 404 } 405 mtx_unlock_spin(&msgmap_lock); 406 return (0); 407} 408 409/* 410 * Initialize the messaging subsystem. 411 * 412 * Message Stations are shared among all threads in a cpu core, this 413 * has to be called once from every core which is online. 414 */ 415static void 416xlp_msgring_config(void *arg) 417{ 418 void *cookie; 419 unsigned int thrmask, mask; 420 int i; 421 422 /* used polled handler for Ax silion */ 423 if (nlm_is_xlp8xx_ax()) 424 polled = 1; 425 426 /* Don't poll on all threads, if polled */ 427 if (polled) 428 xlp_msg_threads_per_core -= 1; 429 430 mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN); 431 if (xlp_threads_per_core < xlp_msg_threads_per_core) 432 xlp_msg_threads_per_core = xlp_threads_per_core; 433 thrmask = ((1 << xlp_msg_threads_per_core) - 1); 434 mask = 0; 435 for (i = 0; i < XLP_MAX_CORES; i++) { 436 mask <<= XLP_MAX_THREADS; 437 mask |= thrmask; 438 } 439 xlp_msg_thread_mask = xlp_hw_thread_mask & mask; 440#if 0 441 printf("CMS Message handler thread mask %#jx\n", 442 (uintmax_t)xlp_msg_thread_mask); 443#endif 444 xlp_cms_credit_setup(CMS_DEFAULT_CREDIT); 445 create_msgring_thread(0); 446 cpu_establish_hardintr("msgring", msgring_process_fast_intr, NULL, 447 NULL, IRQ_MSGRING, INTR_TYPE_NET, &cookie); 448} 449 450/* 451 * Start message ring processing threads on other CPUs, after SMP start 452 */ 453static void 454start_msgring_threads(void *arg) 455{ 456 int hwt; 457 458 for (hwt = 1; hwt < XLP_MAX_CORES * XLP_MAX_THREADS; hwt++) { 459 if ((xlp_msg_thread_mask & (1 << hwt)) == 0) 460 continue; 461 create_msgring_thread(hwt); 462 } 463} 464 465SYSINIT(xlp_msgring_config, SI_SUB_DRIVERS, SI_ORDER_FIRST, 466 xlp_msgring_config, NULL); 467SYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE, 468 start_msgring_threads, NULL); 469 470/* 471 * DEBUG support, XXX: static buffer, not locked 472 */ 473static int 474sys_print_debug(SYSCTL_HANDLER_ARGS) 475{ 476 int error, nb, i, fs; 477 static char xprintb[4096], *buf; 478 479 buf = xprintb; 480 fs = sizeof(xprintb); 481 nb = snprintf(buf, fs, 482 "\nID vc0 vc1 vc2 vc3 loops\n"); 483 buf += nb; 484 fs -= nb; 485 for (i = 0; i < 32; i++) { 486 if ((xlp_hw_thread_mask & (1 << i)) == 0) 487 continue; 488 nb = snprintf(buf, fs, 489 "%2d: %8d %8d %8d %8d %8d\n", i, 490 fmn_msgcount[i][0], fmn_msgcount[i][1], 491 fmn_msgcount[i][2], fmn_msgcount[i][3], 492 fmn_loops[i]); 493 buf += nb; 494 fs -= nb; 495 } 496 error = SYSCTL_OUT(req, xprintb, buf - xprintb); 497 return (error); 498} 499 500SYSCTL_PROC(_debug, OID_AUTO, msgring, CTLTYPE_STRING | CTLFLAG_RD, 0, 0, 501 sys_print_debug, "A", "msgring debug info"); 502