fmn.c revision 213443
1/*- 2 * Copyright (c) 2003-2009 RMI Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. Neither the name of RMI Corporation, nor the names of its contributors, 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * RMI_BSD */ 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD: head/sys/mips/rmi/fmn.c 213443 2010-10-05 06:44:47Z jchandra $"); 32#include <sys/types.h> 33#include <sys/systm.h> 34#include <sys/param.h> 35#include <sys/lock.h> 36#include <sys/mutex.h> 37#include <sys/proc.h> 38#include <sys/limits.h> 39#include <sys/bus.h> 40 41#include <sys/ktr.h> 42#include <sys/kernel.h> 43#include <sys/kthread.h> 44#include <sys/proc.h> 45#include <sys/resourcevar.h> 46#include <sys/sched.h> 47#include <sys/unistd.h> 48#include <sys/sysctl.h> 49#include <sys/malloc.h> 50 51#include <machine/reg.h> 52#include <machine/cpu.h> 53#include <machine/hwfunc.h> 54#include <machine/mips_opcode.h> 55 56#include <machine/param.h> 57#include <machine/intr_machdep.h> 58#include <mips/rmi/interrupt.h> 59#include <mips/rmi/msgring.h> 60#include <mips/rmi/pic.h> 61#include <mips/rmi/board.h> 62 63#define MSGRNG_CC_INIT_CPU_DEST(dest, counter) \ 64do { \ 65 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][0], 0 ); \ 66 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][1], 1 ); \ 67 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][2], 2 ); \ 68 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][3], 3 ); \ 69 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][4], 4 ); \ 70 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][5], 5 ); \ 71 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][6], 6 ); \ 72 msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][7], 7 ); \ 73} while(0) 74 75 76/* 77 * Keep track of our message ring handler threads, each core has a 78 * different message station. Ideally we will need to start a few 79 * message handling threads every core, and wake them up depending on 80 * load 81 */ 82struct msgring_thread { 83 struct { 84 struct thread *thread; /* msgring handler threads */ 85 int needed; /* thread needs to wake up */ 86 } threads[XLR_NTHREADS]; 87 int running; /* number of threads running */ 88 int nthreads; /* number of threads started */ 89 struct mtx lock; /* for changing running/active */ 90}; 91static struct msgring_thread msgring_threads[XLR_MAX_CORES]; 92static struct proc *msgring_proc; /* all threads are under a proc */ 93 94/* 95 * The maximum number of software message handler threads to be started 96 * per core. Default is 3 per core 97 */ 98static int msgring_maxthreads = 3; 99TUNABLE_INT("hw.fmn.maxthreads", &msgring_maxthreads); 100 101/* 102 * The device drivers can register a handler for the the messages sent 103 * from a station (corresponding to the device). 104 */ 105struct tx_stn_handler { 106 msgring_handler action; 107 void *arg; 108}; 109static struct tx_stn_handler msgmap[MSGRNG_NSTATIONS]; 110static struct mtx msgmap_lock; 111 112/* 113 * Initialize the messaging subsystem. 114 * 115 * Message Stations are shared among all threads in a cpu core, this 116 * has to be called once from every core which is online. 117 */ 118void 119xlr_msgring_cpu_init(void) 120{ 121 struct stn_cc *cc_config; 122 struct bucket_size *bucket_sizes; 123 uint32_t flags; 124 int id; 125 126 KASSERT(xlr_thr_id() == 0, 127 ("xlr_msgring_cpu_init from non-zero thread")); 128 id = xlr_core_id(); 129 bucket_sizes = xlr_board_info.bucket_sizes; 130 cc_config = xlr_board_info.credit_configs[id]; 131 132 flags = msgrng_access_enable(); 133 134 /* 135 * FMN messages are received in 8 buckets per core, set up 136 * the bucket sizes for each bucket 137 */ 138 msgrng_write_bucksize(0, bucket_sizes->bucket[id * 8 + 0]); 139 msgrng_write_bucksize(1, bucket_sizes->bucket[id * 8 + 1]); 140 msgrng_write_bucksize(2, bucket_sizes->bucket[id * 8 + 2]); 141 msgrng_write_bucksize(3, bucket_sizes->bucket[id * 8 + 3]); 142 msgrng_write_bucksize(4, bucket_sizes->bucket[id * 8 + 4]); 143 msgrng_write_bucksize(5, bucket_sizes->bucket[id * 8 + 5]); 144 msgrng_write_bucksize(6, bucket_sizes->bucket[id * 8 + 6]); 145 msgrng_write_bucksize(7, bucket_sizes->bucket[id * 8 + 7]); 146 147 /* 148 * For sending FMN messages, we need credits on the destination 149 * bucket. Program the credits this core has on the 128 possible 150 * destination buckets. 151 * We cannot use a loop here, because the the first argument has 152 * to be a constant integer value. 153 */ 154 MSGRNG_CC_INIT_CPU_DEST(0, cc_config->counters); 155 MSGRNG_CC_INIT_CPU_DEST(1, cc_config->counters); 156 MSGRNG_CC_INIT_CPU_DEST(2, cc_config->counters); 157 MSGRNG_CC_INIT_CPU_DEST(3, cc_config->counters); 158 MSGRNG_CC_INIT_CPU_DEST(4, cc_config->counters); 159 MSGRNG_CC_INIT_CPU_DEST(5, cc_config->counters); 160 MSGRNG_CC_INIT_CPU_DEST(6, cc_config->counters); 161 MSGRNG_CC_INIT_CPU_DEST(7, cc_config->counters); 162 MSGRNG_CC_INIT_CPU_DEST(8, cc_config->counters); 163 MSGRNG_CC_INIT_CPU_DEST(9, cc_config->counters); 164 MSGRNG_CC_INIT_CPU_DEST(10, cc_config->counters); 165 MSGRNG_CC_INIT_CPU_DEST(11, cc_config->counters); 166 MSGRNG_CC_INIT_CPU_DEST(12, cc_config->counters); 167 MSGRNG_CC_INIT_CPU_DEST(13, cc_config->counters); 168 MSGRNG_CC_INIT_CPU_DEST(14, cc_config->counters); 169 MSGRNG_CC_INIT_CPU_DEST(15, cc_config->counters); 170 msgrng_restore(flags); 171} 172 173/* 174 * Boot time init, called only once 175 */ 176void 177xlr_msgring_config(void) 178{ 179 mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN); 180 181 /* check value */ 182 if (msgring_maxthreads < 0 || msgring_maxthreads > XLR_NTHREADS) 183 msgring_maxthreads = XLR_NTHREADS; 184} 185 186/* 187 * Drain out max_messages for the buckets set in the bucket mask. 188 * Use max_messages = 0 to drain out all messages. 189 */ 190uint32_t 191xlr_msgring_handler(uint8_t bucket_mask, uint32_t max_messages) 192{ 193 int bucket = 0; 194 int size = 0, code = 0, rx_stid = 0; 195 struct msgrng_msg msg; 196 struct tx_stn_handler *he; 197 unsigned int status = 0; 198 unsigned long mflags; 199 uint32_t n_msgs; 200 uint32_t msgbuckets; 201 202 n_msgs = 0; 203 mflags = msgrng_access_enable(); 204 for (;;) { 205 msgbuckets = (~msgrng_read_status() >> 24) & bucket_mask; 206 207 /* all buckets empty, break */ 208 if (msgbuckets == 0) 209 break; 210 211 for (bucket = 0; bucket < 8; bucket++) { 212 if ((msgbuckets & (1 << bucket)) == 0) /* empty */ 213 continue; 214 215 status = message_receive(bucket, &size, &code, 216 &rx_stid, &msg); 217 if (status != 0) 218 continue; 219 n_msgs++; 220 he = &msgmap[rx_stid]; 221 if (he->action == NULL) { 222 printf("[%s]: No Handler for message from " 223 "stn_id=%d, bucket=%d, size=%d, msg0=%jx\n", 224 __func__, rx_stid, bucket, size, 225 (uintmax_t)msg.msg0); 226 } else { 227 msgrng_restore(mflags); 228 (*he->action)(bucket, size, code, rx_stid, 229 &msg, he->arg); 230 mflags = msgrng_access_enable(); 231 } 232 if (max_messages > 0 && n_msgs >= max_messages) 233 goto done; 234 } 235 } 236 237done: 238 msgrng_restore(mflags); 239 return (n_msgs); 240} 241 242/* 243 * XLR COP2 supports watermark interrupts based on the number of 244 * messages pending in all the buckets in the core. We increase 245 * the watermark until all the possible handler threads in the core 246 * are woken up. 247 */ 248static void 249msgrng_setconfig(int running, int nthr) 250{ 251 uint32_t config, mflags; 252 int watermark = 1; /* non zero needed */ 253 int wm_intr_value; 254 255 KASSERT(nthr >= 0 && nthr <= msgring_maxthreads, 256 ("Bad value of nthr %d", nthr)); 257 KASSERT(running <= nthr, ("Bad value of running %d", running)); 258 259 if (running == nthr) { 260 wm_intr_value = 0; 261 } else { 262 switch (running) { 263 case 0: break; /* keep default */ 264 case 1: 265 watermark = 32; break; 266 case 2: 267 watermark = 48; break; 268 case 3: 269 watermark = 56; break; 270 } 271 wm_intr_value = 0x2; /* set watermark enable interrupt */ 272 } 273 mflags = msgrng_access_enable(); 274 config = (watermark << 24) | (IRQ_MSGRING << 16) | (1 << 8) | 275 wm_intr_value; 276 /* clear pending interrupts, they will get re-raised if still valid */ 277 write_c0_eirr64(1ULL << IRQ_MSGRING); 278 msgrng_write_config(config); 279 msgrng_restore(mflags); 280} 281 282/* Debug counters */ 283static int msgring_nintr[XLR_MAX_CORES]; 284static int msgring_badintr[XLR_MAX_CORES]; 285static int msgring_wakeup_sleep[XLR_MAX_CORES * XLR_NTHREADS]; 286static int msgring_wakeup_nosleep[XLR_MAX_CORES * XLR_NTHREADS]; 287static int msgring_nmsgs[XLR_MAX_CORES * XLR_NTHREADS]; 288 289static int 290msgring_process_fast_intr(void *arg) 291{ 292 struct msgring_thread *mthd; 293 struct thread *td; 294 uint32_t mflags; 295 int core, nt; 296 297 core = xlr_core_id(); 298 mthd = &msgring_threads[core]; 299 msgring_nintr[core]++; 300 mtx_lock_spin(&mthd->lock); 301 nt = mthd->running; 302 if(nt >= mthd->nthreads) { 303 msgring_badintr[core]++; 304 mtx_unlock_spin(&mthd->lock); 305 return (FILTER_HANDLED); 306 } 307 308 td = mthd->threads[nt].thread; 309 mflags = msgrng_access_enable(); 310 311 /* default value with interrupts disabled */ 312 msgrng_write_config((1 << 24) | (IRQ_MSGRING << 16) | (1 << 8)); 313 /* clear pending interrupts */ 314 write_c0_eirr64(1ULL << IRQ_MSGRING); 315 msgrng_restore(mflags); 316 mtx_unlock_spin(&mthd->lock); 317 318 /* wake up the target thread */ 319 mthd->threads[nt].needed = 1; 320 thread_lock(td); 321 if (TD_AWAITING_INTR(td)) { 322 msgring_wakeup_sleep[core*4+nt]++; 323 TD_CLR_IWAIT(td); 324 sched_add(td, SRQ_INTR); 325 } else 326 msgring_wakeup_nosleep[core*4+nt]++; 327 thread_unlock(td); 328 return (FILTER_HANDLED); 329} 330 331static void 332msgring_process(void *arg) 333{ 334 struct msgring_thread *mthd; 335 struct thread *td; 336 int hwtid, tid, core; 337 int nmsgs; 338 339 hwtid = (intptr_t)arg; 340 core = hwtid / 4; 341 tid = hwtid % 4; 342 mthd = &msgring_threads[core]; 343 td = mthd->threads[tid].thread; 344 KASSERT(curthread == td, 345 ("Incorrect thread core %d, thread %d", core, hwtid)); 346 347 /* First bind this thread to the right CPU */ 348 thread_lock(td); 349 sched_bind(td, xlr_hwtid_to_cpuid[hwtid]); 350 thread_unlock(td); 351 352 mtx_lock_spin(&mthd->lock); 353 ++mthd->nthreads; /* Active thread count */ 354 mtx_unlock_spin(&mthd->lock); 355 356 /* start processing messages */ 357 for(;;) { 358 mtx_lock_spin(&mthd->lock); 359 ++mthd->running; 360 msgrng_setconfig(mthd->running, mthd->nthreads); 361 mtx_unlock_spin(&mthd->lock); 362 363 atomic_store_rel_int(&mthd->threads[tid].needed, 0); 364 nmsgs = xlr_msgring_handler(0xff, 0); 365 msgring_nmsgs[hwtid] += nmsgs; 366 367 mtx_lock_spin(&mthd->lock); 368 --mthd->running; 369 msgrng_setconfig(mthd->running, mthd->nthreads); 370 mtx_unlock_spin(&mthd->lock); 371 372 /* sleep */ 373 thread_lock(td); 374 if (mthd->threads[tid].needed) { 375 thread_unlock(td); 376 continue; 377 } 378 sched_class(td, PRI_ITHD); 379 TD_SET_IWAIT(td); 380 mi_switch(SW_VOL, NULL); 381 thread_unlock(td); 382 } 383} 384 385static void 386create_msgring_thread(int hwtid) 387{ 388 struct msgring_thread *mthd; 389 struct thread *td; 390 int tid, core; 391 int error; 392 393 core = hwtid / 4; 394 tid = hwtid % 4; 395 mthd = &msgring_threads[core]; 396 if (tid == 0) { 397 mtx_init(&mthd->lock, "msgrngcore", NULL, MTX_SPIN); 398 mthd->running = mthd->nthreads = 0; 399 } 400 error = kproc_kthread_add(msgring_process, (void *)hwtid, 401 &msgring_proc, &td, RFSTOPPED, 2, "msgrngproc", 402 "msgthr%d", hwtid); 403 if (error) 404 panic("kproc_kthread_add() failed with %d", error); 405 mthd->threads[tid].thread = td; 406 407 thread_lock(td); 408 sched_class(td, PRI_ITHD); 409 sched_add(td, SRQ_INTR); 410 thread_unlock(td); 411 CTR2(KTR_INTR, "%s: created %s", __func__, td->td_name); 412} 413 414int 415register_msgring_handler(int startb, int endb, msgring_handler action, 416 void *arg) 417{ 418 void *cookie; 419 int i; 420 static int msgring_int_enabled = 0; 421 422 KASSERT(startb >= 0 && startb <= endb && endb < MSGRNG_NSTATIONS, 423 ("Invalid value for for bucket range %d,%d", startb, endb)); 424 425 mtx_lock_spin(&msgmap_lock); 426 for (i = startb; i <= endb; i++) { 427 KASSERT(msgmap[i].action == NULL, 428 ("Bucket %d already used [action %p]", i, msgmap[i].action)); 429 msgmap[i].action = action; 430 msgmap[i].arg = arg; 431 } 432 mtx_unlock_spin(&msgmap_lock); 433 434 if (xlr_test_and_set(&msgring_int_enabled)) { 435 create_msgring_thread(0); 436 if (msgring_maxthreads > xlr_threads_per_core) 437 msgring_maxthreads = xlr_threads_per_core; 438 cpu_establish_hardintr("msgring", msgring_process_fast_intr, 439 NULL, NULL, IRQ_MSGRING, 440 INTR_TYPE_NET | INTR_FAST, &cookie); 441 } 442 return (0); 443} 444 445/* 446 * Start message ring processing threads on other CPUs, after SMP start 447 */ 448static void 449start_msgring_threads(void *arg) 450{ 451 int hwt, tid; 452 453 for (hwt = 1; hwt < XLR_MAX_CORES * XLR_NTHREADS; hwt++) { 454 if ((xlr_hw_thread_mask & (1 << hwt)) == 0) 455 continue; 456 tid = hwt % XLR_NTHREADS; 457 if (tid >= msgring_maxthreads) 458 continue; 459 create_msgring_thread(hwt); 460 } 461} 462 463SYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE, 464 start_msgring_threads, NULL); 465 466/* 467 * DEBUG support, XXX: static buffer, not locked 468 */ 469static int 470sys_print_debug(SYSCTL_HANDLER_ARGS) 471{ 472 int error, nb, i, fs; 473 static char xprintb[4096], *buf; 474 475 buf = xprintb; 476 fs = sizeof(xprintb); 477 nb = snprintf(buf, fs, 478 "\nID INTR ER WU-SLP WU-ERR MSGS\n"); 479 buf += nb; 480 fs -= nb; 481 for (i = 0; i < 32; i++) { 482 if ((xlr_hw_thread_mask & (1 << i)) == 0) 483 continue; 484 nb = snprintf(buf, fs, 485 "%2d: %8d %4d %8d %8d %8d\n", i, 486 msgring_nintr[i/4], msgring_badintr[i/4], 487 msgring_wakeup_sleep[i], msgring_wakeup_nosleep[i], 488 msgring_nmsgs[i]); 489 buf += nb; 490 fs -= nb; 491 } 492 error = SYSCTL_OUT(req, xprintb, buf - xprintb); 493 return (error); 494} 495 496SYSCTL_PROC(_debug, OID_AUTO, msgring, CTLTYPE_STRING | CTLFLAG_RD, 0, 0, 497 sys_print_debug, "A", "msgring debug info"); 498