fmn.c revision 280013
1211809Sjchandra/*- 2198160Srrs * Copyright (c) 2003-2009 RMI Corporation 3198160Srrs * All rights reserved. 4198160Srrs * 5198160Srrs * Redistribution and use in source and binary forms, with or without 6198160Srrs * modification, are permitted provided that the following conditions 7198160Srrs * are met: 8198160Srrs * 1. Redistributions of source code must retain the above copyright 9198160Srrs * notice, this list of conditions and the following disclaimer. 10198160Srrs * 2. Redistributions in binary form must reproduce the above copyright 11198160Srrs * notice, this list of conditions and the following disclaimer in the 12198160Srrs * documentation and/or other materials provided with the distribution. 13198160Srrs * 3. Neither the name of RMI Corporation, nor the names of its contributors, 14198160Srrs * may be used to endorse or promote products derived from this software 15198160Srrs * without specific prior written permission. 16198160Srrs * 17198160Srrs * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18198160Srrs * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19198160Srrs * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20198160Srrs * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21198160Srrs * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22198160Srrs * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23198160Srrs * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24198160Srrs * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25198160Srrs * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26198160Srrs * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27198160Srrs * SUCH DAMAGE. 28198160Srrs * 29198160Srrs * RMI_BSD */ 30203112Srrs#include <sys/cdefs.h> 31203112Srrs__FBSDID("$FreeBSD: head/sys/mips/rmi/fmn.c 280013 2015-03-14 23:30:03Z ian $"); 32198160Srrs#include <sys/types.h> 33198160Srrs#include <sys/systm.h> 34198160Srrs#include <sys/param.h> 35198160Srrs#include <sys/lock.h> 36198160Srrs#include <sys/mutex.h> 37198160Srrs#include <sys/proc.h> 38198160Srrs#include <sys/limits.h> 39198160Srrs#include <sys/bus.h> 40280013Sian#include <sys/sbuf.h> 41198160Srrs 42208165Srrs#include <sys/ktr.h> 43208165Srrs#include <sys/kernel.h> 44208165Srrs#include <sys/kthread.h> 45208165Srrs#include <sys/resourcevar.h> 46208165Srrs#include <sys/sched.h> 47208165Srrs#include <sys/unistd.h> 48208165Srrs#include <sys/sysctl.h> 49208165Srrs#include <sys/malloc.h> 50208165Srrs 51198160Srrs#include <machine/reg.h> 52198160Srrs#include <machine/cpu.h> 53208165Srrs#include <machine/hwfunc.h> 54198160Srrs#include <machine/mips_opcode.h> 55198160Srrs 56198160Srrs#include <machine/param.h> 57198160Srrs#include <machine/intr_machdep.h> 58198607Srrs#include <mips/rmi/interrupt.h> 59198607Srrs#include <mips/rmi/msgring.h> 60198607Srrs#include <mips/rmi/pic.h> 61198607Srrs#include <mips/rmi/board.h> 62198160Srrs 63198160Srrs#define MSGRNG_CC_INIT_CPU_DEST(dest, counter) \ 64198160Srrsdo { \ 65198160Srrs msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][0], 0 ); \ 66198160Srrs msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][1], 1 ); \ 67198160Srrs msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][2], 2 ); \ 68198160Srrs msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][3], 3 ); \ 69198160Srrs msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][4], 4 ); \ 70198160Srrs msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][5], 5 ); \ 71198160Srrs msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][6], 6 ); \ 72198160Srrs msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][7], 7 ); \ 73198160Srrs} while(0) 74198160Srrs 75198160Srrs 76213377Sjchandra/* 77213377Sjchandra * Keep track of our message ring handler threads, each core has a 78213377Sjchandra * different message station. Ideally we will need to start a few 79213377Sjchandra * message handling threads every core, and wake them up depending on 80213377Sjchandra * load 81213377Sjchandra */ 82213377Sjchandrastruct msgring_thread { 83213377Sjchandra struct { 84213377Sjchandra struct thread *thread; /* msgring handler threads */ 85213377Sjchandra int needed; /* thread needs to wake up */ 86213377Sjchandra } threads[XLR_NTHREADS]; 87213377Sjchandra int running; /* number of threads running */ 88213377Sjchandra int nthreads; /* number of threads started */ 89213377Sjchandra struct mtx lock; /* for changing running/active */ 90213377Sjchandra}; 91213377Sjchandrastatic struct msgring_thread msgring_threads[XLR_MAX_CORES]; 92213377Sjchandrastatic struct proc *msgring_proc; /* all threads are under a proc */ 93198160Srrs 94213377Sjchandra/* 95213377Sjchandra * The maximum number of software message handler threads to be started 96213443Sjchandra * per core. Default is 3 per core 97213377Sjchandra */ 98213443Sjchandrastatic int msgring_maxthreads = 3; 99213377SjchandraTUNABLE_INT("hw.fmn.maxthreads", &msgring_maxthreads); 100213377Sjchandra 101213377Sjchandra/* 102218909Sbrucec * The device drivers can register a handler for the messages sent 103213377Sjchandra * from a station (corresponding to the device). 104213377Sjchandra */ 105213377Sjchandrastruct tx_stn_handler { 106213377Sjchandra msgring_handler action; 107213377Sjchandra void *arg; 108213377Sjchandra}; 109213377Sjchandrastatic struct tx_stn_handler msgmap[MSGRNG_NSTATIONS]; 110213377Sjchandrastatic struct mtx msgmap_lock; 111213377Sjchandra 112213377Sjchandra/* 113213377Sjchandra * Initialize the messaging subsystem. 114213377Sjchandra * 115213377Sjchandra * Message Stations are shared among all threads in a cpu core, this 116213377Sjchandra * has to be called once from every core which is online. 117213377Sjchandra */ 118198625Srrsvoid 119198625Srrsxlr_msgring_cpu_init(void) 120198160Srrs{ 121198160Srrs struct stn_cc *cc_config; 122198160Srrs struct bucket_size *bucket_sizes; 123213377Sjchandra uint32_t flags; 124198160Srrs int id; 125198160Srrs 126208165Srrs KASSERT(xlr_thr_id() == 0, 127213377Sjchandra ("xlr_msgring_cpu_init from non-zero thread")); 128208165Srrs id = xlr_core_id(); 129198160Srrs bucket_sizes = xlr_board_info.bucket_sizes; 130198160Srrs cc_config = xlr_board_info.credit_configs[id]; 131198160Srrs 132213377Sjchandra flags = msgrng_access_enable(); 133198625Srrs 134198625Srrs /* 135213377Sjchandra * FMN messages are received in 8 buckets per core, set up 136213377Sjchandra * the bucket sizes for each bucket 137198160Srrs */ 138198625Srrs msgrng_write_bucksize(0, bucket_sizes->bucket[id * 8 + 0]); 139198625Srrs msgrng_write_bucksize(1, bucket_sizes->bucket[id * 8 + 1]); 140198625Srrs msgrng_write_bucksize(2, bucket_sizes->bucket[id * 8 + 2]); 141198625Srrs msgrng_write_bucksize(3, bucket_sizes->bucket[id * 8 + 3]); 142198625Srrs msgrng_write_bucksize(4, bucket_sizes->bucket[id * 8 + 4]); 143198625Srrs msgrng_write_bucksize(5, bucket_sizes->bucket[id * 8 + 5]); 144198625Srrs msgrng_write_bucksize(6, bucket_sizes->bucket[id * 8 + 6]); 145198625Srrs msgrng_write_bucksize(7, bucket_sizes->bucket[id * 8 + 7]); 146198160Srrs 147213377Sjchandra /* 148213377Sjchandra * For sending FMN messages, we need credits on the destination 149213377Sjchandra * bucket. Program the credits this core has on the 128 possible 150213377Sjchandra * destination buckets. 151218909Sbrucec * We cannot use a loop here, because the first argument has 152213443Sjchandra * to be a constant integer value. 153213377Sjchandra */ 154213377Sjchandra MSGRNG_CC_INIT_CPU_DEST(0, cc_config->counters); 155213377Sjchandra MSGRNG_CC_INIT_CPU_DEST(1, cc_config->counters); 156213377Sjchandra MSGRNG_CC_INIT_CPU_DEST(2, cc_config->counters); 157213377Sjchandra MSGRNG_CC_INIT_CPU_DEST(3, cc_config->counters); 158213377Sjchandra MSGRNG_CC_INIT_CPU_DEST(4, cc_config->counters); 159213377Sjchandra MSGRNG_CC_INIT_CPU_DEST(5, cc_config->counters); 160213377Sjchandra MSGRNG_CC_INIT_CPU_DEST(6, cc_config->counters); 161213377Sjchandra MSGRNG_CC_INIT_CPU_DEST(7, cc_config->counters); 162213377Sjchandra MSGRNG_CC_INIT_CPU_DEST(8, cc_config->counters); 163213377Sjchandra MSGRNG_CC_INIT_CPU_DEST(9, cc_config->counters); 164198160Srrs MSGRNG_CC_INIT_CPU_DEST(10, cc_config->counters); 165198160Srrs MSGRNG_CC_INIT_CPU_DEST(11, cc_config->counters); 166198160Srrs MSGRNG_CC_INIT_CPU_DEST(12, cc_config->counters); 167198160Srrs MSGRNG_CC_INIT_CPU_DEST(13, cc_config->counters); 168198160Srrs MSGRNG_CC_INIT_CPU_DEST(14, cc_config->counters); 169198160Srrs MSGRNG_CC_INIT_CPU_DEST(15, cc_config->counters); 170212321Sjchandra msgrng_restore(flags); 171198160Srrs} 172198160Srrs 173213377Sjchandra/* 174213377Sjchandra * Boot time init, called only once 175213377Sjchandra */ 176198625Srrsvoid 177198625Srrsxlr_msgring_config(void) 178198160Srrs{ 179213377Sjchandra mtx_init(&msgmap_lock, "msgring", NULL, MTX_SPIN); 180213377Sjchandra 181213377Sjchandra /* check value */ 182213377Sjchandra if (msgring_maxthreads < 0 || msgring_maxthreads > XLR_NTHREADS) 183213377Sjchandra msgring_maxthreads = XLR_NTHREADS; 184198160Srrs} 185198160Srrs 186212790Sjchandra/* 187212790Sjchandra * Drain out max_messages for the buckets set in the bucket mask. 188212790Sjchandra * Use max_messages = 0 to drain out all messages. 189212790Sjchandra */ 190212790Sjchandrauint32_t 191212790Sjchandraxlr_msgring_handler(uint8_t bucket_mask, uint32_t max_messages) 192198160Srrs{ 193198625Srrs int bucket = 0; 194213377Sjchandra int size = 0, code = 0, rx_stid = 0; 195198160Srrs struct msgrng_msg msg; 196213377Sjchandra struct tx_stn_handler *he; 197198625Srrs unsigned int status = 0; 198212790Sjchandra unsigned long mflags; 199213377Sjchandra uint32_t n_msgs; 200213377Sjchandra uint32_t msgbuckets; 201198160Srrs 202212790Sjchandra n_msgs = 0; 203212321Sjchandra mflags = msgrng_access_enable(); 204198625Srrs for (;;) { 205213377Sjchandra msgbuckets = (~msgrng_read_status() >> 24) & bucket_mask; 206198160Srrs 207198625Srrs /* all buckets empty, break */ 208213377Sjchandra if (msgbuckets == 0) 209198625Srrs break; 210198160Srrs 211213377Sjchandra for (bucket = 0; bucket < 8; bucket++) { 212213377Sjchandra if ((msgbuckets & (1 << bucket)) == 0) /* empty */ 213198625Srrs continue; 214198625Srrs 215213377Sjchandra status = message_receive(bucket, &size, &code, 216213377Sjchandra &rx_stid, &msg); 217213377Sjchandra if (status != 0) 218198625Srrs continue; 219212790Sjchandra n_msgs++; 220213377Sjchandra he = &msgmap[rx_stid]; 221213377Sjchandra if (he->action == NULL) { 222213377Sjchandra printf("[%s]: No Handler for message from " 223213377Sjchandra "stn_id=%d, bucket=%d, size=%d, msg0=%jx\n", 224213377Sjchandra __func__, rx_stid, bucket, size, 225213377Sjchandra (uintmax_t)msg.msg0); 226198625Srrs } else { 227212321Sjchandra msgrng_restore(mflags); 228213377Sjchandra (*he->action)(bucket, size, code, rx_stid, 229213377Sjchandra &msg, he->arg); 230212321Sjchandra mflags = msgrng_access_enable(); 231198625Srrs } 232212790Sjchandra if (max_messages > 0 && n_msgs >= max_messages) 233212790Sjchandra goto done; 234198160Srrs } 235198160Srrs } 236212790Sjchandra 237212790Sjchandradone: 238212321Sjchandra msgrng_restore(mflags); 239212790Sjchandra return (n_msgs); 240198160Srrs} 241198160Srrs 242213377Sjchandra/* 243213377Sjchandra * XLR COP2 supports watermark interrupts based on the number of 244213377Sjchandra * messages pending in all the buckets in the core. We increase 245213377Sjchandra * the watermark until all the possible handler threads in the core 246213377Sjchandra * are woken up. 247213377Sjchandra */ 248213377Sjchandrastatic void 249213377Sjchandramsgrng_setconfig(int running, int nthr) 250198160Srrs{ 251212321Sjchandra uint32_t config, mflags; 252213377Sjchandra int watermark = 1; /* non zero needed */ 253213377Sjchandra int wm_intr_value; 254198160Srrs 255213377Sjchandra KASSERT(nthr >= 0 && nthr <= msgring_maxthreads, 256213377Sjchandra ("Bad value of nthr %d", nthr)); 257213377Sjchandra KASSERT(running <= nthr, ("Bad value of running %d", running)); 258198160Srrs 259213377Sjchandra if (running == nthr) { 260213377Sjchandra wm_intr_value = 0; 261213377Sjchandra } else { 262213377Sjchandra switch (running) { 263213377Sjchandra case 0: break; /* keep default */ 264213377Sjchandra case 1: 265213443Sjchandra watermark = 32; break; 266213377Sjchandra case 2: 267213443Sjchandra watermark = 48; break; 268213377Sjchandra case 3: 269213443Sjchandra watermark = 56; break; 270213377Sjchandra } 271213377Sjchandra wm_intr_value = 0x2; /* set watermark enable interrupt */ 272213377Sjchandra } 273212321Sjchandra mflags = msgrng_access_enable(); 274213377Sjchandra config = (watermark << 24) | (IRQ_MSGRING << 16) | (1 << 8) | 275213377Sjchandra wm_intr_value; 276213443Sjchandra /* clear pending interrupts, they will get re-raised if still valid */ 277213377Sjchandra write_c0_eirr64(1ULL << IRQ_MSGRING); 278198160Srrs msgrng_write_config(config); 279212321Sjchandra msgrng_restore(mflags); 280198160Srrs} 281198160Srrs 282213443Sjchandra/* Debug counters */ 283213443Sjchandrastatic int msgring_nintr[XLR_MAX_CORES]; 284213443Sjchandrastatic int msgring_badintr[XLR_MAX_CORES]; 285213443Sjchandrastatic int msgring_wakeup_sleep[XLR_MAX_CORES * XLR_NTHREADS]; 286213443Sjchandrastatic int msgring_wakeup_nosleep[XLR_MAX_CORES * XLR_NTHREADS]; 287213443Sjchandrastatic int msgring_nmsgs[XLR_MAX_CORES * XLR_NTHREADS]; 288213443Sjchandra 289208165Srrsstatic int 290208165Srrsmsgring_process_fast_intr(void *arg) 291208165Srrs{ 292213377Sjchandra struct msgring_thread *mthd; 293213377Sjchandra struct thread *td; 294213377Sjchandra uint32_t mflags; 295213377Sjchandra int core, nt; 296198160Srrs 297213377Sjchandra core = xlr_core_id(); 298213377Sjchandra mthd = &msgring_threads[core]; 299213443Sjchandra msgring_nintr[core]++; 300213377Sjchandra mtx_lock_spin(&mthd->lock); 301213377Sjchandra nt = mthd->running; 302213377Sjchandra if(nt >= mthd->nthreads) { 303213443Sjchandra msgring_badintr[core]++; 304213377Sjchandra mtx_unlock_spin(&mthd->lock); 305213377Sjchandra return (FILTER_HANDLED); 306213377Sjchandra } 307213377Sjchandra 308213377Sjchandra td = mthd->threads[nt].thread; 309213377Sjchandra mflags = msgrng_access_enable(); 310213377Sjchandra 311213377Sjchandra /* default value with interrupts disabled */ 312213377Sjchandra msgrng_write_config((1 << 24) | (IRQ_MSGRING << 16) | (1 << 8)); 313213443Sjchandra /* clear pending interrupts */ 314213443Sjchandra write_c0_eirr64(1ULL << IRQ_MSGRING); 315213377Sjchandra msgrng_restore(mflags); 316213377Sjchandra mtx_unlock_spin(&mthd->lock); 317213377Sjchandra 318213377Sjchandra /* wake up the target thread */ 319213377Sjchandra mthd->threads[nt].needed = 1; 320208165Srrs thread_lock(td); 321208165Srrs if (TD_AWAITING_INTR(td)) { 322213443Sjchandra msgring_wakeup_sleep[core*4+nt]++; 323208165Srrs TD_CLR_IWAIT(td); 324208165Srrs sched_add(td, SRQ_INTR); 325213443Sjchandra } else 326213443Sjchandra msgring_wakeup_nosleep[core*4+nt]++; 327208165Srrs thread_unlock(td); 328213377Sjchandra return (FILTER_HANDLED); 329208165Srrs} 330208165Srrs 331208165Srrsstatic void 332208165Srrsmsgring_process(void *arg) 333208165Srrs{ 334213377Sjchandra struct msgring_thread *mthd; 335213377Sjchandra struct thread *td; 336213377Sjchandra int hwtid, tid, core; 337213377Sjchandra int nmsgs; 338208165Srrs 339213377Sjchandra hwtid = (intptr_t)arg; 340213377Sjchandra core = hwtid / 4; 341213377Sjchandra tid = hwtid % 4; 342213377Sjchandra mthd = &msgring_threads[core]; 343213377Sjchandra td = mthd->threads[tid].thread; 344213377Sjchandra KASSERT(curthread == td, 345213377Sjchandra ("Incorrect thread core %d, thread %d", core, hwtid)); 346208165Srrs 347208165Srrs /* First bind this thread to the right CPU */ 348208165Srrs thread_lock(td); 349213377Sjchandra sched_bind(td, xlr_hwtid_to_cpuid[hwtid]); 350208165Srrs thread_unlock(td); 351208165Srrs 352213377Sjchandra mtx_lock_spin(&mthd->lock); 353213443Sjchandra ++mthd->nthreads; /* Active thread count */ 354213377Sjchandra mtx_unlock_spin(&mthd->lock); 355213377Sjchandra 356213377Sjchandra /* start processing messages */ 357213377Sjchandra for(;;) { 358213377Sjchandra mtx_lock_spin(&mthd->lock); 359213377Sjchandra ++mthd->running; 360213377Sjchandra msgrng_setconfig(mthd->running, mthd->nthreads); 361213377Sjchandra mtx_unlock_spin(&mthd->lock); 362213377Sjchandra 363213377Sjchandra atomic_store_rel_int(&mthd->threads[tid].needed, 0); 364213377Sjchandra nmsgs = xlr_msgring_handler(0xff, 0); 365213443Sjchandra msgring_nmsgs[hwtid] += nmsgs; 366213377Sjchandra 367213377Sjchandra mtx_lock_spin(&mthd->lock); 368213377Sjchandra --mthd->running; 369213377Sjchandra msgrng_setconfig(mthd->running, mthd->nthreads); 370213377Sjchandra mtx_unlock_spin(&mthd->lock); 371213377Sjchandra 372213377Sjchandra /* sleep */ 373213377Sjchandra thread_lock(td); 374213377Sjchandra if (mthd->threads[tid].needed) { 375208165Srrs thread_unlock(td); 376213377Sjchandra continue; 377208165Srrs } 378213377Sjchandra sched_class(td, PRI_ITHD); 379213377Sjchandra TD_SET_IWAIT(td); 380213377Sjchandra mi_switch(SW_VOL, NULL); 381213377Sjchandra thread_unlock(td); 382208165Srrs } 383208165Srrs} 384208165Srrs 385208165Srrsstatic void 386213377Sjchandracreate_msgring_thread(int hwtid) 387208165Srrs{ 388213377Sjchandra struct msgring_thread *mthd; 389208165Srrs struct thread *td; 390213377Sjchandra int tid, core; 391213377Sjchandra int error; 392208165Srrs 393213377Sjchandra core = hwtid / 4; 394213377Sjchandra tid = hwtid % 4; 395213377Sjchandra mthd = &msgring_threads[core]; 396213377Sjchandra if (tid == 0) { 397213377Sjchandra mtx_init(&mthd->lock, "msgrngcore", NULL, MTX_SPIN); 398213377Sjchandra mthd->running = mthd->nthreads = 0; 399213377Sjchandra } 400213474Sjchandra error = kproc_kthread_add(msgring_process, (void *)(uintptr_t)hwtid, 401213377Sjchandra &msgring_proc, &td, RFSTOPPED, 2, "msgrngproc", 402213377Sjchandra "msgthr%d", hwtid); 403208165Srrs if (error) 404213377Sjchandra panic("kproc_kthread_add() failed with %d", error); 405213377Sjchandra mthd->threads[tid].thread = td; 406208165Srrs 407208165Srrs thread_lock(td); 408208165Srrs sched_class(td, PRI_ITHD); 409208165Srrs sched_add(td, SRQ_INTR); 410208165Srrs thread_unlock(td); 411210845Sjchandra CTR2(KTR_INTR, "%s: created %s", __func__, td->td_name); 412208165Srrs} 413208165Srrs 414198625Srrsint 415213377Sjchandraregister_msgring_handler(int startb, int endb, msgring_handler action, 416213377Sjchandra void *arg) 417198160Srrs{ 418213377Sjchandra void *cookie; 419213377Sjchandra int i; 420213377Sjchandra static int msgring_int_enabled = 0; 421198160Srrs 422213377Sjchandra KASSERT(startb >= 0 && startb <= endb && endb < MSGRNG_NSTATIONS, 423213377Sjchandra ("Invalid value for for bucket range %d,%d", startb, endb)); 424198160Srrs 425213377Sjchandra mtx_lock_spin(&msgmap_lock); 426213377Sjchandra for (i = startb; i <= endb; i++) { 427213377Sjchandra KASSERT(msgmap[i].action == NULL, 428213377Sjchandra ("Bucket %d already used [action %p]", i, msgmap[i].action)); 429213377Sjchandra msgmap[i].action = action; 430213377Sjchandra msgmap[i].arg = arg; 431213377Sjchandra } 432213377Sjchandra mtx_unlock_spin(&msgmap_lock); 433198625Srrs 434198160Srrs if (xlr_test_and_set(&msgring_int_enabled)) { 435213377Sjchandra create_msgring_thread(0); 436213377Sjchandra if (msgring_maxthreads > xlr_threads_per_core) 437213377Sjchandra msgring_maxthreads = xlr_threads_per_core; 438213377Sjchandra cpu_establish_hardintr("msgring", msgring_process_fast_intr, 439203112Srrs NULL, NULL, IRQ_MSGRING, 440217072Sjhb INTR_TYPE_NET, &cookie); 441198160Srrs } 442213377Sjchandra return (0); 443198160Srrs} 444198160Srrs 445213443Sjchandra/* 446213443Sjchandra * Start message ring processing threads on other CPUs, after SMP start 447213443Sjchandra */ 448208165Srrsstatic void 449208165Srrsstart_msgring_threads(void *arg) 450208165Srrs{ 451213377Sjchandra int hwt, tid; 452208165Srrs 453213377Sjchandra for (hwt = 1; hwt < XLR_MAX_CORES * XLR_NTHREADS; hwt++) { 454213377Sjchandra if ((xlr_hw_thread_mask & (1 << hwt)) == 0) 455213377Sjchandra continue; 456213377Sjchandra tid = hwt % XLR_NTHREADS; 457213377Sjchandra if (tid >= msgring_maxthreads) 458213377Sjchandra continue; 459213377Sjchandra create_msgring_thread(hwt); 460208369Sjchandra } 461198160Srrs} 462208165Srrs 463213443SjchandraSYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE, 464213443Sjchandra start_msgring_threads, NULL); 465213443Sjchandra 466213443Sjchandra/* 467213443Sjchandra * DEBUG support, XXX: static buffer, not locked 468213443Sjchandra */ 469213443Sjchandrastatic int 470213443Sjchandrasys_print_debug(SYSCTL_HANDLER_ARGS) 471213443Sjchandra{ 472280013Sian struct sbuf sb; 473280013Sian int error, i; 474213443Sjchandra 475280013Sian sbuf_new_for_sysctl(&sb, NULL, 64, req); 476280013Sian sbuf_printf(&sb, 477213443Sjchandra "\nID INTR ER WU-SLP WU-ERR MSGS\n"); 478213443Sjchandra for (i = 0; i < 32; i++) { 479213443Sjchandra if ((xlr_hw_thread_mask & (1 << i)) == 0) 480213443Sjchandra continue; 481280013Sian sbuf_printf(&sb, "%2d: %8d %4d %8d %8d %8d\n", i, 482213443Sjchandra msgring_nintr[i/4], msgring_badintr[i/4], 483213443Sjchandra msgring_wakeup_sleep[i], msgring_wakeup_nosleep[i], 484213443Sjchandra msgring_nmsgs[i]); 485213443Sjchandra } 486280013Sian error = sbuf_finish(&sb); 487280013Sian sbuf_delete(&sb); 488213443Sjchandra return (error); 489213443Sjchandra} 490213443Sjchandra 491213443SjchandraSYSCTL_PROC(_debug, OID_AUTO, msgring, CTLTYPE_STRING | CTLFLAG_RD, 0, 0, 492213443Sjchandra sys_print_debug, "A", "msgring debug info"); 493