fmn.c revision 208369
139215Sgibbs/*- 2107178Snjl * Copyright (c) 2003-2009 RMI Corporation 339215Sgibbs * All rights reserved. 4107178Snjl * 539215Sgibbs * Redistribution and use in source and binary forms, with or without 639215Sgibbs * modification, are permitted provided that the following conditions 739215Sgibbs * are met: 839215Sgibbs * 1. Redistributions of source code must retain the above copyright 939215Sgibbs * notice, this list of conditions and the following disclaimer. 1039215Sgibbs * 2. Redistributions in binary form must reproduce the above copyright 1139215Sgibbs * notice, this list of conditions and the following disclaimer in the 1239215Sgibbs * documentation and/or other materials provided with the distribution. 1339215Sgibbs * 3. Neither the name of RMI Corporation, nor the names of its contributors, 1439215Sgibbs * may be used to endorse or promote products derived from this software 1539215Sgibbs * without specific prior written permission. 1639215Sgibbs * 1739215Sgibbs * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1839215Sgibbs * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1939215Sgibbs * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2039215Sgibbs * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2139215Sgibbs * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2239215Sgibbs * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2339215Sgibbs * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2439215Sgibbs * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2539215Sgibbs * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2639215Sgibbs * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2739215Sgibbs * SUCH DAMAGE. 2850476Speter * 2939215Sgibbs * RMI_BSD */ 3039215Sgibbs#include <sys/cdefs.h> 3139215Sgibbs__FBSDID("$FreeBSD: head/sys/mips/rmi/on_chip.c 208369 2010-05-21 05:34:19Z jchandra $"); 32121184Ssimokawa#include <sys/types.h> 3344498Sgibbs#include <sys/systm.h> 34107178Snjl#include <sys/param.h> 3539215Sgibbs#include <sys/lock.h> 3644498Sgibbs#include <sys/mutex.h> 3739215Sgibbs#include <sys/proc.h> 3839215Sgibbs#include <sys/limits.h> 3939215Sgibbs#include <sys/bus.h> 40107178Snjl 4139215Sgibbs#include <sys/ktr.h> 4239215Sgibbs#include <sys/kernel.h> 43107178Snjl#include <sys/kthread.h> 44109161Snjl#include <sys/proc.h> 45107178Snjl#include <sys/resourcevar.h> 46107178Snjl#include <sys/sched.h> 47107178Snjl#include <sys/unistd.h> 48107178Snjl#include <sys/sysctl.h> 49120428Ssimokawa#include <sys/malloc.h> 50107178Snjl 5139215Sgibbs#include <machine/reg.h> 52107178Snjl#include <machine/cpu.h> 5339215Sgibbs#include <machine/hwfunc.h> 54107178Snjl#include <machine/mips_opcode.h> 5539215Sgibbs 56107178Snjl#include <machine/param.h> 57107178Snjl#include <machine/intr_machdep.h> 58107178Snjl#include <mips/rmi/interrupt.h> 59162704Smjacob#include <mips/rmi/msgring.h> 60107178Snjl#include <mips/rmi/iomap.h> 61107178Snjl#include <mips/rmi/debug.h> 62107178Snjl#include <mips/rmi/pic.h> 63107178Snjl#include <mips/rmi/board.h> 64107178Snjl 65162704Smjacobvoid 66121184Ssimokawadisable_msgring_int(void *arg); 67121184Ssimokawavoid 68107178Snjlenable_msgring_int(void *arg); 69107178Snjl 70107178Snjl/* definitions */ 71107178Snjlstruct tx_stn_handler { 72107178Snjl void (*action) (int, int, int, int, struct msgrng_msg *, void *); 73107178Snjl void *dev_id; 74107178Snjl}; 75107178Snjl 76107178Snjlstruct msgring_ithread { 77107178Snjl struct thread *i_thread; 7844498Sgibbs u_int i_pending; 7944498Sgibbs u_int i_flags; 8044498Sgibbs int i_cpu; 8144498Sgibbs int i_core; 8239215Sgibbs}; 83107178Snjl 84107178Snjlstruct msgring_ithread *msgring_ithreads[MAXCPU]; 85107178Snjl 86107178Snjl/* globals */ 87107178Snjlstatic struct tx_stn_handler tx_stn_handlers[MAX_TX_STNS]; 88107178Snjl 89107178Snjl#define MSGRNG_CC_INIT_CPU_DEST(dest, counter) \ 90162704Smjacobdo { \ 91107178Snjl msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][0], 0 ); \ 92107178Snjl msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][1], 1 ); \ 93107178Snjl msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][2], 2 ); \ 94107178Snjl msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][3], 3 ); \ 95107178Snjl msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][4], 4 ); \ 96107178Snjl msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][5], 5 ); \ 97107178Snjl msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][6], 6 ); \ 98107178Snjl msgrng_write_cc(MSGRNG_CC_##dest##_REG, counter[dest][7], 7 ); \ 9939215Sgibbs} while(0) 10039215Sgibbs 10139215Sgibbs 10239215Sgibbs/* make this a read/write spinlock */ 103228481Sedstatic struct mtx msgrng_lock; 104228481Sedstatic int msgring_int_enabled; 105107178Snjlstruct mtx xlr_pic_lock; 106107178Snjl 10739215Sgibbsstatic int msgring_pop_num_buckets; 108107178Snjlstatic uint32_t msgring_pop_bucket_mask; 109107178Snjlstatic int msgring_int_type; 110107178Snjlstatic int msgring_watermark_count; 111107178Snjlstatic uint32_t msgring_thread_mask; 112107178Snjl 113107178Snjluint32_t msgrng_msg_cycles = 0; 114107178Snjl 115107178Snjlvoid xlr_msgring_handler(struct trapframe *); 116107178Snjl 117107178Snjlvoid 118107178Snjlxlr_msgring_cpu_init(void) 119107178Snjl{ 120107178Snjl struct stn_cc *cc_config; 121162704Smjacob struct bucket_size *bucket_sizes; 12239215Sgibbs int id; 123107178Snjl unsigned long flags; 124107178Snjl 12539215Sgibbs KASSERT(xlr_thr_id() == 0, 126107178Snjl ("xlr_msgring_cpu_init from non-zero thread\n")); 127107178Snjl 12839215Sgibbs id = xlr_core_id(); 129107178Snjl 130107178Snjl bucket_sizes = xlr_board_info.bucket_sizes; 13144498Sgibbs cc_config = xlr_board_info.credit_configs[id]; 132107178Snjl 133107178Snjl msgrng_flags_save(flags); 13444498Sgibbs 135107178Snjl /* 136107178Snjl * Message Stations are shared among all threads in a cpu core 137107178Snjl * Assume, thread 0 on all cores are always active when more than 1 138107178Snjl * thread is active in a core 13944498Sgibbs */ 140107178Snjl msgrng_write_bucksize(0, bucket_sizes->bucket[id * 8 + 0]); 141107178Snjl msgrng_write_bucksize(1, bucket_sizes->bucket[id * 8 + 1]); 142107178Snjl msgrng_write_bucksize(2, bucket_sizes->bucket[id * 8 + 2]); 143107178Snjl msgrng_write_bucksize(3, bucket_sizes->bucket[id * 8 + 3]); 14463185Smjacob msgrng_write_bucksize(4, bucket_sizes->bucket[id * 8 + 4]); 145107178Snjl msgrng_write_bucksize(5, bucket_sizes->bucket[id * 8 + 5]); 146121184Ssimokawa msgrng_write_bucksize(6, bucket_sizes->bucket[id * 8 + 6]); 147121184Ssimokawa msgrng_write_bucksize(7, bucket_sizes->bucket[id * 8 + 7]); 148121184Ssimokawa 149121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(0, cc_config->counters); 150121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(1, cc_config->counters); 151121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(2, cc_config->counters); 152121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(3, cc_config->counters); 153121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(4, cc_config->counters); 154121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(5, cc_config->counters); 155121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(6, cc_config->counters); 156121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(7, cc_config->counters); 157121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(8, cc_config->counters); 158121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(9, cc_config->counters); 159121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(10, cc_config->counters); 160121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(11, cc_config->counters); 161121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(12, cc_config->counters); 162121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(13, cc_config->counters); 163121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(14, cc_config->counters); 164121184Ssimokawa MSGRNG_CC_INIT_CPU_DEST(15, cc_config->counters); 165121184Ssimokawa 166121184Ssimokawa msgrng_flags_restore(flags); 167121184Ssimokawa} 168121184Ssimokawa 169121184Ssimokawavoid 170121184Ssimokawaxlr_msgring_config(void) 171121184Ssimokawa{ 172121184Ssimokawa msgring_int_type = 0x02; 173107178Snjl msgring_pop_num_buckets = 8; 174121184Ssimokawa msgring_pop_bucket_mask = 0xff; 175107178Snjl 176107178Snjl msgring_watermark_count = 1; 177107178Snjl msgring_thread_mask = 0x01; 178121184Ssimokawa} 179107178Snjl 180107178Snjlvoid 181107178Snjlxlr_msgring_handler(struct trapframe *tf) 182107178Snjl{ 183107178Snjl unsigned long mflags; 184107178Snjl int bucket = 0; 185107178Snjl int size = 0, code = 0, rx_stid = 0, tx_stid = 0; 186107178Snjl struct msgrng_msg msg; 187107178Snjl unsigned int bucket_empty_bm = 0; 188107178Snjl unsigned int status = 0; 189107178Snjl 190107178Snjl /* TODO: not necessary to disable preemption */ 191107178Snjl msgrng_flags_save(mflags); 192107178Snjl 193107178Snjl /* First Drain all the high priority messages */ 194107178Snjl for (;;) { 195107178Snjl bucket_empty_bm = (msgrng_read_status() >> 24) & msgring_pop_bucket_mask; 196107178Snjl 197162704Smjacob /* all buckets empty, break */ 198162704Smjacob if (bucket_empty_bm == msgring_pop_bucket_mask) 199162704Smjacob break; 20039215Sgibbs 20139215Sgibbs for (bucket = 0; bucket < msgring_pop_num_buckets; bucket++) { 20239215Sgibbs if ((bucket_empty_bm & (1 << bucket)) /* empty */ ) 20339215Sgibbs continue; 20439215Sgibbs 20539215Sgibbs status = message_receive(bucket, &size, &code, &rx_stid, &msg); 20639215Sgibbs if (status) 207107178Snjl continue; 208107178Snjl 20939215Sgibbs tx_stid = xlr_board_info.msgmap[rx_stid]; 21039215Sgibbs 211107178Snjl if (!tx_stn_handlers[tx_stid].action) { 212107178Snjl printf("[%s]: No Handler for message from stn_id=%d, bucket=%d, " 213107178Snjl "size=%d, msg0=%llx, dropping message\n", 214107178Snjl __FUNCTION__, tx_stid, bucket, size, msg.msg0); 215107178Snjl } else { 216107178Snjl //printf("[%s]: rx_stid = %d\n", __FUNCTION__, rx_stid); 217107178Snjl msgrng_flags_restore(mflags); 218107178Snjl (*tx_stn_handlers[tx_stid].action) (bucket, size, code, rx_stid, 21944498Sgibbs &msg, tx_stn_handlers[tx_stid].dev_id); 22044498Sgibbs msgrng_flags_save(mflags); 22144498Sgibbs } 222107178Snjl } 223107178Snjl } 224107178Snjl 22544498Sgibbs xlr_set_counter(MSGRNG_EXIT_STATUS, msgrng_read_status()); 226107178Snjl 227107178Snjl msgrng_flags_restore(mflags); 228107178Snjl} 229196955Ssbruno 23044498Sgibbsvoid 231107178Snjlenable_msgring_int(void *arg) 232107178Snjl{ 233107178Snjl unsigned long mflags = 0; 234107178Snjl 235107178Snjl msgrng_access_save(&msgrng_lock, mflags); 236107178Snjl /* enable the message ring interrupts */ 237120428Ssimokawa msgrng_write_config((msgring_watermark_count << 24) | (IRQ_MSGRING << 16) 238120428Ssimokawa | (msgring_thread_mask << 8) | msgring_int_type); 239120428Ssimokawa msgrng_access_restore(&msgrng_lock, mflags); 240120428Ssimokawa} 241120428Ssimokawa 242120428Ssimokawavoid 243120428Ssimokawadisable_msgring_int(void *arg) 244120428Ssimokawa{ 245120428Ssimokawa unsigned long mflags = 0; 246120428Ssimokawa uint32_t config; 247120428Ssimokawa 248120428Ssimokawa msgrng_access_save(&msgrng_lock, mflags); 249107178Snjl config = msgrng_read_config(); 250107178Snjl config &= ~0x3; 25144498Sgibbs msgrng_write_config(config); 252121184Ssimokawa msgrng_access_restore(&msgrng_lock, mflags); 253162704Smjacob} 254121184Ssimokawa 255121184Ssimokawastatic int 256107178Snjlmsgring_process_fast_intr(void *arg) 257107178Snjl{ 25844498Sgibbs int core = xlr_core_id(); 259162704Smjacob volatile struct msgring_ithread *it; 260109161Snjl struct thread *td; 261109161Snjl 262162704Smjacob /* wakeup an appropriate intr_thread for processing this interrupt */ 263109161Snjl it = (volatile struct msgring_ithread *)msgring_ithreads[core]; 264109161Snjl KASSERT(it != NULL, ("No interrupt thread on cpu %d", core)); 265109161Snjl td = it->i_thread; 266109161Snjl 267109161Snjl /* 268109161Snjl * Interrupt thread will enable the interrupts after processing all 269109161Snjl * messages 270109161Snjl */ 271109161Snjl disable_msgring_int(NULL); 272162704Smjacob atomic_store_rel_int(&it->i_pending, 1); 273162704Smjacob thread_lock(td); 274162704Smjacob if (TD_AWAITING_INTR(td)) { 275162704Smjacob TD_CLR_IWAIT(td); 276162704Smjacob sched_add(td, SRQ_INTR); 277162704Smjacob } 278162704Smjacob thread_unlock(td); 279162704Smjacob return FILTER_HANDLED; 280109161Snjl} 281109161Snjl 282162704Smjacobstatic void 283109161Snjlmsgring_process(void *arg) 284109161Snjl{ 285109161Snjl volatile struct msgring_ithread *ithd; 286228481Sed struct thread *td; 287107178Snjl struct proc *p; 288228481Sed 289196955Ssbruno td = curthread; 290228481Sed p = td->td_proc; 29163185Smjacob ithd = (volatile struct msgring_ithread *)arg; 292107178Snjl KASSERT(ithd->i_thread == td, 293107178Snjl ("%s:msg_ithread and proc linkage out of sync", __func__)); 294107178Snjl 295107178Snjl /* First bind this thread to the right CPU */ 296107178Snjl thread_lock(td); 297107178Snjl 298107178Snjl sched_bind(td, ithd->i_cpu); 299107178Snjl thread_unlock(td); 30039215Sgibbs 301107178Snjl atomic_store_rel_ptr((volatile uintptr_t *)&msgring_ithreads[ithd->i_core], 302107178Snjl (uintptr_t)arg); 303107178Snjl enable_msgring_int(NULL); 304107178Snjl 305107178Snjl while (1) { 306107178Snjl while (ithd->i_pending) { 307107178Snjl /* 308107178Snjl * This might need a full read and write barrier to 309107178Snjl * make sure that this write posts before any of the 310107178Snjl * memory or device accesses in the handlers. 311162704Smjacob */ 31239215Sgibbs xlr_msgring_handler(NULL); 31339215Sgibbs atomic_store_rel_int(&ithd->i_pending, 0); 314107178Snjl enable_msgring_int(NULL); 315107178Snjl } 316107178Snjl if (!ithd->i_pending) { 317196955Ssbruno thread_lock(td); 318107178Snjl if (ithd->i_pending) { 319107178Snjl thread_unlock(td); 32044498Sgibbs continue; 321107178Snjl } 322107178Snjl sched_class(td, PRI_ITHD); 323107178Snjl TD_SET_IWAIT(td); 32449935Sgibbs mi_switch(SW_VOL, NULL); 325107178Snjl thread_unlock(td); 326107178Snjl } 327196955Ssbruno } 328107178Snjl 32939215Sgibbs} 330107178Snjl 33149935Sgibbsstatic void 33249935Sgibbscreate_msgring_thread(int core, int cpu) 33349935Sgibbs{ 33449935Sgibbs struct msgring_ithread *ithd; 33549935Sgibbs struct thread *td; 336107178Snjl struct proc *p; 337107178Snjl int error; 33863290Smjacob 339107178Snjl /* Create kernel thread for message ring interrupt processing */ 34063290Smjacob /* Currently create one task for thread 0 of each core */ 341107178Snjl ithd = malloc(sizeof(struct msgring_ithread), 34263290Smjacob M_DEVBUF, M_WAITOK | M_ZERO); 343107178Snjl error = kproc_create(msgring_process, (void *)ithd, &p, 344107178Snjl RFSTOPPED | RFHIGHPID, 2, "msg_intr%d", cpu); 345107178Snjl 346107178Snjl if (error) 347107178Snjl panic("kproc_create() failed with %d", error); 348107178Snjl td = FIRST_THREAD_IN_PROC(p); /* XXXKSE */ 34944498Sgibbs 350107178Snjl ithd->i_thread = td; 351107178Snjl ithd->i_pending = 0; 352107178Snjl ithd->i_cpu = cpu; 353107178Snjl ithd->i_core = core; 354107178Snjl 355107178Snjl thread_lock(td); 356107178Snjl sched_class(td, PRI_ITHD); 35739215Sgibbs sched_add(td, SRQ_INTR); 35839215Sgibbs thread_unlock(td); 359107178Snjl CTR2(KTR_INTR, "%s: created %s", __func__, ithd_name[core]); 360107178Snjl} 361107178Snjl 362107178Snjlint 363107178Snjlregister_msgring_handler(int major, 364107178Snjl void (*action) (int, int, int, int, struct msgrng_msg *, void *), 365107178Snjl void *dev_id) 366107178Snjl{ 367107178Snjl void *cookie; /* FIXME - use? */ 368107178Snjl 369107178Snjl if (major >= MAX_TX_STNS) 370107178Snjl return 1; 371107178Snjl 372107178Snjl //dbg_msg("major=%d, action=%p, dev_id=%p\n", major, action, dev_id); 373107178Snjl 374107178Snjl if (rmi_spin_mutex_safe) 375107178Snjl mtx_lock_spin(&msgrng_lock); 376107178Snjl tx_stn_handlers[major].action = action; 377107178Snjl tx_stn_handlers[major].dev_id = dev_id; 378107178Snjl if (rmi_spin_mutex_safe) 379107178Snjl mtx_unlock_spin(&msgrng_lock); 380107178Snjl 381107178Snjl if (xlr_test_and_set(&msgring_int_enabled)) { 382107178Snjl create_msgring_thread(0, 0); 383107178Snjl cpu_establish_hardintr("msgring", (driver_filter_t *) msgring_process_fast_intr, 384107178Snjl NULL, NULL, IRQ_MSGRING, 385107178Snjl INTR_TYPE_NET | INTR_FAST, &cookie); 386107178Snjl } 387107178Snjl return 0; 388107178Snjl} 389107178Snjl 390107178Snjlstatic void 391107178Snjlpic_init(void) 392107178Snjl{ 393107178Snjl xlr_reg_t *mmio = xlr_io_mmio(XLR_IO_PIC_OFFSET); 394107178Snjl int i = 0; 395107178Snjl int level; 396107178Snjl 39739215Sgibbs dbg_msg("Initializing PIC...\n"); 398107178Snjl for (i = 0; i < PIC_NUM_IRTS; i++) { 39939215Sgibbs 400107178Snjl level = PIC_IRQ_IS_EDGE_TRIGGERED(i); 401107178Snjl 402107178Snjl /* Bind all PIC irqs to cpu 0 */ 40339215Sgibbs xlr_write_reg(mmio, PIC_IRT_0_BASE + i, 0x01); 404107178Snjl 405107178Snjl /* 406107178Snjl * Use local scheduling and high polarity for all IRTs 40739215Sgibbs * Invalidate all IRTs, by default 408107178Snjl */ 409107178Snjl xlr_write_reg(mmio, PIC_IRT_1_BASE + i, (level << 30) | (1 << 6) | 410107178Snjl (PIC_IRQ_BASE + i)); 411107178Snjl } 412107178Snjl dbg_msg("PIC init now done\n"); 413107178Snjl} 414107178Snjl 415107178Snjlvoid 416107178Snjlon_chip_init(void) 417107178Snjl{ 418107178Snjl /* Set xlr_io_base to the run time value */ 419107178Snjl mtx_init(&msgrng_lock, "msgring", NULL, MTX_SPIN | MTX_RECURSE); 420107178Snjl mtx_init(&xlr_pic_lock, "pic", NULL, MTX_SPIN); 421107178Snjl 42244498Sgibbs xlr_board_info_setup(); 423162704Smjacob 424107178Snjl msgring_int_enabled = 0; 42539215Sgibbs 426107178Snjl xlr_msgring_config(); 427107178Snjl pic_init(); 428107178Snjl 429107178Snjl xlr_msgring_cpu_init(); 430107178Snjl} 431107178Snjl 43244498Sgibbsstatic void 433107178Snjlstart_msgring_threads(void *arg) 434107178Snjl{ 435107178Snjl int core, cpu; 436107178Snjl 437107178Snjl for (core = 1; core < XLR_MAX_CORES; core++) { 438107178Snjl if ((xlr_hw_thread_mask >> (4 * core)) & 0xf) { 43939215Sgibbs /* start one thread for an enabled core */ 44039215Sgibbs cpu = xlr_hwtid_to_cpuid[4 * core]; 441107178Snjl create_msgring_thread(core, cpu); 442162704Smjacob } 443107178Snjl } 444107178Snjl} 44539215Sgibbs 446107178SnjlSYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE, start_msgring_threads, NULL); 447107178Snjl