1/*- 2 * Copyright (c) 1991 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * William Jolitz. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 4. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32/*- 33 * Copyright (c) 2001 Jake Burkholder. 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 * 57 * from: @(#)isa.c 7.2 (Berkeley) 5/13/91 58 * form: src/sys/i386/isa/intr_machdep.c,v 1.57 2001/07/20 59 */ 60 61#include <sys/cdefs.h> 62__FBSDID("$FreeBSD: stable/11/sys/sparc64/sparc64/intr_machdep.c 331722 2018-03-29 02:50:57Z eadler $"); 63 64#include <sys/param.h> 65#include <sys/systm.h> 66#include <sys/bus.h> 67#include <sys/errno.h> 68#include <sys/interrupt.h> 69#include <sys/kernel.h> 70#include <sys/lock.h> 71#include <sys/mutex.h> 72#include <sys/pcpu.h> 73#include <sys/proc.h> 74#include <sys/smp.h> 75#include <sys/sx.h> 76#include <sys/vmmeter.h> 77 78#include <machine/frame.h> 79#include <machine/intr_machdep.h> 80 81#define MAX_STRAY_LOG 5 82 83CTASSERT((1 << IV_SHIFT) == sizeof(struct intr_vector)); 84 85ih_func_t *intr_handlers[PIL_MAX]; 86uint16_t pil_countp[PIL_MAX]; 87static uint16_t pil_stray_count[PIL_MAX]; 88 89struct intr_vector intr_vectors[IV_MAX]; 90uint16_t intr_countp[IV_MAX]; 91static uint16_t intr_stray_count[IV_MAX]; 92 93static const char *const pil_names[] = { 94 "stray", 95 "low", /* PIL_LOW */ 96 "preempt", /* PIL_PREEMPT */ 97 "ithrd", /* PIL_ITHREAD */ 98 "rndzvs", /* PIL_RENDEZVOUS */ 99 "ast", /* PIL_AST */ 100 "hardclock", /* PIL_HARDCLOCK */ 101 "stray", "stray", "stray", "stray", 102 "filter", /* PIL_FILTER */ 103 "bridge", /* PIL_BRIDGE */ 104 "stop", /* PIL_STOP */ 105 "tick", /* PIL_TICK */ 106}; 107 108/* protect the intr_vectors table */ 109static struct sx intr_table_lock; 110/* protect intrcnt_index */ 111static struct mtx intrcnt_lock; 112 113#ifdef SMP 114static int assign_cpu; 115 116static void intr_assign_next_cpu(struct intr_vector *iv); 117static void intr_shuffle_irqs(void *arg __unused); 118#endif 119 120static int intr_assign_cpu(void *arg, int cpu); 121static void intr_execute_handlers(void *); 122static void intr_stray_level(struct trapframe *); 123static void intr_stray_vector(void *); 124static int intrcnt_setname(const char *, int); 125static void intrcnt_updatename(int, const char *, int); 126 127static void 128intrcnt_updatename(int vec, const char *name, int ispil) 129{ 130 static int intrcnt_index, stray_pil_index, stray_vec_index; 131 int name_index; 132 133 mtx_lock_spin(&intrcnt_lock); 134 if (intrnames[0] == '\0') { 135 /* for bitbucket */ 136 if (bootverbose) 137 printf("initalizing intr_countp\n"); 138 intrcnt_setname("???", intrcnt_index++); 139 140 stray_vec_index = intrcnt_index++; 141 intrcnt_setname("stray", stray_vec_index); 142 for (name_index = 0; name_index < IV_MAX; name_index++) 143 intr_countp[name_index] = stray_vec_index; 144 145 stray_pil_index = intrcnt_index++; 146 intrcnt_setname("pil", stray_pil_index); 147 for (name_index = 0; name_index < PIL_MAX; name_index++) 148 pil_countp[name_index] = stray_pil_index; 149 } 150 151 if (name == NULL) 152 name = "???"; 153 154 if (!ispil && intr_countp[vec] != stray_vec_index) 155 name_index = intr_countp[vec]; 156 else if (ispil && pil_countp[vec] != stray_pil_index) 157 name_index = pil_countp[vec]; 158 else 159 name_index = intrcnt_index++; 160 161 if (intrcnt_setname(name, name_index)) 162 name_index = 0; 163 164 if (!ispil) 165 intr_countp[vec] = name_index; 166 else 167 pil_countp[vec] = name_index; 168 mtx_unlock_spin(&intrcnt_lock); 169} 170 171static int 172intrcnt_setname(const char *name, int index) 173{ 174 175 if ((MAXCOMLEN + 1) * index >= sintrnames) 176 return (E2BIG); 177 snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s", 178 MAXCOMLEN, name); 179 return (0); 180} 181 182void 183intr_setup(int pri, ih_func_t *ihf, int vec, iv_func_t *ivf, void *iva) 184{ 185 char pilname[MAXCOMLEN + 1]; 186 register_t s; 187 188 s = intr_disable(); 189 if (vec != -1) { 190 intr_vectors[vec].iv_func = ivf; 191 intr_vectors[vec].iv_arg = iva; 192 intr_vectors[vec].iv_pri = pri; 193 intr_vectors[vec].iv_vec = vec; 194 } 195 intr_handlers[pri] = ihf; 196 intr_restore(s); 197 snprintf(pilname, MAXCOMLEN + 1, "pil%d: %s", pri, pil_names[pri]); 198 intrcnt_updatename(pri, pilname, 1); 199} 200 201static void 202intr_stray_level(struct trapframe *tf) 203{ 204 uint64_t level; 205 206 level = tf->tf_level; 207 if (pil_stray_count[level] < MAX_STRAY_LOG) { 208 printf("stray level interrupt %ld\n", level); 209 pil_stray_count[level]++; 210 if (pil_stray_count[level] >= MAX_STRAY_LOG) 211 printf("got %d stray level interrupt %ld's: not " 212 "logging anymore\n", MAX_STRAY_LOG, level); 213 } 214} 215 216static void 217intr_stray_vector(void *cookie) 218{ 219 struct intr_vector *iv; 220 u_int vec; 221 222 iv = cookie; 223 vec = iv->iv_vec; 224 if (intr_stray_count[vec] < MAX_STRAY_LOG) { 225 printf("stray vector interrupt %d\n", vec); 226 intr_stray_count[vec]++; 227 if (intr_stray_count[vec] >= MAX_STRAY_LOG) 228 printf("got %d stray vector interrupt %d's: not " 229 "logging anymore\n", MAX_STRAY_LOG, vec); 230 } 231} 232 233void 234intr_init1() 235{ 236 int i; 237 238 /* Mark all interrupts as being stray. */ 239 for (i = 0; i < PIL_MAX; i++) 240 intr_handlers[i] = intr_stray_level; 241 for (i = 0; i < IV_MAX; i++) { 242 intr_vectors[i].iv_func = intr_stray_vector; 243 intr_vectors[i].iv_arg = &intr_vectors[i]; 244 intr_vectors[i].iv_pri = PIL_LOW; 245 intr_vectors[i].iv_vec = i; 246 intr_vectors[i].iv_refcnt = 0; 247 } 248 intr_handlers[PIL_LOW] = intr_fast; 249} 250 251void 252intr_init2() 253{ 254 255 sx_init(&intr_table_lock, "intr sources"); 256 mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN); 257} 258 259static int 260intr_assign_cpu(void *arg, int cpu) 261{ 262#ifdef SMP 263 struct pcpu *pc; 264 struct intr_vector *iv; 265 266 /* 267 * Don't do anything during early boot. We will pick up the 268 * assignment once the APs are started. 269 */ 270 if (assign_cpu && cpu != NOCPU) { 271 pc = pcpu_find(cpu); 272 if (pc == NULL) 273 return (EINVAL); 274 iv = arg; 275 sx_xlock(&intr_table_lock); 276 iv->iv_mid = pc->pc_mid; 277 iv->iv_ic->ic_assign(iv); 278 sx_xunlock(&intr_table_lock); 279 } 280 return (0); 281#else 282 return (EOPNOTSUPP); 283#endif 284} 285 286static void 287intr_execute_handlers(void *cookie) 288{ 289 struct intr_vector *iv; 290 291 iv = cookie; 292 if (__predict_false(intr_event_handle(iv->iv_event, NULL) != 0)) 293 intr_stray_vector(iv); 294} 295 296int 297intr_controller_register(int vec, const struct intr_controller *ic, 298 void *icarg) 299{ 300 struct intr_event *ie; 301 struct intr_vector *iv; 302 int error; 303 304 if (vec < 0 || vec >= IV_MAX) 305 return (EINVAL); 306 sx_xlock(&intr_table_lock); 307 iv = &intr_vectors[vec]; 308 ie = iv->iv_event; 309 sx_xunlock(&intr_table_lock); 310 if (ie != NULL) 311 return (EEXIST); 312 error = intr_event_create(&ie, iv, 0, vec, NULL, ic->ic_clear, 313 ic->ic_clear, intr_assign_cpu, "vec%d:", vec); 314 if (error != 0) 315 return (error); 316 sx_xlock(&intr_table_lock); 317 if (iv->iv_event != NULL) { 318 sx_xunlock(&intr_table_lock); 319 intr_event_destroy(ie); 320 return (EEXIST); 321 } 322 iv->iv_ic = ic; 323 iv->iv_icarg = icarg; 324 iv->iv_event = ie; 325 iv->iv_mid = PCPU_GET(mid); 326 sx_xunlock(&intr_table_lock); 327 return (0); 328} 329 330int 331inthand_add(const char *name, int vec, driver_filter_t *filt, 332 driver_intr_t *handler, void *arg, int flags, void **cookiep) 333{ 334 const struct intr_controller *ic; 335 struct intr_event *ie; 336 struct intr_handler *ih; 337 struct intr_vector *iv; 338 int error, filter; 339 340 if (vec < 0 || vec >= IV_MAX) 341 return (EINVAL); 342 /* 343 * INTR_BRIDGE filters/handlers are special purpose only, allowing 344 * them to be shared just would complicate things unnecessarily. 345 */ 346 if ((flags & INTR_BRIDGE) != 0 && (flags & INTR_EXCL) == 0) 347 return (EINVAL); 348 sx_xlock(&intr_table_lock); 349 iv = &intr_vectors[vec]; 350 ic = iv->iv_ic; 351 ie = iv->iv_event; 352 sx_xunlock(&intr_table_lock); 353 if (ic == NULL || ie == NULL) 354 return (EINVAL); 355 error = intr_event_add_handler(ie, name, filt, handler, arg, 356 intr_priority(flags), flags, cookiep); 357 if (error != 0) 358 return (error); 359 sx_xlock(&intr_table_lock); 360 /* Disable the interrupt while we fiddle with it. */ 361 ic->ic_disable(iv); 362 iv->iv_refcnt++; 363 if (iv->iv_refcnt == 1) 364 intr_setup((flags & INTR_BRIDGE) != 0 ? PIL_BRIDGE : 365 filt != NULL ? PIL_FILTER : PIL_ITHREAD, intr_fast, 366 vec, intr_execute_handlers, iv); 367 else if (filt != NULL) { 368 /* 369 * Check if we need to upgrade from PIL_ITHREAD to PIL_FILTER. 370 * Given that apart from the on-board SCCs and UARTs shared 371 * interrupts are rather uncommon on sparc64 this should be 372 * pretty rare in practice. 373 */ 374 filter = 0; 375 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 376 if (ih->ih_filter != NULL && ih->ih_filter != filt) { 377 filter = 1; 378 break; 379 } 380 } 381 if (filter == 0) 382 intr_setup(PIL_FILTER, intr_fast, vec, 383 intr_execute_handlers, iv); 384 } 385 intr_stray_count[vec] = 0; 386 intrcnt_updatename(vec, ie->ie_fullname, 0); 387#ifdef SMP 388 if (assign_cpu) 389 intr_assign_next_cpu(iv); 390#endif 391 ic->ic_enable(iv); 392 /* Ensure the interrupt is cleared, it might have triggered before. */ 393 if (ic->ic_clear != NULL) 394 ic->ic_clear(iv); 395 sx_xunlock(&intr_table_lock); 396 return (0); 397} 398 399int 400inthand_remove(int vec, void *cookie) 401{ 402 struct intr_vector *iv; 403 int error; 404 405 if (vec < 0 || vec >= IV_MAX) 406 return (EINVAL); 407 error = intr_event_remove_handler(cookie); 408 if (error == 0) { 409 /* 410 * XXX: maybe this should be done regardless of whether 411 * intr_event_remove_handler() succeeded? 412 */ 413 sx_xlock(&intr_table_lock); 414 iv = &intr_vectors[vec]; 415 iv->iv_refcnt--; 416 if (iv->iv_refcnt == 0) { 417 /* 418 * Don't disable the interrupt for now, so that 419 * stray interrupts get detected... 420 */ 421 intr_setup(PIL_LOW, intr_fast, vec, 422 intr_stray_vector, iv); 423 } 424 sx_xunlock(&intr_table_lock); 425 } 426 return (error); 427} 428 429/* Add a description to an active interrupt handler. */ 430int 431intr_describe(int vec, void *ih, const char *descr) 432{ 433 struct intr_vector *iv; 434 int error; 435 436 if (vec < 0 || vec >= IV_MAX) 437 return (EINVAL); 438 sx_xlock(&intr_table_lock); 439 iv = &intr_vectors[vec]; 440 if (iv == NULL) { 441 sx_xunlock(&intr_table_lock); 442 return (EINVAL); 443 } 444 error = intr_event_describe_handler(iv->iv_event, ih, descr); 445 if (error) { 446 sx_xunlock(&intr_table_lock); 447 return (error); 448 } 449 intrcnt_updatename(vec, iv->iv_event->ie_fullname, 0); 450 sx_xunlock(&intr_table_lock); 451 return (error); 452} 453 454#ifdef SMP 455/* 456 * Support for balancing interrupt sources across CPUs. For now we just 457 * allocate CPUs round-robin. 458 */ 459 460static cpuset_t intr_cpus = CPUSET_T_INITIALIZER(0x1); 461static int current_cpu; 462 463static void 464intr_assign_next_cpu(struct intr_vector *iv) 465{ 466 struct pcpu *pc; 467 468 sx_assert(&intr_table_lock, SA_XLOCKED); 469 470 /* 471 * Assign this source to a CPU in a round-robin fashion. 472 */ 473 pc = pcpu_find(current_cpu); 474 if (pc == NULL) 475 return; 476 iv->iv_mid = pc->pc_mid; 477 iv->iv_ic->ic_assign(iv); 478 do { 479 current_cpu++; 480 if (current_cpu > mp_maxid) 481 current_cpu = 0; 482 } while (!CPU_ISSET(current_cpu, &intr_cpus)); 483} 484 485/* Attempt to bind the specified IRQ to the specified CPU. */ 486int 487intr_bind(int vec, u_char cpu) 488{ 489 struct intr_vector *iv; 490 int error; 491 492 if (vec < 0 || vec >= IV_MAX) 493 return (EINVAL); 494 sx_xlock(&intr_table_lock); 495 iv = &intr_vectors[vec]; 496 if (iv == NULL) { 497 sx_xunlock(&intr_table_lock); 498 return (EINVAL); 499 } 500 error = intr_event_bind(iv->iv_event, cpu); 501 sx_xunlock(&intr_table_lock); 502 return (error); 503} 504 505/* 506 * Add a CPU to our mask of valid CPUs that can be destinations of 507 * interrupts. 508 */ 509void 510intr_add_cpu(u_int cpu) 511{ 512 513 if (cpu >= MAXCPU) 514 panic("%s: Invalid CPU ID", __func__); 515 if (bootverbose) 516 printf("INTR: Adding CPU %d as a target\n", cpu); 517 518 CPU_SET(cpu, &intr_cpus); 519} 520 521/* 522 * Distribute all the interrupt sources among the available CPUs once the 523 * APs have been launched. 524 */ 525static void 526intr_shuffle_irqs(void *arg __unused) 527{ 528 struct pcpu *pc; 529 struct intr_vector *iv; 530 int i; 531 532 /* Don't bother on UP. */ 533 if (mp_ncpus == 1) 534 return; 535 536 sx_xlock(&intr_table_lock); 537 assign_cpu = 1; 538 for (i = 0; i < IV_MAX; i++) { 539 iv = &intr_vectors[i]; 540 if (iv != NULL && iv->iv_refcnt > 0) { 541 /* 542 * If this event is already bound to a CPU, 543 * then assign the source to that CPU instead 544 * of picking one via round-robin. 545 */ 546 if (iv->iv_event->ie_cpu != NOCPU && 547 (pc = pcpu_find(iv->iv_event->ie_cpu)) != NULL) { 548 iv->iv_mid = pc->pc_mid; 549 iv->iv_ic->ic_assign(iv); 550 } else 551 intr_assign_next_cpu(iv); 552 } 553 } 554 sx_xunlock(&intr_table_lock); 555} 556SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, 557 NULL); 558#endif 559