1/*- 2 * Copyright (c) 1991 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * William Jolitz. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 4. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32/*- 33 * Copyright (c) 2001 Jake Burkholder. 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 * 57 * from: @(#)isa.c 7.2 (Berkeley) 5/13/91 58 * form: src/sys/i386/isa/intr_machdep.c,v 1.57 2001/07/20 59 */ 60 61#include <sys/cdefs.h> 62__FBSDID("$FreeBSD$"); 63 64#include <sys/param.h> 65#include <sys/systm.h> 66#include <sys/bus.h> 67#include <sys/errno.h> 68#include <sys/interrupt.h> 69#include <sys/kernel.h> 70#include <sys/lock.h> 71#include <sys/mutex.h> 72#include <sys/pcpu.h> 73#include <sys/proc.h> 74#include <sys/smp.h> 75#include <sys/sx.h> 76 77#include <machine/frame.h> 78#include <machine/intr_machdep.h> 79 80#define MAX_STRAY_LOG 5 81 82CTASSERT((1 << IV_SHIFT) == sizeof(struct intr_vector)); 83 84ih_func_t *intr_handlers[PIL_MAX]; 85uint16_t pil_countp[PIL_MAX]; 86static uint16_t pil_stray_count[PIL_MAX]; 87 88struct intr_vector intr_vectors[IV_MAX]; 89uint16_t intr_countp[IV_MAX]; 90static uint16_t intr_stray_count[IV_MAX]; 91 92static const char *const pil_names[] = { 93 "stray", 94 "low", /* PIL_LOW */ 95 "preempt", /* PIL_PREEMPT */ 96 "ithrd", /* PIL_ITHREAD */ 97 "rndzvs", /* PIL_RENDEZVOUS */ 98 "ast", /* PIL_AST */ 99 "hardclock", /* PIL_HARDCLOCK */ 100 "stray", "stray", "stray", "stray", 101 "filter", /* PIL_FILTER */ 102 "bridge", /* PIL_BRIDGE */ 103 "stop", /* PIL_STOP */ 104 "tick", /* PIL_TICK */ 105}; 106 107/* protect the intr_vectors table */ 108static struct sx intr_table_lock; 109/* protect intrcnt_index */ 110static struct mtx intrcnt_lock; 111 112#ifdef SMP 113static int assign_cpu; 114 115static void intr_assign_next_cpu(struct intr_vector *iv); 116static void intr_shuffle_irqs(void *arg __unused); 117#endif 118 119static int intr_assign_cpu(void *arg, u_char cpu); 120static void intr_execute_handlers(void *); 121static void intr_stray_level(struct trapframe *); 122static void intr_stray_vector(void *); 123static int intrcnt_setname(const char *, int); 124static void intrcnt_updatename(int, const char *, int); 125 126static void 127intrcnt_updatename(int vec, const char *name, int ispil) 128{ 129 static int intrcnt_index, stray_pil_index, stray_vec_index; 130 int name_index; 131 132 mtx_lock_spin(&intrcnt_lock); 133 if (intrnames[0] == '\0') { 134 /* for bitbucket */ 135 if (bootverbose) 136 printf("initalizing intr_countp\n"); 137 intrcnt_setname("???", intrcnt_index++); 138 139 stray_vec_index = intrcnt_index++; 140 intrcnt_setname("stray", stray_vec_index); 141 for (name_index = 0; name_index < IV_MAX; name_index++) 142 intr_countp[name_index] = stray_vec_index; 143 144 stray_pil_index = intrcnt_index++; 145 intrcnt_setname("pil", stray_pil_index); 146 for (name_index = 0; name_index < PIL_MAX; name_index++) 147 pil_countp[name_index] = stray_pil_index; 148 } 149 150 if (name == NULL) 151 name = "???"; 152 153 if (!ispil && intr_countp[vec] != stray_vec_index) 154 name_index = intr_countp[vec]; 155 else if (ispil && pil_countp[vec] != stray_pil_index) 156 name_index = pil_countp[vec]; 157 else 158 name_index = intrcnt_index++; 159 160 if (intrcnt_setname(name, name_index)) 161 name_index = 0; 162 163 if (!ispil) 164 intr_countp[vec] = name_index; 165 else 166 pil_countp[vec] = name_index; 167 mtx_unlock_spin(&intrcnt_lock); 168} 169 170static int 171intrcnt_setname(const char *name, int index) 172{ 173 174 if ((MAXCOMLEN + 1) * index >= sintrnames) 175 return (E2BIG); 176 snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s", 177 MAXCOMLEN, name); 178 return (0); 179} 180 181void 182intr_setup(int pri, ih_func_t *ihf, int vec, iv_func_t *ivf, void *iva) 183{ 184 char pilname[MAXCOMLEN + 1]; 185 register_t s; 186 187 s = intr_disable(); 188 if (vec != -1) { 189 intr_vectors[vec].iv_func = ivf; 190 intr_vectors[vec].iv_arg = iva; 191 intr_vectors[vec].iv_pri = pri; 192 intr_vectors[vec].iv_vec = vec; 193 } 194 intr_handlers[pri] = ihf; 195 intr_restore(s); 196 snprintf(pilname, MAXCOMLEN + 1, "pil%d: %s", pri, pil_names[pri]); 197 intrcnt_updatename(pri, pilname, 1); 198} 199 200static void 201intr_stray_level(struct trapframe *tf) 202{ 203 uint64_t level; 204 205 level = tf->tf_level; 206 if (pil_stray_count[level] < MAX_STRAY_LOG) { 207 printf("stray level interrupt %ld\n", level); 208 pil_stray_count[level]++; 209 if (pil_stray_count[level] >= MAX_STRAY_LOG) 210 printf("got %d stray level interrupt %ld's: not " 211 "logging anymore\n", MAX_STRAY_LOG, level); 212 } 213} 214 215static void 216intr_stray_vector(void *cookie) 217{ 218 struct intr_vector *iv; 219 u_int vec; 220 221 iv = cookie; 222 vec = iv->iv_vec; 223 if (intr_stray_count[vec] < MAX_STRAY_LOG) { 224 printf("stray vector interrupt %d\n", vec); 225 intr_stray_count[vec]++; 226 if (intr_stray_count[vec] >= MAX_STRAY_LOG) 227 printf("got %d stray vector interrupt %d's: not " 228 "logging anymore\n", MAX_STRAY_LOG, vec); 229 } 230} 231 232void 233intr_init1() 234{ 235 int i; 236 237 /* Mark all interrupts as being stray. */ 238 for (i = 0; i < PIL_MAX; i++) 239 intr_handlers[i] = intr_stray_level; 240 for (i = 0; i < IV_MAX; i++) { 241 intr_vectors[i].iv_func = intr_stray_vector; 242 intr_vectors[i].iv_arg = &intr_vectors[i]; 243 intr_vectors[i].iv_pri = PIL_LOW; 244 intr_vectors[i].iv_vec = i; 245 intr_vectors[i].iv_refcnt = 0; 246 } 247 intr_handlers[PIL_LOW] = intr_fast; 248} 249 250void 251intr_init2() 252{ 253 254 sx_init(&intr_table_lock, "intr sources"); 255 mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN); 256} 257 258static int 259intr_assign_cpu(void *arg, u_char cpu) 260{ 261#ifdef SMP 262 struct pcpu *pc; 263 struct intr_vector *iv; 264 265 /* 266 * Don't do anything during early boot. We will pick up the 267 * assignment once the APs are started. 268 */ 269 if (assign_cpu && cpu != NOCPU) { 270 pc = pcpu_find(cpu); 271 if (pc == NULL) 272 return (EINVAL); 273 iv = arg; 274 sx_xlock(&intr_table_lock); 275 iv->iv_mid = pc->pc_mid; 276 iv->iv_ic->ic_assign(iv); 277 sx_xunlock(&intr_table_lock); 278 } 279 return (0); 280#else 281 return (EOPNOTSUPP); 282#endif 283} 284 285static void 286intr_execute_handlers(void *cookie) 287{ 288 struct intr_vector *iv; 289 290 iv = cookie; 291 if (__predict_false(intr_event_handle(iv->iv_event, NULL) != 0)) 292 intr_stray_vector(iv); 293} 294 295int 296intr_controller_register(int vec, const struct intr_controller *ic, 297 void *icarg) 298{ 299 struct intr_event *ie; 300 struct intr_vector *iv; 301 int error; 302 303 if (vec < 0 || vec >= IV_MAX) 304 return (EINVAL); 305 sx_xlock(&intr_table_lock); 306 iv = &intr_vectors[vec]; 307 ie = iv->iv_event; 308 sx_xunlock(&intr_table_lock); 309 if (ie != NULL) 310 return (EEXIST); 311 error = intr_event_create(&ie, iv, 0, vec, NULL, ic->ic_clear, 312 ic->ic_clear, intr_assign_cpu, "vec%d:", vec); 313 if (error != 0) 314 return (error); 315 sx_xlock(&intr_table_lock); 316 if (iv->iv_event != NULL) { 317 sx_xunlock(&intr_table_lock); 318 intr_event_destroy(ie); 319 return (EEXIST); 320 } 321 iv->iv_ic = ic; 322 iv->iv_icarg = icarg; 323 iv->iv_event = ie; 324 iv->iv_mid = PCPU_GET(mid); 325 sx_xunlock(&intr_table_lock); 326 return (0); 327} 328 329int 330inthand_add(const char *name, int vec, driver_filter_t *filt, 331 driver_intr_t *handler, void *arg, int flags, void **cookiep) 332{ 333 const struct intr_controller *ic; 334 struct intr_event *ie; 335 struct intr_handler *ih; 336 struct intr_vector *iv; 337 int error, filter; 338 339 if (vec < 0 || vec >= IV_MAX) 340 return (EINVAL); 341 /* 342 * INTR_BRIDGE filters/handlers are special purpose only, allowing 343 * them to be shared just would complicate things unnecessarily. 344 */ 345 if ((flags & INTR_BRIDGE) != 0 && (flags & INTR_EXCL) == 0) 346 return (EINVAL); 347 sx_xlock(&intr_table_lock); 348 iv = &intr_vectors[vec]; 349 ic = iv->iv_ic; 350 ie = iv->iv_event; 351 sx_xunlock(&intr_table_lock); 352 if (ic == NULL || ie == NULL) 353 return (EINVAL); 354 error = intr_event_add_handler(ie, name, filt, handler, arg, 355 intr_priority(flags), flags, cookiep); 356 if (error != 0) 357 return (error); 358 sx_xlock(&intr_table_lock); 359 /* Disable the interrupt while we fiddle with it. */ 360 ic->ic_disable(iv); 361 iv->iv_refcnt++; 362 if (iv->iv_refcnt == 1) 363 intr_setup((flags & INTR_BRIDGE) != 0 ? PIL_BRIDGE : 364 filt != NULL ? PIL_FILTER : PIL_ITHREAD, intr_fast, 365 vec, intr_execute_handlers, iv); 366 else if (filt != NULL) { 367 /* 368 * Check if we need to upgrade from PIL_ITHREAD to PIL_FILTER. 369 * Given that apart from the on-board SCCs and UARTs shared 370 * interrupts are rather uncommon on sparc64 this sould be 371 * pretty rare in practice. 372 */ 373 filter = 0; 374 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 375 if (ih->ih_filter != NULL && ih->ih_filter != filt) { 376 filter = 1; 377 break; 378 } 379 } 380 if (filter == 0) 381 intr_setup(PIL_FILTER, intr_fast, vec, 382 intr_execute_handlers, iv); 383 } 384 intr_stray_count[vec] = 0; 385 intrcnt_updatename(vec, ie->ie_fullname, 0); 386#ifdef SMP 387 if (assign_cpu) 388 intr_assign_next_cpu(iv); 389#endif 390 ic->ic_enable(iv); 391 /* Ensure the interrupt is cleared, it might have triggered before. */ 392 if (ic->ic_clear != NULL) 393 ic->ic_clear(iv); 394 sx_xunlock(&intr_table_lock); 395 return (0); 396} 397 398int 399inthand_remove(int vec, void *cookie) 400{ 401 struct intr_vector *iv; 402 int error; 403 404 if (vec < 0 || vec >= IV_MAX) 405 return (EINVAL); 406 error = intr_event_remove_handler(cookie); 407 if (error == 0) { 408 /* 409 * XXX: maybe this should be done regardless of whether 410 * intr_event_remove_handler() succeeded? 411 */ 412 sx_xlock(&intr_table_lock); 413 iv = &intr_vectors[vec]; 414 iv->iv_refcnt--; 415 if (iv->iv_refcnt == 0) { 416 /* 417 * Don't disable the interrupt for now, so that 418 * stray interrupts get detected... 419 */ 420 intr_setup(PIL_LOW, intr_fast, vec, 421 intr_stray_vector, iv); 422 } 423 sx_xunlock(&intr_table_lock); 424 } 425 return (error); 426} 427 428/* Add a description to an active interrupt handler. */ 429int 430intr_describe(int vec, void *ih, const char *descr) 431{ 432 struct intr_vector *iv; 433 int error; 434 435 if (vec < 0 || vec >= IV_MAX) 436 return (EINVAL); 437 sx_xlock(&intr_table_lock); 438 iv = &intr_vectors[vec]; 439 if (iv == NULL) { 440 sx_xunlock(&intr_table_lock); 441 return (EINVAL); 442 } 443 error = intr_event_describe_handler(iv->iv_event, ih, descr); 444 if (error) { 445 sx_xunlock(&intr_table_lock); 446 return (error); 447 } 448 intrcnt_updatename(vec, iv->iv_event->ie_fullname, 0); 449 sx_xunlock(&intr_table_lock); 450 return (error); 451} 452 453#ifdef SMP 454/* 455 * Support for balancing interrupt sources across CPUs. For now we just 456 * allocate CPUs round-robin. 457 */ 458 459static cpuset_t intr_cpus; 460static int current_cpu; 461 462static void 463intr_assign_next_cpu(struct intr_vector *iv) 464{ 465 struct pcpu *pc; 466 467 sx_assert(&intr_table_lock, SA_XLOCKED); 468 469 /* 470 * Assign this source to a CPU in a round-robin fashion. 471 */ 472 pc = pcpu_find(current_cpu); 473 if (pc == NULL) 474 return; 475 iv->iv_mid = pc->pc_mid; 476 iv->iv_ic->ic_assign(iv); 477 do { 478 current_cpu++; 479 if (current_cpu > mp_maxid) 480 current_cpu = 0; 481 } while (!CPU_ISSET(current_cpu, &intr_cpus)); 482} 483 484/* Attempt to bind the specified IRQ to the specified CPU. */ 485int 486intr_bind(int vec, u_char cpu) 487{ 488 struct intr_vector *iv; 489 int error; 490 491 if (vec < 0 || vec >= IV_MAX) 492 return (EINVAL); 493 sx_xlock(&intr_table_lock); 494 iv = &intr_vectors[vec]; 495 if (iv == NULL) { 496 sx_xunlock(&intr_table_lock); 497 return (EINVAL); 498 } 499 error = intr_event_bind(iv->iv_event, cpu); 500 sx_xunlock(&intr_table_lock); 501 return (error); 502} 503 504/* 505 * Add a CPU to our mask of valid CPUs that can be destinations of 506 * interrupts. 507 */ 508void 509intr_add_cpu(u_int cpu) 510{ 511 512 if (cpu >= MAXCPU) 513 panic("%s: Invalid CPU ID", __func__); 514 if (bootverbose) 515 printf("INTR: Adding CPU %d as a target\n", cpu); 516 517 CPU_SET(cpu, &intr_cpus); 518} 519 520/* 521 * Distribute all the interrupt sources among the available CPUs once the 522 * APs have been launched. 523 */ 524static void 525intr_shuffle_irqs(void *arg __unused) 526{ 527 struct pcpu *pc; 528 struct intr_vector *iv; 529 int i; 530 531 /* Don't bother on UP. */ 532 if (mp_ncpus == 1) 533 return; 534 535 sx_xlock(&intr_table_lock); 536 assign_cpu = 1; 537 for (i = 0; i < IV_MAX; i++) { 538 iv = &intr_vectors[i]; 539 if (iv != NULL && iv->iv_refcnt > 0) { 540 /* 541 * If this event is already bound to a CPU, 542 * then assign the source to that CPU instead 543 * of picking one via round-robin. 544 */ 545 if (iv->iv_event->ie_cpu != NOCPU && 546 (pc = pcpu_find(iv->iv_event->ie_cpu)) != NULL) { 547 iv->iv_mid = pc->pc_mid; 548 iv->iv_ic->ic_assign(iv); 549 } else 550 intr_assign_next_cpu(iv); 551 } 552 } 553 sx_xunlock(&intr_table_lock); 554} 555SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, 556 NULL); 557#else /* !SMP */ 558/* Use an empty stub for compatibility. */ 559void 560intr_add_cpu(u_int cpu __unused) 561{ 562 563} 564#endif 565