1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Copyright (C) 2008-2014 Mathieu Desnoyers 4 */ 5#include <linux/module.h> 6#include <linux/mutex.h> 7#include <linux/types.h> 8#include <linux/jhash.h> 9#include <linux/list.h> 10#include <linux/rcupdate.h> 11#include <linux/tracepoint.h> 12#include <linux/err.h> 13#include <linux/slab.h> 14#include <linux/sched/signal.h> 15#include <linux/sched/task.h> 16#include <linux/static_key.h> 17 18enum tp_func_state { 19 TP_FUNC_0, 20 TP_FUNC_1, 21 TP_FUNC_2, 22 TP_FUNC_N, 23}; 24 25extern tracepoint_ptr_t __start___tracepoints_ptrs[]; 26extern tracepoint_ptr_t __stop___tracepoints_ptrs[]; 27 28DEFINE_SRCU(tracepoint_srcu); 29EXPORT_SYMBOL_GPL(tracepoint_srcu); 30 31enum tp_transition_sync { 32 TP_TRANSITION_SYNC_1_0_1, 33 TP_TRANSITION_SYNC_N_2_1, 34 35 _NR_TP_TRANSITION_SYNC, 36}; 37 38struct tp_transition_snapshot { 39 unsigned long rcu; 40 unsigned long srcu; 41 bool ongoing; 42}; 43 44/* Protected by tracepoints_mutex */ 45static struct tp_transition_snapshot tp_transition_snapshot[_NR_TP_TRANSITION_SYNC]; 46 47static void tp_rcu_get_state(enum tp_transition_sync sync) 48{ 49 struct tp_transition_snapshot *snapshot = &tp_transition_snapshot[sync]; 50 51 /* Keep the latest get_state snapshot. */ 52 snapshot->rcu = get_state_synchronize_rcu(); 53 snapshot->srcu = start_poll_synchronize_srcu(&tracepoint_srcu); 54 snapshot->ongoing = true; 55} 56 57static void tp_rcu_cond_sync(enum tp_transition_sync sync) 58{ 59 struct tp_transition_snapshot *snapshot = &tp_transition_snapshot[sync]; 60 61 if (!snapshot->ongoing) 62 return; 63 cond_synchronize_rcu(snapshot->rcu); 64 if (!poll_state_synchronize_srcu(&tracepoint_srcu, snapshot->srcu)) 65 synchronize_srcu(&tracepoint_srcu); 66 snapshot->ongoing = false; 67} 68 69/* Set to 1 to enable tracepoint debug output */ 70static const int tracepoint_debug; 71 72#ifdef CONFIG_MODULES 73/* 74 * Tracepoint module list mutex protects the local module list. 75 */ 76static DEFINE_MUTEX(tracepoint_module_list_mutex); 77 78/* Local list of struct tp_module */ 79static LIST_HEAD(tracepoint_module_list); 80#endif /* CONFIG_MODULES */ 81 82/* 83 * tracepoints_mutex protects the builtin and module tracepoints. 84 * tracepoints_mutex nests inside tracepoint_module_list_mutex. 85 */ 86static DEFINE_MUTEX(tracepoints_mutex); 87 88static struct rcu_head *early_probes; 89static bool ok_to_free_tracepoints; 90 91/* 92 * Note about RCU : 93 * It is used to delay the free of multiple probes array until a quiescent 94 * state is reached. 95 */ 96struct tp_probes { 97 struct rcu_head rcu; 98 struct tracepoint_func probes[]; 99}; 100 101/* Called in removal of a func but failed to allocate a new tp_funcs */ 102static void tp_stub_func(void) 103{ 104 return; 105} 106 107static inline void *allocate_probes(int count) 108{ 109 struct tp_probes *p = kmalloc(struct_size(p, probes, count), 110 GFP_KERNEL); 111 return p == NULL ? NULL : p->probes; 112} 113 114static void srcu_free_old_probes(struct rcu_head *head) 115{ 116 kfree(container_of(head, struct tp_probes, rcu)); 117} 118 119static void rcu_free_old_probes(struct rcu_head *head) 120{ 121 call_srcu(&tracepoint_srcu, head, srcu_free_old_probes); 122} 123 124static __init int release_early_probes(void) 125{ 126 struct rcu_head *tmp; 127 128 ok_to_free_tracepoints = true; 129 130 while (early_probes) { 131 tmp = early_probes; 132 early_probes = tmp->next; 133 call_rcu(tmp, rcu_free_old_probes); 134 } 135 136 return 0; 137} 138 139/* SRCU is initialized at core_initcall */ 140postcore_initcall(release_early_probes); 141 142static inline void release_probes(struct tracepoint_func *old) 143{ 144 if (old) { 145 struct tp_probes *tp_probes = container_of(old, 146 struct tp_probes, probes[0]); 147 148 /* 149 * We can't free probes if SRCU is not initialized yet. 150 * Postpone the freeing till after SRCU is initialized. 151 */ 152 if (unlikely(!ok_to_free_tracepoints)) { 153 tp_probes->rcu.next = early_probes; 154 early_probes = &tp_probes->rcu; 155 return; 156 } 157 158 /* 159 * Tracepoint probes are protected by both sched RCU and SRCU, 160 * by calling the SRCU callback in the sched RCU callback we 161 * cover both cases. So let us chain the SRCU and sched RCU 162 * callbacks to wait for both grace periods. 163 */ 164 call_rcu(&tp_probes->rcu, rcu_free_old_probes); 165 } 166} 167 168static void debug_print_probes(struct tracepoint_func *funcs) 169{ 170 int i; 171 172 if (!tracepoint_debug || !funcs) 173 return; 174 175 for (i = 0; funcs[i].func; i++) 176 printk(KERN_DEBUG "Probe %d : %p\n", i, funcs[i].func); 177} 178 179static struct tracepoint_func * 180func_add(struct tracepoint_func **funcs, struct tracepoint_func *tp_func, 181 int prio) 182{ 183 struct tracepoint_func *old, *new; 184 int iter_probes; /* Iterate over old probe array. */ 185 int nr_probes = 0; /* Counter for probes */ 186 int pos = -1; /* Insertion position into new array */ 187 188 if (WARN_ON(!tp_func->func)) 189 return ERR_PTR(-EINVAL); 190 191 debug_print_probes(*funcs); 192 old = *funcs; 193 if (old) { 194 /* (N -> N+1), (N != 0, 1) probes */ 195 for (iter_probes = 0; old[iter_probes].func; iter_probes++) { 196 if (old[iter_probes].func == tp_stub_func) 197 continue; /* Skip stub functions. */ 198 if (old[iter_probes].func == tp_func->func && 199 old[iter_probes].data == tp_func->data) 200 return ERR_PTR(-EEXIST); 201 nr_probes++; 202 } 203 } 204 /* + 2 : one for new probe, one for NULL func */ 205 new = allocate_probes(nr_probes + 2); 206 if (new == NULL) 207 return ERR_PTR(-ENOMEM); 208 if (old) { 209 nr_probes = 0; 210 for (iter_probes = 0; old[iter_probes].func; iter_probes++) { 211 if (old[iter_probes].func == tp_stub_func) 212 continue; 213 /* Insert before probes of lower priority */ 214 if (pos < 0 && old[iter_probes].prio < prio) 215 pos = nr_probes++; 216 new[nr_probes++] = old[iter_probes]; 217 } 218 if (pos < 0) 219 pos = nr_probes++; 220 /* nr_probes now points to the end of the new array */ 221 } else { 222 pos = 0; 223 nr_probes = 1; /* must point at end of array */ 224 } 225 new[pos] = *tp_func; 226 new[nr_probes].func = NULL; 227 *funcs = new; 228 debug_print_probes(*funcs); 229 return old; 230} 231 232static void *func_remove(struct tracepoint_func **funcs, 233 struct tracepoint_func *tp_func) 234{ 235 int nr_probes = 0, nr_del = 0, i; 236 struct tracepoint_func *old, *new; 237 238 old = *funcs; 239 240 if (!old) 241 return ERR_PTR(-ENOENT); 242 243 debug_print_probes(*funcs); 244 /* (N -> M), (N > 1, M >= 0) probes */ 245 if (tp_func->func) { 246 for (nr_probes = 0; old[nr_probes].func; nr_probes++) { 247 if ((old[nr_probes].func == tp_func->func && 248 old[nr_probes].data == tp_func->data) || 249 old[nr_probes].func == tp_stub_func) 250 nr_del++; 251 } 252 } 253 254 /* 255 * If probe is NULL, then nr_probes = nr_del = 0, and then the 256 * entire entry will be removed. 257 */ 258 if (nr_probes - nr_del == 0) { 259 /* N -> 0, (N > 1) */ 260 *funcs = NULL; 261 debug_print_probes(*funcs); 262 return old; 263 } else { 264 int j = 0; 265 /* N -> M, (N > 1, M > 0) */ 266 /* + 1 for NULL */ 267 new = allocate_probes(nr_probes - nr_del + 1); 268 if (new) { 269 for (i = 0; old[i].func; i++) { 270 if ((old[i].func != tp_func->func || 271 old[i].data != tp_func->data) && 272 old[i].func != tp_stub_func) 273 new[j++] = old[i]; 274 } 275 new[nr_probes - nr_del].func = NULL; 276 *funcs = new; 277 } else { 278 /* 279 * Failed to allocate, replace the old function 280 * with calls to tp_stub_func. 281 */ 282 for (i = 0; old[i].func; i++) { 283 if (old[i].func == tp_func->func && 284 old[i].data == tp_func->data) 285 WRITE_ONCE(old[i].func, tp_stub_func); 286 } 287 *funcs = old; 288 } 289 } 290 debug_print_probes(*funcs); 291 return old; 292} 293 294/* 295 * Count the number of functions (enum tp_func_state) in a tp_funcs array. 296 */ 297static enum tp_func_state nr_func_state(const struct tracepoint_func *tp_funcs) 298{ 299 if (!tp_funcs) 300 return TP_FUNC_0; 301 if (!tp_funcs[1].func) 302 return TP_FUNC_1; 303 if (!tp_funcs[2].func) 304 return TP_FUNC_2; 305 return TP_FUNC_N; /* 3 or more */ 306} 307 308static void tracepoint_update_call(struct tracepoint *tp, struct tracepoint_func *tp_funcs) 309{ 310 void *func = tp->iterator; 311 312 /* Synthetic events do not have static call sites */ 313 if (!tp->static_call_key) 314 return; 315 if (nr_func_state(tp_funcs) == TP_FUNC_1) 316 func = tp_funcs[0].func; 317 __static_call_update(tp->static_call_key, tp->static_call_tramp, func); 318} 319 320/* 321 * Add the probe function to a tracepoint. 322 */ 323static int tracepoint_add_func(struct tracepoint *tp, 324 struct tracepoint_func *func, int prio, 325 bool warn) 326{ 327 struct tracepoint_func *old, *tp_funcs; 328 int ret; 329 330 if (tp->regfunc && !static_key_enabled(&tp->key)) { 331 ret = tp->regfunc(); 332 if (ret < 0) 333 return ret; 334 } 335 336 tp_funcs = rcu_dereference_protected(tp->funcs, 337 lockdep_is_held(&tracepoints_mutex)); 338 old = func_add(&tp_funcs, func, prio); 339 if (IS_ERR(old)) { 340 WARN_ON_ONCE(warn && PTR_ERR(old) != -ENOMEM); 341 return PTR_ERR(old); 342 } 343 344 /* 345 * rcu_assign_pointer has as smp_store_release() which makes sure 346 * that the new probe callbacks array is consistent before setting 347 * a pointer to it. This array is referenced by __DO_TRACE from 348 * include/linux/tracepoint.h using rcu_dereference_sched(). 349 */ 350 switch (nr_func_state(tp_funcs)) { 351 case TP_FUNC_1: /* 0->1 */ 352 /* 353 * Make sure new static func never uses old data after a 354 * 1->0->1 transition sequence. 355 */ 356 tp_rcu_cond_sync(TP_TRANSITION_SYNC_1_0_1); 357 /* Set static call to first function */ 358 tracepoint_update_call(tp, tp_funcs); 359 /* Both iterator and static call handle NULL tp->funcs */ 360 rcu_assign_pointer(tp->funcs, tp_funcs); 361 static_key_enable(&tp->key); 362 break; 363 case TP_FUNC_2: /* 1->2 */ 364 /* Set iterator static call */ 365 tracepoint_update_call(tp, tp_funcs); 366 /* 367 * Iterator callback installed before updating tp->funcs. 368 * Requires ordering between RCU assign/dereference and 369 * static call update/call. 370 */ 371 fallthrough; 372 case TP_FUNC_N: /* N->N+1 (N>1) */ 373 rcu_assign_pointer(tp->funcs, tp_funcs); 374 /* 375 * Make sure static func never uses incorrect data after a 376 * N->...->2->1 (N>1) transition sequence. 377 */ 378 if (tp_funcs[0].data != old[0].data) 379 tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1); 380 break; 381 default: 382 WARN_ON_ONCE(1); 383 break; 384 } 385 386 release_probes(old); 387 return 0; 388} 389 390/* 391 * Remove a probe function from a tracepoint. 392 * Note: only waiting an RCU period after setting elem->call to the empty 393 * function insures that the original callback is not used anymore. This insured 394 * by preempt_disable around the call site. 395 */ 396static int tracepoint_remove_func(struct tracepoint *tp, 397 struct tracepoint_func *func) 398{ 399 struct tracepoint_func *old, *tp_funcs; 400 401 tp_funcs = rcu_dereference_protected(tp->funcs, 402 lockdep_is_held(&tracepoints_mutex)); 403 old = func_remove(&tp_funcs, func); 404 if (WARN_ON_ONCE(IS_ERR(old))) 405 return PTR_ERR(old); 406 407 if (tp_funcs == old) 408 /* Failed allocating new tp_funcs, replaced func with stub */ 409 return 0; 410 411 switch (nr_func_state(tp_funcs)) { 412 case TP_FUNC_0: /* 1->0 */ 413 /* Removed last function */ 414 if (tp->unregfunc && static_key_enabled(&tp->key)) 415 tp->unregfunc(); 416 417 static_key_disable(&tp->key); 418 /* Set iterator static call */ 419 tracepoint_update_call(tp, tp_funcs); 420 /* Both iterator and static call handle NULL tp->funcs */ 421 rcu_assign_pointer(tp->funcs, NULL); 422 /* 423 * Make sure new static func never uses old data after a 424 * 1->0->1 transition sequence. 425 */ 426 tp_rcu_get_state(TP_TRANSITION_SYNC_1_0_1); 427 break; 428 case TP_FUNC_1: /* 2->1 */ 429 rcu_assign_pointer(tp->funcs, tp_funcs); 430 /* 431 * Make sure static func never uses incorrect data after a 432 * N->...->2->1 (N>2) transition sequence. If the first 433 * element's data has changed, then force the synchronization 434 * to prevent current readers that have loaded the old data 435 * from calling the new function. 436 */ 437 if (tp_funcs[0].data != old[0].data) 438 tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1); 439 tp_rcu_cond_sync(TP_TRANSITION_SYNC_N_2_1); 440 /* Set static call to first function */ 441 tracepoint_update_call(tp, tp_funcs); 442 break; 443 case TP_FUNC_2: /* N->N-1 (N>2) */ 444 fallthrough; 445 case TP_FUNC_N: 446 rcu_assign_pointer(tp->funcs, tp_funcs); 447 /* 448 * Make sure static func never uses incorrect data after a 449 * N->...->2->1 (N>2) transition sequence. 450 */ 451 if (tp_funcs[0].data != old[0].data) 452 tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1); 453 break; 454 default: 455 WARN_ON_ONCE(1); 456 break; 457 } 458 release_probes(old); 459 return 0; 460} 461 462/** 463 * tracepoint_probe_register_prio_may_exist - Connect a probe to a tracepoint with priority 464 * @tp: tracepoint 465 * @probe: probe handler 466 * @data: tracepoint data 467 * @prio: priority of this function over other registered functions 468 * 469 * Same as tracepoint_probe_register_prio() except that it will not warn 470 * if the tracepoint is already registered. 471 */ 472int tracepoint_probe_register_prio_may_exist(struct tracepoint *tp, void *probe, 473 void *data, int prio) 474{ 475 struct tracepoint_func tp_func; 476 int ret; 477 478 mutex_lock(&tracepoints_mutex); 479 tp_func.func = probe; 480 tp_func.data = data; 481 tp_func.prio = prio; 482 ret = tracepoint_add_func(tp, &tp_func, prio, false); 483 mutex_unlock(&tracepoints_mutex); 484 return ret; 485} 486EXPORT_SYMBOL_GPL(tracepoint_probe_register_prio_may_exist); 487 488/** 489 * tracepoint_probe_register_prio - Connect a probe to a tracepoint with priority 490 * @tp: tracepoint 491 * @probe: probe handler 492 * @data: tracepoint data 493 * @prio: priority of this function over other registered functions 494 * 495 * Returns 0 if ok, error value on error. 496 * Note: if @tp is within a module, the caller is responsible for 497 * unregistering the probe before the module is gone. This can be 498 * performed either with a tracepoint module going notifier, or from 499 * within module exit functions. 500 */ 501int tracepoint_probe_register_prio(struct tracepoint *tp, void *probe, 502 void *data, int prio) 503{ 504 struct tracepoint_func tp_func; 505 int ret; 506 507 mutex_lock(&tracepoints_mutex); 508 tp_func.func = probe; 509 tp_func.data = data; 510 tp_func.prio = prio; 511 ret = tracepoint_add_func(tp, &tp_func, prio, true); 512 mutex_unlock(&tracepoints_mutex); 513 return ret; 514} 515EXPORT_SYMBOL_GPL(tracepoint_probe_register_prio); 516 517/** 518 * tracepoint_probe_register - Connect a probe to a tracepoint 519 * @tp: tracepoint 520 * @probe: probe handler 521 * @data: tracepoint data 522 * 523 * Returns 0 if ok, error value on error. 524 * Note: if @tp is within a module, the caller is responsible for 525 * unregistering the probe before the module is gone. This can be 526 * performed either with a tracepoint module going notifier, or from 527 * within module exit functions. 528 */ 529int tracepoint_probe_register(struct tracepoint *tp, void *probe, void *data) 530{ 531 return tracepoint_probe_register_prio(tp, probe, data, TRACEPOINT_DEFAULT_PRIO); 532} 533EXPORT_SYMBOL_GPL(tracepoint_probe_register); 534 535/** 536 * tracepoint_probe_unregister - Disconnect a probe from a tracepoint 537 * @tp: tracepoint 538 * @probe: probe function pointer 539 * @data: tracepoint data 540 * 541 * Returns 0 if ok, error value on error. 542 */ 543int tracepoint_probe_unregister(struct tracepoint *tp, void *probe, void *data) 544{ 545 struct tracepoint_func tp_func; 546 int ret; 547 548 mutex_lock(&tracepoints_mutex); 549 tp_func.func = probe; 550 tp_func.data = data; 551 ret = tracepoint_remove_func(tp, &tp_func); 552 mutex_unlock(&tracepoints_mutex); 553 return ret; 554} 555EXPORT_SYMBOL_GPL(tracepoint_probe_unregister); 556 557static void for_each_tracepoint_range( 558 tracepoint_ptr_t *begin, tracepoint_ptr_t *end, 559 void (*fct)(struct tracepoint *tp, void *priv), 560 void *priv) 561{ 562 tracepoint_ptr_t *iter; 563 564 if (!begin) 565 return; 566 for (iter = begin; iter < end; iter++) 567 fct(tracepoint_ptr_deref(iter), priv); 568} 569 570#ifdef CONFIG_MODULES 571bool trace_module_has_bad_taint(struct module *mod) 572{ 573 return mod->taints & ~((1 << TAINT_OOT_MODULE) | (1 << TAINT_CRAP) | 574 (1 << TAINT_UNSIGNED_MODULE) | (1 << TAINT_TEST) | 575 (1 << TAINT_LIVEPATCH)); 576} 577 578static BLOCKING_NOTIFIER_HEAD(tracepoint_notify_list); 579 580/** 581 * register_tracepoint_module_notifier - register tracepoint coming/going notifier 582 * @nb: notifier block 583 * 584 * Notifiers registered with this function are called on module 585 * coming/going with the tracepoint_module_list_mutex held. 586 * The notifier block callback should expect a "struct tp_module" data 587 * pointer. 588 */ 589int register_tracepoint_module_notifier(struct notifier_block *nb) 590{ 591 struct tp_module *tp_mod; 592 int ret; 593 594 mutex_lock(&tracepoint_module_list_mutex); 595 ret = blocking_notifier_chain_register(&tracepoint_notify_list, nb); 596 if (ret) 597 goto end; 598 list_for_each_entry(tp_mod, &tracepoint_module_list, list) 599 (void) nb->notifier_call(nb, MODULE_STATE_COMING, tp_mod); 600end: 601 mutex_unlock(&tracepoint_module_list_mutex); 602 return ret; 603} 604EXPORT_SYMBOL_GPL(register_tracepoint_module_notifier); 605 606/** 607 * unregister_tracepoint_module_notifier - unregister tracepoint coming/going notifier 608 * @nb: notifier block 609 * 610 * The notifier block callback should expect a "struct tp_module" data 611 * pointer. 612 */ 613int unregister_tracepoint_module_notifier(struct notifier_block *nb) 614{ 615 struct tp_module *tp_mod; 616 int ret; 617 618 mutex_lock(&tracepoint_module_list_mutex); 619 ret = blocking_notifier_chain_unregister(&tracepoint_notify_list, nb); 620 if (ret) 621 goto end; 622 list_for_each_entry(tp_mod, &tracepoint_module_list, list) 623 (void) nb->notifier_call(nb, MODULE_STATE_GOING, tp_mod); 624end: 625 mutex_unlock(&tracepoint_module_list_mutex); 626 return ret; 627 628} 629EXPORT_SYMBOL_GPL(unregister_tracepoint_module_notifier); 630 631/* 632 * Ensure the tracer unregistered the module's probes before the module 633 * teardown is performed. Prevents leaks of probe and data pointers. 634 */ 635static void tp_module_going_check_quiescent(struct tracepoint *tp, void *priv) 636{ 637 WARN_ON_ONCE(tp->funcs); 638} 639 640static int tracepoint_module_coming(struct module *mod) 641{ 642 struct tp_module *tp_mod; 643 644 if (!mod->num_tracepoints) 645 return 0; 646 647 /* 648 * We skip modules that taint the kernel, especially those with different 649 * module headers (for forced load), to make sure we don't cause a crash. 650 * Staging, out-of-tree, unsigned GPL, and test modules are fine. 651 */ 652 if (trace_module_has_bad_taint(mod)) 653 return 0; 654 655 tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL); 656 if (!tp_mod) 657 return -ENOMEM; 658 tp_mod->mod = mod; 659 660 mutex_lock(&tracepoint_module_list_mutex); 661 list_add_tail(&tp_mod->list, &tracepoint_module_list); 662 blocking_notifier_call_chain(&tracepoint_notify_list, 663 MODULE_STATE_COMING, tp_mod); 664 mutex_unlock(&tracepoint_module_list_mutex); 665 return 0; 666} 667 668static void tracepoint_module_going(struct module *mod) 669{ 670 struct tp_module *tp_mod; 671 672 if (!mod->num_tracepoints) 673 return; 674 675 mutex_lock(&tracepoint_module_list_mutex); 676 list_for_each_entry(tp_mod, &tracepoint_module_list, list) { 677 if (tp_mod->mod == mod) { 678 blocking_notifier_call_chain(&tracepoint_notify_list, 679 MODULE_STATE_GOING, tp_mod); 680 list_del(&tp_mod->list); 681 kfree(tp_mod); 682 /* 683 * Called the going notifier before checking for 684 * quiescence. 685 */ 686 for_each_tracepoint_range(mod->tracepoints_ptrs, 687 mod->tracepoints_ptrs + mod->num_tracepoints, 688 tp_module_going_check_quiescent, NULL); 689 break; 690 } 691 } 692 /* 693 * In the case of modules that were tainted at "coming", we'll simply 694 * walk through the list without finding it. We cannot use the "tainted" 695 * flag on "going", in case a module taints the kernel only after being 696 * loaded. 697 */ 698 mutex_unlock(&tracepoint_module_list_mutex); 699} 700 701static int tracepoint_module_notify(struct notifier_block *self, 702 unsigned long val, void *data) 703{ 704 struct module *mod = data; 705 int ret = 0; 706 707 switch (val) { 708 case MODULE_STATE_COMING: 709 ret = tracepoint_module_coming(mod); 710 break; 711 case MODULE_STATE_LIVE: 712 break; 713 case MODULE_STATE_GOING: 714 tracepoint_module_going(mod); 715 break; 716 case MODULE_STATE_UNFORMED: 717 break; 718 } 719 return notifier_from_errno(ret); 720} 721 722static struct notifier_block tracepoint_module_nb = { 723 .notifier_call = tracepoint_module_notify, 724 .priority = 0, 725}; 726 727static __init int init_tracepoints(void) 728{ 729 int ret; 730 731 ret = register_module_notifier(&tracepoint_module_nb); 732 if (ret) 733 pr_warn("Failed to register tracepoint module enter notifier\n"); 734 735 return ret; 736} 737__initcall(init_tracepoints); 738#endif /* CONFIG_MODULES */ 739 740/** 741 * for_each_kernel_tracepoint - iteration on all kernel tracepoints 742 * @fct: callback 743 * @priv: private data 744 */ 745void for_each_kernel_tracepoint(void (*fct)(struct tracepoint *tp, void *priv), 746 void *priv) 747{ 748 for_each_tracepoint_range(__start___tracepoints_ptrs, 749 __stop___tracepoints_ptrs, fct, priv); 750} 751EXPORT_SYMBOL_GPL(for_each_kernel_tracepoint); 752 753#ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS 754 755/* NB: reg/unreg are called while guarded with the tracepoints_mutex */ 756static int sys_tracepoint_refcount; 757 758int syscall_regfunc(void) 759{ 760 struct task_struct *p, *t; 761 762 if (!sys_tracepoint_refcount) { 763 read_lock(&tasklist_lock); 764 for_each_process_thread(p, t) { 765 set_task_syscall_work(t, SYSCALL_TRACEPOINT); 766 } 767 read_unlock(&tasklist_lock); 768 } 769 sys_tracepoint_refcount++; 770 771 return 0; 772} 773 774void syscall_unregfunc(void) 775{ 776 struct task_struct *p, *t; 777 778 sys_tracepoint_refcount--; 779 if (!sys_tracepoint_refcount) { 780 read_lock(&tasklist_lock); 781 for_each_process_thread(p, t) { 782 clear_task_syscall_work(t, SYSCALL_TRACEPOINT); 783 } 784 read_unlock(&tasklist_lock); 785 } 786} 787#endif 788