1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2008-2014 Mathieu Desnoyers
4 */
5#include <linux/module.h>
6#include <linux/mutex.h>
7#include <linux/types.h>
8#include <linux/jhash.h>
9#include <linux/list.h>
10#include <linux/rcupdate.h>
11#include <linux/tracepoint.h>
12#include <linux/err.h>
13#include <linux/slab.h>
14#include <linux/sched/signal.h>
15#include <linux/sched/task.h>
16#include <linux/static_key.h>
17
18enum tp_func_state {
19	TP_FUNC_0,
20	TP_FUNC_1,
21	TP_FUNC_2,
22	TP_FUNC_N,
23};
24
25extern tracepoint_ptr_t __start___tracepoints_ptrs[];
26extern tracepoint_ptr_t __stop___tracepoints_ptrs[];
27
28DEFINE_SRCU(tracepoint_srcu);
29EXPORT_SYMBOL_GPL(tracepoint_srcu);
30
31enum tp_transition_sync {
32	TP_TRANSITION_SYNC_1_0_1,
33	TP_TRANSITION_SYNC_N_2_1,
34
35	_NR_TP_TRANSITION_SYNC,
36};
37
38struct tp_transition_snapshot {
39	unsigned long rcu;
40	unsigned long srcu;
41	bool ongoing;
42};
43
44/* Protected by tracepoints_mutex */
45static struct tp_transition_snapshot tp_transition_snapshot[_NR_TP_TRANSITION_SYNC];
46
47static void tp_rcu_get_state(enum tp_transition_sync sync)
48{
49	struct tp_transition_snapshot *snapshot = &tp_transition_snapshot[sync];
50
51	/* Keep the latest get_state snapshot. */
52	snapshot->rcu = get_state_synchronize_rcu();
53	snapshot->srcu = start_poll_synchronize_srcu(&tracepoint_srcu);
54	snapshot->ongoing = true;
55}
56
57static void tp_rcu_cond_sync(enum tp_transition_sync sync)
58{
59	struct tp_transition_snapshot *snapshot = &tp_transition_snapshot[sync];
60
61	if (!snapshot->ongoing)
62		return;
63	cond_synchronize_rcu(snapshot->rcu);
64	if (!poll_state_synchronize_srcu(&tracepoint_srcu, snapshot->srcu))
65		synchronize_srcu(&tracepoint_srcu);
66	snapshot->ongoing = false;
67}
68
69/* Set to 1 to enable tracepoint debug output */
70static const int tracepoint_debug;
71
72#ifdef CONFIG_MODULES
73/*
74 * Tracepoint module list mutex protects the local module list.
75 */
76static DEFINE_MUTEX(tracepoint_module_list_mutex);
77
78/* Local list of struct tp_module */
79static LIST_HEAD(tracepoint_module_list);
80#endif /* CONFIG_MODULES */
81
82/*
83 * tracepoints_mutex protects the builtin and module tracepoints.
84 * tracepoints_mutex nests inside tracepoint_module_list_mutex.
85 */
86static DEFINE_MUTEX(tracepoints_mutex);
87
88static struct rcu_head *early_probes;
89static bool ok_to_free_tracepoints;
90
91/*
92 * Note about RCU :
93 * It is used to delay the free of multiple probes array until a quiescent
94 * state is reached.
95 */
96struct tp_probes {
97	struct rcu_head rcu;
98	struct tracepoint_func probes[];
99};
100
101/* Called in removal of a func but failed to allocate a new tp_funcs */
102static void tp_stub_func(void)
103{
104	return;
105}
106
107static inline void *allocate_probes(int count)
108{
109	struct tp_probes *p  = kmalloc(struct_size(p, probes, count),
110				       GFP_KERNEL);
111	return p == NULL ? NULL : p->probes;
112}
113
114static void srcu_free_old_probes(struct rcu_head *head)
115{
116	kfree(container_of(head, struct tp_probes, rcu));
117}
118
119static void rcu_free_old_probes(struct rcu_head *head)
120{
121	call_srcu(&tracepoint_srcu, head, srcu_free_old_probes);
122}
123
124static __init int release_early_probes(void)
125{
126	struct rcu_head *tmp;
127
128	ok_to_free_tracepoints = true;
129
130	while (early_probes) {
131		tmp = early_probes;
132		early_probes = tmp->next;
133		call_rcu(tmp, rcu_free_old_probes);
134	}
135
136	return 0;
137}
138
139/* SRCU is initialized at core_initcall */
140postcore_initcall(release_early_probes);
141
142static inline void release_probes(struct tracepoint_func *old)
143{
144	if (old) {
145		struct tp_probes *tp_probes = container_of(old,
146			struct tp_probes, probes[0]);
147
148		/*
149		 * We can't free probes if SRCU is not initialized yet.
150		 * Postpone the freeing till after SRCU is initialized.
151		 */
152		if (unlikely(!ok_to_free_tracepoints)) {
153			tp_probes->rcu.next = early_probes;
154			early_probes = &tp_probes->rcu;
155			return;
156		}
157
158		/*
159		 * Tracepoint probes are protected by both sched RCU and SRCU,
160		 * by calling the SRCU callback in the sched RCU callback we
161		 * cover both cases. So let us chain the SRCU and sched RCU
162		 * callbacks to wait for both grace periods.
163		 */
164		call_rcu(&tp_probes->rcu, rcu_free_old_probes);
165	}
166}
167
168static void debug_print_probes(struct tracepoint_func *funcs)
169{
170	int i;
171
172	if (!tracepoint_debug || !funcs)
173		return;
174
175	for (i = 0; funcs[i].func; i++)
176		printk(KERN_DEBUG "Probe %d : %p\n", i, funcs[i].func);
177}
178
179static struct tracepoint_func *
180func_add(struct tracepoint_func **funcs, struct tracepoint_func *tp_func,
181	 int prio)
182{
183	struct tracepoint_func *old, *new;
184	int iter_probes;	/* Iterate over old probe array. */
185	int nr_probes = 0;	/* Counter for probes */
186	int pos = -1;		/* Insertion position into new array */
187
188	if (WARN_ON(!tp_func->func))
189		return ERR_PTR(-EINVAL);
190
191	debug_print_probes(*funcs);
192	old = *funcs;
193	if (old) {
194		/* (N -> N+1), (N != 0, 1) probes */
195		for (iter_probes = 0; old[iter_probes].func; iter_probes++) {
196			if (old[iter_probes].func == tp_stub_func)
197				continue;	/* Skip stub functions. */
198			if (old[iter_probes].func == tp_func->func &&
199			    old[iter_probes].data == tp_func->data)
200				return ERR_PTR(-EEXIST);
201			nr_probes++;
202		}
203	}
204	/* + 2 : one for new probe, one for NULL func */
205	new = allocate_probes(nr_probes + 2);
206	if (new == NULL)
207		return ERR_PTR(-ENOMEM);
208	if (old) {
209		nr_probes = 0;
210		for (iter_probes = 0; old[iter_probes].func; iter_probes++) {
211			if (old[iter_probes].func == tp_stub_func)
212				continue;
213			/* Insert before probes of lower priority */
214			if (pos < 0 && old[iter_probes].prio < prio)
215				pos = nr_probes++;
216			new[nr_probes++] = old[iter_probes];
217		}
218		if (pos < 0)
219			pos = nr_probes++;
220		/* nr_probes now points to the end of the new array */
221	} else {
222		pos = 0;
223		nr_probes = 1; /* must point at end of array */
224	}
225	new[pos] = *tp_func;
226	new[nr_probes].func = NULL;
227	*funcs = new;
228	debug_print_probes(*funcs);
229	return old;
230}
231
232static void *func_remove(struct tracepoint_func **funcs,
233		struct tracepoint_func *tp_func)
234{
235	int nr_probes = 0, nr_del = 0, i;
236	struct tracepoint_func *old, *new;
237
238	old = *funcs;
239
240	if (!old)
241		return ERR_PTR(-ENOENT);
242
243	debug_print_probes(*funcs);
244	/* (N -> M), (N > 1, M >= 0) probes */
245	if (tp_func->func) {
246		for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
247			if ((old[nr_probes].func == tp_func->func &&
248			     old[nr_probes].data == tp_func->data) ||
249			    old[nr_probes].func == tp_stub_func)
250				nr_del++;
251		}
252	}
253
254	/*
255	 * If probe is NULL, then nr_probes = nr_del = 0, and then the
256	 * entire entry will be removed.
257	 */
258	if (nr_probes - nr_del == 0) {
259		/* N -> 0, (N > 1) */
260		*funcs = NULL;
261		debug_print_probes(*funcs);
262		return old;
263	} else {
264		int j = 0;
265		/* N -> M, (N > 1, M > 0) */
266		/* + 1 for NULL */
267		new = allocate_probes(nr_probes - nr_del + 1);
268		if (new) {
269			for (i = 0; old[i].func; i++) {
270				if ((old[i].func != tp_func->func ||
271				     old[i].data != tp_func->data) &&
272				    old[i].func != tp_stub_func)
273					new[j++] = old[i];
274			}
275			new[nr_probes - nr_del].func = NULL;
276			*funcs = new;
277		} else {
278			/*
279			 * Failed to allocate, replace the old function
280			 * with calls to tp_stub_func.
281			 */
282			for (i = 0; old[i].func; i++) {
283				if (old[i].func == tp_func->func &&
284				    old[i].data == tp_func->data)
285					WRITE_ONCE(old[i].func, tp_stub_func);
286			}
287			*funcs = old;
288		}
289	}
290	debug_print_probes(*funcs);
291	return old;
292}
293
294/*
295 * Count the number of functions (enum tp_func_state) in a tp_funcs array.
296 */
297static enum tp_func_state nr_func_state(const struct tracepoint_func *tp_funcs)
298{
299	if (!tp_funcs)
300		return TP_FUNC_0;
301	if (!tp_funcs[1].func)
302		return TP_FUNC_1;
303	if (!tp_funcs[2].func)
304		return TP_FUNC_2;
305	return TP_FUNC_N;	/* 3 or more */
306}
307
308static void tracepoint_update_call(struct tracepoint *tp, struct tracepoint_func *tp_funcs)
309{
310	void *func = tp->iterator;
311
312	/* Synthetic events do not have static call sites */
313	if (!tp->static_call_key)
314		return;
315	if (nr_func_state(tp_funcs) == TP_FUNC_1)
316		func = tp_funcs[0].func;
317	__static_call_update(tp->static_call_key, tp->static_call_tramp, func);
318}
319
320/*
321 * Add the probe function to a tracepoint.
322 */
323static int tracepoint_add_func(struct tracepoint *tp,
324			       struct tracepoint_func *func, int prio,
325			       bool warn)
326{
327	struct tracepoint_func *old, *tp_funcs;
328	int ret;
329
330	if (tp->regfunc && !static_key_enabled(&tp->key)) {
331		ret = tp->regfunc();
332		if (ret < 0)
333			return ret;
334	}
335
336	tp_funcs = rcu_dereference_protected(tp->funcs,
337			lockdep_is_held(&tracepoints_mutex));
338	old = func_add(&tp_funcs, func, prio);
339	if (IS_ERR(old)) {
340		WARN_ON_ONCE(warn && PTR_ERR(old) != -ENOMEM);
341		return PTR_ERR(old);
342	}
343
344	/*
345	 * rcu_assign_pointer has as smp_store_release() which makes sure
346	 * that the new probe callbacks array is consistent before setting
347	 * a pointer to it.  This array is referenced by __DO_TRACE from
348	 * include/linux/tracepoint.h using rcu_dereference_sched().
349	 */
350	switch (nr_func_state(tp_funcs)) {
351	case TP_FUNC_1:		/* 0->1 */
352		/*
353		 * Make sure new static func never uses old data after a
354		 * 1->0->1 transition sequence.
355		 */
356		tp_rcu_cond_sync(TP_TRANSITION_SYNC_1_0_1);
357		/* Set static call to first function */
358		tracepoint_update_call(tp, tp_funcs);
359		/* Both iterator and static call handle NULL tp->funcs */
360		rcu_assign_pointer(tp->funcs, tp_funcs);
361		static_key_enable(&tp->key);
362		break;
363	case TP_FUNC_2:		/* 1->2 */
364		/* Set iterator static call */
365		tracepoint_update_call(tp, tp_funcs);
366		/*
367		 * Iterator callback installed before updating tp->funcs.
368		 * Requires ordering between RCU assign/dereference and
369		 * static call update/call.
370		 */
371		fallthrough;
372	case TP_FUNC_N:		/* N->N+1 (N>1) */
373		rcu_assign_pointer(tp->funcs, tp_funcs);
374		/*
375		 * Make sure static func never uses incorrect data after a
376		 * N->...->2->1 (N>1) transition sequence.
377		 */
378		if (tp_funcs[0].data != old[0].data)
379			tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
380		break;
381	default:
382		WARN_ON_ONCE(1);
383		break;
384	}
385
386	release_probes(old);
387	return 0;
388}
389
390/*
391 * Remove a probe function from a tracepoint.
392 * Note: only waiting an RCU period after setting elem->call to the empty
393 * function insures that the original callback is not used anymore. This insured
394 * by preempt_disable around the call site.
395 */
396static int tracepoint_remove_func(struct tracepoint *tp,
397		struct tracepoint_func *func)
398{
399	struct tracepoint_func *old, *tp_funcs;
400
401	tp_funcs = rcu_dereference_protected(tp->funcs,
402			lockdep_is_held(&tracepoints_mutex));
403	old = func_remove(&tp_funcs, func);
404	if (WARN_ON_ONCE(IS_ERR(old)))
405		return PTR_ERR(old);
406
407	if (tp_funcs == old)
408		/* Failed allocating new tp_funcs, replaced func with stub */
409		return 0;
410
411	switch (nr_func_state(tp_funcs)) {
412	case TP_FUNC_0:		/* 1->0 */
413		/* Removed last function */
414		if (tp->unregfunc && static_key_enabled(&tp->key))
415			tp->unregfunc();
416
417		static_key_disable(&tp->key);
418		/* Set iterator static call */
419		tracepoint_update_call(tp, tp_funcs);
420		/* Both iterator and static call handle NULL tp->funcs */
421		rcu_assign_pointer(tp->funcs, NULL);
422		/*
423		 * Make sure new static func never uses old data after a
424		 * 1->0->1 transition sequence.
425		 */
426		tp_rcu_get_state(TP_TRANSITION_SYNC_1_0_1);
427		break;
428	case TP_FUNC_1:		/* 2->1 */
429		rcu_assign_pointer(tp->funcs, tp_funcs);
430		/*
431		 * Make sure static func never uses incorrect data after a
432		 * N->...->2->1 (N>2) transition sequence. If the first
433		 * element's data has changed, then force the synchronization
434		 * to prevent current readers that have loaded the old data
435		 * from calling the new function.
436		 */
437		if (tp_funcs[0].data != old[0].data)
438			tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
439		tp_rcu_cond_sync(TP_TRANSITION_SYNC_N_2_1);
440		/* Set static call to first function */
441		tracepoint_update_call(tp, tp_funcs);
442		break;
443	case TP_FUNC_2:		/* N->N-1 (N>2) */
444		fallthrough;
445	case TP_FUNC_N:
446		rcu_assign_pointer(tp->funcs, tp_funcs);
447		/*
448		 * Make sure static func never uses incorrect data after a
449		 * N->...->2->1 (N>2) transition sequence.
450		 */
451		if (tp_funcs[0].data != old[0].data)
452			tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
453		break;
454	default:
455		WARN_ON_ONCE(1);
456		break;
457	}
458	release_probes(old);
459	return 0;
460}
461
462/**
463 * tracepoint_probe_register_prio_may_exist -  Connect a probe to a tracepoint with priority
464 * @tp: tracepoint
465 * @probe: probe handler
466 * @data: tracepoint data
467 * @prio: priority of this function over other registered functions
468 *
469 * Same as tracepoint_probe_register_prio() except that it will not warn
470 * if the tracepoint is already registered.
471 */
472int tracepoint_probe_register_prio_may_exist(struct tracepoint *tp, void *probe,
473					     void *data, int prio)
474{
475	struct tracepoint_func tp_func;
476	int ret;
477
478	mutex_lock(&tracepoints_mutex);
479	tp_func.func = probe;
480	tp_func.data = data;
481	tp_func.prio = prio;
482	ret = tracepoint_add_func(tp, &tp_func, prio, false);
483	mutex_unlock(&tracepoints_mutex);
484	return ret;
485}
486EXPORT_SYMBOL_GPL(tracepoint_probe_register_prio_may_exist);
487
488/**
489 * tracepoint_probe_register_prio -  Connect a probe to a tracepoint with priority
490 * @tp: tracepoint
491 * @probe: probe handler
492 * @data: tracepoint data
493 * @prio: priority of this function over other registered functions
494 *
495 * Returns 0 if ok, error value on error.
496 * Note: if @tp is within a module, the caller is responsible for
497 * unregistering the probe before the module is gone. This can be
498 * performed either with a tracepoint module going notifier, or from
499 * within module exit functions.
500 */
501int tracepoint_probe_register_prio(struct tracepoint *tp, void *probe,
502				   void *data, int prio)
503{
504	struct tracepoint_func tp_func;
505	int ret;
506
507	mutex_lock(&tracepoints_mutex);
508	tp_func.func = probe;
509	tp_func.data = data;
510	tp_func.prio = prio;
511	ret = tracepoint_add_func(tp, &tp_func, prio, true);
512	mutex_unlock(&tracepoints_mutex);
513	return ret;
514}
515EXPORT_SYMBOL_GPL(tracepoint_probe_register_prio);
516
517/**
518 * tracepoint_probe_register -  Connect a probe to a tracepoint
519 * @tp: tracepoint
520 * @probe: probe handler
521 * @data: tracepoint data
522 *
523 * Returns 0 if ok, error value on error.
524 * Note: if @tp is within a module, the caller is responsible for
525 * unregistering the probe before the module is gone. This can be
526 * performed either with a tracepoint module going notifier, or from
527 * within module exit functions.
528 */
529int tracepoint_probe_register(struct tracepoint *tp, void *probe, void *data)
530{
531	return tracepoint_probe_register_prio(tp, probe, data, TRACEPOINT_DEFAULT_PRIO);
532}
533EXPORT_SYMBOL_GPL(tracepoint_probe_register);
534
535/**
536 * tracepoint_probe_unregister -  Disconnect a probe from a tracepoint
537 * @tp: tracepoint
538 * @probe: probe function pointer
539 * @data: tracepoint data
540 *
541 * Returns 0 if ok, error value on error.
542 */
543int tracepoint_probe_unregister(struct tracepoint *tp, void *probe, void *data)
544{
545	struct tracepoint_func tp_func;
546	int ret;
547
548	mutex_lock(&tracepoints_mutex);
549	tp_func.func = probe;
550	tp_func.data = data;
551	ret = tracepoint_remove_func(tp, &tp_func);
552	mutex_unlock(&tracepoints_mutex);
553	return ret;
554}
555EXPORT_SYMBOL_GPL(tracepoint_probe_unregister);
556
557static void for_each_tracepoint_range(
558		tracepoint_ptr_t *begin, tracepoint_ptr_t *end,
559		void (*fct)(struct tracepoint *tp, void *priv),
560		void *priv)
561{
562	tracepoint_ptr_t *iter;
563
564	if (!begin)
565		return;
566	for (iter = begin; iter < end; iter++)
567		fct(tracepoint_ptr_deref(iter), priv);
568}
569
570#ifdef CONFIG_MODULES
571bool trace_module_has_bad_taint(struct module *mod)
572{
573	return mod->taints & ~((1 << TAINT_OOT_MODULE) | (1 << TAINT_CRAP) |
574			       (1 << TAINT_UNSIGNED_MODULE) |
575			       (1 << TAINT_TEST));
576}
577
578static BLOCKING_NOTIFIER_HEAD(tracepoint_notify_list);
579
580/**
581 * register_tracepoint_module_notifier - register tracepoint coming/going notifier
582 * @nb: notifier block
583 *
584 * Notifiers registered with this function are called on module
585 * coming/going with the tracepoint_module_list_mutex held.
586 * The notifier block callback should expect a "struct tp_module" data
587 * pointer.
588 */
589int register_tracepoint_module_notifier(struct notifier_block *nb)
590{
591	struct tp_module *tp_mod;
592	int ret;
593
594	mutex_lock(&tracepoint_module_list_mutex);
595	ret = blocking_notifier_chain_register(&tracepoint_notify_list, nb);
596	if (ret)
597		goto end;
598	list_for_each_entry(tp_mod, &tracepoint_module_list, list)
599		(void) nb->notifier_call(nb, MODULE_STATE_COMING, tp_mod);
600end:
601	mutex_unlock(&tracepoint_module_list_mutex);
602	return ret;
603}
604EXPORT_SYMBOL_GPL(register_tracepoint_module_notifier);
605
606/**
607 * unregister_tracepoint_module_notifier - unregister tracepoint coming/going notifier
608 * @nb: notifier block
609 *
610 * The notifier block callback should expect a "struct tp_module" data
611 * pointer.
612 */
613int unregister_tracepoint_module_notifier(struct notifier_block *nb)
614{
615	struct tp_module *tp_mod;
616	int ret;
617
618	mutex_lock(&tracepoint_module_list_mutex);
619	ret = blocking_notifier_chain_unregister(&tracepoint_notify_list, nb);
620	if (ret)
621		goto end;
622	list_for_each_entry(tp_mod, &tracepoint_module_list, list)
623		(void) nb->notifier_call(nb, MODULE_STATE_GOING, tp_mod);
624end:
625	mutex_unlock(&tracepoint_module_list_mutex);
626	return ret;
627
628}
629EXPORT_SYMBOL_GPL(unregister_tracepoint_module_notifier);
630
631/*
632 * Ensure the tracer unregistered the module's probes before the module
633 * teardown is performed. Prevents leaks of probe and data pointers.
634 */
635static void tp_module_going_check_quiescent(struct tracepoint *tp, void *priv)
636{
637	WARN_ON_ONCE(tp->funcs);
638}
639
640static int tracepoint_module_coming(struct module *mod)
641{
642	struct tp_module *tp_mod;
643	int ret = 0;
644
645	if (!mod->num_tracepoints)
646		return 0;
647
648	/*
649	 * We skip modules that taint the kernel, especially those with different
650	 * module headers (for forced load), to make sure we don't cause a crash.
651	 * Staging, out-of-tree, unsigned GPL, and test modules are fine.
652	 */
653	if (trace_module_has_bad_taint(mod))
654		return 0;
655	mutex_lock(&tracepoint_module_list_mutex);
656	tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL);
657	if (!tp_mod) {
658		ret = -ENOMEM;
659		goto end;
660	}
661	tp_mod->mod = mod;
662	list_add_tail(&tp_mod->list, &tracepoint_module_list);
663	blocking_notifier_call_chain(&tracepoint_notify_list,
664			MODULE_STATE_COMING, tp_mod);
665end:
666	mutex_unlock(&tracepoint_module_list_mutex);
667	return ret;
668}
669
670static void tracepoint_module_going(struct module *mod)
671{
672	struct tp_module *tp_mod;
673
674	if (!mod->num_tracepoints)
675		return;
676
677	mutex_lock(&tracepoint_module_list_mutex);
678	list_for_each_entry(tp_mod, &tracepoint_module_list, list) {
679		if (tp_mod->mod == mod) {
680			blocking_notifier_call_chain(&tracepoint_notify_list,
681					MODULE_STATE_GOING, tp_mod);
682			list_del(&tp_mod->list);
683			kfree(tp_mod);
684			/*
685			 * Called the going notifier before checking for
686			 * quiescence.
687			 */
688			for_each_tracepoint_range(mod->tracepoints_ptrs,
689				mod->tracepoints_ptrs + mod->num_tracepoints,
690				tp_module_going_check_quiescent, NULL);
691			break;
692		}
693	}
694	/*
695	 * In the case of modules that were tainted at "coming", we'll simply
696	 * walk through the list without finding it. We cannot use the "tainted"
697	 * flag on "going", in case a module taints the kernel only after being
698	 * loaded.
699	 */
700	mutex_unlock(&tracepoint_module_list_mutex);
701}
702
703static int tracepoint_module_notify(struct notifier_block *self,
704		unsigned long val, void *data)
705{
706	struct module *mod = data;
707	int ret = 0;
708
709	switch (val) {
710	case MODULE_STATE_COMING:
711		ret = tracepoint_module_coming(mod);
712		break;
713	case MODULE_STATE_LIVE:
714		break;
715	case MODULE_STATE_GOING:
716		tracepoint_module_going(mod);
717		break;
718	case MODULE_STATE_UNFORMED:
719		break;
720	}
721	return notifier_from_errno(ret);
722}
723
724static struct notifier_block tracepoint_module_nb = {
725	.notifier_call = tracepoint_module_notify,
726	.priority = 0,
727};
728
729static __init int init_tracepoints(void)
730{
731	int ret;
732
733	ret = register_module_notifier(&tracepoint_module_nb);
734	if (ret)
735		pr_warn("Failed to register tracepoint module enter notifier\n");
736
737	return ret;
738}
739__initcall(init_tracepoints);
740#endif /* CONFIG_MODULES */
741
742/**
743 * for_each_kernel_tracepoint - iteration on all kernel tracepoints
744 * @fct: callback
745 * @priv: private data
746 */
747void for_each_kernel_tracepoint(void (*fct)(struct tracepoint *tp, void *priv),
748		void *priv)
749{
750	for_each_tracepoint_range(__start___tracepoints_ptrs,
751		__stop___tracepoints_ptrs, fct, priv);
752}
753EXPORT_SYMBOL_GPL(for_each_kernel_tracepoint);
754
755#ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
756
757/* NB: reg/unreg are called while guarded with the tracepoints_mutex */
758static int sys_tracepoint_refcount;
759
760int syscall_regfunc(void)
761{
762	struct task_struct *p, *t;
763
764	if (!sys_tracepoint_refcount) {
765		read_lock(&tasklist_lock);
766		for_each_process_thread(p, t) {
767			set_task_syscall_work(t, SYSCALL_TRACEPOINT);
768		}
769		read_unlock(&tasklist_lock);
770	}
771	sys_tracepoint_refcount++;
772
773	return 0;
774}
775
776void syscall_unregfunc(void)
777{
778	struct task_struct *p, *t;
779
780	sys_tracepoint_refcount--;
781	if (!sys_tracepoint_refcount) {
782		read_lock(&tasklist_lock);
783		for_each_process_thread(p, t) {
784			clear_task_syscall_work(t, SYSCALL_TRACEPOINT);
785		}
786		read_unlock(&tasklist_lock);
787	}
788}
789#endif
790