1// SPDX-License-Identifier: GPL-2.0 2/* Copyright (c) 2021 Facebook */ 3 4#define _GNU_SOURCE /* See feature_test_macros(7) */ 5#include <unistd.h> 6#include <sched.h> 7#include <pthread.h> 8#include <sys/syscall.h> /* For SYS_xxx definitions */ 9#include <sys/types.h> 10#include <test_progs.h> 11#include "task_local_storage_helpers.h" 12#include "task_local_storage.skel.h" 13#include "task_local_storage_exit_creds.skel.h" 14#include "task_ls_recursion.skel.h" 15#include "task_storage_nodeadlock.skel.h" 16 17static void test_sys_enter_exit(void) 18{ 19 struct task_local_storage *skel; 20 int err; 21 22 skel = task_local_storage__open_and_load(); 23 if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) 24 return; 25 26 skel->bss->target_pid = syscall(SYS_gettid); 27 28 err = task_local_storage__attach(skel); 29 if (!ASSERT_OK(err, "skel_attach")) 30 goto out; 31 32 syscall(SYS_gettid); 33 syscall(SYS_gettid); 34 35 /* 3x syscalls: 1x attach and 2x gettid */ 36 ASSERT_EQ(skel->bss->enter_cnt, 3, "enter_cnt"); 37 ASSERT_EQ(skel->bss->exit_cnt, 3, "exit_cnt"); 38 ASSERT_EQ(skel->bss->mismatch_cnt, 0, "mismatch_cnt"); 39out: 40 task_local_storage__destroy(skel); 41} 42 43static void test_exit_creds(void) 44{ 45 struct task_local_storage_exit_creds *skel; 46 int err, run_count, sync_rcu_calls = 0; 47 const int MAX_SYNC_RCU_CALLS = 1000; 48 49 skel = task_local_storage_exit_creds__open_and_load(); 50 if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) 51 return; 52 53 err = task_local_storage_exit_creds__attach(skel); 54 if (!ASSERT_OK(err, "skel_attach")) 55 goto out; 56 57 /* trigger at least one exit_creds() */ 58 if (CHECK_FAIL(system("ls > /dev/null"))) 59 goto out; 60 61 /* kern_sync_rcu is not enough on its own as the read section we want 62 * to wait for may start after we enter synchronize_rcu, so our call 63 * won't wait for the section to finish. Loop on the run counter 64 * as well to ensure the program has run. 65 */ 66 do { 67 kern_sync_rcu(); 68 run_count = __atomic_load_n(&skel->bss->run_count, __ATOMIC_SEQ_CST); 69 } while (run_count == 0 && ++sync_rcu_calls < MAX_SYNC_RCU_CALLS); 70 71 ASSERT_NEQ(sync_rcu_calls, MAX_SYNC_RCU_CALLS, 72 "sync_rcu count too high"); 73 ASSERT_NEQ(run_count, 0, "run_count"); 74 ASSERT_EQ(skel->bss->valid_ptr_count, 0, "valid_ptr_count"); 75 ASSERT_NEQ(skel->bss->null_ptr_count, 0, "null_ptr_count"); 76out: 77 task_local_storage_exit_creds__destroy(skel); 78} 79 80static void test_recursion(void) 81{ 82 int err, map_fd, prog_fd, task_fd; 83 struct task_ls_recursion *skel; 84 struct bpf_prog_info info; 85 __u32 info_len = sizeof(info); 86 long value; 87 88 task_fd = sys_pidfd_open(getpid(), 0); 89 if (!ASSERT_NEQ(task_fd, -1, "sys_pidfd_open")) 90 return; 91 92 skel = task_ls_recursion__open_and_load(); 93 if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) 94 goto out; 95 96 err = task_ls_recursion__attach(skel); 97 if (!ASSERT_OK(err, "skel_attach")) 98 goto out; 99 100 /* trigger sys_enter, make sure it does not cause deadlock */ 101 skel->bss->test_pid = getpid(); 102 syscall(SYS_gettid); 103 skel->bss->test_pid = 0; 104 task_ls_recursion__detach(skel); 105 106 /* Refer to the comment in BPF_PROG(on_update) for 107 * the explanation on the value 201 and 100. 108 */ 109 map_fd = bpf_map__fd(skel->maps.map_a); 110 err = bpf_map_lookup_elem(map_fd, &task_fd, &value); 111 ASSERT_OK(err, "lookup map_a"); 112 ASSERT_EQ(value, 201, "map_a value"); 113 ASSERT_EQ(skel->bss->nr_del_errs, 1, "bpf_task_storage_delete busy"); 114 115 map_fd = bpf_map__fd(skel->maps.map_b); 116 err = bpf_map_lookup_elem(map_fd, &task_fd, &value); 117 ASSERT_OK(err, "lookup map_b"); 118 ASSERT_EQ(value, 100, "map_b value"); 119 120 prog_fd = bpf_program__fd(skel->progs.on_update); 121 memset(&info, 0, sizeof(info)); 122 err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); 123 ASSERT_OK(err, "get prog info"); 124 ASSERT_EQ(info.recursion_misses, 0, "on_update prog recursion"); 125 126 prog_fd = bpf_program__fd(skel->progs.on_enter); 127 memset(&info, 0, sizeof(info)); 128 err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); 129 ASSERT_OK(err, "get prog info"); 130 ASSERT_EQ(info.recursion_misses, 0, "on_enter prog recursion"); 131 132out: 133 close(task_fd); 134 task_ls_recursion__destroy(skel); 135} 136 137static bool stop; 138 139static void waitall(const pthread_t *tids, int nr) 140{ 141 int i; 142 143 stop = true; 144 for (i = 0; i < nr; i++) 145 pthread_join(tids[i], NULL); 146} 147 148static void *sock_create_loop(void *arg) 149{ 150 struct task_storage_nodeadlock *skel = arg; 151 int fd; 152 153 while (!stop) { 154 fd = socket(AF_INET, SOCK_STREAM, 0); 155 close(fd); 156 if (skel->bss->nr_get_errs || skel->bss->nr_del_errs) 157 stop = true; 158 } 159 160 return NULL; 161} 162 163static void test_nodeadlock(void) 164{ 165 struct task_storage_nodeadlock *skel; 166 struct bpf_prog_info info = {}; 167 __u32 info_len = sizeof(info); 168 const int nr_threads = 32; 169 pthread_t tids[nr_threads]; 170 int i, prog_fd, err; 171 cpu_set_t old, new; 172 173 /* Pin all threads to one cpu to increase the chance of preemption 174 * in a sleepable bpf prog. 175 */ 176 CPU_ZERO(&new); 177 CPU_SET(0, &new); 178 err = sched_getaffinity(getpid(), sizeof(old), &old); 179 if (!ASSERT_OK(err, "getaffinity")) 180 return; 181 err = sched_setaffinity(getpid(), sizeof(new), &new); 182 if (!ASSERT_OK(err, "setaffinity")) 183 return; 184 185 skel = task_storage_nodeadlock__open_and_load(); 186 if (!ASSERT_OK_PTR(skel, "open_and_load")) 187 goto done; 188 189 /* Unnecessary recursion and deadlock detection are reproducible 190 * in the preemptible kernel. 191 */ 192 if (!skel->kconfig->CONFIG_PREEMPT) { 193 test__skip(); 194 goto done; 195 } 196 197 err = task_storage_nodeadlock__attach(skel); 198 ASSERT_OK(err, "attach prog"); 199 200 for (i = 0; i < nr_threads; i++) { 201 err = pthread_create(&tids[i], NULL, sock_create_loop, skel); 202 if (err) { 203 /* Only assert once here to avoid excessive 204 * PASS printing during test failure. 205 */ 206 ASSERT_OK(err, "pthread_create"); 207 waitall(tids, i); 208 goto done; 209 } 210 } 211 212 /* With 32 threads, 1s is enough to reproduce the issue */ 213 sleep(1); 214 waitall(tids, nr_threads); 215 216 info_len = sizeof(info); 217 prog_fd = bpf_program__fd(skel->progs.socket_post_create); 218 err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); 219 ASSERT_OK(err, "get prog info"); 220 ASSERT_EQ(info.recursion_misses, 0, "prog recursion"); 221 222 ASSERT_EQ(skel->bss->nr_get_errs, 0, "bpf_task_storage_get busy"); 223 ASSERT_EQ(skel->bss->nr_del_errs, 0, "bpf_task_storage_delete busy"); 224 225done: 226 task_storage_nodeadlock__destroy(skel); 227 sched_setaffinity(getpid(), sizeof(old), &old); 228} 229 230void test_task_local_storage(void) 231{ 232 if (test__start_subtest("sys_enter_exit")) 233 test_sys_enter_exit(); 234 if (test__start_subtest("exit_creds")) 235 test_exit_creds(); 236 if (test__start_subtest("recursion")) 237 test_recursion(); 238 if (test__start_subtest("nodeadlock")) 239 test_nodeadlock(); 240} 241