1// SPDX-License-Identifier: GPL-2.0 2// Copyright (c) 2019 Facebook 3 4#include <stdint.h> 5#include <stddef.h> 6#include <stdbool.h> 7#include <linux/bpf.h> 8#include <linux/ptrace.h> 9#include <linux/sched.h> 10#include <linux/types.h> 11#include <bpf/bpf_helpers.h> 12 13#include "bpf_compiler.h" 14 15typedef uint32_t pid_t; 16struct task_struct {}; 17 18#define TASK_COMM_LEN 16 19#define PERF_MAX_STACK_DEPTH 127 20 21#define STROBE_TYPE_INVALID 0 22#define STROBE_TYPE_INT 1 23#define STROBE_TYPE_STR 2 24#define STROBE_TYPE_MAP 3 25 26#define STACK_TABLE_EPOCH_SHIFT 20 27#define STROBE_MAX_STR_LEN 1 28#define STROBE_MAX_CFGS 32 29#define READ_MAP_VAR_PAYLOAD_CAP \ 30 ((1 + STROBE_MAX_MAP_ENTRIES * 2) * STROBE_MAX_STR_LEN) 31#define STROBE_MAX_PAYLOAD \ 32 (STROBE_MAX_STRS * STROBE_MAX_STR_LEN + \ 33 STROBE_MAX_MAPS * READ_MAP_VAR_PAYLOAD_CAP) 34 35struct strobe_value_header { 36 /* 37 * meaning depends on type: 38 * 1. int: 0, if value not set, 1 otherwise 39 * 2. str: 1 always, whether value is set or not is determined by ptr 40 * 3. map: 1 always, pointer points to additional struct with number 41 * of entries (up to STROBE_MAX_MAP_ENTRIES) 42 */ 43 uint16_t len; 44 /* 45 * _reserved might be used for some future fields/flags, but we always 46 * want to keep strobe_value_header to be 8 bytes, so BPF can read 16 47 * bytes in one go and get both header and value 48 */ 49 uint8_t _reserved[6]; 50}; 51 52/* 53 * strobe_value_generic is used from BPF probe only, but needs to be a union 54 * of strobe_value_int/strobe_value_str/strobe_value_map 55 */ 56struct strobe_value_generic { 57 struct strobe_value_header header; 58 union { 59 int64_t val; 60 void *ptr; 61 }; 62}; 63 64struct strobe_value_int { 65 struct strobe_value_header header; 66 int64_t value; 67}; 68 69struct strobe_value_str { 70 struct strobe_value_header header; 71 const char* value; 72}; 73 74struct strobe_value_map { 75 struct strobe_value_header header; 76 const struct strobe_map_raw* value; 77}; 78 79struct strobe_map_entry { 80 const char* key; 81 const char* val; 82}; 83 84/* 85 * Map of C-string key/value pairs with fixed maximum capacity. Each map has 86 * corresponding int64 ID, which application can use (or ignore) in whatever 87 * way appropriate. Map is "write-only", there is no way to get data out of 88 * map. Map is intended to be used to provide metadata for profilers and is 89 * not to be used for internal in-app communication. All methods are 90 * thread-safe. 91 */ 92struct strobe_map_raw { 93 /* 94 * general purpose unique ID that's up to application to decide 95 * whether and how to use; for request metadata use case id is unique 96 * request ID that's used to match metadata with stack traces on 97 * Strobelight backend side 98 */ 99 int64_t id; 100 /* number of used entries in map */ 101 int64_t cnt; 102 /* 103 * having volatile doesn't change anything on BPF side, but clang 104 * emits warnings for passing `volatile const char *` into 105 * bpf_probe_read_user_str that expects just `const char *` 106 */ 107 const char* tag; 108 /* 109 * key/value entries, each consisting of 2 pointers to key and value 110 * C strings 111 */ 112 struct strobe_map_entry entries[STROBE_MAX_MAP_ENTRIES]; 113}; 114 115/* Following values define supported values of TLS mode */ 116#define TLS_NOT_SET -1 117#define TLS_LOCAL_EXEC 0 118#define TLS_IMM_EXEC 1 119#define TLS_GENERAL_DYN 2 120 121/* 122 * structure that universally represents TLS location (both for static 123 * executables and shared libraries) 124 */ 125struct strobe_value_loc { 126 /* 127 * tls_mode defines what TLS mode was used for particular metavariable: 128 * - -1 (TLS_NOT_SET) - no metavariable; 129 * - 0 (TLS_LOCAL_EXEC) - Local Executable mode; 130 * - 1 (TLS_IMM_EXEC) - Immediate Executable mode; 131 * - 2 (TLS_GENERAL_DYN) - General Dynamic mode; 132 * Local Dynamic mode is not yet supported, because never seen in 133 * practice. Mode defines how offset field is interpreted. See 134 * calc_location() in below for details. 135 */ 136 int64_t tls_mode; 137 /* 138 * TLS_LOCAL_EXEC: offset from thread pointer (fs:0 for x86-64, 139 * tpidr_el0 for aarch64). 140 * TLS_IMM_EXEC: absolute address of GOT entry containing offset 141 * from thread pointer; 142 * TLS_GENERAL_DYN: absolute address of double GOT entry 143 * containing tls_index_t struct; 144 */ 145 int64_t offset; 146}; 147 148struct strobemeta_cfg { 149 int64_t req_meta_idx; 150 struct strobe_value_loc int_locs[STROBE_MAX_INTS]; 151 struct strobe_value_loc str_locs[STROBE_MAX_STRS]; 152 struct strobe_value_loc map_locs[STROBE_MAX_MAPS]; 153}; 154 155struct strobe_map_descr { 156 uint64_t id; 157 int16_t tag_len; 158 /* 159 * cnt <0 - map value isn't set; 160 * 0 - map has id set, but no key/value entries 161 */ 162 int16_t cnt; 163 /* 164 * both key_lens[i] and val_lens[i] should be >0 for present key/value 165 * entry 166 */ 167 uint16_t key_lens[STROBE_MAX_MAP_ENTRIES]; 168 uint16_t val_lens[STROBE_MAX_MAP_ENTRIES]; 169}; 170 171struct strobemeta_payload { 172 /* req_id has valid request ID, if req_meta_valid == 1 */ 173 int64_t req_id; 174 uint8_t req_meta_valid; 175 /* 176 * mask has Nth bit set to 1, if Nth metavar was present and 177 * successfully read 178 */ 179 uint64_t int_vals_set_mask; 180 int64_t int_vals[STROBE_MAX_INTS]; 181 /* len is >0 for present values */ 182 uint16_t str_lens[STROBE_MAX_STRS]; 183 /* if map_descrs[i].cnt == -1, metavar is not present/set */ 184 struct strobe_map_descr map_descrs[STROBE_MAX_MAPS]; 185 /* 186 * payload has compactly packed values of str and map variables in the 187 * form: strval1\0strval2\0map1key1\0map1val1\0map2key1\0map2val1\0 188 * (and so on); str_lens[i], key_lens[i] and val_lens[i] determines 189 * value length 190 */ 191 char payload[STROBE_MAX_PAYLOAD]; 192}; 193 194struct strobelight_bpf_sample { 195 uint64_t ktime; 196 char comm[TASK_COMM_LEN]; 197 pid_t pid; 198 int user_stack_id; 199 int kernel_stack_id; 200 int has_meta; 201 struct strobemeta_payload metadata; 202 /* 203 * makes it possible to pass (<real payload size> + 1) as data size to 204 * perf_submit() to avoid perf_submit's paranoia about passing zero as 205 * size, as it deduces that <real payload size> might be 206 * **theoretically** zero 207 */ 208 char dummy_safeguard; 209}; 210 211struct { 212 __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); 213 __uint(max_entries, 32); 214 __uint(key_size, sizeof(int)); 215 __uint(value_size, sizeof(int)); 216} samples SEC(".maps"); 217 218struct { 219 __uint(type, BPF_MAP_TYPE_STACK_TRACE); 220 __uint(max_entries, 16); 221 __uint(key_size, sizeof(uint32_t)); 222 __uint(value_size, sizeof(uint64_t) * PERF_MAX_STACK_DEPTH); 223} stacks_0 SEC(".maps"); 224 225struct { 226 __uint(type, BPF_MAP_TYPE_STACK_TRACE); 227 __uint(max_entries, 16); 228 __uint(key_size, sizeof(uint32_t)); 229 __uint(value_size, sizeof(uint64_t) * PERF_MAX_STACK_DEPTH); 230} stacks_1 SEC(".maps"); 231 232struct { 233 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 234 __uint(max_entries, 1); 235 __type(key, uint32_t); 236 __type(value, struct strobelight_bpf_sample); 237} sample_heap SEC(".maps"); 238 239struct { 240 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 241 __uint(max_entries, STROBE_MAX_CFGS); 242 __type(key, pid_t); 243 __type(value, struct strobemeta_cfg); 244} strobemeta_cfgs SEC(".maps"); 245 246/* Type for the dtv. */ 247/* https://github.com/lattera/glibc/blob/master/nptl/sysdeps/x86_64/tls.h#L34 */ 248typedef union dtv { 249 size_t counter; 250 struct { 251 void* val; 252 bool is_static; 253 } pointer; 254} dtv_t; 255 256/* Partial definition for tcbhead_t */ 257/* https://github.com/bminor/glibc/blob/master/sysdeps/x86_64/nptl/tls.h#L42 */ 258struct tcbhead { 259 void* tcb; 260 dtv_t* dtv; 261}; 262 263/* 264 * TLS module/offset information for shared library case. 265 * For x86-64, this is mapped onto two entries in GOT. 266 * For aarch64, this is pointed to by second GOT entry. 267 */ 268struct tls_index { 269 uint64_t module; 270 uint64_t offset; 271}; 272 273#ifdef SUBPROGS 274__noinline 275#else 276__always_inline 277#endif 278static void *calc_location(struct strobe_value_loc *loc, void *tls_base) 279{ 280 /* 281 * tls_mode value is: 282 * - -1 (TLS_NOT_SET), if no metavar is present; 283 * - 0 (TLS_LOCAL_EXEC), if metavar uses Local Executable mode of TLS 284 * (offset from fs:0 for x86-64 or tpidr_el0 for aarch64); 285 * - 1 (TLS_IMM_EXEC), if metavar uses Immediate Executable mode of TLS; 286 * - 2 (TLS_GENERAL_DYN), if metavar uses General Dynamic mode of TLS; 287 * This schema allows to use something like: 288 * (tls_mode + 1) * (tls_base + offset) 289 * to get NULL for "no metavar" location, or correct pointer for local 290 * executable mode without doing extra ifs. 291 */ 292 if (loc->tls_mode <= TLS_LOCAL_EXEC) { 293 /* static executable is simple, we just have offset from 294 * tls_base */ 295 void *addr = tls_base + loc->offset; 296 /* multiply by (tls_mode + 1) to get NULL, if we have no 297 * metavar in this slot */ 298 return (void *)((loc->tls_mode + 1) * (int64_t)addr); 299 } 300 /* 301 * Other modes are more complicated, we need to jump through few hoops. 302 * 303 * For immediate executable mode (currently supported only for aarch64): 304 * - loc->offset is pointing to a GOT entry containing fixed offset 305 * relative to tls_base; 306 * 307 * For general dynamic mode: 308 * - loc->offset is pointing to a beginning of double GOT entries; 309 * - (for aarch64 only) second entry points to tls_index_t struct; 310 * - (for x86-64 only) two GOT entries are already tls_index_t; 311 * - tls_index_t->module is used to find start of TLS section in 312 * which variable resides; 313 * - tls_index_t->offset provides offset within that TLS section, 314 * pointing to value of variable. 315 */ 316 struct tls_index tls_index; 317 dtv_t *dtv; 318 void *tls_ptr; 319 320 bpf_probe_read_user(&tls_index, sizeof(struct tls_index), 321 (void *)loc->offset); 322 /* valid module index is always positive */ 323 if (tls_index.module > 0) { 324 /* dtv = ((struct tcbhead *)tls_base)->dtv[tls_index.module] */ 325 bpf_probe_read_user(&dtv, sizeof(dtv), 326 &((struct tcbhead *)tls_base)->dtv); 327 dtv += tls_index.module; 328 } else { 329 dtv = NULL; 330 } 331 bpf_probe_read_user(&tls_ptr, sizeof(void *), dtv); 332 /* if pointer has (void *)-1 value, then TLS wasn't initialized yet */ 333 return tls_ptr && tls_ptr != (void *)-1 334 ? tls_ptr + tls_index.offset 335 : NULL; 336} 337 338#ifdef SUBPROGS 339__noinline 340#else 341__always_inline 342#endif 343static void read_int_var(struct strobemeta_cfg *cfg, 344 size_t idx, void *tls_base, 345 struct strobe_value_generic *value, 346 struct strobemeta_payload *data) 347{ 348 void *location = calc_location(&cfg->int_locs[idx], tls_base); 349 if (!location) 350 return; 351 352 bpf_probe_read_user(value, sizeof(struct strobe_value_generic), location); 353 data->int_vals[idx] = value->val; 354 if (value->header.len) 355 data->int_vals_set_mask |= (1 << idx); 356} 357 358static __always_inline uint64_t read_str_var(struct strobemeta_cfg *cfg, 359 size_t idx, void *tls_base, 360 struct strobe_value_generic *value, 361 struct strobemeta_payload *data, 362 size_t off) 363{ 364 void *location; 365 uint64_t len; 366 367 data->str_lens[idx] = 0; 368 location = calc_location(&cfg->str_locs[idx], tls_base); 369 if (!location) 370 return 0; 371 372 bpf_probe_read_user(value, sizeof(struct strobe_value_generic), location); 373 len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN, value->ptr); 374 /* 375 * if bpf_probe_read_user_str returns error (<0), due to casting to 376 * unsinged int, it will become big number, so next check is 377 * sufficient to check for errors AND prove to BPF verifier, that 378 * bpf_probe_read_user_str won't return anything bigger than 379 * STROBE_MAX_STR_LEN 380 */ 381 if (len > STROBE_MAX_STR_LEN) 382 return 0; 383 384 data->str_lens[idx] = len; 385 return off + len; 386} 387 388static __always_inline uint64_t read_map_var(struct strobemeta_cfg *cfg, 389 size_t idx, void *tls_base, 390 struct strobe_value_generic *value, 391 struct strobemeta_payload *data, 392 size_t off) 393{ 394 struct strobe_map_descr* descr = &data->map_descrs[idx]; 395 struct strobe_map_raw map; 396 void *location; 397 uint64_t len; 398 399 descr->tag_len = 0; /* presume no tag is set */ 400 descr->cnt = -1; /* presume no value is set */ 401 402 location = calc_location(&cfg->map_locs[idx], tls_base); 403 if (!location) 404 return off; 405 406 bpf_probe_read_user(value, sizeof(struct strobe_value_generic), location); 407 if (bpf_probe_read_user(&map, sizeof(struct strobe_map_raw), value->ptr)) 408 return off; 409 410 descr->id = map.id; 411 descr->cnt = map.cnt; 412 if (cfg->req_meta_idx == idx) { 413 data->req_id = map.id; 414 data->req_meta_valid = 1; 415 } 416 417 len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN, map.tag); 418 if (len <= STROBE_MAX_STR_LEN) { 419 descr->tag_len = len; 420 off += len; 421 } 422 423#ifdef NO_UNROLL 424 __pragma_loop_no_unroll 425#else 426 __pragma_loop_unroll 427#endif 428 for (int i = 0; i < STROBE_MAX_MAP_ENTRIES; ++i) { 429 if (i >= map.cnt) 430 break; 431 432 descr->key_lens[i] = 0; 433 len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN, 434 map.entries[i].key); 435 if (len <= STROBE_MAX_STR_LEN) { 436 descr->key_lens[i] = len; 437 off += len; 438 } 439 descr->val_lens[i] = 0; 440 len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN, 441 map.entries[i].val); 442 if (len <= STROBE_MAX_STR_LEN) { 443 descr->val_lens[i] = len; 444 off += len; 445 } 446 } 447 448 return off; 449} 450 451#ifdef USE_BPF_LOOP 452enum read_type { 453 READ_INT_VAR, 454 READ_MAP_VAR, 455 READ_STR_VAR, 456}; 457 458struct read_var_ctx { 459 struct strobemeta_payload *data; 460 void *tls_base; 461 struct strobemeta_cfg *cfg; 462 size_t payload_off; 463 /* value gets mutated */ 464 struct strobe_value_generic *value; 465 enum read_type type; 466}; 467 468static int read_var_callback(__u64 index, struct read_var_ctx *ctx) 469{ 470 /* lose precision info for ctx->payload_off, verifier won't track 471 * double xor, barrier_var() is needed to force clang keep both xors. 472 */ 473 ctx->payload_off ^= index; 474 barrier_var(ctx->payload_off); 475 ctx->payload_off ^= index; 476 switch (ctx->type) { 477 case READ_INT_VAR: 478 if (index >= STROBE_MAX_INTS) 479 return 1; 480 read_int_var(ctx->cfg, index, ctx->tls_base, ctx->value, ctx->data); 481 break; 482 case READ_MAP_VAR: 483 if (index >= STROBE_MAX_MAPS) 484 return 1; 485 if (ctx->payload_off > sizeof(ctx->data->payload) - READ_MAP_VAR_PAYLOAD_CAP) 486 return 1; 487 ctx->payload_off = read_map_var(ctx->cfg, index, ctx->tls_base, 488 ctx->value, ctx->data, ctx->payload_off); 489 break; 490 case READ_STR_VAR: 491 if (index >= STROBE_MAX_STRS) 492 return 1; 493 if (ctx->payload_off > sizeof(ctx->data->payload) - STROBE_MAX_STR_LEN) 494 return 1; 495 ctx->payload_off = read_str_var(ctx->cfg, index, ctx->tls_base, 496 ctx->value, ctx->data, ctx->payload_off); 497 break; 498 } 499 return 0; 500} 501#endif /* USE_BPF_LOOP */ 502 503/* 504 * read_strobe_meta returns NULL, if no metadata was read; otherwise returns 505 * pointer to *right after* payload ends 506 */ 507#ifdef SUBPROGS 508__noinline 509#else 510__always_inline 511#endif 512static void *read_strobe_meta(struct task_struct *task, 513 struct strobemeta_payload *data) 514{ 515 pid_t pid = bpf_get_current_pid_tgid() >> 32; 516 struct strobe_value_generic value = {0}; 517 struct strobemeta_cfg *cfg; 518 size_t payload_off; 519 void *tls_base; 520 521 cfg = bpf_map_lookup_elem(&strobemeta_cfgs, &pid); 522 if (!cfg) 523 return NULL; 524 525 data->int_vals_set_mask = 0; 526 data->req_meta_valid = 0; 527 payload_off = 0; 528 /* 529 * we don't have struct task_struct definition, it should be: 530 * tls_base = (void *)task->thread.fsbase; 531 */ 532 tls_base = (void *)task; 533 534#ifdef USE_BPF_LOOP 535 struct read_var_ctx ctx = { 536 .cfg = cfg, 537 .tls_base = tls_base, 538 .value = &value, 539 .data = data, 540 .payload_off = 0, 541 }; 542 int err; 543 544 ctx.type = READ_INT_VAR; 545 err = bpf_loop(STROBE_MAX_INTS, read_var_callback, &ctx, 0); 546 if (err != STROBE_MAX_INTS) 547 return NULL; 548 549 ctx.type = READ_STR_VAR; 550 err = bpf_loop(STROBE_MAX_STRS, read_var_callback, &ctx, 0); 551 if (err != STROBE_MAX_STRS) 552 return NULL; 553 554 ctx.type = READ_MAP_VAR; 555 err = bpf_loop(STROBE_MAX_MAPS, read_var_callback, &ctx, 0); 556 if (err != STROBE_MAX_MAPS) 557 return NULL; 558 559 payload_off = ctx.payload_off; 560 /* this should not really happen, here only to satisfy verifer */ 561 if (payload_off > sizeof(data->payload)) 562 payload_off = sizeof(data->payload); 563#else 564#ifdef NO_UNROLL 565 __pragma_loop_no_unroll 566#else 567 __pragma_loop_unroll 568#endif /* NO_UNROLL */ 569 for (int i = 0; i < STROBE_MAX_INTS; ++i) { 570 read_int_var(cfg, i, tls_base, &value, data); 571 } 572#ifdef NO_UNROLL 573 __pragma_loop_no_unroll 574#else 575 __pragma_loop_unroll 576#endif /* NO_UNROLL */ 577 for (int i = 0; i < STROBE_MAX_STRS; ++i) { 578 payload_off = read_str_var(cfg, i, tls_base, &value, data, payload_off); 579 } 580#ifdef NO_UNROLL 581 __pragma_loop_no_unroll 582#else 583 __pragma_loop_unroll 584#endif /* NO_UNROLL */ 585 for (int i = 0; i < STROBE_MAX_MAPS; ++i) { 586 payload_off = read_map_var(cfg, i, tls_base, &value, data, payload_off); 587 } 588#endif /* USE_BPF_LOOP */ 589 590 /* 591 * return pointer right after end of payload, so it's possible to 592 * calculate exact amount of useful data that needs to be sent 593 */ 594 return &data->payload[payload_off]; 595} 596 597SEC("raw_tracepoint/kfree_skb") 598int on_event(struct pt_regs *ctx) { 599 pid_t pid = bpf_get_current_pid_tgid() >> 32; 600 struct strobelight_bpf_sample* sample; 601 struct task_struct *task; 602 uint32_t zero = 0; 603 uint64_t ktime_ns; 604 void *sample_end; 605 606 sample = bpf_map_lookup_elem(&sample_heap, &zero); 607 if (!sample) 608 return 0; /* this will never happen */ 609 610 sample->pid = pid; 611 bpf_get_current_comm(&sample->comm, TASK_COMM_LEN); 612 ktime_ns = bpf_ktime_get_ns(); 613 sample->ktime = ktime_ns; 614 615 task = (struct task_struct *)bpf_get_current_task(); 616 sample_end = read_strobe_meta(task, &sample->metadata); 617 sample->has_meta = sample_end != NULL; 618 sample_end = sample_end ? : &sample->metadata; 619 620 if ((ktime_ns >> STACK_TABLE_EPOCH_SHIFT) & 1) { 621 sample->kernel_stack_id = bpf_get_stackid(ctx, &stacks_1, 0); 622 sample->user_stack_id = bpf_get_stackid(ctx, &stacks_1, BPF_F_USER_STACK); 623 } else { 624 sample->kernel_stack_id = bpf_get_stackid(ctx, &stacks_0, 0); 625 sample->user_stack_id = bpf_get_stackid(ctx, &stacks_0, BPF_F_USER_STACK); 626 } 627 628 uint64_t sample_size = sample_end - (void *)sample; 629 /* should always be true */ 630 if (sample_size < sizeof(struct strobelight_bpf_sample)) 631 bpf_perf_event_output(ctx, &samples, 0, sample, 1 + sample_size); 632 return 0; 633} 634 635char _license[] SEC("license") = "GPL"; 636