1// SPDX-License-Identifier: GPL-2.0-only 2/* Copyright (c) 2022 Google */ 3#include <linux/bpf.h> 4#include <linux/btf_ids.h> 5#include <linux/cgroup.h> 6#include <linux/kernel.h> 7#include <linux/seq_file.h> 8 9#include "../cgroup/cgroup-internal.h" /* cgroup_mutex and cgroup_is_dead */ 10 11/* cgroup_iter provides four modes of traversal to the cgroup hierarchy. 12 * 13 * 1. Walk the descendants of a cgroup in pre-order. 14 * 2. Walk the descendants of a cgroup in post-order. 15 * 3. Walk the ancestors of a cgroup. 16 * 4. Show the given cgroup only. 17 * 18 * For walking descendants, cgroup_iter can walk in either pre-order or 19 * post-order. For walking ancestors, the iter walks up from a cgroup to 20 * the root. 21 * 22 * The iter program can terminate the walk early by returning 1. Walk 23 * continues if prog returns 0. 24 * 25 * The prog can check (seq->num == 0) to determine whether this is 26 * the first element. The prog may also be passed a NULL cgroup, 27 * which means the walk has completed and the prog has a chance to 28 * do post-processing, such as outputting an epilogue. 29 * 30 * Note: the iter_prog is called with cgroup_mutex held. 31 * 32 * Currently only one session is supported, which means, depending on the 33 * volume of data bpf program intends to send to user space, the number 34 * of cgroups that can be walked is limited. For example, given the current 35 * buffer size is 8 * PAGE_SIZE, if the program sends 64B data for each 36 * cgroup, assuming PAGE_SIZE is 4kb, the total number of cgroups that can 37 * be walked is 512. This is a limitation of cgroup_iter. If the output data 38 * is larger than the kernel buffer size, after all data in the kernel buffer 39 * is consumed by user space, the subsequent read() syscall will signal 40 * EOPNOTSUPP. In order to work around, the user may have to update their 41 * program to reduce the volume of data sent to output. For example, skip 42 * some uninteresting cgroups. 43 */ 44 45struct bpf_iter__cgroup { 46 __bpf_md_ptr(struct bpf_iter_meta *, meta); 47 __bpf_md_ptr(struct cgroup *, cgroup); 48}; 49 50struct cgroup_iter_priv { 51 struct cgroup_subsys_state *start_css; 52 bool visited_all; 53 bool terminate; 54 int order; 55}; 56 57static void *cgroup_iter_seq_start(struct seq_file *seq, loff_t *pos) 58{ 59 struct cgroup_iter_priv *p = seq->private; 60 61 cgroup_lock(); 62 63 /* cgroup_iter doesn't support read across multiple sessions. */ 64 if (*pos > 0) { 65 if (p->visited_all) 66 return NULL; 67 68 /* Haven't visited all, but because cgroup_mutex has dropped, 69 * return -EOPNOTSUPP to indicate incomplete iteration. 70 */ 71 return ERR_PTR(-EOPNOTSUPP); 72 } 73 74 ++*pos; 75 p->terminate = false; 76 p->visited_all = false; 77 if (p->order == BPF_CGROUP_ITER_DESCENDANTS_PRE) 78 return css_next_descendant_pre(NULL, p->start_css); 79 else if (p->order == BPF_CGROUP_ITER_DESCENDANTS_POST) 80 return css_next_descendant_post(NULL, p->start_css); 81 else /* BPF_CGROUP_ITER_SELF_ONLY and BPF_CGROUP_ITER_ANCESTORS_UP */ 82 return p->start_css; 83} 84 85static int __cgroup_iter_seq_show(struct seq_file *seq, 86 struct cgroup_subsys_state *css, int in_stop); 87 88static void cgroup_iter_seq_stop(struct seq_file *seq, void *v) 89{ 90 struct cgroup_iter_priv *p = seq->private; 91 92 cgroup_unlock(); 93 94 /* pass NULL to the prog for post-processing */ 95 if (!v) { 96 __cgroup_iter_seq_show(seq, NULL, true); 97 p->visited_all = true; 98 } 99} 100 101static void *cgroup_iter_seq_next(struct seq_file *seq, void *v, loff_t *pos) 102{ 103 struct cgroup_subsys_state *curr = (struct cgroup_subsys_state *)v; 104 struct cgroup_iter_priv *p = seq->private; 105 106 ++*pos; 107 if (p->terminate) 108 return NULL; 109 110 if (p->order == BPF_CGROUP_ITER_DESCENDANTS_PRE) 111 return css_next_descendant_pre(curr, p->start_css); 112 else if (p->order == BPF_CGROUP_ITER_DESCENDANTS_POST) 113 return css_next_descendant_post(curr, p->start_css); 114 else if (p->order == BPF_CGROUP_ITER_ANCESTORS_UP) 115 return curr->parent; 116 else /* BPF_CGROUP_ITER_SELF_ONLY */ 117 return NULL; 118} 119 120static int __cgroup_iter_seq_show(struct seq_file *seq, 121 struct cgroup_subsys_state *css, int in_stop) 122{ 123 struct cgroup_iter_priv *p = seq->private; 124 struct bpf_iter__cgroup ctx; 125 struct bpf_iter_meta meta; 126 struct bpf_prog *prog; 127 int ret = 0; 128 129 /* cgroup is dead, skip this element */ 130 if (css && cgroup_is_dead(css->cgroup)) 131 return 0; 132 133 ctx.meta = &meta; 134 ctx.cgroup = css ? css->cgroup : NULL; 135 meta.seq = seq; 136 prog = bpf_iter_get_info(&meta, in_stop); 137 if (prog) 138 ret = bpf_iter_run_prog(prog, &ctx); 139 140 /* if prog returns > 0, terminate after this element. */ 141 if (ret != 0) 142 p->terminate = true; 143 144 return 0; 145} 146 147static int cgroup_iter_seq_show(struct seq_file *seq, void *v) 148{ 149 return __cgroup_iter_seq_show(seq, (struct cgroup_subsys_state *)v, 150 false); 151} 152 153static const struct seq_operations cgroup_iter_seq_ops = { 154 .start = cgroup_iter_seq_start, 155 .next = cgroup_iter_seq_next, 156 .stop = cgroup_iter_seq_stop, 157 .show = cgroup_iter_seq_show, 158}; 159 160BTF_ID_LIST_GLOBAL_SINGLE(bpf_cgroup_btf_id, struct, cgroup) 161 162static int cgroup_iter_seq_init(void *priv, struct bpf_iter_aux_info *aux) 163{ 164 struct cgroup_iter_priv *p = (struct cgroup_iter_priv *)priv; 165 struct cgroup *cgrp = aux->cgroup.start; 166 167 /* bpf_iter_attach_cgroup() has already acquired an extra reference 168 * for the start cgroup, but the reference may be released after 169 * cgroup_iter_seq_init(), so acquire another reference for the 170 * start cgroup. 171 */ 172 p->start_css = &cgrp->self; 173 css_get(p->start_css); 174 p->terminate = false; 175 p->visited_all = false; 176 p->order = aux->cgroup.order; 177 return 0; 178} 179 180static void cgroup_iter_seq_fini(void *priv) 181{ 182 struct cgroup_iter_priv *p = (struct cgroup_iter_priv *)priv; 183 184 css_put(p->start_css); 185} 186 187static const struct bpf_iter_seq_info cgroup_iter_seq_info = { 188 .seq_ops = &cgroup_iter_seq_ops, 189 .init_seq_private = cgroup_iter_seq_init, 190 .fini_seq_private = cgroup_iter_seq_fini, 191 .seq_priv_size = sizeof(struct cgroup_iter_priv), 192}; 193 194static int bpf_iter_attach_cgroup(struct bpf_prog *prog, 195 union bpf_iter_link_info *linfo, 196 struct bpf_iter_aux_info *aux) 197{ 198 int fd = linfo->cgroup.cgroup_fd; 199 u64 id = linfo->cgroup.cgroup_id; 200 int order = linfo->cgroup.order; 201 struct cgroup *cgrp; 202 203 if (order != BPF_CGROUP_ITER_DESCENDANTS_PRE && 204 order != BPF_CGROUP_ITER_DESCENDANTS_POST && 205 order != BPF_CGROUP_ITER_ANCESTORS_UP && 206 order != BPF_CGROUP_ITER_SELF_ONLY) 207 return -EINVAL; 208 209 if (fd && id) 210 return -EINVAL; 211 212 if (fd) 213 cgrp = cgroup_v1v2_get_from_fd(fd); 214 else if (id) 215 cgrp = cgroup_get_from_id(id); 216 else /* walk the entire hierarchy by default. */ 217 cgrp = cgroup_get_from_path("/"); 218 219 if (IS_ERR(cgrp)) 220 return PTR_ERR(cgrp); 221 222 aux->cgroup.start = cgrp; 223 aux->cgroup.order = order; 224 return 0; 225} 226 227static void bpf_iter_detach_cgroup(struct bpf_iter_aux_info *aux) 228{ 229 cgroup_put(aux->cgroup.start); 230} 231 232static void bpf_iter_cgroup_show_fdinfo(const struct bpf_iter_aux_info *aux, 233 struct seq_file *seq) 234{ 235 char *buf; 236 237 buf = kzalloc(PATH_MAX, GFP_KERNEL); 238 if (!buf) { 239 seq_puts(seq, "cgroup_path:\t<unknown>\n"); 240 goto show_order; 241 } 242 243 /* If cgroup_path_ns() fails, buf will be an empty string, cgroup_path 244 * will print nothing. 245 * 246 * Path is in the calling process's cgroup namespace. 247 */ 248 cgroup_path_ns(aux->cgroup.start, buf, PATH_MAX, 249 current->nsproxy->cgroup_ns); 250 seq_printf(seq, "cgroup_path:\t%s\n", buf); 251 kfree(buf); 252 253show_order: 254 if (aux->cgroup.order == BPF_CGROUP_ITER_DESCENDANTS_PRE) 255 seq_puts(seq, "order: descendants_pre\n"); 256 else if (aux->cgroup.order == BPF_CGROUP_ITER_DESCENDANTS_POST) 257 seq_puts(seq, "order: descendants_post\n"); 258 else if (aux->cgroup.order == BPF_CGROUP_ITER_ANCESTORS_UP) 259 seq_puts(seq, "order: ancestors_up\n"); 260 else /* BPF_CGROUP_ITER_SELF_ONLY */ 261 seq_puts(seq, "order: self_only\n"); 262} 263 264static int bpf_iter_cgroup_fill_link_info(const struct bpf_iter_aux_info *aux, 265 struct bpf_link_info *info) 266{ 267 info->iter.cgroup.order = aux->cgroup.order; 268 info->iter.cgroup.cgroup_id = cgroup_id(aux->cgroup.start); 269 return 0; 270} 271 272DEFINE_BPF_ITER_FUNC(cgroup, struct bpf_iter_meta *meta, 273 struct cgroup *cgroup) 274 275static struct bpf_iter_reg bpf_cgroup_reg_info = { 276 .target = "cgroup", 277 .feature = BPF_ITER_RESCHED, 278 .attach_target = bpf_iter_attach_cgroup, 279 .detach_target = bpf_iter_detach_cgroup, 280 .show_fdinfo = bpf_iter_cgroup_show_fdinfo, 281 .fill_link_info = bpf_iter_cgroup_fill_link_info, 282 .ctx_arg_info_size = 1, 283 .ctx_arg_info = { 284 { offsetof(struct bpf_iter__cgroup, cgroup), 285 PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED }, 286 }, 287 .seq_info = &cgroup_iter_seq_info, 288}; 289 290static int __init bpf_cgroup_iter_init(void) 291{ 292 bpf_cgroup_reg_info.ctx_arg_info[0].btf_id = bpf_cgroup_btf_id[0]; 293 return bpf_iter_reg_target(&bpf_cgroup_reg_info); 294} 295 296late_initcall(bpf_cgroup_iter_init); 297 298struct bpf_iter_css { 299 __u64 __opaque[3]; 300} __attribute__((aligned(8))); 301 302struct bpf_iter_css_kern { 303 struct cgroup_subsys_state *start; 304 struct cgroup_subsys_state *pos; 305 unsigned int flags; 306} __attribute__((aligned(8))); 307 308__bpf_kfunc_start_defs(); 309 310__bpf_kfunc int bpf_iter_css_new(struct bpf_iter_css *it, 311 struct cgroup_subsys_state *start, unsigned int flags) 312{ 313 struct bpf_iter_css_kern *kit = (void *)it; 314 315 BUILD_BUG_ON(sizeof(struct bpf_iter_css_kern) > sizeof(struct bpf_iter_css)); 316 BUILD_BUG_ON(__alignof__(struct bpf_iter_css_kern) != __alignof__(struct bpf_iter_css)); 317 318 kit->start = NULL; 319 switch (flags) { 320 case BPF_CGROUP_ITER_DESCENDANTS_PRE: 321 case BPF_CGROUP_ITER_DESCENDANTS_POST: 322 case BPF_CGROUP_ITER_ANCESTORS_UP: 323 break; 324 default: 325 return -EINVAL; 326 } 327 328 kit->start = start; 329 kit->pos = NULL; 330 kit->flags = flags; 331 return 0; 332} 333 334__bpf_kfunc struct cgroup_subsys_state *bpf_iter_css_next(struct bpf_iter_css *it) 335{ 336 struct bpf_iter_css_kern *kit = (void *)it; 337 338 if (!kit->start) 339 return NULL; 340 341 switch (kit->flags) { 342 case BPF_CGROUP_ITER_DESCENDANTS_PRE: 343 kit->pos = css_next_descendant_pre(kit->pos, kit->start); 344 break; 345 case BPF_CGROUP_ITER_DESCENDANTS_POST: 346 kit->pos = css_next_descendant_post(kit->pos, kit->start); 347 break; 348 case BPF_CGROUP_ITER_ANCESTORS_UP: 349 kit->pos = kit->pos ? kit->pos->parent : kit->start; 350 } 351 352 return kit->pos; 353} 354 355__bpf_kfunc void bpf_iter_css_destroy(struct bpf_iter_css *it) 356{ 357} 358 359__bpf_kfunc_end_defs(); 360