kernel/bpf/local_storage.c

78 		struct bpf_cgroup_storage *storage;
80 		storage = container_of(node, struct bpf_cgroup_storage, node);
82 		switch (bpf_cgroup_storage_key_cmp(map, key, &storage->key)) {
92 			return storage;
103 				 struct bpf_cgroup_storage *storage)
114 		switch (bpf_cgroup_storage_key_cmp(map, &storage->key, &this->key)) {
126 	rb_link_node(&storage->node, parent, new);
127 	rb_insert_color(&storage->node, root);
135 	struct bpf_cgroup_storage *storage;
137 	storage = cgroup_storage_lookup(map, key, false);
138 	if (!storage)
141 	return &READ_ONCE(storage->buf)->data[0];
147 	struct bpf_cgroup_storage *storage;
157 	storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map,
159 	if (!storage)
163 		copy_map_value_locked(map, storage->buf->data, value, false);
176 	new = xchg(&storage->buf, new);
186 	struct bpf_cgroup_storage *storage;
191 	storage = cgroup_storage_lookup(map, key, false);
192 	if (!storage) {
204 				per_cpu_ptr(storage->percpu_buf, cpu), size);
215 	struct bpf_cgroup_storage *storage;
223 	storage = cgroup_storage_lookup(map, key, false);
224 	if (!storage) {
237 		bpf_long_memcpy(per_cpu_ptr(storage->percpu_buf, cpu),
249 	struct bpf_cgroup_storage *storage;
257 		storage = cgroup_storage_lookup(map, key, true);
258 		if (!storage)
261 		storage = list_next_entry(storage, list_map);
262 		if (!storage)
265 		storage = list_first_entry(&map->list,
273 		*next = storage->key;
276 		*next = storage->key.cgroup_inode_id;
334 	struct bpf_cgroup_storage *storage, *stmp;
338 	list_for_each_entry_safe(storage, stmp, storages, list_map) {
339 		bpf_cgroup_storage_unlink(storage);
340 		bpf_cgroup_storage_free(storage);
418 	struct bpf_cgroup_storage *storage;
422 	storage = cgroup_storage_lookup(map_to_storage(map), key, false);
423 	if (!storage) {
433 				  &READ_ONCE(storage->buf)->data[0], m);
440 					  per_cpu_ptr(storage->percpu_buf, cpu),
503 	struct bpf_cgroup_storage *storage;
514 	storage = bpf_map_kmalloc_node(map, sizeof(struct bpf_cgroup_storage),
516 	if (!storage)
520 		storage->buf = bpf_map_kmalloc_node(map, size, gfp,
522 		if (!storage->buf)
524 		check_and_init_map_value(map, storage->buf->data);
526 		storage->percpu_buf = bpf_map_alloc_percpu(map, size, 8, gfp);
527 		if (!storage->percpu_buf)
531 	storage->map = (struct bpf_cgroup_storage_map *)map;
533 	return storage;
536 	kfree(storage);
542 	struct bpf_cgroup_storage *storage =
545 	kfree(storage->buf);
546 	kfree(storage);
551 	struct bpf_cgroup_storage *storage =
554 	free_percpu(storage->percpu_buf);
555 	kfree(storage);
558 void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage)
563 	if (!storage)
566 	map = &storage->map->map;
569 		call_rcu(&storage->rcu, free_shared_cgroup_storage_rcu);
571 		call_rcu(&storage->rcu, free_percpu_cgroup_storage_rcu);
574 void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,
580 	if (!storage)
583 	storage->key.attach_type = type;
584 	storage->key.cgroup_inode_id = cgroup_id(cgroup);
586 	map = storage->map;
589 	WARN_ON(cgroup_storage_insert(map, storage));
590 	list_add(&storage->list_map, &map->list);
591 	list_add(&storage->list_cg, &cgroup->bpf.storages);
595 void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage)
600 	if (!storage)
603 	map = storage->map;
607 	rb_erase(&storage->node, root);
609 	list_del(&storage->list_map);
610 	list_del(&storage->list_cg);