kernel/bpf/memalloc.c

513 int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)
526 	ma->percpu = percpu;
541 		ma->objcg = objcg;
552 		ma->cache = pc;
562 	ma->objcg = objcg;
577 	ma->caches = pcc;
581 int bpf_mem_alloc_percpu_init(struct bpf_mem_alloc *ma, struct obj_cgroup *objcg)
589 	ma->caches = pcc;
590 	ma->objcg = objcg;
591 	ma->percpu = true;
595 int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size)
610 	objcg = ma->objcg;
611 	pcc = ma->caches;
662 static void check_leaked_objs(struct bpf_mem_alloc *ma)
668 	if (ma->cache) {
670 			c = per_cpu_ptr(ma->cache, cpu);
674 	if (ma->caches) {
676 			cc = per_cpu_ptr(ma->caches, cpu);
685 static void free_mem_alloc_no_barrier(struct bpf_mem_alloc *ma)
687 	check_leaked_objs(ma);
688 	free_percpu(ma->cache);
689 	free_percpu(ma->caches);
690 	ma->cache = NULL;
691 	ma->caches = NULL;
694 static void free_mem_alloc(struct bpf_mem_alloc *ma)
710 	free_mem_alloc_no_barrier(ma);
715 	struct bpf_mem_alloc *ma = container_of(work, struct bpf_mem_alloc, work);
717 	free_mem_alloc(ma);
718 	kfree(ma);
721 static void destroy_mem_alloc(struct bpf_mem_alloc *ma, int rcu_in_progress)
729 		free_mem_alloc_no_barrier(ma);
733 	copy = kmemdup(ma, sizeof(*ma), GFP_KERNEL);
736 		free_mem_alloc(ma);
741 	memset(ma, 0, sizeof(*ma));
746 void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma)
752 	if (ma->cache) {
755 			c = per_cpu_ptr(ma->cache, cpu);
762 		if (ma->objcg)
763 			obj_cgroup_put(ma->objcg);
764 		destroy_mem_alloc(ma, rcu_in_progress);
766 	if (ma->caches) {
769 			cc = per_cpu_ptr(ma->caches, cpu);
779 		if (ma->objcg)
780 			obj_cgroup_put(ma->objcg);
781 		destroy_mem_alloc(ma, rcu_in_progress);
895 void notrace *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size)
903 	if (!ma->percpu)
909 	ret = unit_alloc(this_cpu_ptr(ma->caches)->cache + idx);
913 void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr)
926 	unit_free(this_cpu_ptr(ma->caches)->cache + idx, ptr);
929 void notrace bpf_mem_free_rcu(struct bpf_mem_alloc *ma, void *ptr)
942 	unit_free_rcu(this_cpu_ptr(ma->caches)->cache + idx, ptr);
945 void notrace *bpf_mem_cache_alloc(struct bpf_mem_alloc *ma)
949 	ret = unit_alloc(this_cpu_ptr(ma->cache));
953 void notrace bpf_mem_cache_free(struct bpf_mem_alloc *ma, void *ptr)
958 	unit_free(this_cpu_ptr(ma->cache), ptr);
961 void notrace bpf_mem_cache_free_rcu(struct bpf_mem_alloc *ma, void *ptr)
966 	unit_free_rcu(this_cpu_ptr(ma->cache), ptr);
991 void notrace *bpf_mem_cache_alloc_flags(struct bpf_mem_alloc *ma, gfp_t flags)
996 	c = this_cpu_ptr(ma->cache);