kernel/bpf/stackmap.c

45 static int prealloc_elems_and_freelist(struct bpf_stack_map *smap)
48 			(u64)smap->map.value_size;
51 	smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries,
52 					 smap->map.numa_node);
53 	if (!smap->elems)
56 	err = pcpu_freelist_init(&smap->freelist);
60 	pcpu_freelist_populate(&smap->freelist, smap->elems, elem_size,
61 			       smap->map.max_entries);
65 	bpf_map_area_free(smap->elems);
73 	struct bpf_stack_map *smap;
102 	cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap);
103 	smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr));
104 	if (!smap)
107 	bpf_map_init_from_attr(&smap->map, attr);
108 	smap->n_buckets = n_buckets;
114 	err = prealloc_elems_and_freelist(smap);
118 	return &smap->map;
123 	bpf_map_area_free(smap);
216 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
232 	id = hash & (smap->n_buckets - 1);
233 	bucket = READ_ONCE(smap->buckets[id]);
243 			pcpu_freelist_pop(&smap->freelist);
253 			pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
257 			pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
268 			pcpu_freelist_pop(&smap->freelist);
277 	old_bucket = xchg(&smap->buckets[id], new_bucket);
279 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
580 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
584 	if (unlikely(id >= smap->n_buckets))
587 	bucket = xchg(&smap->buckets[id], NULL);
595 	old_bucket = xchg(&smap->buckets[id], bucket);
597 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
604 	struct bpf_stack_map *smap = container_of(map,
614 		if (id >= smap->n_buckets || !smap->buckets[id])
620 	while (id < smap->n_buckets && !smap->buckets[id])
623 	if (id >= smap->n_buckets)
639 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
643 	if (unlikely(id >= smap->n_buckets))
646 	old_bucket = xchg(&smap->buckets[id], NULL);
648 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
658 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
660 	bpf_map_area_free(smap->elems);
661 	pcpu_freelist_destroy(&smap->freelist);
662 	bpf_map_area_free(smap);
668 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
670 	u64 n_buckets = smap->n_buckets;
672 	u64 usage = sizeof(*smap);