net/core/sock_map.c

862 static struct bpf_shtab_bucket *sock_hash_select_bucket(struct bpf_shtab *htab,
865 	return &htab->buckets[hash & (htab->buckets_num - 1)];
885 	struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
893 	bucket = sock_hash_select_bucket(htab, hash);
899 static void sock_hash_free_elem(struct bpf_shtab *htab,
902 	atomic_dec(&htab->count);
909 	struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
914 	bucket = sock_hash_select_bucket(htab, elem->hash);
926 		sock_hash_free_elem(htab, elem);
933 	struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
943 	bucket = sock_hash_select_bucket(htab, hash);
950 		sock_hash_free_elem(htab, elem);
957 static struct bpf_shtab_elem *sock_hash_alloc_elem(struct bpf_shtab *htab,
964 	if (atomic_inc_return(&htab->count) > htab->map.max_entries) {
966 			atomic_dec(&htab->count);
971 	new = bpf_map_kmalloc_node(&htab->map, htab->elem_size,
973 				   htab->map.numa_node);
975 		atomic_dec(&htab->count);
987 	struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
1011 	bucket = sock_hash_select_bucket(htab, hash);
1023 	elem_new = sock_hash_alloc_elem(htab, key, key_size, hash, sk, elem);
1037 		sock_hash_free_elem(htab, elem);
1052 	struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
1061 	head = &sock_hash_select_bucket(htab, hash)->head;
1073 	i = hash & (htab->buckets_num - 1);
1076 	for (; i < htab->buckets_num; i++) {
1077 		head = &sock_hash_select_bucket(htab, i)->head;
1091 	struct bpf_shtab *htab;
1103 	htab = bpf_map_area_alloc(sizeof(*htab), NUMA_NO_NODE);
1104 	if (!htab)
1107 	bpf_map_init_from_attr(&htab->map, attr);
1109 	htab->buckets_num = roundup_pow_of_two(htab->map.max_entries);
1110 	htab->elem_size = sizeof(struct bpf_shtab_elem) +
1111 			  round_up(htab->map.key_size, 8);
1112 	if (htab->buckets_num == 0 ||
1113 	    htab->buckets_num > U32_MAX / sizeof(struct bpf_shtab_bucket)) {
1118 	htab->buckets = bpf_map_area_alloc(htab->buckets_num *
1120 					   htab->map.numa_node);
1121 	if (!htab->buckets) {
1126 	for (i = 0; i < htab->buckets_num; i++) {
1127 		INIT_HLIST_HEAD(&htab->buckets[i].head);
1128 		spin_lock_init(&htab->buckets[i].lock);
1131 	return &htab->map;
1133 	bpf_map_area_free(htab);
1139 	struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
1151 	for (i = 0; i < htab->buckets_num; i++) {
1152 		bucket = sock_hash_select_bucket(htab, i);
1178 			sock_hash_free_elem(htab, elem);
1185 	bpf_map_area_free(htab->buckets);
1186 	bpf_map_area_free(htab);
1300 	struct bpf_shtab *htab;
1307 	const struct bpf_shtab *htab = info->htab;
1323 	for (; info->bucket_id < htab->buckets_num; info->bucket_id++) {
1324 		bucket = &htab->buckets[info->bucket_id];
1404 	info->htab = container_of(aux->map, struct bpf_shtab, map);
1417 	struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
1418 	u64 usage = sizeof(*htab);
1420 	usage += htab->buckets_num * sizeof(struct bpf_shtab_bucket);
1421 	usage += atomic_read(&htab->count) * (u64)htab->elem_size;