Lines Matching refs:slab
3 * SLUB: A slab allocator that limits cache line use instead of queuing
6 * The allocator synchronizes using per slab locks or atomic operations
20 #include <linux/slab.h>
21 #include "slab.h"
56 * 4. slab_lock(slab) (Only on some arches)
62 * and to synchronize major metadata changes to slab cache structures.
73 * A. slab->freelist -> List of free objects in a slab
74 * B. slab->inuse -> Number of objects in use
75 * C. slab->objects -> Number of objects in slab
76 * D. slab->frozen -> frozen state
80 * If a slab is frozen then it is exempt from list management. It is
81 * the cpu slab which is actively allocated from by the processor that
83 * slab is the one who can perform list operations on the slab. Other
85 * froze the slab is the only one that can retrieve the objects from the
86 * slab's freelist.
97 * - node partial slab: PG_Workingset && !frozen
98 * - cpu partial slab: !PG_Workingset && !frozen
99 * - cpu slab: !PG_Workingset && frozen
100 * - full slab: !PG_Workingset && !frozen
105 * the partial slab counter. If taken then no new slabs may be added or
134 * are fully lockless when satisfied from the percpu slab (and when
143 * around the slab_lock operation, in order to make the slab allocator safe
151 * SLUB assigns one slab for allocation to each processor.
155 * operations no list for full slabs is used. If an object in a full slab is
156 * freed then the slab will show up again on the partial lists.
164 * slab->frozen The slab is frozen and exempt from list processing.
165 * This means that the slab is dedicated to a purpose
167 * processor. Objects may be freed in the slab while
170 * the slab to integrate the slab into the slab lists
171 * when the slab is no longer needed.
174 * used for allocations. Then such a slab becomes a cpu
175 * slab. The cpu slab may be equipped with an additional
178 * that requires the slab lock.
181 * options set. This moves slab handling out of
297 * Debugging flags that require metadata to be stored in the slab. These get
305 #define MAX_OBJS_PER_PAGE 32767 /* since slab.objects is u15 */
319 * Tracking user of a slab.
350 ALLOC_FASTPATH, /* Allocation from cpu slab */
351 ALLOC_SLOWPATH, /* Allocation by getting a new cpu slab */
352 FREE_FASTPATH, /* Free to cpu slab */
353 FREE_SLOWPATH, /* Freeing not to cpu slab */
354 FREE_FROZEN, /* Freeing to frozen slab */
355 FREE_ADD_PARTIAL, /* Freeing moves slab to partial list */
357 ALLOC_FROM_PARTIAL, /* Cpu slab acquired from node partial list */
358 ALLOC_SLAB, /* Cpu slab acquired from page allocator */
359 ALLOC_REFILL, /* Refill cpu slab from slab freelist */
360 ALLOC_NODE_MISMATCH, /* Switching cpu slab */
362 CPUSLAB_FLUSH, /* Abandoning of the cpu slab */
363 DEACTIVATE_FULL, /* Cpu slab was full when deactivated */
364 DEACTIVATE_EMPTY, /* Cpu slab was empty when deactivated */
365 DEACTIVATE_TO_HEAD, /* Cpu slab was moved to the head of partials */
366 DEACTIVATE_TO_TAIL, /* Cpu slab was moved to the tail of partials */
371 CMPXCHG_DOUBLE_FAIL, /* Failures of slab freelist update */
392 struct slab *slab; /* The slab from which we are allocating */
394 struct slab *partial; /* Partially allocated slabs */
423 * The slab lists for all objects.
465 * Core slab cache functions
580 /* Loop over all objects in a slab */
635 * Per slab locking using the pagelock
637 static __always_inline void slab_lock(struct slab *slab)
639 struct page *page = slab_page(slab);
645 static __always_inline void slab_unlock(struct slab *slab)
647 struct page *page = slab_page(slab);
654 __update_freelist_fast(struct slab *slab,
662 return try_cmpxchg_freelist(&slab->freelist_counter.full, &old.full, new.full);
669 __update_freelist_slow(struct slab *slab,
675 slab_lock(slab);
676 if (slab->freelist == freelist_old &&
677 slab->counters == counters_old) {
678 slab->freelist = freelist_new;
679 slab->counters = counters_new;
682 slab_unlock(slab);
694 static inline bool __slab_update_freelist(struct kmem_cache *s, struct slab *slab,
705 ret = __update_freelist_fast(slab, freelist_old, counters_old,
708 ret = __update_freelist_slow(slab, freelist_old, counters_old,
724 static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab,
732 ret = __update_freelist_fast(slab, freelist_old, counters_old,
738 ret = __update_freelist_slow(slab, freelist_old, counters_old,
760 struct slab *slab)
762 void *addr = slab_address(slab);
765 bitmap_zero(obj_map, slab->objects);
767 for (p = slab->freelist; p; p = get_freepointer(s, p))
839 /* Verify that a pointer has an address that is valid within a slab page */
841 struct slab *slab, void *object)
848 base = slab_address(slab);
851 if (object < base || object >= base + slab->objects * s->size ||
959 static void print_slab_info(const struct slab *slab)
961 struct folio *folio = (struct folio *)slab_folio(slab);
964 slab, slab->objects, slab->inuse, slab->freelist,
1047 static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)
1050 u8 *addr = slab_address(slab);
1054 print_slab_info(slab);
1089 static void object_err(struct kmem_cache *s, struct slab *slab,
1096 print_trailer(s, slab, object);
1100 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
1104 !check_valid_pointer(s, slab, nextfree) && freelist) {
1105 object_err(s, slab, *freelist, "Freechain corrupt");
1114 static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab,
1127 print_slab_info(slab);
1166 static int check_bytes_and_report(struct kmem_cache *s, struct slab *slab,
1172 u8 *addr = slab_address(slab);
1191 print_trailer(s, slab, object);
1234 * ignored. And therefore no slab options that rely on these boundaries
1238 static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)
1255 return check_bytes_and_report(s, slab, p, "Object padding",
1259 /* Check the pad bytes at the end of a slab page */
1260 static void slab_pad_check(struct kmem_cache *s, struct slab *slab)
1272 start = slab_address(slab);
1273 length = slab_size(slab);
1288 slab_err(s, slab, "Padding overwritten. 0x%p-0x%p @offset=%tu",
1292 restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
1295 static int check_object(struct kmem_cache *s, struct slab *slab,
1303 if (!check_bytes_and_report(s, slab, object, "Left Redzone",
1307 if (!check_bytes_and_report(s, slab, object, "Right Redzone",
1315 !check_bytes_and_report(s, slab, object,
1323 check_bytes_and_report(s, slab, p, "Alignment padding",
1338 !check_bytes_and_report(s, slab, p, "Poison",
1343 !check_bytes_and_report(s, slab, p, "End Poison",
1350 check_pad_bytes(s, slab, p);
1361 if (!check_valid_pointer(s, slab, get_freepointer(s, p))) {
1362 object_err(s, slab, p, "Freepointer corrupt");
1365 * of the free objects in this slab. May cause
1374 static int check_slab(struct kmem_cache *s, struct slab *slab)
1378 if (!folio_test_slab(slab_folio(slab))) {
1379 slab_err(s, slab, "Not a valid slab page");
1383 maxobj = order_objects(slab_order(slab), s->size);
1384 if (slab->objects > maxobj) {
1385 slab_err(s, slab, "objects %u > max %u",
1386 slab->objects, maxobj);
1389 if (slab->inuse > slab->objects) {
1390 slab_err(s, slab, "inuse %u > max %u",
1391 slab->inuse, slab->objects);
1395 slab_pad_check(s, slab);
1400 * Determine if a certain object in a slab is on the freelist. Must hold the
1401 * slab lock to guarantee that the chains are in a consistent state.
1403 static int on_freelist(struct kmem_cache *s, struct slab *slab, void *search)
1410 fp = slab->freelist;
1411 while (fp && nr <= slab->objects) {
1414 if (!check_valid_pointer(s, slab, fp)) {
1416 object_err(s, slab, object,
1420 slab_err(s, slab, "Freepointer corrupt");
1421 slab->freelist = NULL;
1422 slab->inuse = slab->objects;
1433 max_objects = order_objects(slab_order(slab), s->size);
1437 if (slab->objects != max_objects) {
1438 slab_err(s, slab, "Wrong number of objects. Found %d but should be %d",
1439 slab->objects, max_objects);
1440 slab->objects = max_objects;
1443 if (slab->inuse != slab->objects - nr) {
1444 slab_err(s, slab, "Wrong object count. Counter is %d but counted were %d",
1445 slab->inuse, slab->objects - nr);
1446 slab->inuse = slab->objects - nr;
1452 static void trace(struct kmem_cache *s, struct slab *slab, void *object,
1459 object, slab->inuse,
1460 slab->freelist);
1474 struct kmem_cache_node *n, struct slab *slab)
1480 list_add(&slab->slab_list, &n->full);
1483 static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab)
1489 list_del(&slab->slab_list);
1523 void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr)
1529 memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab));
1534 struct slab *slab, void *object)
1536 if (!check_slab(s, slab))
1539 if (!check_valid_pointer(s, slab, object)) {
1540 object_err(s, slab, object, "Freelist Pointer check fails");
1544 if (!check_object(s, slab, object, SLUB_RED_INACTIVE))
1551 struct slab *slab, void *object, int orig_size)
1554 if (!alloc_consistency_checks(s, slab, object))
1559 trace(s, slab, object, 1);
1565 if (folio_test_slab(slab_folio(slab))) {
1567 * If this is a slab page then lets do the best we can
1572 slab->inuse = slab->objects;
1573 slab->freelist = NULL;
1579 struct slab *slab, void *object, unsigned long addr)
1581 if (!check_valid_pointer(s, slab, object)) {
1582 slab_err(s, slab, "Invalid object pointer 0x%p", object);
1586 if (on_freelist(s, slab, object)) {
1587 object_err(s, slab, object, "Object already free");
1591 if (!check_object(s, slab, object, SLUB_RED_ACTIVE))
1594 if (unlikely(s != slab->slab_cache)) {
1595 if (!folio_test_slab(slab_folio(slab))) {
1596 slab_err(s, slab, "Attempt to free object(0x%p) outside of slab",
1598 } else if (!slab->slab_cache) {
1599 pr_err("SLUB <none>: no slab for object 0x%p.\n",
1603 object_err(s, slab, object,
1604 "page slab pointer corrupt.");
1681 /* Skip over the slab list */
1733 * long as there is no option specifying flags without a slab list.
1764 * option(s), if a slab name (or multiple) is specified i.e.
1765 * slab_debug=<Debug-Options>,<slab name1>,<slab name2> ...
1780 * If the slab cache is for debugging (e.g. kmemleak) then
1789 /* Go through all blocks of debug options, see if any matches our slab's name */
1794 /* Found a block that has a slab list, search it */
1825 void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {}
1828 struct slab *slab, void *object, int orig_size) { return true; }
1831 struct slab *slab, void *head, void *tail, int *bulk_cnt,
1834 static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {}
1835 static inline int check_object(struct kmem_cache *s, struct slab *slab,
1841 struct slab *slab) {}
1843 struct slab *slab) {}
1860 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
1875 static inline void memcg_free_slab_cgroups(struct slab *slab)
1877 kfree(slab_objcgs(slab));
1878 slab->memcg_data = 0;
1949 struct slab *slab;
1957 slab = virt_to_slab(p[i]);
1959 if (!slab_objcgs(slab) &&
1960 memcg_alloc_slab_cgroups(slab, s, flags, false)) {
1965 off = obj_to_index(s, slab, p[i]);
1967 slab_objcgs(slab)[off] = objcg;
1968 mod_objcg_state(objcg, slab_pgdat(slab),
1986 static void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
1994 off = obj_to_index(s, slab, p[i]);
2001 mod_objcg_state(objcg, slab_pgdat(slab), cache_vmstat_idx(s),
2008 void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
2016 objcgs = slab_objcgs(slab);
2020 __memcg_slab_free_hook(s, slab, p, objects, objcgs);
2031 static inline void memcg_free_slab_cgroups(struct slab *slab)
2050 static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
2173 static inline struct slab *alloc_slab_page(gfp_t flags, int node,
2177 struct slab *slab;
2184 slab = folio_slab(folio);
2189 slab_set_pfmemalloc(slab);
2191 return slab;
2258 static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
2265 if (slab->objects < 2 || !s->random_seq)
2271 page_limit = slab->objects * s->size;
2272 start = fixup_red_left(s, slab_address(slab));
2277 slab->freelist = cur;
2279 for (idx = 1; idx < slab->objects; idx++) {
2296 static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
2302 static __always_inline void account_slab(struct slab *slab, int order,
2306 memcg_alloc_slab_cgroups(slab, s, gfp, true);
2308 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
2312 static __always_inline void unaccount_slab(struct slab *slab, int order,
2316 memcg_free_slab_cgroups(slab);
2318 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
2322 static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
2324 struct slab *slab;
2343 slab = alloc_slab_page(alloc_gfp, node, oo);
2344 if (unlikely(!slab)) {
2351 slab = alloc_slab_page(alloc_gfp, node, oo);
2352 if (unlikely(!slab))
2357 slab->objects = oo_objects(oo);
2358 slab->inuse = 0;
2359 slab->frozen = 0;
2361 account_slab(slab, oo_order(oo), s, flags);
2363 slab->slab_cache = s;
2365 kasan_poison_slab(slab);
2367 start = slab_address(slab);
2369 setup_slab_debug(s, slab, start);
2371 shuffle = shuffle_freelist(s, slab);
2376 slab->freelist = start;
2377 for (idx = 0, p = start; idx < slab->objects - 1; idx++) {
2386 return slab;
2389 static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)
2400 static void __free_slab(struct kmem_cache *s, struct slab *slab)
2402 struct folio *folio = slab_folio(slab);
2406 __slab_clear_pfmemalloc(slab);
2412 unaccount_slab(slab, order, s);
2418 struct slab *slab = container_of(h, struct slab, rcu_head);
2420 __free_slab(slab->slab_cache, slab);
2423 static void free_slab(struct kmem_cache *s, struct slab *slab)
2428 slab_pad_check(s, slab);
2429 for_each_object(p, s, slab_address(slab), slab->objects)
2430 check_object(s, slab, p, SLUB_RED_INACTIVE);
2434 call_rcu(&slab->rcu_head, rcu_free_slab);
2436 __free_slab(s, slab);
2439 static void discard_slab(struct kmem_cache *s, struct slab *slab)
2441 dec_slabs_node(s, slab_nid(slab), slab->objects);
2442 free_slab(s, slab);
2449 static inline bool slab_test_node_partial(const struct slab *slab)
2451 return folio_test_workingset((struct folio *)slab_folio(slab));
2454 static inline void slab_set_node_partial(struct slab *slab)
2456 set_bit(PG_workingset, folio_flags(slab_folio(slab), 0));
2459 static inline void slab_clear_node_partial(struct slab *slab)
2461 clear_bit(PG_workingset, folio_flags(slab_folio(slab), 0));
2468 __add_partial(struct kmem_cache_node *n, struct slab *slab, int tail)
2472 list_add_tail(&slab->slab_list, &n->partial);
2474 list_add(&slab->slab_list, &n->partial);
2475 slab_set_node_partial(slab);
2479 struct slab *slab, int tail)
2482 __add_partial(n, slab, tail);
2486 struct slab *slab)
2489 list_del(&slab->slab_list);
2490 slab_clear_node_partial(slab);
2496 * slab from the n->partial list. Remove only a single object from the slab, do
2497 * the alloc_debug_processing() checks and leave the slab on the list, or move
2501 struct kmem_cache_node *n, struct slab *slab, int orig_size)
2507 object = slab->freelist;
2508 slab->freelist = get_freepointer(s, object);
2509 slab->inuse++;
2511 if (!alloc_debug_processing(s, slab, object, orig_size)) {
2512 remove_partial(n, slab);
2516 if (slab->inuse == slab->objects) {
2517 remove_partial(n, slab);
2518 add_full(s, n, slab);
2526 * allocated slab. Allocate a single object instead of whole freelist
2527 * and put the slab to the partial (or full) list.
2530 struct slab *slab, int orig_size)
2532 int nid = slab_nid(slab);
2538 object = slab->freelist;
2539 slab->freelist = get_freepointer(s, object);
2540 slab->inuse = 1;
2542 if (!alloc_debug_processing(s, slab, object, orig_size))
2545 * freshly allocated slab, but a concurrent memory
2552 if (slab->inuse == slab->objects)
2553 add_full(s, n, slab);
2555 add_partial(n, slab, DEACTIVATE_TO_HEAD);
2557 inc_slabs_node(s, nid, slab->objects);
2564 static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain);
2566 static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab,
2569 static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags);
2572 * Try to allocate a partial slab from a specific node.
2574 static struct slab *get_partial_node(struct kmem_cache *s,
2578 struct slab *slab, *slab2, *partial = NULL;
2584 * just allocate an empty slab. If we mistakenly try to get a
2585 * partial slab and there is none available then get_partial()
2592 list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) {
2593 if (!pfmemalloc_match(slab, pc->flags))
2597 void *object = alloc_single_from_partial(s, n, slab,
2600 partial = slab;
2607 remove_partial(n, slab);
2610 partial = slab;
2613 put_cpu_partial(s, slab, 0);
2631 * Get a slab from somewhere. Search in increasing NUMA distances.
2633 static struct slab *get_any_partial(struct kmem_cache *s,
2641 struct slab *slab;
2655 * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100
2657 * allocation will first attempt to defrag slab caches on other nodes.
2659 * may be expensive if we do it every time we are trying to find a slab
2676 slab = get_partial_node(s, n, pc);
2677 if (slab) {
2685 return slab;
2695 * Get a partial slab, lock it and return it.
2697 static struct slab *get_partial(struct kmem_cache *s, int node,
2700 struct slab *slab;
2706 slab = get_partial_node(s, get_node(s, searchnode), pc);
2707 if (slab || node != NUMA_NO_NODE)
2708 return slab;
2789 * Finishes removing the cpu slab. Merges cpu's freelist with slab's freelist,
2791 * Assumes the slab has been already safely taken away from kmem_cache_cpu
2794 static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
2797 struct kmem_cache_node *n = get_node(s, slab_nid(slab));
2802 struct slab new;
2803 struct slab old;
2805 if (slab->freelist) {
2824 if (freelist_corrupted(s, slab, &freelist_iter, nextfree))
2834 * Stage two: Unfreeze the slab while splicing the per-cpu
2835 * freelist to the head of slab's freelist.
2838 old.freelist = READ_ONCE(slab->freelist);
2839 old.counters = READ_ONCE(slab->counters);
2842 /* Determine target state of the slab */
2852 } while (!slab_update_freelist(s, slab,
2855 "unfreezing slab"));
2858 * Stage three: Manipulate the slab list based on the updated state.
2862 discard_slab(s, slab);
2866 add_partial(n, slab, tail);
2875 static void __put_partials(struct kmem_cache *s, struct slab *partial_slab)
2878 struct slab *slab, *slab_to_discard = NULL;
2882 slab = partial_slab;
2883 partial_slab = slab->next;
2885 n2 = get_node(s, slab_nid(slab));
2894 if (unlikely(!slab->inuse && n->nr_partial >= s->min_partial)) {
2895 slab->next = slab_to_discard;
2896 slab_to_discard = slab;
2898 add_partial(n, slab, DEACTIVATE_TO_TAIL);
2907 slab = slab_to_discard;
2911 discard_slab(s, slab);
2921 struct slab *partial_slab;
2936 struct slab *partial_slab;
2946 * Put a slab into a partial slab slot if available.
2951 static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain)
2953 struct slab *oldslab;
2954 struct slab *slab_to_put = NULL;
2978 slab->slabs = slabs;
2979 slab->next = oldslab;
2981 this_cpu_write(s->cpu_slab->partial, slab);
3002 struct slab *slab;
3007 slab = c->slab;
3010 c->slab = NULL;
3016 if (slab) {
3017 deactivate_slab(s, slab, freelist);
3026 struct slab *slab = c->slab;
3028 c->slab = NULL;
3032 if (slab) {
3033 deactivate_slab(s, slab, freelist);
3047 * Flush cpu slab.
3062 if (c->slab)
3072 return c->slab || slub_percpu_partial(c);
3141 static inline int node_match(struct slab *slab, int node)
3144 if (node != NUMA_NO_NODE && slab_nid(slab) != node)
3151 static int count_free(struct slab *slab)
3153 return slab->objects - slab->inuse;
3163 struct slab *slab, void *head, void *tail, int *bulk_cnt,
3171 if (!check_slab(s, slab))
3175 if (slab->inuse < *bulk_cnt) {
3176 slab_err(s, slab, "Slab has %d allocated objects but %d are to be freed\n",
3177 slab->inuse, *bulk_cnt);
3187 if (!free_consistency_checks(s, slab, object, addr))
3193 trace(s, slab, object, 0);
3206 slab_err(s, slab, "Bulk free expected %d objects but found %d\n",
3222 int (*get_count)(struct slab *))
3226 struct slab *slab;
3229 list_for_each_entry(slab, &n->partial, slab_list)
3230 x += get_count(slab);
3276 static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags)
3278 if (unlikely(slab_test_pfmemalloc(slab)))
3298 * Check the slab->freelist and either transfer the freelist to the
3299 * per cpu freelist or deactivate the slab.
3301 * The slab is still frozen if the return value is not NULL.
3303 * If this function returns NULL then the slab has been unfrozen.
3305 static inline void *get_freelist(struct kmem_cache *s, struct slab *slab)
3307 struct slab new;
3314 freelist = slab->freelist;
3315 counters = slab->counters;
3319 new.inuse = slab->objects;
3322 } while (!__slab_update_freelist(s, slab,
3331 * Freeze the partial slab and return the pointer to the freelist.
3333 static inline void *freeze_slab(struct kmem_cache *s, struct slab *slab)
3335 struct slab new;
3340 freelist = slab->freelist;
3341 counters = slab->counters;
3346 new.inuse = slab->objects;
3349 } while (!slab_update_freelist(s, slab,
3369 * And if we were unable to get a new slab from the partial slab lists then
3370 * we need to allocate a new slab. This is the slowest path since it involves
3371 * a call to the page allocator and the setup of a new slab.
3380 struct slab *slab;
3388 slab = READ_ONCE(c->slab);
3389 if (!slab) {
3400 if (unlikely(!node_match(slab, node))) {
3414 * By rights, we should be searching for a slab page that was
3418 if (unlikely(!pfmemalloc_match(slab, gfpflags)))
3421 /* must check again c->slab in case we got preempted and it changed */
3423 if (unlikely(slab != c->slab)) {
3431 freelist = get_freelist(s, slab);
3434 c->slab = NULL;
3449 * slab is pointing to the slab from which the objects are obtained.
3450 * That slab must be frozen for per cpu allocations to work.
3452 VM_BUG_ON(!c->slab->frozen);
3461 if (slab != c->slab) {
3466 c->slab = NULL;
3470 deactivate_slab(s, slab, freelist);
3477 if (unlikely(c->slab)) {
3487 slab = slub_percpu_partial(c);
3488 slub_set_percpu_partial(c, slab);
3490 if (likely(node_match(slab, node) &&
3491 pfmemalloc_match(slab, gfpflags))) {
3492 c->slab = slab;
3493 freelist = get_freelist(s, slab);
3501 slab->next = NULL;
3502 __put_partials(s, slab);
3510 slab = get_partial(s, node, &pc);
3511 if (slab) {
3525 freelist = freeze_slab(s, slab);
3530 slab = new_slab(s, gfpflags, node);
3533 if (unlikely(!slab)) {
3541 freelist = alloc_single_from_new_slab(s, slab, orig_size);
3553 * No other reference to the slab yet so we can
3556 freelist = slab->freelist;
3557 slab->freelist = NULL;
3558 slab->inuse = slab->objects;
3559 slab->frozen = 1;
3561 inc_slabs_node(s, slab_nid(slab), slab->objects);
3563 if (unlikely(!pfmemalloc_match(slab, gfpflags))) {
3568 deactivate_slab(s, slab, get_freepointer(s, freelist));
3575 if (unlikely(c->slab)) {
3577 struct slab *flush_slab = c->slab;
3579 c->slab = NULL;
3591 c->slab = slab;
3626 struct slab *slab;
3649 * on c to guarantee that object and slab associated with previous tid
3651 * slab could be one associated with next tid and our alloc/free
3664 slab = c->slab;
3667 unlikely(!object || !slab || !node_match(slab, node))) {
3701 struct slab *slab;
3706 slab = get_partial(s, node, &pc);
3708 if (slab)
3711 slab = new_slab(s, gfpflags, node);
3712 if (unlikely(!slab)) {
3717 object = alloc_single_from_new_slab(s, slab, orig_size);
4021 struct kmem_cache *s, struct slab *slab,
4025 struct kmem_cache_node *n = get_node(s, slab_nid(slab));
4026 struct slab *slab_free = NULL;
4036 if (free_debug_processing(s, slab, head, tail, &cnt, addr, handle)) {
4037 void *prior = slab->freelist;
4040 slab->inuse -= cnt;
4042 slab->freelist = head;
4045 * If the slab is empty, and node's partial list is full,
4049 if (slab->inuse == 0 && n->nr_partial >= s->min_partial)
4050 slab_free = slab;
4054 remove_full(s, n, slab);
4056 add_partial(n, slab, DEACTIVATE_TO_TAIL);
4060 remove_partial(n, slab);
4085 * So we still attempt to reduce cache line usage. Just take the slab
4086 * lock and free the item. If there is no additional partial slab
4089 static void __slab_free(struct kmem_cache *s, struct slab *slab,
4096 struct slab new;
4105 free_to_partial_list(s, slab, head, tail, cnt, addr);
4114 prior = slab->freelist;
4115 counters = slab->counters;
4124 n = get_node(s, slab_nid(slab));
4135 on_node_partial = slab_test_node_partial(slab);
4139 } while (!slab_update_freelist(s, slab,
4154 * If we started with a full slab then put it onto the
4157 put_cpu_partial(s, slab, 1);
4165 * This slab was partially empty but not on the per-node partial list,
4177 * Objects left in the slab. If it was not on the partial list before
4181 add_partial(n, slab, DEACTIVATE_TO_TAIL);
4192 remove_partial(n, slab);
4198 discard_slab(s, slab);
4206 * The fastpath is only possible if we are freeing to the current cpu slab
4214 * same slab) possible by specifying head and tail ptr, plus objects
4218 struct slab *slab, void *head, void *tail,
4227 * Determine the currently cpus per cpu slab.
4238 if (unlikely(slab != c->slab)) {
4239 __slab_free(s, slab, head, tail, cnt, addr);
4256 if (unlikely(slab != c->slab)) {
4273 struct slab *slab, void *head, void *tail,
4276 __slab_free(s, slab, head, tail, cnt, addr);
4281 void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
4284 memcg_slab_free_hook(s, slab, &object, 1);
4287 do_slab_free(s, slab, object, object, 1, addr);
4291 void slab_free_bulk(struct kmem_cache *s, struct slab *slab, void *head,
4294 memcg_slab_free_hook(s, slab, p, cnt);
4300 do_slab_free(s, slab, head, tail, cnt, addr);
4312 struct slab *slab;
4314 slab = virt_to_slab(obj);
4315 if (WARN_ONCE(!slab, "%s: Object is not a Slab page!\n", __func__))
4317 return slab->slab_cache;
4330 "%s: Wrong slab cache. %s but object is from %s\n",
4379 struct slab *slab;
4394 slab = folio_slab(folio);
4395 s = slab->slab_cache;
4396 slab_free(s, slab, x, _RET_IP_);
4401 struct slab *slab;
4411 * slab. It builds a detached freelist directly within the given
4412 * slab/objects. This can happen without any need for
4435 df->slab = NULL;
4439 df->slab = folio_slab(folio);
4440 df->s = df->slab->slab_cache;
4442 df->slab = folio_slab(folio);
4459 /* df->slab is always set at this point */
4460 if (df->slab == virt_to_slab(object)) {
4492 if (!df.slab)
4495 do_slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt,
4510 if (!df.slab)
4513 slab_free_bulk(df.s, df.slab, df.freelist, df.tail, &p[size],
4529 * Drain objects in the per cpu slab, while disabling local
4654 * Object placement in a slab is made very easy because we always start at
4660 * caches. Each processor has always one slab available for allocations.
4667 * Minimum / Maximum order of slab pages. This influences locking overhead
4668 * and slab fragmentation. A higher order reduces the number of partial slabs
4678 * Calculate the order of allocation given an slab object size.
4684 * unused space left. We go to a higher order if more than 1/16th of the slab
4688 * number of objects is in one slab. Otherwise we may generate too much
4693 * number of objects in a slab as critical. If we reach slab_max_order then
4698 * slab and thereby reduce object handling overhead. If the user has
4755 * Attempt to find best configuration for a slab. This works by first
4777 * Doh this slab cannot be placed using slab_max_order.
4830 * slab on the node for this slabcache. There are no concurrent accesses
4835 * memory on a fresh node that has no slab structures yet.
4839 struct slab *slab;
4844 slab = new_slab(kmem_cache_node, GFP_NOWAIT, node);
4846 BUG_ON(!slab);
4847 if (slab_nid(slab) != node) {
4852 n = slab->freelist;
4859 slab->freelist = get_freepointer(kmem_cache_node, n);
4860 slab->inuse = 1;
4863 inc_slabs_node(kmem_cache_node, node, slab->objects);
4869 __add_partial(n, slab, DEACTIVATE_TO_HEAD);
4928 * filled up again with minimal effort. The slab will never hit the
4952 * a slab object.
4970 * the slab may touch the object after free or before allocation
5083 * Determine the number of objects per slab
5102 * Disable debugging flags that store metadata if the min slab
5133 /* Initialize the pre-computed randomized freelist if slab is up */
5150 static void list_slab_objects(struct kmem_cache *s, struct slab *slab,
5154 void *addr = slab_address(slab);
5157 slab_err(s, slab, text, s->name);
5160 __fill_map(object_map, s, slab);
5162 for_each_object(p, s, addr, slab->objects) {
5181 struct slab *slab, *h;
5185 list_for_each_entry_safe(slab, h, &n->partial, slab_list) {
5186 if (!slab->inuse) {
5187 remove_partial(n, slab);
5188 list_add(&slab->slab_list, &discard);
5190 list_slab_objects(s, slab,
5196 list_for_each_entry_safe(slab, h, &discard, slab_list)
5197 discard_slab(s, slab);
5212 * Release all resources used by a slab cache.
5230 void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
5237 struct kmem_cache *s = slab->slab_cache;
5241 kpp->kp_slab = slab;
5243 base = slab_address(slab);
5250 objnr = obj_to_index(s, slab, objp);
5254 if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size
5333 * to/from userspace but do not fall entirely within the containing slab
5340 const struct slab *slab, bool to_user)
5349 s = slab->slab_cache;
5352 if (ptr < slab_address(slab))
5360 offset = (ptr - slab_address(slab)) % s->size;
5396 struct slab *slab;
5397 struct slab *t;
5414 * list_lock. slab->inuse here is the upper limit.
5416 list_for_each_entry_safe(slab, t, &n->partial, slab_list) {
5417 int free = slab->objects - slab->inuse;
5419 /* Do not reread slab->inuse */
5425 if (free == slab->objects) {
5426 list_move(&slab->slab_list, &discard);
5427 slab_clear_node_partial(slab);
5429 dec_slabs_node(s, node, slab->objects);
5431 list_move(&slab->slab_list, promote + free - 1);
5444 list_for_each_entry_safe(slab, t, &discard, slab_list)
5445 free_slab(s, slab);
5601 struct slab *p;
5727 static int count_inuse(struct slab *slab)
5729 return slab->inuse;
5732 static int count_total(struct slab *slab)
5734 return slab->objects;
5739 static void validate_slab(struct kmem_cache *s, struct slab *slab,
5743 void *addr = slab_address(slab);
5745 if (!check_slab(s, slab) || !on_freelist(s, slab, NULL))
5749 __fill_map(obj_map, s, slab);
5750 for_each_object(p, s, addr, slab->objects) {
5754 if (!check_object(s, slab, p, val))
5763 struct slab *slab;
5768 list_for_each_entry(slab, &n->partial, slab_list) {
5769 validate_slab(s, slab, obj_map);
5781 list_for_each_entry(slab, &n->full, slab_list) {
5782 validate_slab(s, slab, obj_map);
5966 struct slab *slab, enum track_item alloc,
5969 void *addr = slab_address(slab);
5973 __fill_map(obj_map, s, slab);
5975 for_each_object(p, s, addr, slab->objects)
6019 struct slab *slab;
6021 slab = READ_ONCE(c->slab);
6022 if (!slab)
6025 node = slab_nid(slab);
6027 x = slab->objects;
6029 x = slab->inuse;
6037 slab = slub_percpu_partial_read_once(c);
6038 if (slab) {
6039 node = slab_nid(slab);
6045 x = slab->slabs;
6060 * slab_mem_going_offline_callback) here because slab's memory hot
6245 struct slab *slab;
6247 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
6249 if (slab)
6250 slabs += slab->slabs;
6260 struct slab *slab;
6262 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
6263 if (slab) {
6264 slabs = READ_ONCE(slab->slabs);
6690 /* Create a unique string id for a slab cache:
6753 * Create a unique name for the slab as a target
6836 slab_kset = kset_create_and_add("slab", NULL, kernel_kobj);
6839 pr_err("Cannot register slab subsystem.\n");
6848 pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
6858 pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
7010 struct slab *slab;
7016 list_for_each_entry(slab, &n->partial, slab_list)
7017 process_slab(t, s, slab, alloc, obj_map);
7018 list_for_each_entry(slab, &n->full, slab_list)
7019 process_slab(t, s, slab, alloc, obj_map);
7072 slab_debugfs_root = debugfs_create_dir("slab", NULL);