linux-master/mm/mempool.c

24 static void poison_error(mempool_t *pool, void *element, size_t size,
32 	pr_err("BUG: mempool element poison mismatch\n");
34 	pr_err(" nr=%d @ %p: %s0x", nr, element, start > 0 ? "... " : "");
36 		pr_cont("%x ", *(u8 *)(element + i));
41 static void __check_element(mempool_t *pool, void *element, size_t size)
43 	u8 *obj = element;
50 			poison_error(pool, element, size, i);
57 static void check_element(mempool_t *pool, void *element)
59 	/* Skip checking: KASAN might save its metadata in the element. */
65 		__check_element(pool, element, (size_t)pool->pool_data);
67 		__check_element(pool, element, kmem_cache_size(pool->pool_data));
71 		void *addr = kmap_local_page((struct page *)element);
78 static void __poison_element(void *element, size_t size)
80 	u8 *obj = element;
86 static void poison_element(mempool_t *pool, void *element)
88 	/* Skip poisoning: KASAN might save its metadata in the element. */
94 		__poison_element(element, (size_t)pool->pool_data);
96 		__poison_element(element, kmem_cache_size(pool->pool_data));
100 		void *addr = kmap_local_page((struct page *)element);
107 static inline void check_element(mempool_t *pool, void *element)
110 static inline void poison_element(mempool_t *pool, void *element)
115 static __always_inline bool kasan_poison_element(mempool_t *pool, void *element)
118 		return kasan_mempool_poison_object(element);
120 		return kasan_mempool_poison_pages(element,
125 static void kasan_unpoison_element(mempool_t *pool, void *element)
128 		kasan_mempool_unpoison_object(element, (size_t)pool->pool_data);
130 		kasan_mempool_unpoison_object(element,
133 		kasan_mempool_unpoison_pages(element,
137 static __always_inline void add_element(mempool_t *pool, void *element)
140 	poison_element(pool, element);
141 	if (kasan_poison_element(pool, element))
142 		pool->elements[pool->curr_nr++] = element;
147 	void *element = pool->elements[--pool->curr_nr];
150 	kasan_unpoison_element(pool, element);
151 	check_element(pool, element);
152 	return element;
169 		void *element = remove_element(pool);
170 		pool->free(element, pool->pool_data);
215 		void *element;
217 		element = pool->alloc(gfp_mask, pool->pool_data);
218 		if (unlikely(!element)) {
222 		add_element(pool, element);
234  * @alloc_fn:  user-defined element-allocation function.
235  * @free_fn:   user-defined element-freeing function.
256  * @alloc_fn:  user-defined element-allocation function.
257  * @free_fn:   user-defined element-freeing function.
316 	void *element;
326 			element = remove_element(pool);
328 			pool->free(element, pool->pool_data);
357 		element = pool->alloc(GFP_KERNEL, pool->pool_data);
358 		if (!element)
362 			add_element(pool, element);
365 			pool->free(element, pool->pool_data);	/* Raced */
377  * mempool_alloc - allocate an element from a specific memory pool
388  * Return: pointer to the allocated element or %NULL on error.
392 	void *element;
408 	element = pool->alloc(gfp_temp, pool->pool_data);
409 	if (likely(element != NULL))
410 		return element;
414 		element = remove_element(pool);
422 		kmemleak_update_trace(element);
423 		return element;
442 	/* Let's wait for someone else to return an element to @pool */
460  * mempool_alloc_preallocated - allocate an element from preallocated elements
466  * an element from the preallocated elements. It does not sleep and immediately
469  * Return: pointer to the allocated element or %NULL if no elements are
474 	void *element;
479 		element = remove_element(pool);
487 		kmemleak_update_trace(element);
488 		return element;
497  * mempool_free - return an element to the pool.
498  * @element:   pool element pointer.
504 void mempool_free(void *element, mempool_t *pool)
508 	if (unlikely(element == NULL))
513 	 * for @element and the following @pool->curr_nr.  This ensures
515 	 * allocation of @element.  This is necessary for fringe cases
516 	 * where @element was passed to this task without going through
534 	 * allocation of @element, any task which decremented curr_nr below
537 	 * to min_nr after the allocation of @element, the elements
547 			add_element(pool, element);
554 	pool->free(element, pool->pool_data);
569 void mempool_free_slab(void *element, void *pool_data)
572 	kmem_cache_free(mem, element);
587 void mempool_kfree(void *element, void *pool_data)
589 	kfree(element);
600 void mempool_kvfree(void *element, void *pool_data)
602 	kvfree(element);
617 void mempool_free_pages(void *element, void *pool_data)
620 	__free_pages(element, order);