1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _MM_PERCPU_INTERNAL_H 3#define _MM_PERCPU_INTERNAL_H 4 5#include <linux/types.h> 6#include <linux/percpu.h> 7#include <linux/memcontrol.h> 8 9/* 10 * pcpu_block_md is the metadata block struct. 11 * Each chunk's bitmap is split into a number of full blocks. 12 * All units are in terms of bits. 13 * 14 * The scan hint is the largest known contiguous area before the contig hint. 15 * It is not necessarily the actual largest contig hint though. There is an 16 * invariant that the scan_hint_start > contig_hint_start iff 17 * scan_hint == contig_hint. This is necessary because when scanning forward, 18 * we don't know if a new contig hint would be better than the current one. 19 */ 20struct pcpu_block_md { 21 int scan_hint; /* scan hint for block */ 22 int scan_hint_start; /* block relative starting 23 position of the scan hint */ 24 int contig_hint; /* contig hint for block */ 25 int contig_hint_start; /* block relative starting 26 position of the contig hint */ 27 int left_free; /* size of free space along 28 the left side of the block */ 29 int right_free; /* size of free space along 30 the right side of the block */ 31 int first_free; /* block position of first free */ 32 int nr_bits; /* total bits responsible for */ 33}; 34 35struct pcpu_chunk { 36#ifdef CONFIG_PERCPU_STATS 37 int nr_alloc; /* # of allocations */ 38 size_t max_alloc_size; /* largest allocation size */ 39#endif 40 41 struct list_head list; /* linked to pcpu_slot lists */ 42 int free_bytes; /* free bytes in the chunk */ 43 struct pcpu_block_md chunk_md; 44 unsigned long *bound_map; /* boundary map */ 45 46 /* 47 * base_addr is the base address of this chunk. 48 * To reduce false sharing, current layout is optimized to make sure 49 * base_addr locate in the different cacheline with free_bytes and 50 * chunk_md. 51 */ 52 void *base_addr ____cacheline_aligned_in_smp; 53 54 unsigned long *alloc_map; /* allocation map */ 55 struct pcpu_block_md *md_blocks; /* metadata blocks */ 56 57 void *data; /* chunk data */ 58 bool immutable; /* no [de]population allowed */ 59 bool isolated; /* isolated from active chunk 60 slots */ 61 int start_offset; /* the overlap with the previous 62 region to have a page aligned 63 base_addr */ 64 int end_offset; /* additional area required to 65 have the region end page 66 aligned */ 67#ifdef CONFIG_MEMCG_KMEM 68 struct obj_cgroup **obj_cgroups; /* vector of object cgroups */ 69#endif 70 71 int nr_pages; /* # of pages served by this chunk */ 72 int nr_populated; /* # of populated pages */ 73 int nr_empty_pop_pages; /* # of empty populated pages */ 74 unsigned long populated[]; /* populated bitmap */ 75}; 76 77extern spinlock_t pcpu_lock; 78 79extern struct list_head *pcpu_chunk_lists; 80extern int pcpu_nr_slots; 81extern int pcpu_sidelined_slot; 82extern int pcpu_to_depopulate_slot; 83extern int pcpu_nr_empty_pop_pages; 84 85extern struct pcpu_chunk *pcpu_first_chunk; 86extern struct pcpu_chunk *pcpu_reserved_chunk; 87 88/** 89 * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks 90 * @chunk: chunk of interest 91 * 92 * This conversion is from the number of physical pages that the chunk 93 * serves to the number of bitmap blocks used. 94 */ 95static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk) 96{ 97 return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE; 98} 99 100/** 101 * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap 102 * @pages: number of physical pages 103 * 104 * This conversion is from physical pages to the number of bits 105 * required in the bitmap. 106 */ 107static inline int pcpu_nr_pages_to_map_bits(int pages) 108{ 109 return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE; 110} 111 112/** 113 * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap 114 * @chunk: chunk of interest 115 * 116 * This conversion is from the number of physical pages that the chunk 117 * serves to the number of bits in the bitmap. 118 */ 119static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk) 120{ 121 return pcpu_nr_pages_to_map_bits(chunk->nr_pages); 122} 123 124/** 125 * pcpu_obj_full_size - helper to calculate size of each accounted object 126 * @size: size of area to allocate in bytes 127 * 128 * For each accounted object there is an extra space which is used to store 129 * obj_cgroup membership if kmemcg is not disabled. Charge it too. 130 */ 131static inline size_t pcpu_obj_full_size(size_t size) 132{ 133 size_t extra_size = 0; 134 135#ifdef CONFIG_MEMCG_KMEM 136 if (!mem_cgroup_kmem_disabled()) 137 extra_size += size / PCPU_MIN_ALLOC_SIZE * sizeof(struct obj_cgroup *); 138#endif 139 140 return size * num_possible_cpus() + extra_size; 141} 142 143#ifdef CONFIG_PERCPU_STATS 144 145#include <linux/spinlock.h> 146 147struct percpu_stats { 148 u64 nr_alloc; /* lifetime # of allocations */ 149 u64 nr_dealloc; /* lifetime # of deallocations */ 150 u64 nr_cur_alloc; /* current # of allocations */ 151 u64 nr_max_alloc; /* max # of live allocations */ 152 u32 nr_chunks; /* current # of live chunks */ 153 u32 nr_max_chunks; /* max # of live chunks */ 154 size_t min_alloc_size; /* min allocation size */ 155 size_t max_alloc_size; /* max allocation size */ 156}; 157 158extern struct percpu_stats pcpu_stats; 159extern struct pcpu_alloc_info pcpu_stats_ai; 160 161/* 162 * For debug purposes. We don't care about the flexible array. 163 */ 164static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 165{ 166 memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info)); 167 168 /* initialize min_alloc_size to unit_size */ 169 pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size; 170} 171 172/* 173 * pcpu_stats_area_alloc - increment area allocation stats 174 * @chunk: the location of the area being allocated 175 * @size: size of area to allocate in bytes 176 * 177 * CONTEXT: 178 * pcpu_lock. 179 */ 180static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 181{ 182 lockdep_assert_held(&pcpu_lock); 183 184 pcpu_stats.nr_alloc++; 185 pcpu_stats.nr_cur_alloc++; 186 pcpu_stats.nr_max_alloc = 187 max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc); 188 pcpu_stats.min_alloc_size = 189 min(pcpu_stats.min_alloc_size, size); 190 pcpu_stats.max_alloc_size = 191 max(pcpu_stats.max_alloc_size, size); 192 193 chunk->nr_alloc++; 194 chunk->max_alloc_size = max(chunk->max_alloc_size, size); 195} 196 197/* 198 * pcpu_stats_area_dealloc - decrement allocation stats 199 * @chunk: the location of the area being deallocated 200 * 201 * CONTEXT: 202 * pcpu_lock. 203 */ 204static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 205{ 206 lockdep_assert_held(&pcpu_lock); 207 208 pcpu_stats.nr_dealloc++; 209 pcpu_stats.nr_cur_alloc--; 210 211 chunk->nr_alloc--; 212} 213 214/* 215 * pcpu_stats_chunk_alloc - increment chunk stats 216 */ 217static inline void pcpu_stats_chunk_alloc(void) 218{ 219 unsigned long flags; 220 spin_lock_irqsave(&pcpu_lock, flags); 221 222 pcpu_stats.nr_chunks++; 223 pcpu_stats.nr_max_chunks = 224 max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks); 225 226 spin_unlock_irqrestore(&pcpu_lock, flags); 227} 228 229/* 230 * pcpu_stats_chunk_dealloc - decrement chunk stats 231 */ 232static inline void pcpu_stats_chunk_dealloc(void) 233{ 234 unsigned long flags; 235 spin_lock_irqsave(&pcpu_lock, flags); 236 237 pcpu_stats.nr_chunks--; 238 239 spin_unlock_irqrestore(&pcpu_lock, flags); 240} 241 242#else 243 244static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 245{ 246} 247 248static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 249{ 250} 251 252static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 253{ 254} 255 256static inline void pcpu_stats_chunk_alloc(void) 257{ 258} 259 260static inline void pcpu_stats_chunk_dealloc(void) 261{ 262} 263 264#endif /* !CONFIG_PERCPU_STATS */ 265 266#endif 267