1/* 2 * mm/percpu-vm.c - vmalloc area based chunk allocation 3 * 4 * Copyright (C) 2010 SUSE Linux Products GmbH 5 * Copyright (C) 2010 Tejun Heo <tj@kernel.org> 6 * 7 * This file is released under the GPLv2. 8 * 9 * Chunks are mapped into vmalloc areas and populated page by page. 10 * This is the default chunk allocator. 11 */ 12 13static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk, 14 unsigned int cpu, int page_idx) 15{ 16 /* must not be used on pre-mapped chunk */ 17 WARN_ON(chunk->immutable); 18 19 return vmalloc_to_page((void *)pcpu_chunk_addr(chunk, cpu, page_idx)); 20} 21 22/** 23 * pcpu_get_pages_and_bitmap - get temp pages array and bitmap 24 * @chunk: chunk of interest 25 * @bitmapp: output parameter for bitmap 26 * @may_alloc: may allocate the array 27 * 28 * Returns pointer to array of pointers to struct page and bitmap, 29 * both of which can be indexed with pcpu_page_idx(). The returned 30 * array is cleared to zero and *@bitmapp is copied from 31 * @chunk->populated. Note that there is only one array and bitmap 32 * and access exclusion is the caller's responsibility. 33 * 34 * CONTEXT: 35 * pcpu_alloc_mutex and does GFP_KERNEL allocation if @may_alloc. 36 * Otherwise, don't care. 37 * 38 * RETURNS: 39 * Pointer to temp pages array on success, NULL on failure. 40 */ 41static struct page **pcpu_get_pages_and_bitmap(struct pcpu_chunk *chunk, 42 unsigned long **bitmapp, 43 bool may_alloc) 44{ 45 static struct page **pages; 46 static unsigned long *bitmap; 47 size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]); 48 size_t bitmap_size = BITS_TO_LONGS(pcpu_unit_pages) * 49 sizeof(unsigned long); 50 51 if (!pages || !bitmap) { 52 if (may_alloc && !pages) 53 pages = pcpu_mem_alloc(pages_size); 54 if (may_alloc && !bitmap) 55 bitmap = pcpu_mem_alloc(bitmap_size); 56 if (!pages || !bitmap) 57 return NULL; 58 } 59 60 memset(pages, 0, pages_size); 61 bitmap_copy(bitmap, chunk->populated, pcpu_unit_pages); 62 63 *bitmapp = bitmap; 64 return pages; 65} 66 67/** 68 * pcpu_free_pages - free pages which were allocated for @chunk 69 * @chunk: chunk pages were allocated for 70 * @pages: array of pages to be freed, indexed by pcpu_page_idx() 71 * @populated: populated bitmap 72 * @page_start: page index of the first page to be freed 73 * @page_end: page index of the last page to be freed + 1 74 * 75 * Free pages [@page_start and @page_end) in @pages for all units. 76 * The pages were allocated for @chunk. 77 */ 78static void pcpu_free_pages(struct pcpu_chunk *chunk, 79 struct page **pages, unsigned long *populated, 80 int page_start, int page_end) 81{ 82 unsigned int cpu; 83 int i; 84 85 for_each_possible_cpu(cpu) { 86 for (i = page_start; i < page_end; i++) { 87 struct page *page = pages[pcpu_page_idx(cpu, i)]; 88 89 if (page) 90 __free_page(page); 91 } 92 } 93} 94 95/** 96 * pcpu_alloc_pages - allocates pages for @chunk 97 * @chunk: target chunk 98 * @pages: array to put the allocated pages into, indexed by pcpu_page_idx() 99 * @populated: populated bitmap 100 * @page_start: page index of the first page to be allocated 101 * @page_end: page index of the last page to be allocated + 1 102 * 103 * Allocate pages [@page_start,@page_end) into @pages for all units. 104 * The allocation is for @chunk. Percpu core doesn't care about the 105 * content of @pages and will pass it verbatim to pcpu_map_pages(). 106 */ 107static int pcpu_alloc_pages(struct pcpu_chunk *chunk, 108 struct page **pages, unsigned long *populated, 109 int page_start, int page_end) 110{ 111 const gfp_t gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_COLD; 112 unsigned int cpu; 113 int i; 114 115 for_each_possible_cpu(cpu) { 116 for (i = page_start; i < page_end; i++) { 117 struct page **pagep = &pages[pcpu_page_idx(cpu, i)]; 118 119 *pagep = alloc_pages_node(cpu_to_node(cpu), gfp, 0); 120 if (!*pagep) { 121 pcpu_free_pages(chunk, pages, populated, 122 page_start, page_end); 123 return -ENOMEM; 124 } 125 } 126 } 127 return 0; 128} 129 130/** 131 * pcpu_pre_unmap_flush - flush cache prior to unmapping 132 * @chunk: chunk the regions to be flushed belongs to 133 * @page_start: page index of the first page to be flushed 134 * @page_end: page index of the last page to be flushed + 1 135 * 136 * Pages in [@page_start,@page_end) of @chunk are about to be 137 * unmapped. Flush cache. As each flushing trial can be very 138 * expensive, issue flush on the whole region at once rather than 139 * doing it for each cpu. This could be an overkill but is more 140 * scalable. 141 */ 142static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk, 143 int page_start, int page_end) 144{ 145 flush_cache_vunmap( 146 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start), 147 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end)); 148} 149 150static void __pcpu_unmap_pages(unsigned long addr, int nr_pages) 151{ 152 unmap_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT); 153} 154 155/** 156 * pcpu_unmap_pages - unmap pages out of a pcpu_chunk 157 * @chunk: chunk of interest 158 * @pages: pages array which can be used to pass information to free 159 * @populated: populated bitmap 160 * @page_start: page index of the first page to unmap 161 * @page_end: page index of the last page to unmap + 1 162 * 163 * For each cpu, unmap pages [@page_start,@page_end) out of @chunk. 164 * Corresponding elements in @pages were cleared by the caller and can 165 * be used to carry information to pcpu_free_pages() which will be 166 * called after all unmaps are finished. The caller should call 167 * proper pre/post flush functions. 168 */ 169static void pcpu_unmap_pages(struct pcpu_chunk *chunk, 170 struct page **pages, unsigned long *populated, 171 int page_start, int page_end) 172{ 173 unsigned int cpu; 174 int i; 175 176 for_each_possible_cpu(cpu) { 177 for (i = page_start; i < page_end; i++) { 178 struct page *page; 179 180 page = pcpu_chunk_page(chunk, cpu, i); 181 WARN_ON(!page); 182 pages[pcpu_page_idx(cpu, i)] = page; 183 } 184 __pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start), 185 page_end - page_start); 186 } 187 188 for (i = page_start; i < page_end; i++) 189 __clear_bit(i, populated); 190} 191 192/** 193 * pcpu_post_unmap_tlb_flush - flush TLB after unmapping 194 * @chunk: pcpu_chunk the regions to be flushed belong to 195 * @page_start: page index of the first page to be flushed 196 * @page_end: page index of the last page to be flushed + 1 197 * 198 * Pages [@page_start,@page_end) of @chunk have been unmapped. Flush 199 * TLB for the regions. This can be skipped if the area is to be 200 * returned to vmalloc as vmalloc will handle TLB flushing lazily. 201 * 202 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once 203 * for the whole region. 204 */ 205static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk, 206 int page_start, int page_end) 207{ 208 flush_tlb_kernel_range( 209 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start), 210 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end)); 211} 212 213static int __pcpu_map_pages(unsigned long addr, struct page **pages, 214 int nr_pages) 215{ 216 return map_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT, 217 PAGE_KERNEL, pages); 218} 219 220/** 221 * pcpu_map_pages - map pages into a pcpu_chunk 222 * @chunk: chunk of interest 223 * @pages: pages array containing pages to be mapped 224 * @populated: populated bitmap 225 * @page_start: page index of the first page to map 226 * @page_end: page index of the last page to map + 1 227 * 228 * For each cpu, map pages [@page_start,@page_end) into @chunk. The 229 * caller is responsible for calling pcpu_post_map_flush() after all 230 * mappings are complete. 231 * 232 * This function is responsible for setting corresponding bits in 233 * @chunk->populated bitmap and whatever is necessary for reverse 234 * lookup (addr -> chunk). 235 */ 236static int pcpu_map_pages(struct pcpu_chunk *chunk, 237 struct page **pages, unsigned long *populated, 238 int page_start, int page_end) 239{ 240 unsigned int cpu, tcpu; 241 int i, err; 242 243 for_each_possible_cpu(cpu) { 244 err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start), 245 &pages[pcpu_page_idx(cpu, page_start)], 246 page_end - page_start); 247 if (err < 0) 248 goto err; 249 } 250 251 /* mapping successful, link chunk and mark populated */ 252 for (i = page_start; i < page_end; i++) { 253 for_each_possible_cpu(cpu) 254 pcpu_set_page_chunk(pages[pcpu_page_idx(cpu, i)], 255 chunk); 256 __set_bit(i, populated); 257 } 258 259 return 0; 260 261err: 262 for_each_possible_cpu(tcpu) { 263 if (tcpu == cpu) 264 break; 265 __pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start), 266 page_end - page_start); 267 } 268 return err; 269} 270 271/** 272 * pcpu_post_map_flush - flush cache after mapping 273 * @chunk: pcpu_chunk the regions to be flushed belong to 274 * @page_start: page index of the first page to be flushed 275 * @page_end: page index of the last page to be flushed + 1 276 * 277 * Pages [@page_start,@page_end) of @chunk have been mapped. Flush 278 * cache. 279 * 280 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once 281 * for the whole region. 282 */ 283static void pcpu_post_map_flush(struct pcpu_chunk *chunk, 284 int page_start, int page_end) 285{ 286 flush_cache_vmap( 287 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start), 288 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end)); 289} 290 291/** 292 * pcpu_populate_chunk - populate and map an area of a pcpu_chunk 293 * @chunk: chunk of interest 294 * @off: offset to the area to populate 295 * @size: size of the area to populate in bytes 296 * 297 * For each cpu, populate and map pages [@page_start,@page_end) into 298 * @chunk. The area is cleared on return. 299 * 300 * CONTEXT: 301 * pcpu_alloc_mutex, does GFP_KERNEL allocation. 302 */ 303static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size) 304{ 305 int page_start = PFN_DOWN(off); 306 int page_end = PFN_UP(off + size); 307 int free_end = page_start, unmap_end = page_start; 308 struct page **pages; 309 unsigned long *populated; 310 unsigned int cpu; 311 int rs, re, rc; 312 313 /* quick path, check whether all pages are already there */ 314 rs = page_start; 315 pcpu_next_pop(chunk, &rs, &re, page_end); 316 if (rs == page_start && re == page_end) 317 goto clear; 318 319 /* need to allocate and map pages, this chunk can't be immutable */ 320 WARN_ON(chunk->immutable); 321 322 pages = pcpu_get_pages_and_bitmap(chunk, &populated, true); 323 if (!pages) 324 return -ENOMEM; 325 326 /* alloc and map */ 327 pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) { 328 rc = pcpu_alloc_pages(chunk, pages, populated, rs, re); 329 if (rc) 330 goto err_free; 331 free_end = re; 332 } 333 334 pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) { 335 rc = pcpu_map_pages(chunk, pages, populated, rs, re); 336 if (rc) 337 goto err_unmap; 338 unmap_end = re; 339 } 340 pcpu_post_map_flush(chunk, page_start, page_end); 341 342 /* commit new bitmap */ 343 bitmap_copy(chunk->populated, populated, pcpu_unit_pages); 344clear: 345 for_each_possible_cpu(cpu) 346 memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size); 347 return 0; 348 349err_unmap: 350 pcpu_pre_unmap_flush(chunk, page_start, unmap_end); 351 pcpu_for_each_unpop_region(chunk, rs, re, page_start, unmap_end) 352 pcpu_unmap_pages(chunk, pages, populated, rs, re); 353 pcpu_post_unmap_tlb_flush(chunk, page_start, unmap_end); 354err_free: 355 pcpu_for_each_unpop_region(chunk, rs, re, page_start, free_end) 356 pcpu_free_pages(chunk, pages, populated, rs, re); 357 return rc; 358} 359 360/** 361 * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk 362 * @chunk: chunk to depopulate 363 * @off: offset to the area to depopulate 364 * @size: size of the area to depopulate in bytes 365 * @flush: whether to flush cache and tlb or not 366 * 367 * For each cpu, depopulate and unmap pages [@page_start,@page_end) 368 * from @chunk. If @flush is true, vcache is flushed before unmapping 369 * and tlb after. 370 * 371 * CONTEXT: 372 * pcpu_alloc_mutex. 373 */ 374static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size) 375{ 376 int page_start = PFN_DOWN(off); 377 int page_end = PFN_UP(off + size); 378 struct page **pages; 379 unsigned long *populated; 380 int rs, re; 381 382 /* quick path, check whether it's empty already */ 383 rs = page_start; 384 pcpu_next_unpop(chunk, &rs, &re, page_end); 385 if (rs == page_start && re == page_end) 386 return; 387 388 /* immutable chunks can't be depopulated */ 389 WARN_ON(chunk->immutable); 390 391 /* 392 * If control reaches here, there must have been at least one 393 * successful population attempt so the temp pages array must 394 * be available now. 395 */ 396 pages = pcpu_get_pages_and_bitmap(chunk, &populated, false); 397 BUG_ON(!pages); 398 399 /* unmap and free */ 400 pcpu_pre_unmap_flush(chunk, page_start, page_end); 401 402 pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end) 403 pcpu_unmap_pages(chunk, pages, populated, rs, re); 404 405 /* no need to flush tlb, vmalloc will handle it lazily */ 406 407 pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end) 408 pcpu_free_pages(chunk, pages, populated, rs, re); 409 410 /* commit new bitmap */ 411 bitmap_copy(chunk->populated, populated, pcpu_unit_pages); 412} 413 414static struct pcpu_chunk *pcpu_create_chunk(void) 415{ 416 struct pcpu_chunk *chunk; 417 struct vm_struct **vms; 418 419 chunk = pcpu_alloc_chunk(); 420 if (!chunk) 421 return NULL; 422 423 vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes, 424 pcpu_nr_groups, pcpu_atom_size, GFP_KERNEL); 425 if (!vms) { 426 pcpu_free_chunk(chunk); 427 return NULL; 428 } 429 430 chunk->data = vms; 431 chunk->base_addr = vms[0]->addr - pcpu_group_offsets[0]; 432 return chunk; 433} 434 435static void pcpu_destroy_chunk(struct pcpu_chunk *chunk) 436{ 437 if (chunk && chunk->data) 438 pcpu_free_vm_areas(chunk->data, pcpu_nr_groups); 439 pcpu_free_chunk(chunk); 440} 441 442static struct page *pcpu_addr_to_page(void *addr) 443{ 444 return vmalloc_to_page(addr); 445} 446 447static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai) 448{ 449 /* no extra restriction */ 450 return 0; 451} 452