1/* 2 * linux/mm/madvise.c 3 * 4 * Copyright (C) 1999 Linus Torvalds 5 * Copyright (C) 2002 Christoph Hellwig 6 */ 7 8#include <linux/mman.h> 9#include <linux/pagemap.h> 10#include <linux/syscalls.h> 11#include <linux/mempolicy.h> 12#include <linux/page-isolation.h> 13#include <linux/hugetlb.h> 14#include <linux/sched.h> 15#include <linux/ksm.h> 16 17/* 18 * Any behaviour which results in changes to the vma->vm_flags needs to 19 * take mmap_sem for writing. Others, which simply traverse vmas, need 20 * to only take it for reading. 21 */ 22static int madvise_need_mmap_write(int behavior) 23{ 24 switch (behavior) { 25 case MADV_REMOVE: 26 case MADV_WILLNEED: 27 case MADV_DONTNEED: 28 return 0; 29 default: 30 /* be safe, default to 1. list exceptions explicitly */ 31 return 1; 32 } 33} 34 35/* 36 * We can potentially split a vm area into separate 37 * areas, each area with its own behavior. 38 */ 39static long madvise_behavior(struct vm_area_struct * vma, 40 struct vm_area_struct **prev, 41 unsigned long start, unsigned long end, int behavior) 42{ 43 struct mm_struct * mm = vma->vm_mm; 44 int error = 0; 45 pgoff_t pgoff; 46 unsigned long new_flags = vma->vm_flags; 47 48 switch (behavior) { 49 case MADV_NORMAL: 50 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ; 51 break; 52 case MADV_SEQUENTIAL: 53 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ; 54 break; 55 case MADV_RANDOM: 56 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ; 57 break; 58 case MADV_DONTFORK: 59 new_flags |= VM_DONTCOPY; 60 break; 61 case MADV_DOFORK: 62 if (vma->vm_flags & VM_IO) { 63 error = -EINVAL; 64 goto out; 65 } 66 new_flags &= ~VM_DONTCOPY; 67 break; 68 case MADV_MERGEABLE: 69 case MADV_UNMERGEABLE: 70 error = ksm_madvise(vma, start, end, behavior, &new_flags); 71 if (error) 72 goto out; 73 break; 74 } 75 76 if (new_flags == vma->vm_flags) { 77 *prev = vma; 78 goto out; 79 } 80 81 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 82 *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma, 83 vma->vm_file, pgoff, vma_policy(vma)); 84 if (*prev) { 85 vma = *prev; 86 goto success; 87 } 88 89 *prev = vma; 90 91 if (start != vma->vm_start) { 92 error = split_vma(mm, vma, start, 1); 93 if (error) 94 goto out; 95 } 96 97 if (end != vma->vm_end) { 98 error = split_vma(mm, vma, end, 0); 99 if (error) 100 goto out; 101 } 102 103success: 104 /* 105 * vm_flags is protected by the mmap_sem held in write mode. 106 */ 107 vma->vm_flags = new_flags; 108 109out: 110 if (error == -ENOMEM) 111 error = -EAGAIN; 112 return error; 113} 114 115/* 116 * Schedule all required I/O operations. Do not wait for completion. 117 */ 118static long madvise_willneed(struct vm_area_struct * vma, 119 struct vm_area_struct ** prev, 120 unsigned long start, unsigned long end) 121{ 122 struct file *file = vma->vm_file; 123 124 if (!file) 125 return -EBADF; 126 127 if (file->f_mapping->a_ops->get_xip_mem) { 128 /* no bad return value, but ignore advice */ 129 return 0; 130 } 131 132 *prev = vma; 133 start = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 134 if (end > vma->vm_end) 135 end = vma->vm_end; 136 end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 137 138 force_page_cache_readahead(file->f_mapping, file, start, end - start); 139 return 0; 140} 141 142/* 143 * Application no longer needs these pages. If the pages are dirty, 144 * it's OK to just throw them away. The app will be more careful about 145 * data it wants to keep. Be sure to free swap resources too. The 146 * zap_page_range call sets things up for shrink_active_list to actually free 147 * these pages later if no one else has touched them in the meantime, 148 * although we could add these pages to a global reuse list for 149 * shrink_active_list to pick up before reclaiming other pages. 150 * 151 * NB: This interface discards data rather than pushes it out to swap, 152 * as some implementations do. This has performance implications for 153 * applications like large transactional databases which want to discard 154 * pages in anonymous maps after committing to backing store the data 155 * that was kept in them. There is no reason to write this data out to 156 * the swap area if the application is discarding it. 157 * 158 * An interface that causes the system to free clean pages and flush 159 * dirty pages is already available as msync(MS_INVALIDATE). 160 */ 161static long madvise_dontneed(struct vm_area_struct * vma, 162 struct vm_area_struct ** prev, 163 unsigned long start, unsigned long end) 164{ 165 *prev = vma; 166 if (vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP)) 167 return -EINVAL; 168 169 if (unlikely(vma->vm_flags & VM_NONLINEAR)) { 170 struct zap_details details = { 171 .nonlinear_vma = vma, 172 .last_index = ULONG_MAX, 173 }; 174 zap_page_range(vma, start, end - start, &details); 175 } else 176 zap_page_range(vma, start, end - start, NULL); 177 return 0; 178} 179 180/* 181 * Application wants to free up the pages and associated backing store. 182 * This is effectively punching a hole into the middle of a file. 183 * 184 * NOTE: Currently, only shmfs/tmpfs is supported for this operation. 185 * Other filesystems return -ENOSYS. 186 */ 187static long madvise_remove(struct vm_area_struct *vma, 188 struct vm_area_struct **prev, 189 unsigned long start, unsigned long end) 190{ 191 struct address_space *mapping; 192 loff_t offset, endoff; 193 int error; 194 195 *prev = NULL; /* tell sys_madvise we drop mmap_sem */ 196 197 if (vma->vm_flags & (VM_LOCKED|VM_NONLINEAR|VM_HUGETLB)) 198 return -EINVAL; 199 200 if (!vma->vm_file || !vma->vm_file->f_mapping 201 || !vma->vm_file->f_mapping->host) { 202 return -EINVAL; 203 } 204 205 if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE)) 206 return -EACCES; 207 208 mapping = vma->vm_file->f_mapping; 209 210 offset = (loff_t)(start - vma->vm_start) 211 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); 212 endoff = (loff_t)(end - vma->vm_start - 1) 213 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); 214 215 /* vmtruncate_range needs to take i_mutex and i_alloc_sem */ 216 up_read(¤t->mm->mmap_sem); 217 error = vmtruncate_range(mapping->host, offset, endoff); 218 down_read(¤t->mm->mmap_sem); 219 return error; 220} 221 222#ifdef CONFIG_MEMORY_FAILURE 223/* 224 * Error injection support for memory error handling. 225 */ 226static int madvise_hwpoison(int bhv, unsigned long start, unsigned long end) 227{ 228 int ret = 0; 229 230 if (!capable(CAP_SYS_ADMIN)) 231 return -EPERM; 232 for (; start < end; start += PAGE_SIZE) { 233 struct page *p; 234 int ret = get_user_pages_fast(start, 1, 0, &p); 235 if (ret != 1) 236 return ret; 237 if (bhv == MADV_SOFT_OFFLINE) { 238 printk(KERN_INFO "Soft offlining page %lx at %lx\n", 239 page_to_pfn(p), start); 240 ret = soft_offline_page(p, MF_COUNT_INCREASED); 241 if (ret) 242 break; 243 continue; 244 } 245 printk(KERN_INFO "Injecting memory failure for page %lx at %lx\n", 246 page_to_pfn(p), start); 247 /* Ignore return value for now */ 248 __memory_failure(page_to_pfn(p), 0, MF_COUNT_INCREASED); 249 } 250 return ret; 251} 252#endif 253 254static long 255madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev, 256 unsigned long start, unsigned long end, int behavior) 257{ 258 switch (behavior) { 259 case MADV_REMOVE: 260 return madvise_remove(vma, prev, start, end); 261 case MADV_WILLNEED: 262 return madvise_willneed(vma, prev, start, end); 263 case MADV_DONTNEED: 264 return madvise_dontneed(vma, prev, start, end); 265 default: 266 return madvise_behavior(vma, prev, start, end, behavior); 267 } 268} 269 270static int 271madvise_behavior_valid(int behavior) 272{ 273 switch (behavior) { 274 case MADV_DOFORK: 275 case MADV_DONTFORK: 276 case MADV_NORMAL: 277 case MADV_SEQUENTIAL: 278 case MADV_RANDOM: 279 case MADV_REMOVE: 280 case MADV_WILLNEED: 281 case MADV_DONTNEED: 282#ifdef CONFIG_KSM 283 case MADV_MERGEABLE: 284 case MADV_UNMERGEABLE: 285#endif 286 return 1; 287 288 default: 289 return 0; 290 } 291} 292 293/* 294 * The madvise(2) system call. 295 * 296 * Applications can use madvise() to advise the kernel how it should 297 * handle paging I/O in this VM area. The idea is to help the kernel 298 * use appropriate read-ahead and caching techniques. The information 299 * provided is advisory only, and can be safely disregarded by the 300 * kernel without affecting the correct operation of the application. 301 * 302 * behavior values: 303 * MADV_NORMAL - the default behavior is to read clusters. This 304 * results in some read-ahead and read-behind. 305 * MADV_RANDOM - the system should read the minimum amount of data 306 * on any access, since it is unlikely that the appli- 307 * cation will need more than what it asks for. 308 * MADV_SEQUENTIAL - pages in the given range will probably be accessed 309 * once, so they can be aggressively read ahead, and 310 * can be freed soon after they are accessed. 311 * MADV_WILLNEED - the application is notifying the system to read 312 * some pages ahead. 313 * MADV_DONTNEED - the application is finished with the given range, 314 * so the kernel can free resources associated with it. 315 * MADV_REMOVE - the application wants to free up the given range of 316 * pages and associated backing store. 317 * MADV_DONTFORK - omit this area from child's address space when forking: 318 * typically, to avoid COWing pages pinned by get_user_pages(). 319 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking. 320 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in 321 * this area with pages of identical content from other such areas. 322 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others. 323 * 324 * return values: 325 * zero - success 326 * -EINVAL - start + len < 0, start is not page-aligned, 327 * "behavior" is not a valid value, or application 328 * is attempting to release locked or shared pages. 329 * -ENOMEM - addresses in the specified range are not currently 330 * mapped, or are outside the AS of the process. 331 * -EIO - an I/O error occurred while paging in data. 332 * -EBADF - map exists, but area maps something that isn't a file. 333 * -EAGAIN - a kernel resource was temporarily unavailable. 334 */ 335SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior) 336{ 337 unsigned long end, tmp; 338 struct vm_area_struct * vma, *prev; 339 int unmapped_error = 0; 340 int error = -EINVAL; 341 int write; 342 size_t len; 343 344#ifdef CONFIG_MEMORY_FAILURE 345 if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE) 346 return madvise_hwpoison(behavior, start, start+len_in); 347#endif 348 if (!madvise_behavior_valid(behavior)) 349 return error; 350 351 write = madvise_need_mmap_write(behavior); 352 if (write) 353 down_write(¤t->mm->mmap_sem); 354 else 355 down_read(¤t->mm->mmap_sem); 356 357 if (start & ~PAGE_MASK) 358 goto out; 359 len = (len_in + ~PAGE_MASK) & PAGE_MASK; 360 361 /* Check to see whether len was rounded up from small -ve to zero */ 362 if (len_in && !len) 363 goto out; 364 365 end = start + len; 366 if (end < start) 367 goto out; 368 369 error = 0; 370 if (end == start) 371 goto out; 372 373 /* 374 * If the interval [start,end) covers some unmapped address 375 * ranges, just ignore them, but return -ENOMEM at the end. 376 * - different from the way of handling in mlock etc. 377 */ 378 vma = find_vma_prev(current->mm, start, &prev); 379 if (vma && start > vma->vm_start) 380 prev = vma; 381 382 for (;;) { 383 /* Still start < end. */ 384 error = -ENOMEM; 385 if (!vma) 386 goto out; 387 388 /* Here start < (end|vma->vm_end). */ 389 if (start < vma->vm_start) { 390 unmapped_error = -ENOMEM; 391 start = vma->vm_start; 392 if (start >= end) 393 goto out; 394 } 395 396 /* Here vma->vm_start <= start < (end|vma->vm_end) */ 397 tmp = vma->vm_end; 398 if (end < tmp) 399 tmp = end; 400 401 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */ 402 error = madvise_vma(vma, &prev, start, tmp, behavior); 403 if (error) 404 goto out; 405 start = tmp; 406 if (prev && start < prev->vm_end) 407 start = prev->vm_end; 408 error = unmapped_error; 409 if (start >= end) 410 goto out; 411 if (prev) 412 vma = prev->vm_next; 413 else /* madvise_remove dropped mmap_sem */ 414 vma = find_vma(current->mm, start); 415 } 416out: 417 if (write) 418 up_write(¤t->mm->mmap_sem); 419 else 420 up_read(¤t->mm->mmap_sem); 421 422 return error; 423} 424