Deleted Added
sdiff udiff text old ( 246087 ) new ( 247360 )
full compact
uma_core.c (246087) uma_core.c (247360)
1/*-
2 * Copyright (c) 2002-2005, 2009 Jeffrey Roberson <jeff@FreeBSD.org>
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4 * Copyright (c) 2004-2006 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 34 unchanged lines hidden (view full) ---

43
44/*
45 * TODO:
46 * - Improve memory usage for large allocations
47 * - Investigate cache size adjustments
48 */
49
50#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2002-2005, 2009 Jeffrey Roberson <jeff@FreeBSD.org>
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4 * Copyright (c) 2004-2006 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 34 unchanged lines hidden (view full) ---

43
44/*
45 * TODO:
46 * - Improve memory usage for large allocations
47 * - Investigate cache size adjustments
48 */
49
50#include <sys/cdefs.h>
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 246087 2013-01-29 19:06:16Z glebius $");
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 247360 2013-02-26 23:35:27Z attilio $");
52
53/* I should really use ktr.. */
54/*
55#define UMA_DEBUG 1
56#define UMA_DEBUG_ALLOC 1
57#define UMA_DEBUG_ALLOC_1 1
58*/
59

--- 14 unchanged lines hidden (view full) ---

74#include <sys/proc.h>
75#include <sys/sbuf.h>
76#include <sys/smp.h>
77#include <sys/vmmeter.h>
78
79#include <vm/vm.h>
80#include <vm/vm_object.h>
81#include <vm/vm_page.h>
52
53/* I should really use ktr.. */
54/*
55#define UMA_DEBUG 1
56#define UMA_DEBUG_ALLOC 1
57#define UMA_DEBUG_ALLOC_1 1
58*/
59

--- 14 unchanged lines hidden (view full) ---

74#include <sys/proc.h>
75#include <sys/sbuf.h>
76#include <sys/smp.h>
77#include <sys/vmmeter.h>
78
79#include <vm/vm.h>
80#include <vm/vm_object.h>
81#include <vm/vm_page.h>
82#include <vm/vm_pageout.h>
82#include <vm/vm_param.h>
83#include <vm/vm_map.h>
84#include <vm/vm_kern.h>
85#include <vm/vm_extern.h>
86#include <vm/uma.h>
87#include <vm/uma_int.h>
88#include <vm/uma_dbg.h>
89

--- 118 unchanged lines hidden (view full) ---

208 */
209enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI };
210
211#define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */
212#define ZFREE_STATFREE 0x00000002 /* Update zone free statistic. */
213
214/* Prototypes.. */
215
83#include <vm/vm_param.h>
84#include <vm/vm_map.h>
85#include <vm/vm_kern.h>
86#include <vm/vm_extern.h>
87#include <vm/uma.h>
88#include <vm/uma_int.h>
89#include <vm/uma_dbg.h>
90

--- 118 unchanged lines hidden (view full) ---

209 */
210enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI };
211
212#define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */
213#define ZFREE_STATFREE 0x00000002 /* Update zone free statistic. */
214
215/* Prototypes.. */
216
216static void *obj_alloc(uma_zone_t, int, u_int8_t *, int);
217static void *noobj_alloc(uma_zone_t, int, u_int8_t *, int);
217static void *page_alloc(uma_zone_t, int, u_int8_t *, int);
218static void *startup_alloc(uma_zone_t, int, u_int8_t *, int);
219static void page_free(void *, int, u_int8_t);
220static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int);
221static void cache_drain(uma_zone_t);
222static void bucket_drain(uma_zone_t, uma_bucket_t);
223static void bucket_cache_drain(uma_zone_t zone);
224static int keg_ctor(void *, int, void *, int);

--- 800 unchanged lines hidden (view full) ---

1025 * bytes The number of bytes requested
1026 * wait Shall we wait?
1027 *
1028 * Returns:
1029 * A pointer to the alloced memory or possibly
1030 * NULL if M_NOWAIT is set.
1031 */
1032static void *
218static void *page_alloc(uma_zone_t, int, u_int8_t *, int);
219static void *startup_alloc(uma_zone_t, int, u_int8_t *, int);
220static void page_free(void *, int, u_int8_t);
221static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int);
222static void cache_drain(uma_zone_t);
223static void bucket_drain(uma_zone_t, uma_bucket_t);
224static void bucket_cache_drain(uma_zone_t zone);
225static int keg_ctor(void *, int, void *, int);

--- 800 unchanged lines hidden (view full) ---

1026 * bytes The number of bytes requested
1027 * wait Shall we wait?
1028 *
1029 * Returns:
1030 * A pointer to the alloced memory or possibly
1031 * NULL if M_NOWAIT is set.
1032 */
1033static void *
1033obj_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
1034noobj_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
1034{
1035{
1035 vm_object_t object;
1036 TAILQ_HEAD(, vm_page) alloctail;
1037 u_long npages;
1036 vm_offset_t retkva, zkva;
1038 vm_offset_t retkva, zkva;
1037 vm_page_t p;
1038 int pages, startpages;
1039 vm_page_t p, p_next;
1039 uma_keg_t keg;
1040
1040 uma_keg_t keg;
1041
1042 TAILQ_INIT(&alloctail);
1041 keg = zone_first_keg(zone);
1043 keg = zone_first_keg(zone);
1042 object = keg->uk_obj;
1043 retkva = 0;
1044
1044
1045 /*
1046 * This looks a little weird since we're getting one page at a time.
1047 */
1048 VM_OBJECT_LOCK(object);
1049 p = TAILQ_LAST(&object->memq, pglist);
1050 pages = p != NULL ? p->pindex + 1 : 0;
1051 startpages = pages;
1052 zkva = keg->uk_kva + pages * PAGE_SIZE;
1053 for (; bytes > 0; bytes -= PAGE_SIZE) {
1054 p = vm_page_alloc(object, pages,
1055 VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED);
1056 if (p == NULL) {
1057 if (pages != startpages)
1058 pmap_qremove(retkva, pages - startpages);
1059 while (pages != startpages) {
1060 pages--;
1061 p = TAILQ_LAST(&object->memq, pglist);
1062 vm_page_unwire(p, 0);
1063 vm_page_free(p);
1064 }
1065 retkva = 0;
1066 goto done;
1045 npages = howmany(bytes, PAGE_SIZE);
1046 while (npages > 0) {
1047 p = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT |
1048 VM_ALLOC_WIRED | VM_ALLOC_NOOBJ);
1049 if (p != NULL) {
1050 /*
1051 * Since the page does not belong to an object, its
1052 * listq is unused.
1053 */
1054 TAILQ_INSERT_TAIL(&alloctail, p, listq);
1055 npages--;
1056 continue;
1067 }
1057 }
1058 if (wait & M_WAITOK) {
1059 VM_WAIT;
1060 continue;
1061 }
1062
1063 /*
1064 * Page allocation failed, free intermediate pages and
1065 * exit.
1066 */
1067 TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) {
1068 vm_page_unwire(p, 0);
1069 vm_page_free(p);
1070 }
1071 return (NULL);
1072 }
1073 *flags = UMA_SLAB_PRIV;
1074 zkva = keg->uk_kva +
1075 atomic_fetchadd_long(&keg->uk_offset, round_page(bytes));
1076 retkva = zkva;
1077 TAILQ_FOREACH(p, &alloctail, listq) {
1068 pmap_qenter(zkva, &p, 1);
1078 pmap_qenter(zkva, &p, 1);
1069 if (retkva == 0)
1070 retkva = zkva;
1071 zkva += PAGE_SIZE;
1079 zkva += PAGE_SIZE;
1072 pages += 1;
1073 }
1080 }
1074done:
1075 VM_OBJECT_UNLOCK(object);
1076 *flags = UMA_SLAB_PRIV;
1077
1078 return ((void *)retkva);
1079}
1080
1081/*
1082 * Frees a number of pages to the system
1083 *
1084 * Arguments:

--- 1922 unchanged lines hidden (view full) ---

3007 keg = zone_first_keg(zone);
3008 keg->uk_flags |= UMA_ZFLAG_PRIVALLOC;
3009 keg->uk_allocf = allocf;
3010 ZONE_UNLOCK(zone);
3011}
3012
3013/* See uma.h */
3014int
1081
1082 return ((void *)retkva);
1083}
1084
1085/*
1086 * Frees a number of pages to the system
1087 *
1088 * Arguments:

--- 1922 unchanged lines hidden (view full) ---

3011 keg = zone_first_keg(zone);
3012 keg->uk_flags |= UMA_ZFLAG_PRIVALLOC;
3013 keg->uk_allocf = allocf;
3014 ZONE_UNLOCK(zone);
3015}
3016
3017/* See uma.h */
3018int
3015uma_zone_set_obj(uma_zone_t zone, struct vm_object *obj, int count)
3019uma_zone_reserve_kva(uma_zone_t zone, int count)
3016{
3017 uma_keg_t keg;
3018 vm_offset_t kva;
3019 int pages;
3020
3021 keg = zone_first_keg(zone);
3022 pages = count / keg->uk_ipers;
3023
3024 if (pages * keg->uk_ipers < count)
3025 pages++;
3026
3020{
3021 uma_keg_t keg;
3022 vm_offset_t kva;
3023 int pages;
3024
3025 keg = zone_first_keg(zone);
3026 pages = count / keg->uk_ipers;
3027
3028 if (pages * keg->uk_ipers < count)
3029 pages++;
3030
3027 kva = kmem_alloc_nofault(kernel_map, pages * UMA_SLAB_SIZE);
3028
3029 if (kva == 0)
3030 return (0);
3031 if (obj == NULL)
3032 obj = vm_object_allocate(OBJT_PHYS, pages);
3033 else {
3034 VM_OBJECT_LOCK_INIT(obj, "uma object");
3035 _vm_object_allocate(OBJT_PHYS, pages, obj);
3036 }
3031#ifdef UMA_MD_SMALL_ALLOC
3032 if (keg->uk_ppera > 1) {
3033#else
3034 if (1) {
3035#endif
3036 kva = kmem_alloc_nofault(kernel_map, pages * UMA_SLAB_SIZE);
3037 if (kva == 0)
3038 return (0);
3039 } else
3040 kva = 0;
3037 ZONE_LOCK(zone);
3038 keg->uk_kva = kva;
3041 ZONE_LOCK(zone);
3042 keg->uk_kva = kva;
3039 keg->uk_obj = obj;
3043 keg->uk_offset = 0;
3040 keg->uk_maxpages = pages;
3044 keg->uk_maxpages = pages;
3041 keg->uk_allocf = obj_alloc;
3045#ifdef UMA_MD_SMALL_ALLOC
3046 keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc;
3047#else
3048 keg->uk_allocf = noobj_alloc;
3049#endif
3042 keg->uk_flags |= UMA_ZONE_NOFREE | UMA_ZFLAG_PRIVALLOC;
3043 ZONE_UNLOCK(zone);
3044 return (1);
3045}
3046
3047/* See uma.h */
3048void
3049uma_prealloc(uma_zone_t zone, int items)

--- 375 unchanged lines hidden --- 		3050 keg->uk_flags |= UMA_ZONE_NOFREE | UMA_ZFLAG_PRIVALLOC;
3051 ZONE_UNLOCK(zone);
3052 return (1);
3053}
3054
3055/* See uma.h */
3056void
3057uma_prealloc(uma_zone_t zone, int items)

--- 375 unchanged lines hidden ---