Deleted Added
sdiff udiff text old ( 249305 ) new ( 249313 )
full compact
uma_core.c (249305) uma_core.c (249313)
1/*-
2 * Copyright (c) 2002-2005, 2009 Jeffrey Roberson <jeff@FreeBSD.org>
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4 * Copyright (c) 2004-2006 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 34 unchanged lines hidden (view full) ---

43
44/*
45 * TODO:
46 * - Improve memory usage for large allocations
47 * - Investigate cache size adjustments
48 */
49
50#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2002-2005, 2009 Jeffrey Roberson <jeff@FreeBSD.org>
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4 * Copyright (c) 2004-2006 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 34 unchanged lines hidden (view full) ---

43
44/*
45 * TODO:
46 * - Improve memory usage for large allocations
47 * - Investigate cache size adjustments
48 */
49
50#include <sys/cdefs.h>
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 249305 2013-04-09 12:20:44Z glebius $");
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 249313 2013-04-09 17:43:48Z glebius $");
52
53/* I should really use ktr.. */
54/*
55#define UMA_DEBUG 1
56#define UMA_DEBUG_ALLOC 1
57#define UMA_DEBUG_ALLOC_1 1
58*/
59

--- 103 unchanged lines hidden (view full) ---

163 const char *name;
164 size_t size;
165 uma_ctor ctor;
166 uma_dtor dtor;
167 uma_init uminit;
168 uma_fini fini;
169 uma_keg_t keg;
170 int align;
52
53/* I should really use ktr.. */
54/*
55#define UMA_DEBUG 1
56#define UMA_DEBUG_ALLOC 1
57#define UMA_DEBUG_ALLOC_1 1
58*/
59

--- 103 unchanged lines hidden (view full) ---

163 const char *name;
164 size_t size;
165 uma_ctor ctor;
166 uma_dtor dtor;
167 uma_init uminit;
168 uma_fini fini;
169 uma_keg_t keg;
170 int align;
171 u_int32_t flags;
171 uint32_t flags;
172};
173
174struct uma_kctor_args {
175 uma_zone_t zone;
176 size_t size;
177 uma_init uminit;
178 uma_fini fini;
179 int align;
172};
173
174struct uma_kctor_args {
175 uma_zone_t zone;
176 size_t size;
177 uma_init uminit;
178 uma_fini fini;
179 int align;
180 u_int32_t flags;
180 uint32_t flags;
181};
182
183struct uma_bucket_zone {
184 uma_zone_t ubz_zone;
185 char *ubz_name;
186 int ubz_entries;
187};
188

--- 21 unchanged lines hidden (view full) ---

210 */
211enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI };
212
213#define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */
214#define ZFREE_STATFREE 0x00000002 /* Update zone free statistic. */
215
216/* Prototypes.. */
217
181};
182
183struct uma_bucket_zone {
184 uma_zone_t ubz_zone;
185 char *ubz_name;
186 int ubz_entries;
187};
188

--- 21 unchanged lines hidden (view full) ---

210 */
211enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI };
212
213#define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */
214#define ZFREE_STATFREE 0x00000002 /* Update zone free statistic. */
215
216/* Prototypes.. */
217
218static void *noobj_alloc(uma_zone_t, int, u_int8_t *, int);
219static void *page_alloc(uma_zone_t, int, u_int8_t *, int);
220static void *startup_alloc(uma_zone_t, int, u_int8_t *, int);
221static void page_free(void *, int, u_int8_t);
218static void *noobj_alloc(uma_zone_t, int, uint8_t *, int);
219static void *page_alloc(uma_zone_t, int, uint8_t *, int);
220static void *startup_alloc(uma_zone_t, int, uint8_t *, int);
221static void page_free(void *, int, uint8_t);
222static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int);
223static void cache_drain(uma_zone_t);
224static void bucket_drain(uma_zone_t, uma_bucket_t);
225static void bucket_cache_drain(uma_zone_t zone);
226static int keg_ctor(void *, int, void *, int);
227static void keg_dtor(void *, int, void *);
228static int zone_ctor(void *, int, void *, int);
229static void zone_dtor(void *, int, void *);

--- 15 unchanged lines hidden (view full) ---

245static uma_bucket_t bucket_alloc(int, int);
246static void bucket_free(uma_bucket_t);
247static void bucket_zone_drain(void);
248static int zone_alloc_bucket(uma_zone_t zone, int flags);
249static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
250static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
251static void *slab_alloc_item(uma_zone_t zone, uma_slab_t slab);
252static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
222static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int);
223static void cache_drain(uma_zone_t);
224static void bucket_drain(uma_zone_t, uma_bucket_t);
225static void bucket_cache_drain(uma_zone_t zone);
226static int keg_ctor(void *, int, void *, int);
227static void keg_dtor(void *, int, void *);
228static int zone_ctor(void *, int, void *, int);
229static void zone_dtor(void *, int, void *);

--- 15 unchanged lines hidden (view full) ---

245static uma_bucket_t bucket_alloc(int, int);
246static void bucket_free(uma_bucket_t);
247static void bucket_zone_drain(void);
248static int zone_alloc_bucket(uma_zone_t zone, int flags);
249static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
250static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
251static void *slab_alloc_item(uma_zone_t zone, uma_slab_t slab);
252static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
253 uma_fini fini, int align, u_int32_t flags);
253 uma_fini fini, int align, uint32_t flags);
254static inline void zone_relock(uma_zone_t zone, uma_keg_t keg);
255static inline void keg_relock(uma_keg_t keg, uma_zone_t zone);
256
257void uma_print_zone(uma_zone_t);
258void uma_print_stats(void);
259static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
260static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
261

--- 428 unchanged lines hidden (view full) ---

690 * Returns nothing.
691 */
692static void
693keg_drain(uma_keg_t keg)
694{
695 struct slabhead freeslabs = { 0 };
696 uma_slab_t slab;
697 uma_slab_t n;
254static inline void zone_relock(uma_zone_t zone, uma_keg_t keg);
255static inline void keg_relock(uma_keg_t keg, uma_zone_t zone);
256
257void uma_print_zone(uma_zone_t);
258void uma_print_stats(void);
259static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
260static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
261

--- 428 unchanged lines hidden (view full) ---

690 * Returns nothing.
691 */
692static void
693keg_drain(uma_keg_t keg)
694{
695 struct slabhead freeslabs = { 0 };
696 uma_slab_t slab;
697 uma_slab_t n;
698 u_int8_t flags;
699 u_int8_t *mem;
698 uint8_t flags;
699 uint8_t *mem;
700 int i;
701
702 /*
703 * We don't want to take pages from statically allocated kegs at this
704 * time
705 */
706 if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL)
707 return;

--- 115 unchanged lines hidden (view full) ---

823 * caller specified M_NOWAIT.
824 */
825static uma_slab_t
826keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait)
827{
828 uma_slabrefcnt_t slabref;
829 uma_alloc allocf;
830 uma_slab_t slab;
700 int i;
701
702 /*
703 * We don't want to take pages from statically allocated kegs at this
704 * time
705 */
706 if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL)
707 return;

--- 115 unchanged lines hidden (view full) ---

823 * caller specified M_NOWAIT.
824 */
825static uma_slab_t
826keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait)
827{
828 uma_slabrefcnt_t slabref;
829 uma_alloc allocf;
830 uma_slab_t slab;
831 u_int8_t *mem;
832 u_int8_t flags;
831 uint8_t *mem;
832 uint8_t flags;
833 int i;
834
835 mtx_assert(&keg->uk_lock, MA_OWNED);
836 slab = NULL;
837
838#ifdef UMA_DEBUG
839 printf("slab_zalloc: Allocating a new slab for %s\n", keg->uk_name);
840#endif

--- 104 unchanged lines hidden (view full) ---

945}
946
947/*
948 * This function is intended to be used early on in place of page_alloc() so
949 * that we may use the boot time page cache to satisfy allocations before
950 * the VM is ready.
951 */
952static void *
833 int i;
834
835 mtx_assert(&keg->uk_lock, MA_OWNED);
836 slab = NULL;
837
838#ifdef UMA_DEBUG
839 printf("slab_zalloc: Allocating a new slab for %s\n", keg->uk_name);
840#endif

--- 104 unchanged lines hidden (view full) ---

945}
946
947/*
948 * This function is intended to be used early on in place of page_alloc() so
949 * that we may use the boot time page cache to satisfy allocations before
950 * the VM is ready.
951 */
952static void *
953startup_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait)
953startup_alloc(uma_zone_t zone, int bytes, uint8_t *pflag, int wait)
954{
955 uma_keg_t keg;
956 uma_slab_t tmps;
957 int pages, check_pages;
958
959 keg = zone_first_keg(zone);
960 pages = howmany(bytes, PAGE_SIZE);
961 check_pages = pages - 1;

--- 43 unchanged lines hidden (view full) ---

1005 * bytes The number of bytes requested
1006 * wait Shall we wait?
1007 *
1008 * Returns:
1009 * A pointer to the alloced memory or possibly
1010 * NULL if M_NOWAIT is set.
1011 */
1012static void *
954{
955 uma_keg_t keg;
956 uma_slab_t tmps;
957 int pages, check_pages;
958
959 keg = zone_first_keg(zone);
960 pages = howmany(bytes, PAGE_SIZE);
961 check_pages = pages - 1;

--- 43 unchanged lines hidden (view full) ---

1005 * bytes The number of bytes requested
1006 * wait Shall we wait?
1007 *
1008 * Returns:
1009 * A pointer to the alloced memory or possibly
1010 * NULL if M_NOWAIT is set.
1011 */
1012static void *
1013page_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait)
1013page_alloc(uma_zone_t zone, int bytes, uint8_t *pflag, int wait)
1014{
1015 void *p; /* Returned page */
1016
1017 *pflag = UMA_SLAB_KMEM;
1018 p = (void *) kmem_malloc(kmem_map, bytes, wait);
1019
1020 return (p);
1021}

--- 5 unchanged lines hidden (view full) ---

1027 * bytes The number of bytes requested
1028 * wait Shall we wait?
1029 *
1030 * Returns:
1031 * A pointer to the alloced memory or possibly
1032 * NULL if M_NOWAIT is set.
1033 */
1034static void *
1014{
1015 void *p; /* Returned page */
1016
1017 *pflag = UMA_SLAB_KMEM;
1018 p = (void *) kmem_malloc(kmem_map, bytes, wait);
1019
1020 return (p);
1021}

--- 5 unchanged lines hidden (view full) ---

1027 * bytes The number of bytes requested
1028 * wait Shall we wait?
1029 *
1030 * Returns:
1031 * A pointer to the alloced memory or possibly
1032 * NULL if M_NOWAIT is set.
1033 */
1034static void *
1035noobj_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
1035noobj_alloc(uma_zone_t zone, int bytes, uint8_t *flags, int wait)
1036{
1037 TAILQ_HEAD(, vm_page) alloctail;
1038 u_long npages;
1039 vm_offset_t retkva, zkva;
1040 vm_page_t p, p_next;
1041 uma_keg_t keg;
1042
1043 TAILQ_INIT(&alloctail);

--- 46 unchanged lines hidden (view full) ---

1090 * mem A pointer to the memory to be freed
1091 * size The size of the memory being freed
1092 * flags The original p->us_flags field
1093 *
1094 * Returns:
1095 * Nothing
1096 */
1097static void
1036{
1037 TAILQ_HEAD(, vm_page) alloctail;
1038 u_long npages;
1039 vm_offset_t retkva, zkva;
1040 vm_page_t p, p_next;
1041 uma_keg_t keg;
1042
1043 TAILQ_INIT(&alloctail);

--- 46 unchanged lines hidden (view full) ---

1090 * mem A pointer to the memory to be freed
1091 * size The size of the memory being freed
1092 * flags The original p->us_flags field
1093 *
1094 * Returns:
1095 * Nothing
1096 */
1097static void
1098page_free(void *mem, int size, u_int8_t flags)
1098page_free(void *mem, int size, uint8_t flags)
1099{
1100 vm_map_t map;
1101
1102 if (flags & UMA_SLAB_KMEM)
1103 map = kmem_map;
1104 else if (flags & UMA_SLAB_KERNEL)
1105 map = kernel_map;
1106 else

--- 640 unchanged lines hidden (view full) ---

1747 args.flags = UMA_ZFLAG_INTERNAL;
1748 /* The initial zone has no Per cpu queues so it's smaller */
1749 zone_ctor(kegs, sizeof(struct uma_zone), &args, M_WAITOK);
1750
1751#ifdef UMA_DEBUG
1752 printf("Filling boot free list.\n");
1753#endif
1754 for (i = 0; i < boot_pages; i++) {
1099{
1100 vm_map_t map;
1101
1102 if (flags & UMA_SLAB_KMEM)
1103 map = kmem_map;
1104 else if (flags & UMA_SLAB_KERNEL)
1105 map = kernel_map;
1106 else

--- 640 unchanged lines hidden (view full) ---

1747 args.flags = UMA_ZFLAG_INTERNAL;
1748 /* The initial zone has no Per cpu queues so it's smaller */
1749 zone_ctor(kegs, sizeof(struct uma_zone), &args, M_WAITOK);
1750
1751#ifdef UMA_DEBUG
1752 printf("Filling boot free list.\n");
1753#endif
1754 for (i = 0; i < boot_pages; i++) {
1755 slab = (uma_slab_t)((u_int8_t *)bootmem + (i * UMA_SLAB_SIZE));
1756 slab->us_data = (u_int8_t *)slab;
1755 slab = (uma_slab_t)((uint8_t *)bootmem + (i * UMA_SLAB_SIZE));
1756 slab->us_data = (uint8_t *)slab;
1757 slab->us_flags = UMA_SLAB_BOOT;
1758 LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link);
1759 }
1760 mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF);
1761
1762#ifdef UMA_DEBUG
1763 printf("Creating uma zone headers zone and keg.\n");
1764#endif

--- 82 unchanged lines hidden (view full) ---

1847 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
1848#ifdef UMA_DEBUG
1849 printf("UMA startup3 complete.\n");
1850#endif
1851}
1852
1853static uma_keg_t
1854uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini,
1757 slab->us_flags = UMA_SLAB_BOOT;
1758 LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link);
1759 }
1760 mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF);
1761
1762#ifdef UMA_DEBUG
1763 printf("Creating uma zone headers zone and keg.\n");
1764#endif

--- 82 unchanged lines hidden (view full) ---

1847 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
1848#ifdef UMA_DEBUG
1849 printf("UMA startup3 complete.\n");
1850#endif
1851}
1852
1853static uma_keg_t
1854uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini,
1855 int align, u_int32_t flags)
1855 int align, uint32_t flags)
1856{
1857 struct uma_kctor_args args;
1858
1859 args.size = size;
1860 args.uminit = uminit;
1861 args.fini = fini;
1862 args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align;
1863 args.flags = flags;

--- 8 unchanged lines hidden (view full) ---

1872
1873 if (align != UMA_ALIGN_CACHE)
1874 uma_align_cache = align;
1875}
1876
1877/* See uma.h */
1878uma_zone_t
1879uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
1856{
1857 struct uma_kctor_args args;
1858
1859 args.size = size;
1860 args.uminit = uminit;
1861 args.fini = fini;
1862 args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align;
1863 args.flags = flags;

--- 8 unchanged lines hidden (view full) ---

1872
1873 if (align != UMA_ALIGN_CACHE)
1874 uma_align_cache = align;
1875}
1876
1877/* See uma.h */
1878uma_zone_t
1879uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
1880 uma_init uminit, uma_fini fini, int align, u_int32_t flags)
1880 uma_init uminit, uma_fini fini, int align, uint32_t flags)
1881
1882{
1883 struct uma_zctor_args args;
1884
1885 /* This stuff is essential for the zone ctor */
1886 args.name = name;
1887 args.size = size;
1888 args.ctor = ctor;

--- 510 unchanged lines hidden (view full) ---

2399}
2400
2401static void *
2402slab_alloc_item(uma_zone_t zone, uma_slab_t slab)
2403{
2404 uma_keg_t keg;
2405 uma_slabrefcnt_t slabref;
2406 void *item;
1881
1882{
1883 struct uma_zctor_args args;
1884
1885 /* This stuff is essential for the zone ctor */
1886 args.name = name;
1887 args.size = size;
1888 args.ctor = ctor;

--- 510 unchanged lines hidden (view full) ---

2399}
2400
2401static void *
2402slab_alloc_item(uma_zone_t zone, uma_slab_t slab)
2403{
2404 uma_keg_t keg;
2405 uma_slabrefcnt_t slabref;
2406 void *item;
2407 u_int8_t freei;
2407 uint8_t freei;
2408
2409 keg = slab->us_keg;
2410 mtx_assert(&keg->uk_lock, MA_OWNED);
2411
2412 freei = slab->us_firstfree;
2413 if (keg->uk_flags & UMA_ZONE_REFCNT) {
2414 slabref = (uma_slabrefcnt_t)slab;
2415 slab->us_firstfree = slabref->us_freelist[freei].us_item;

--- 387 unchanged lines hidden (view full) ---

2803 */
2804static void
2805zone_free_item(uma_zone_t zone, void *item, void *udata,
2806 enum zfreeskip skip, int flags)
2807{
2808 uma_slab_t slab;
2809 uma_slabrefcnt_t slabref;
2810 uma_keg_t keg;
2408
2409 keg = slab->us_keg;
2410 mtx_assert(&keg->uk_lock, MA_OWNED);
2411
2412 freei = slab->us_firstfree;
2413 if (keg->uk_flags & UMA_ZONE_REFCNT) {
2414 slabref = (uma_slabrefcnt_t)slab;
2415 slab->us_firstfree = slabref->us_freelist[freei].us_item;

--- 387 unchanged lines hidden (view full) ---

2803 */
2804static void
2805zone_free_item(uma_zone_t zone, void *item, void *udata,
2806 enum zfreeskip skip, int flags)
2807{
2808 uma_slab_t slab;
2809 uma_slabrefcnt_t slabref;
2810 uma_keg_t keg;
2811 u_int8_t *mem;
2812 u_int8_t freei;
2811 uint8_t *mem;
2812 uint8_t freei;
2813 int clearfull;
2814
2815 if (skip < SKIP_DTOR && zone->uz_dtor)
2816 zone->uz_dtor(item, zone->uz_size, udata);
2817
2818 if (skip < SKIP_FINI && zone->uz_fini)
2819 zone->uz_fini(item, zone->uz_size);
2820
2821 ZONE_LOCK(zone);
2822
2823 if (flags & ZFREE_STATFAIL)
2824 zone->uz_fails++;
2825 if (flags & ZFREE_STATFREE)
2826 zone->uz_frees++;
2827
2828 if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
2813 int clearfull;
2814
2815 if (skip < SKIP_DTOR && zone->uz_dtor)
2816 zone->uz_dtor(item, zone->uz_size, udata);
2817
2818 if (skip < SKIP_FINI && zone->uz_fini)
2819 zone->uz_fini(item, zone->uz_size);
2820
2821 ZONE_LOCK(zone);
2822
2823 if (flags & ZFREE_STATFAIL)
2824 zone->uz_fails++;
2825 if (flags & ZFREE_STATFREE)
2826 zone->uz_frees++;
2827
2828 if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
2829 mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK));
2829 mem = (uint8_t *)((unsigned long)item & (~UMA_SLAB_MASK));
2830 keg = zone_first_keg(zone); /* Must only be one. */
2831 if (zone->uz_flags & UMA_ZONE_HASH) {
2832 slab = hash_sfind(&keg->uk_hash, mem);
2833 } else {
2834 mem += keg->uk_pgoff;
2835 slab = (uma_slab_t)mem;
2836 }
2837 } else {

--- 259 unchanged lines hidden (view full) ---

3097 MPASS(slab->us_keg == keg);
3098 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
3099 slabs--;
3100 }
3101 ZONE_UNLOCK(zone);
3102}
3103
3104/* See uma.h */
2830 keg = zone_first_keg(zone); /* Must only be one. */
2831 if (zone->uz_flags & UMA_ZONE_HASH) {
2832 slab = hash_sfind(&keg->uk_hash, mem);
2833 } else {
2834 mem += keg->uk_pgoff;
2835 slab = (uma_slab_t)mem;
2836 }
2837 } else {

--- 259 unchanged lines hidden (view full) ---

3097 MPASS(slab->us_keg == keg);
3098 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
3099 slabs--;
3100 }
3101 ZONE_UNLOCK(zone);
3102}
3103
3104/* See uma.h */
3105u_int32_t *
3105uint32_t *
3106uma_find_refcnt(uma_zone_t zone, void *item)
3107{
3108 uma_slabrefcnt_t slabref;
3109 uma_keg_t keg;
3106uma_find_refcnt(uma_zone_t zone, void *item)
3107{
3108 uma_slabrefcnt_t slabref;
3109 uma_keg_t keg;
3110 u_int32_t *refcnt;
3110 uint32_t *refcnt;
3111 int idx;
3112
3113 slabref = (uma_slabrefcnt_t)vtoslab((vm_offset_t)item &
3114 (~UMA_SLAB_MASK));
3115 keg = slabref->us_keg;
3116 KASSERT(slabref != NULL && slabref->us_keg->uk_flags & UMA_ZONE_REFCNT,
3117 ("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT"));
3118 idx = ((unsigned long)item - (unsigned long)slabref->us_data)

--- 39 unchanged lines hidden (view full) ---

3158 return (zone->uz_flags & UMA_ZFLAG_FULL);
3159}
3160
3161void *
3162uma_large_malloc(int size, int wait)
3163{
3164 void *mem;
3165 uma_slab_t slab;
3111 int idx;
3112
3113 slabref = (uma_slabrefcnt_t)vtoslab((vm_offset_t)item &
3114 (~UMA_SLAB_MASK));
3115 keg = slabref->us_keg;
3116 KASSERT(slabref != NULL && slabref->us_keg->uk_flags & UMA_ZONE_REFCNT,
3117 ("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT"));
3118 idx = ((unsigned long)item - (unsigned long)slabref->us_data)

--- 39 unchanged lines hidden (view full) ---

3158 return (zone->uz_flags & UMA_ZFLAG_FULL);
3159}
3160
3161void *
3162uma_large_malloc(int size, int wait)
3163{
3164 void *mem;
3165 uma_slab_t slab;
3166 u_int8_t flags;
3166 uint8_t flags;
3167
3168 slab = zone_alloc_item(slabzone, NULL, wait);
3169 if (slab == NULL)
3170 return (NULL);
3171 mem = page_alloc(NULL, size, &flags, wait);
3172 if (mem) {
3173 vsetslab((vm_offset_t)mem, slab);
3174 slab->us_data = mem;

--- 87 unchanged lines hidden (view full) ---

3262 * Note: does not update the zone statistics, as it can't safely clear the
3263 * per-CPU cache statistic.
3264 *
3265 * XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't
3266 * safe from off-CPU; we should modify the caches to track this information
3267 * directly so that we don't have to.
3268 */
3269static void
3167
3168 slab = zone_alloc_item(slabzone, NULL, wait);
3169 if (slab == NULL)
3170 return (NULL);
3171 mem = page_alloc(NULL, size, &flags, wait);
3172 if (mem) {
3173 vsetslab((vm_offset_t)mem, slab);
3174 slab->us_data = mem;

--- 87 unchanged lines hidden (view full) ---

3262 * Note: does not update the zone statistics, as it can't safely clear the
3263 * per-CPU cache statistic.
3264 *
3265 * XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't
3266 * safe from off-CPU; we should modify the caches to track this information
3267 * directly so that we don't have to.
3268 */
3269static void
3270uma_zone_sumstat(uma_zone_t z, int *cachefreep, u_int64_t *allocsp,
3271 u_int64_t *freesp, u_int64_t *sleepsp)
3270uma_zone_sumstat(uma_zone_t z, int *cachefreep, uint64_t *allocsp,
3271 uint64_t *freesp, uint64_t *sleepsp)
3272{
3273 uma_cache_t cache;
3272{
3273 uma_cache_t cache;
3274 u_int64_t allocs, frees, sleeps;
3274 uint64_t allocs, frees, sleeps;
3275 int cachefree, cpu;
3276
3277 allocs = frees = sleeps = 0;
3278 cachefree = 0;
3279 CPU_FOREACH(cpu) {
3280 cache = &z->uz_cpu[cpu];
3281 if (cache->uc_allocbucket != NULL)
3282 cachefree += cache->uc_allocbucket->ub_cnt;

--- 134 unchanged lines hidden (view full) ---

3417 error = sbuf_finish(&sbuf);
3418 sbuf_delete(&sbuf);
3419 return (error);
3420}
3421
3422#ifdef DDB
3423DB_SHOW_COMMAND(uma, db_show_uma)
3424{
3275 int cachefree, cpu;
3276
3277 allocs = frees = sleeps = 0;
3278 cachefree = 0;
3279 CPU_FOREACH(cpu) {
3280 cache = &z->uz_cpu[cpu];
3281 if (cache->uc_allocbucket != NULL)
3282 cachefree += cache->uc_allocbucket->ub_cnt;

--- 134 unchanged lines hidden (view full) ---

3417 error = sbuf_finish(&sbuf);
3418 sbuf_delete(&sbuf);
3419 return (error);
3420}
3421
3422#ifdef DDB
3423DB_SHOW_COMMAND(uma, db_show_uma)
3424{
3425 u_int64_t allocs, frees, sleeps;
3425 uint64_t allocs, frees, sleeps;
3426 uma_bucket_t bucket;
3427 uma_keg_t kz;
3428 uma_zone_t z;
3429 int cachefree;
3430
3431 db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free",
3432 "Requests", "Sleeps");
3433 LIST_FOREACH(kz, &uma_kegs, uk_link) {

--- 24 unchanged lines hidden --- 		3426 uma_bucket_t bucket;
3427 uma_keg_t kz;
3428 uma_zone_t z;
3429 int cachefree;
3430
3431 db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free",
3432 "Requests", "Sleeps");
3433 LIST_FOREACH(kz, &uma_kegs, uk_link) {

--- 24 unchanged lines hidden ---