1// SPDX-License-Identifier: GPL-2.0-or-later 2 3#define pr_fmt(fmt) "ref_tracker: " fmt 4 5#include <linux/export.h> 6#include <linux/list_sort.h> 7#include <linux/ref_tracker.h> 8#include <linux/slab.h> 9#include <linux/stacktrace.h> 10#include <linux/stackdepot.h> 11 12#define REF_TRACKER_STACK_ENTRIES 16 13#define STACK_BUF_SIZE 1024 14 15struct ref_tracker { 16 struct list_head head; /* anchor into dir->list or dir->quarantine */ 17 bool dead; 18 depot_stack_handle_t alloc_stack_handle; 19 depot_stack_handle_t free_stack_handle; 20}; 21 22struct ref_tracker_dir_stats { 23 int total; 24 int count; 25 struct { 26 depot_stack_handle_t stack_handle; 27 unsigned int count; 28 } stacks[]; 29}; 30 31static struct ref_tracker_dir_stats * 32ref_tracker_get_stats(struct ref_tracker_dir *dir, unsigned int limit) 33{ 34 struct ref_tracker_dir_stats *stats; 35 struct ref_tracker *tracker; 36 37 stats = kmalloc(struct_size(stats, stacks, limit), 38 GFP_NOWAIT | __GFP_NOWARN); 39 if (!stats) 40 return ERR_PTR(-ENOMEM); 41 stats->total = 0; 42 stats->count = 0; 43 44 list_for_each_entry(tracker, &dir->list, head) { 45 depot_stack_handle_t stack = tracker->alloc_stack_handle; 46 int i; 47 48 ++stats->total; 49 for (i = 0; i < stats->count; ++i) 50 if (stats->stacks[i].stack_handle == stack) 51 break; 52 if (i >= limit) 53 continue; 54 if (i >= stats->count) { 55 stats->stacks[i].stack_handle = stack; 56 stats->stacks[i].count = 0; 57 ++stats->count; 58 } 59 ++stats->stacks[i].count; 60 } 61 62 return stats; 63} 64 65struct ostream { 66 char *buf; 67 int size, used; 68}; 69 70#define pr_ostream(stream, fmt, args...) \ 71({ \ 72 struct ostream *_s = (stream); \ 73\ 74 if (!_s->buf) { \ 75 pr_err(fmt, ##args); \ 76 } else { \ 77 int ret, len = _s->size - _s->used; \ 78 ret = snprintf(_s->buf + _s->used, len, pr_fmt(fmt), ##args); \ 79 _s->used += min(ret, len); \ 80 } \ 81}) 82 83static void 84__ref_tracker_dir_pr_ostream(struct ref_tracker_dir *dir, 85 unsigned int display_limit, struct ostream *s) 86{ 87 struct ref_tracker_dir_stats *stats; 88 unsigned int i = 0, skipped; 89 depot_stack_handle_t stack; 90 char *sbuf; 91 92 lockdep_assert_held(&dir->lock); 93 94 if (list_empty(&dir->list)) 95 return; 96 97 stats = ref_tracker_get_stats(dir, display_limit); 98 if (IS_ERR(stats)) { 99 pr_ostream(s, "%s@%pK: couldn't get stats, error %pe\n", 100 dir->name, dir, stats); 101 return; 102 } 103 104 sbuf = kmalloc(STACK_BUF_SIZE, GFP_NOWAIT | __GFP_NOWARN); 105 106 for (i = 0, skipped = stats->total; i < stats->count; ++i) { 107 stack = stats->stacks[i].stack_handle; 108 if (sbuf && !stack_depot_snprint(stack, sbuf, STACK_BUF_SIZE, 4)) 109 sbuf[0] = 0; 110 pr_ostream(s, "%s@%pK has %d/%d users at\n%s\n", dir->name, dir, 111 stats->stacks[i].count, stats->total, sbuf); 112 skipped -= stats->stacks[i].count; 113 } 114 115 if (skipped) 116 pr_ostream(s, "%s@%pK skipped reports about %d/%d users.\n", 117 dir->name, dir, skipped, stats->total); 118 119 kfree(sbuf); 120 121 kfree(stats); 122} 123 124void ref_tracker_dir_print_locked(struct ref_tracker_dir *dir, 125 unsigned int display_limit) 126{ 127 struct ostream os = {}; 128 129 __ref_tracker_dir_pr_ostream(dir, display_limit, &os); 130} 131EXPORT_SYMBOL(ref_tracker_dir_print_locked); 132 133void ref_tracker_dir_print(struct ref_tracker_dir *dir, 134 unsigned int display_limit) 135{ 136 unsigned long flags; 137 138 spin_lock_irqsave(&dir->lock, flags); 139 ref_tracker_dir_print_locked(dir, display_limit); 140 spin_unlock_irqrestore(&dir->lock, flags); 141} 142EXPORT_SYMBOL(ref_tracker_dir_print); 143 144int ref_tracker_dir_snprint(struct ref_tracker_dir *dir, char *buf, size_t size) 145{ 146 struct ostream os = { .buf = buf, .size = size }; 147 unsigned long flags; 148 149 spin_lock_irqsave(&dir->lock, flags); 150 __ref_tracker_dir_pr_ostream(dir, 16, &os); 151 spin_unlock_irqrestore(&dir->lock, flags); 152 153 return os.used; 154} 155EXPORT_SYMBOL(ref_tracker_dir_snprint); 156 157void ref_tracker_dir_exit(struct ref_tracker_dir *dir) 158{ 159 struct ref_tracker *tracker, *n; 160 unsigned long flags; 161 bool leak = false; 162 163 dir->dead = true; 164 spin_lock_irqsave(&dir->lock, flags); 165 list_for_each_entry_safe(tracker, n, &dir->quarantine, head) { 166 list_del(&tracker->head); 167 kfree(tracker); 168 dir->quarantine_avail++; 169 } 170 if (!list_empty(&dir->list)) { 171 ref_tracker_dir_print_locked(dir, 16); 172 leak = true; 173 list_for_each_entry_safe(tracker, n, &dir->list, head) { 174 list_del(&tracker->head); 175 kfree(tracker); 176 } 177 } 178 spin_unlock_irqrestore(&dir->lock, flags); 179 WARN_ON_ONCE(leak); 180 WARN_ON_ONCE(refcount_read(&dir->untracked) != 1); 181 WARN_ON_ONCE(refcount_read(&dir->no_tracker) != 1); 182} 183EXPORT_SYMBOL(ref_tracker_dir_exit); 184 185int ref_tracker_alloc(struct ref_tracker_dir *dir, 186 struct ref_tracker **trackerp, 187 gfp_t gfp) 188{ 189 unsigned long entries[REF_TRACKER_STACK_ENTRIES]; 190 struct ref_tracker *tracker; 191 unsigned int nr_entries; 192 gfp_t gfp_mask = gfp | __GFP_NOWARN; 193 unsigned long flags; 194 195 WARN_ON_ONCE(dir->dead); 196 197 if (!trackerp) { 198 refcount_inc(&dir->no_tracker); 199 return 0; 200 } 201 if (gfp & __GFP_DIRECT_RECLAIM) 202 gfp_mask |= __GFP_NOFAIL; 203 *trackerp = tracker = kzalloc(sizeof(*tracker), gfp_mask); 204 if (unlikely(!tracker)) { 205 pr_err_once("memory allocation failure, unreliable refcount tracker.\n"); 206 refcount_inc(&dir->untracked); 207 return -ENOMEM; 208 } 209 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 210 tracker->alloc_stack_handle = stack_depot_save(entries, nr_entries, gfp); 211 212 spin_lock_irqsave(&dir->lock, flags); 213 list_add(&tracker->head, &dir->list); 214 spin_unlock_irqrestore(&dir->lock, flags); 215 return 0; 216} 217EXPORT_SYMBOL_GPL(ref_tracker_alloc); 218 219int ref_tracker_free(struct ref_tracker_dir *dir, 220 struct ref_tracker **trackerp) 221{ 222 unsigned long entries[REF_TRACKER_STACK_ENTRIES]; 223 depot_stack_handle_t stack_handle; 224 struct ref_tracker *tracker; 225 unsigned int nr_entries; 226 unsigned long flags; 227 228 WARN_ON_ONCE(dir->dead); 229 230 if (!trackerp) { 231 refcount_dec(&dir->no_tracker); 232 return 0; 233 } 234 tracker = *trackerp; 235 if (!tracker) { 236 refcount_dec(&dir->untracked); 237 return -EEXIST; 238 } 239 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 240 stack_handle = stack_depot_save(entries, nr_entries, 241 GFP_NOWAIT | __GFP_NOWARN); 242 243 spin_lock_irqsave(&dir->lock, flags); 244 if (tracker->dead) { 245 pr_err("reference already released.\n"); 246 if (tracker->alloc_stack_handle) { 247 pr_err("allocated in:\n"); 248 stack_depot_print(tracker->alloc_stack_handle); 249 } 250 if (tracker->free_stack_handle) { 251 pr_err("freed in:\n"); 252 stack_depot_print(tracker->free_stack_handle); 253 } 254 spin_unlock_irqrestore(&dir->lock, flags); 255 WARN_ON_ONCE(1); 256 return -EINVAL; 257 } 258 tracker->dead = true; 259 260 tracker->free_stack_handle = stack_handle; 261 262 list_move_tail(&tracker->head, &dir->quarantine); 263 if (!dir->quarantine_avail) { 264 tracker = list_first_entry(&dir->quarantine, struct ref_tracker, head); 265 list_del(&tracker->head); 266 } else { 267 dir->quarantine_avail--; 268 tracker = NULL; 269 } 270 spin_unlock_irqrestore(&dir->lock, flags); 271 272 kfree(tracker); 273 return 0; 274} 275EXPORT_SYMBOL_GPL(ref_tracker_free); 276