Deleted Added
sdiff udiff text old ( 303953 ) new ( 315339 )
full compact
subr_lock.c (303953) subr_lock.c (315339)
1/*-
2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 16 unchanged lines hidden (view full) ---

25 */
26
27/*
28 * This module holds the global variables and functions used to maintain
29 * lock_object structures.
30 */
31
32#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 16 unchanged lines hidden (view full) ---

25 */
26
27/*
28 * This module holds the global variables and functions used to maintain
29 * lock_object structures.
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD: stable/11/sys/kern/subr_lock.c 303953 2016-08-11 09:28:49Z mjg $");
33__FBSDID("$FreeBSD: stable/11/sys/kern/subr_lock.c 315339 2017-03-16 00:51:24Z mjg $");
34
35#include "opt_ddb.h"
36#include "opt_mprof.h"
37
38#include <sys/param.h>
39#include <sys/systm.h>
40#include <sys/kernel.h>
41#include <sys/ktr.h>

--- 9 unchanged lines hidden (view full) ---

51#include <sys/sysctl.h>
52
53#ifdef DDB
54#include <ddb/ddb.h>
55#endif
56
57#include <machine/cpufunc.h>
58
34
35#include "opt_ddb.h"
36#include "opt_mprof.h"
37
38#include <sys/param.h>
39#include <sys/systm.h>
40#include <sys/kernel.h>
41#include <sys/ktr.h>

--- 9 unchanged lines hidden (view full) ---

51#include <sys/sysctl.h>
52
53#ifdef DDB
54#include <ddb/ddb.h>
55#endif
56
57#include <machine/cpufunc.h>
58
59SDT_PROVIDER_DEFINE(lock);
60SDT_PROBE_DEFINE1(lock, , , starvation, "u_int");
61
59CTASSERT(LOCK_CLASS_MAX == 15);
60
61struct lock_class *lock_classes[LOCK_CLASS_MAX + 1] = {
62 &lock_class_mtx_spin,
63 &lock_class_mtx_sleep,
64 &lock_class_sx,
65 &lock_class_rm,
66 &lock_class_rm_sleepable,

--- 31 unchanged lines hidden (view full) ---

98{
99
100 KASSERT(lock_initialized(lock), ("lock %p is not initialized", lock));
101 WITNESS_DESTROY(lock);
102 LOCK_LOG_DESTROY(lock, 0);
103 lock->lo_flags &= ~LO_INITIALIZED;
104}
105
62CTASSERT(LOCK_CLASS_MAX == 15);
63
64struct lock_class *lock_classes[LOCK_CLASS_MAX + 1] = {
65 &lock_class_mtx_spin,
66 &lock_class_mtx_sleep,
67 &lock_class_sx,
68 &lock_class_rm,
69 &lock_class_rm_sleepable,

--- 31 unchanged lines hidden (view full) ---

101{
102
103 KASSERT(lock_initialized(lock), ("lock %p is not initialized", lock));
104 WITNESS_DESTROY(lock);
105 LOCK_LOG_DESTROY(lock, 0);
106 lock->lo_flags &= ~LO_INITIALIZED;
107}
108
109static SYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD, NULL, "lock debugging");
110static SYSCTL_NODE(_debug_lock, OID_AUTO, delay, CTLFLAG_RD, NULL,
111 "lock delay");
112
113static u_int __read_mostly starvation_limit = 131072;
114SYSCTL_INT(_debug_lock_delay, OID_AUTO, starvation_limit, CTLFLAG_RW,
115 &starvation_limit, 0, "");
116
117static u_int __read_mostly restrict_starvation = 0;
118SYSCTL_INT(_debug_lock_delay, OID_AUTO, restrict_starvation, CTLFLAG_RW,
119 &restrict_starvation, 0, "");
120
106void
107lock_delay(struct lock_delay_arg *la)
108{
121void
122lock_delay(struct lock_delay_arg *la)
123{
109 u_int i, delay, backoff, min, max;
110 struct lock_delay_config *lc = la->config;
124 struct lock_delay_config *lc = la->config;
125 u_int i;
111
126
112 delay = la->delay;
127 la->delay <<= 1;
128 if (__predict_false(la->delay > lc->max))
129 la->delay = lc->max;
113
130
114 if (delay == 0)
115 delay = lc->initial;
116 else {
117 delay += lc->step;
118 max = lc->max;
119 if (delay > max)
120 delay = max;
131 for (i = la->delay; i > 0; i--)
132 cpu_spinwait();
133
134 la->spin_cnt += la->delay;
135 if (__predict_false(la->spin_cnt > starvation_limit)) {
136 SDT_PROBE1(lock, , , starvation, la->delay);
137 if (restrict_starvation)
138 la->delay = lc->base;
121 }
139 }
140}
122
141
123 backoff = cpu_ticks() % delay;
124 min = lc->min;
125 if (backoff < min)
126 backoff = min;
127 for (i = 0; i < backoff; i++)
128 cpu_spinwait();
142static u_int
143lock_roundup_2(u_int val)
144{
145 u_int res;
129
146
130 la->delay = delay;
131 la->spin_cnt += backoff;
147 for (res = 1; res <= val; res <<= 1)
148 continue;
149
150 return (res);
132}
133
151}
152
153void
154lock_delay_default_init(struct lock_delay_config *lc)
155{
156
157 lc->base = lock_roundup_2(mp_ncpus) / 4;
158 lc->max = lc->base * 1024;
159}
160
134#ifdef DDB
135DB_SHOW_COMMAND(lock, db_show_lock)
136{
137 struct lock_object *lock;
138 struct lock_class *class;
139
140 if (!have_addr)
141 return;

--- 66 unchanged lines hidden (view full) ---

208};
209
210struct lock_prof_cpu {
211 struct lock_prof_type lpc_types[2]; /* One for spin one for other. */
212};
213
214struct lock_prof_cpu *lp_cpu[MAXCPU];
215
161#ifdef DDB
162DB_SHOW_COMMAND(lock, db_show_lock)
163{
164 struct lock_object *lock;
165 struct lock_class *class;
166
167 if (!have_addr)
168 return;

--- 66 unchanged lines hidden (view full) ---

235};
236
237struct lock_prof_cpu {
238 struct lock_prof_type lpc_types[2]; /* One for spin one for other. */
239};
240
241struct lock_prof_cpu *lp_cpu[MAXCPU];
242
216volatile int lock_prof_enable = 0;
243volatile int __read_mostly lock_prof_enable;
217static volatile int lock_prof_resetting;
218
219#define LPROF_SBUF_SIZE 256
220
221static int lock_prof_rejected;
222static int lock_prof_skipspin;
223static int lock_prof_skipcount;
224

--- 425 unchanged lines hidden (view full) ---

650release:
651 LIST_REMOVE(l, lpo_link);
652 type = &lp_cpu[PCPU_GET(cpuid)]->lpc_types[spin];
653 LIST_INSERT_HEAD(&type->lpt_lpoalloc, l, lpo_link);
654out:
655 critical_exit();
656}
657
244static volatile int lock_prof_resetting;
245
246#define LPROF_SBUF_SIZE 256
247
248static int lock_prof_rejected;
249static int lock_prof_skipspin;
250static int lock_prof_skipcount;
251

--- 425 unchanged lines hidden (view full) ---

677release:
678 LIST_REMOVE(l, lpo_link);
679 type = &lp_cpu[PCPU_GET(cpuid)]->lpc_types[spin];
680 LIST_INSERT_HEAD(&type->lpt_lpoalloc, l, lpo_link);
681out:
682 critical_exit();
683}
684
658static SYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD, NULL, "lock debugging");
659static SYSCTL_NODE(_debug_lock, OID_AUTO, prof, CTLFLAG_RD, NULL,
660 "lock profiling");
661SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipspin, CTLFLAG_RW,
662 &lock_prof_skipspin, 0, "Skip profiling on spinlocks.");
663SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipcount, CTLFLAG_RW,
664 &lock_prof_skipcount, 0, "Sample approximately every N lock acquisitions.");
665SYSCTL_INT(_debug_lock_prof, OID_AUTO, rejected, CTLFLAG_RD,
666 &lock_prof_rejected, 0, "Number of rejected profiling records");
667SYSCTL_PROC(_debug_lock_prof, OID_AUTO, stats, CTLTYPE_STRING | CTLFLAG_RD,
668 NULL, 0, dump_lock_prof_stats, "A", "Lock profiling statistics");
669SYSCTL_PROC(_debug_lock_prof, OID_AUTO, reset, CTLTYPE_INT | CTLFLAG_RW,
670 NULL, 0, reset_lock_prof_stats, "I", "Reset lock profiling statistics");
671SYSCTL_PROC(_debug_lock_prof, OID_AUTO, enable, CTLTYPE_INT | CTLFLAG_RW,
672 NULL, 0, enable_lock_prof, "I", "Enable lock profiling");
673
674#endif
685static SYSCTL_NODE(_debug_lock, OID_AUTO, prof, CTLFLAG_RD, NULL,
686 "lock profiling");
687SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipspin, CTLFLAG_RW,
688 &lock_prof_skipspin, 0, "Skip profiling on spinlocks.");
689SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipcount, CTLFLAG_RW,
690 &lock_prof_skipcount, 0, "Sample approximately every N lock acquisitions.");
691SYSCTL_INT(_debug_lock_prof, OID_AUTO, rejected, CTLFLAG_RD,
692 &lock_prof_rejected, 0, "Number of rejected profiling records");
693SYSCTL_PROC(_debug_lock_prof, OID_AUTO, stats, CTLTYPE_STRING | CTLFLAG_RD,
694 NULL, 0, dump_lock_prof_stats, "A", "Lock profiling statistics");
695SYSCTL_PROC(_debug_lock_prof, OID_AUTO, reset, CTLTYPE_INT | CTLFLAG_RW,
696 NULL, 0, reset_lock_prof_stats, "I", "Reset lock profiling statistics");
697SYSCTL_PROC(_debug_lock_prof, OID_AUTO, enable, CTLTYPE_INT | CTLFLAG_RW,
698 NULL, 0, enable_lock_prof, "I", "Enable lock profiling");
699
700#endif