Deleted Added
sdiff udiff text old ( 322042 ) new ( 322345 )
full compact
__vdso_gettc.c (322042) __vdso_gettc.c (322345)
1/*-
2 * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
3 * Copyright (c) 2016, 2017 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * Portions of this software were developed by Konstantin Belousov
7 * under sponsorship from the FreeBSD Foundation.
8 *

--- 15 unchanged lines hidden (view full) ---

24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
3 * Copyright (c) 2016, 2017 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * Portions of this software were developed by Konstantin Belousov
7 * under sponsorship from the FreeBSD Foundation.
8 *

--- 15 unchanged lines hidden (view full) ---

24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD: stable/11/lib/libc/x86/sys/__vdso_gettc.c 322042 2017-08-04 08:12:19Z kib $");
32__FBSDID("$FreeBSD: stable/11/lib/libc/x86/sys/__vdso_gettc.c 322345 2017-08-10 09:00:15Z kib $");
33
34#include <sys/param.h>
35#include "namespace.h"
36#include <sys/capsicum.h>
37#include <sys/elf.h>
38#include <sys/fcntl.h>
39#include <sys/mman.h>
40#include <sys/time.h>

--- 6 unchanged lines hidden (view full) ---

47#include <machine/cpufunc.h>
48#include <machine/specialreg.h>
49#include <dev/acpica/acpi_hpet.h>
50#ifdef WANT_HYPERV
51#include <dev/hyperv/hyperv.h>
52#endif
53#include "libc_private.h"
54
33
34#include <sys/param.h>
35#include "namespace.h"
36#include <sys/capsicum.h>
37#include <sys/elf.h>
38#include <sys/fcntl.h>
39#include <sys/mman.h>
40#include <sys/time.h>

--- 6 unchanged lines hidden (view full) ---

47#include <machine/cpufunc.h>
48#include <machine/specialreg.h>
49#include <dev/acpica/acpi_hpet.h>
50#ifdef WANT_HYPERV
51#include <dev/hyperv/hyperv.h>
52#endif
53#include "libc_private.h"
54
55static enum LMB {
56 LMB_UNKNOWN,
57 LMB_NONE,
58 LMB_MFENCE,
59 LMB_LFENCE
60} lfence_works = LMB_UNKNOWN;
61
55static void
62static void
56lfence_mb(void)
63cpuidp(u_int leaf, u_int p[4])
57{
64{
65
66 __asm __volatile(
58#if defined(__i386__)
67#if defined(__i386__)
59 static int lfence_works = -1;
68 " pushl %%ebx\n"
69#endif
70 " cpuid\n"
71#if defined(__i386__)
72 " movl %%ebx,%1\n"
73 " popl %%ebx"
74#endif
75 : "=a" (p[0]),
76#if defined(__i386__)
77 "=r" (p[1]),
78#elif defined(__amd64__)
79 "=b" (p[1]),
80#else
81#error "Arch"
82#endif
83 "=c" (p[2]), "=d" (p[3])
84 : "0" (leaf));
85}
86
87static enum LMB
88select_lmb(void)
89{
90 u_int p[4];
91 static const char intel_id[] = "GenuntelineI";
92
93 cpuidp(0, p);
94 return (memcmp(p + 1, intel_id, sizeof(intel_id) - 1) == 0 ?
95 LMB_LFENCE : LMB_MFENCE);
96}
97
98static void
99init_fence(void)
100{
101#if defined(__i386__)
60 u_int cpuid_supported, p[4];
61
102 u_int cpuid_supported, p[4];
103
62 if (lfence_works == -1) {
63 __asm __volatile(
64 " pushfl\n"
65 " popl %%eax\n"
66 " movl %%eax,%%ecx\n"
67 " xorl $0x200000,%%eax\n"
68 " pushl %%eax\n"
69 " popfl\n"
70 " pushfl\n"
71 " popl %%eax\n"
72 " xorl %%eax,%%ecx\n"
73 " je 1f\n"
74 " movl $1,%0\n"
75 " jmp 2f\n"
76 "1: movl $0,%0\n"
77 "2:\n"
78 : "=r" (cpuid_supported) : : "eax", "ecx", "cc");
79 if (cpuid_supported) {
80 __asm __volatile(
81 " pushl %%ebx\n"
82 " cpuid\n"
83 " movl %%ebx,%1\n"
84 " popl %%ebx\n"
85 : "=a" (p[0]), "=r" (p[1]), "=c" (p[2]), "=d" (p[3])
86 : "0" (0x1));
87 lfence_works = (p[3] & CPUID_SSE2) != 0;
88 } else
89 lfence_works = 0;
90 }
91 if (lfence_works == 1)
92 lfence();
104 __asm __volatile(
105 " pushfl\n"
106 " popl %%eax\n"
107 " movl %%eax,%%ecx\n"
108 " xorl $0x200000,%%eax\n"
109 " pushl %%eax\n"
110 " popfl\n"
111 " pushfl\n"
112 " popl %%eax\n"
113 " xorl %%eax,%%ecx\n"
114 " je 1f\n"
115 " movl $1,%0\n"
116 " jmp 2f\n"
117 "1: movl $0,%0\n"
118 "2:\n"
119 : "=r" (cpuid_supported) : : "eax", "ecx", "cc");
120 if (cpuid_supported) {
121 cpuidp(0x1, p);
122 if ((p[3] & CPUID_SSE2) != 0)
123 lfence_works = select_lmb();
124 } else
125 lfence_works = LMB_NONE;
93#elif defined(__amd64__)
126#elif defined(__amd64__)
94 lfence();
127 lfence_works = select_lmb();
95#else
128#else
96#error "arch"
129#error "Arch"
97#endif
98}
99
130#endif
131}
132
133static void
134rdtsc_mb(void)
135{
136
137again:
138 if (__predict_true(lfence_works == LMB_LFENCE)) {
139 lfence();
140 return;
141 } else if (lfence_works == LMB_MFENCE) {
142 mfence();
143 return;
144 } else if (lfence_works == LMB_NONE) {
145 return;
146 }
147 init_fence();
148 goto again;
149}
150
100static u_int
101__vdso_gettc_rdtsc_low(const struct vdso_timehands *th)
102{
103 u_int rv;
104
151static u_int
152__vdso_gettc_rdtsc_low(const struct vdso_timehands *th)
153{
154 u_int rv;
155
105 lfence_mb();
156 rdtsc_mb();
106 __asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
107 : "=a" (rv) : "c" (th->th_x86_shift) : "edx");
108 return (rv);
109}
110
111static u_int
112__vdso_rdtsc32(void)
113{
114
157 __asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
158 : "=a" (rv) : "c" (th->th_x86_shift) : "edx");
159 return (rv);
160}
161
162static u_int
163__vdso_rdtsc32(void)
164{
165
115 lfence_mb();
166 rdtsc_mb();
116 return (rdtsc32());
117}
118
119#define HPET_DEV_MAP_MAX 10
120static volatile char *hpet_dev_map[HPET_DEV_MAP_MAX];
121
122static void
123__vdso_init_hpet(uint32_t u)

--- 82 unchanged lines hidden (view full) ---

206 uint64_t disc, ret, tsc, scale;
207 uint32_t seq;
208 int64_t ofs;
209
210 while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) {
211 scale = tsc_ref->tsc_scale;
212 ofs = tsc_ref->tsc_ofs;
213
167 return (rdtsc32());
168}
169
170#define HPET_DEV_MAP_MAX 10
171static volatile char *hpet_dev_map[HPET_DEV_MAP_MAX];
172
173static void
174__vdso_init_hpet(uint32_t u)

--- 82 unchanged lines hidden (view full) ---

257 uint64_t disc, ret, tsc, scale;
258 uint32_t seq;
259 int64_t ofs;
260
261 while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) {
262 scale = tsc_ref->tsc_scale;
263 ofs = tsc_ref->tsc_ofs;
264
214 lfence_mb();
265 rdtsc_mb();
215 tsc = rdtsc();
216
217 /* ret = ((tsc * scale) >> 64) + ofs */
218 __asm__ __volatile__ ("mulq %3" :
219 "=d" (ret), "=a" (disc) :
220 "a" (tsc), "r" (scale));
221 ret += ofs;
222

--- 60 unchanged lines hidden --- 		266 tsc = rdtsc();
267
268 /* ret = ((tsc * scale) >> 64) + ofs */
269 __asm__ __volatile__ ("mulq %3" :
270 "=d" (ret), "=a" (disc) :
271 "a" (tsc), "r" (scale));
272 ret += ofs;
273

--- 60 unchanged lines hidden ---