1/* 2 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE 3 * Copyright 2003 Andi Kleen, SuSE Labs. 4 * 5 * Thanks to hpa@transmeta.com for some useful hint. 6 * Special thanks to Ingo Molnar for his early experience with 7 * a different vsyscall implementation for Linux/IA32 and for the name. 8 * 9 * vsyscall 1 is located at -10Mbyte, vsyscall 2 is located 10 * at virtual address -10Mbyte+1024bytes etc... There are at max 4 11 * vsyscalls. One vsyscall can reserve more than 1 slot to avoid 12 * jumping out of line if necessary. We cannot add more with this 13 * mechanism because older kernels won't return -ENOSYS. 14 * If we want more than four we need a vDSO. 15 * 16 * Note: the concept clashes with user mode linux. If you use UML and 17 * want per guest time just set the kernel.vsyscall64 sysctl to 0. 18 */ 19 20/* Disable profiling for userspace code: */ 21#define DISABLE_BRANCH_PROFILING 22 23#include <linux/time.h> 24#include <linux/init.h> 25#include <linux/kernel.h> 26#include <linux/timer.h> 27#include <linux/seqlock.h> 28#include <linux/jiffies.h> 29#include <linux/sysctl.h> 30#include <linux/clocksource.h> 31#include <linux/getcpu.h> 32#include <linux/cpu.h> 33#include <linux/smp.h> 34#include <linux/notifier.h> 35 36#include <asm/vsyscall.h> 37#include <asm/pgtable.h> 38#include <asm/page.h> 39#include <asm/unistd.h> 40#include <asm/fixmap.h> 41#include <asm/errno.h> 42#include <asm/io.h> 43#include <asm/segment.h> 44#include <asm/desc.h> 45#include <asm/topology.h> 46#include <asm/vgtod.h> 47 48#define __vsyscall(nr) \ 49 __attribute__ ((unused, __section__(".vsyscall_" #nr))) notrace 50#define __syscall_clobber "r11","cx","memory" 51 52/* 53 * vsyscall_gtod_data contains data that is : 54 * - readonly from vsyscalls 55 * - written by timer interrupt or systcl (/proc/sys/kernel/vsyscall64) 56 * Try to keep this structure as small as possible to avoid cache line ping pongs 57 */ 58int __vgetcpu_mode __section_vgetcpu_mode; 59 60struct vsyscall_gtod_data __vsyscall_gtod_data __section_vsyscall_gtod_data = 61{ 62 .lock = SEQLOCK_UNLOCKED, 63 .sysctl_enabled = 1, 64}; 65 66void update_vsyscall_tz(void) 67{ 68 unsigned long flags; 69 70 write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags); 71 /* sys_tz has changed */ 72 vsyscall_gtod_data.sys_tz = sys_tz; 73 write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags); 74} 75 76void update_vsyscall(struct timespec *wall_time, struct timespec *wtm, 77 struct clocksource *clock, u32 mult) 78{ 79 unsigned long flags; 80 81 write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags); 82 /* copy vsyscall data */ 83 vsyscall_gtod_data.clock.vread = clock->vread; 84 vsyscall_gtod_data.clock.cycle_last = clock->cycle_last; 85 vsyscall_gtod_data.clock.mask = clock->mask; 86 vsyscall_gtod_data.clock.mult = mult; 87 vsyscall_gtod_data.clock.shift = clock->shift; 88 vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec; 89 vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec; 90 vsyscall_gtod_data.wall_to_monotonic = *wtm; 91 vsyscall_gtod_data.wall_time_coarse = __current_kernel_time(); 92 write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags); 93} 94 95/* RED-PEN may want to readd seq locking, but then the variable should be 96 * write-once. 97 */ 98static __always_inline void do_get_tz(struct timezone * tz) 99{ 100 *tz = __vsyscall_gtod_data.sys_tz; 101} 102 103static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz) 104{ 105 int ret; 106 asm volatile("syscall" 107 : "=a" (ret) 108 : "0" (__NR_gettimeofday),"D" (tv),"S" (tz) 109 : __syscall_clobber ); 110 return ret; 111} 112 113static __always_inline long time_syscall(long *t) 114{ 115 long secs; 116 asm volatile("syscall" 117 : "=a" (secs) 118 : "0" (__NR_time),"D" (t) : __syscall_clobber); 119 return secs; 120} 121 122static __always_inline void do_vgettimeofday(struct timeval * tv) 123{ 124 cycle_t now, base, mask, cycle_delta; 125 unsigned seq; 126 unsigned long mult, shift, nsec; 127 cycle_t (*vread)(void); 128 do { 129 seq = read_seqbegin(&__vsyscall_gtod_data.lock); 130 131 vread = __vsyscall_gtod_data.clock.vread; 132 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) { 133 gettimeofday(tv,NULL); 134 return; 135 } 136 137 now = vread(); 138 base = __vsyscall_gtod_data.clock.cycle_last; 139 mask = __vsyscall_gtod_data.clock.mask; 140 mult = __vsyscall_gtod_data.clock.mult; 141 shift = __vsyscall_gtod_data.clock.shift; 142 143 tv->tv_sec = __vsyscall_gtod_data.wall_time_sec; 144 nsec = __vsyscall_gtod_data.wall_time_nsec; 145 } while (read_seqretry(&__vsyscall_gtod_data.lock, seq)); 146 147 /* calculate interval: */ 148 cycle_delta = (now - base) & mask; 149 /* convert to nsecs: */ 150 nsec += (cycle_delta * mult) >> shift; 151 152 while (nsec >= NSEC_PER_SEC) { 153 tv->tv_sec += 1; 154 nsec -= NSEC_PER_SEC; 155 } 156 tv->tv_usec = nsec / NSEC_PER_USEC; 157} 158 159int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz) 160{ 161 if (tv) 162 do_vgettimeofday(tv); 163 if (tz) 164 do_get_tz(tz); 165 return 0; 166} 167 168/* This will break when the xtime seconds get inaccurate, but that is 169 * unlikely */ 170time_t __vsyscall(1) vtime(time_t *t) 171{ 172 unsigned seq; 173 time_t result; 174 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled)) 175 return time_syscall(t); 176 177 do { 178 seq = read_seqbegin(&__vsyscall_gtod_data.lock); 179 180 result = __vsyscall_gtod_data.wall_time_sec; 181 182 } while (read_seqretry(&__vsyscall_gtod_data.lock, seq)); 183 184 if (t) 185 *t = result; 186 return result; 187} 188 189/* Fast way to get current CPU and node. 190 This helps to do per node and per CPU caches in user space. 191 The result is not guaranteed without CPU affinity, but usually 192 works out because the scheduler tries to keep a thread on the same 193 CPU. 194 195 tcache must point to a two element sized long array. 196 All arguments can be NULL. */ 197long __vsyscall(2) 198vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache) 199{ 200 unsigned int p; 201 unsigned long j = 0; 202 203 /* Fast cache - only recompute value once per jiffies and avoid 204 relatively costly rdtscp/cpuid otherwise. 205 This works because the scheduler usually keeps the process 206 on the same CPU and this syscall doesn't guarantee its 207 results anyways. 208 We do this here because otherwise user space would do it on 209 its own in a likely inferior way (no access to jiffies). 210 If you don't like it pass NULL. */ 211 if (tcache && tcache->blob[0] == (j = __jiffies)) { 212 p = tcache->blob[1]; 213 } else if (__vgetcpu_mode == VGETCPU_RDTSCP) { 214 /* Load per CPU data from RDTSCP */ 215 native_read_tscp(&p); 216 } else { 217 /* Load per CPU data from GDT */ 218 asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG)); 219 } 220 if (tcache) { 221 tcache->blob[0] = j; 222 tcache->blob[1] = p; 223 } 224 if (cpu) 225 *cpu = p & 0xfff; 226 if (node) 227 *node = p >> 12; 228 return 0; 229} 230 231static long __vsyscall(3) venosys_1(void) 232{ 233 return -ENOSYS; 234} 235 236#ifdef CONFIG_SYSCTL 237static ctl_table kernel_table2[] = { 238 { .procname = "vsyscall64", 239 .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int), 240 .mode = 0644, 241 .proc_handler = proc_dointvec }, 242 {} 243}; 244 245static ctl_table kernel_root_table2[] = { 246 { .procname = "kernel", .mode = 0555, 247 .child = kernel_table2 }, 248 {} 249}; 250#endif 251 252/* Assume __initcall executes before all user space. Hopefully kmod 253 doesn't violate that. We'll find out if it does. */ 254static void __cpuinit vsyscall_set_cpu(int cpu) 255{ 256 unsigned long d; 257 unsigned long node = 0; 258#ifdef CONFIG_NUMA 259 node = cpu_to_node(cpu); 260#endif 261 if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP)) 262 write_rdtscp_aux((node << 12) | cpu); 263 264 /* Store cpu number in limit so that it can be loaded quickly 265 in user space in vgetcpu. 266 12 bits for the CPU and 8 bits for the node. */ 267 d = 0x0f40000000000ULL; 268 d |= cpu; 269 d |= (node & 0xf) << 12; 270 d |= (node >> 4) << 48; 271 write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S); 272} 273 274static void __cpuinit cpu_vsyscall_init(void *arg) 275{ 276 /* preemption should be already off */ 277 vsyscall_set_cpu(raw_smp_processor_id()); 278} 279 280static int __cpuinit 281cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg) 282{ 283 long cpu = (long)arg; 284 if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) 285 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 1); 286 return NOTIFY_DONE; 287} 288 289void __init map_vsyscall(void) 290{ 291 extern char __vsyscall_0; 292 unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0); 293 294 /* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */ 295 __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL); 296} 297 298static int __init vsyscall_init(void) 299{ 300 BUG_ON(((unsigned long) &vgettimeofday != 301 VSYSCALL_ADDR(__NR_vgettimeofday))); 302 BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime)); 303 BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE))); 304 BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu)); 305#ifdef CONFIG_SYSCTL 306 register_sysctl_table(kernel_root_table2); 307#endif 308 on_each_cpu(cpu_vsyscall_init, NULL, 1); 309 /* notifier priority > KVM */ 310 hotcpu_notifier(cpu_vsyscall_notifier, 30); 311 return 0; 312} 313 314__initcall(vsyscall_init); 315