1/**
2 * \file
3 * \brief Timer calibration and setting functions.
4 */
5
6/*
7 * Copyright (c) 2007, 2008, 2009, 2010, 2011, ETH Zurich.
8 * All rights reserved.
9 *
10 * This file is distributed under the terms in the attached LICENSE file.
11 * If you do not find this file, copies can be found by writing to:
12 * ETH Zurich D-INFK, Universitaetstrasse 6, CH-8092 Zurich. Attn: Systems Group.
13 */
14
15#include <string.h>
16#include <kernel.h>
17#include <systime.h>
18#include <arch/x86/apic.h>
19#include <arch/x86/rtc.h>
20#include <arch/x86/pit.h>
21#include <arch/x86/global.h>
22#include <arch/x86/timing.h>
23
24#ifdef __k1om__
25#include <xeon_phi.h>
26#endif
27
28static uint32_t apic_frequency = 0;
29static uint64_t tscperms = 0;
30
31#if !defined(__k1om__)
32/**
33 * \brief Calibrates local APIC timer against RTC.
34 * \return Local APIC timer ticks per RTC second.
35 */
36static uint32_t __attribute__((unused)) calibrate_apic_timer_rtc(systime_t *systime_freq)
37{
38    // Set APIC timer to one-shot mode
39    apic_timer_init(true, false);
40    apic_timer_set_divide(xapic_by1);
41    // Wait until start of new second
42
43    uint64_t tsc_start, tsc_end;
44    uint8_t start = rtc_read_secs(), now;
45    do {
46        now = rtc_read_secs();
47    } while(now == start);
48
49    apic_timer_set_count(UINT32_MAX);
50    tsc_start = rdtsc();
51
52    // Wait a second
53    //uint32_t oldcount = UINT32_MAX;
54    int reads = 0;
55    start = now;
56    do {
57        now = rtc_read_secs();
58#if 0
59        // Assert timer never underflows
60        // XXX: this fires on QEMU, because the APIC timer is driven directly by
61        // gettimeofday() and may go backwards if ntpd is adjusting the clock
62        uint32_t curc = apic_timer_get_count();
63        assert(curc <= oldcount);
64        oldcount = curc;
65#endif
66        reads++;
67    } while(start == now);
68
69    // Get new count
70    uint32_t curcount = apic_timer_get_count();
71    tsc_end = rdtsc();
72    assert(curcount != 0);
73
74    *systime_freq = tsc_end - tsc_start;
75    uint32_t tps = UINT32_MAX - curcount;
76    printk(LOG_NOTE, "Measured APIC frequency: %u Hz in one RTC second,"
77        "systime frequency: %lu Hz\n", tps, *systime_freq);
78    return tps;
79}
80#endif
81
82#if defined(__k1om__)
83/**
84 * \brief Calibrates local APIC timer against RTC.
85 * \return Local APIC timer ticks per RTC second.
86 */
87static uint32_t calibrate_apic_timer_k1om(void)
88{
89    // Set APIC timer to one-shot mode
90    apic_timer_init(true, false);
91    apic_timer_set_divide(xapic_by1);
92    // Wait until start of new second
93
94    /*
95     * The Intel Xeon PhiTM coprocessor has a SBox MMIO register that provides
96     * the current CPU frequency, which can be used to calibrate the LAPIC timer.
97     * The TOD clock has to be emulated in software to query the host OS for the
98     * time at bootup and then using the LAPIC timer interrupt to update it.
99     *
100     * COREFREQ = COREFREQ
101    */
102
103    /* TODO: mackerel */
104    volatile uint32_t *freq;
105    freq = (uint32_t*)local_phys_to_mem(XEON_PHI_SBOX_BASE + 0x4100);
106    printf("Core Frequency: %u\n", (*freq) & 0xFFF );
107    return (((*freq)&0xFFF) * 1000*1000 );
108}
109#endif
110
111
112#if 0
113/**
114 * \brief Calibrates PIT against RTC.
115 * \return PIT ticks per RTC second.
116 */
117static uint32_t calibrate_pit_rtc(void)
118{
119    // Calibrate against RTC
120    rtc_init();
121    pit_init();
122
123    // Wait until start of new second
124    rtc_wait_next_second();
125
126    pit_timer0_set(0xffff, false);
127
128    // Wait a second
129    uint16_t oldcnt = pit_timer0_read();
130    uint32_t ticks = 0;
131    int reads = 0;
132    do {
133        uint16_t cnt = pit_timer0_read();
134        if(cnt <= oldcnt) {
135            ticks += oldcnt - cnt;
136        } else {
137            ticks += oldcnt + (0xffff - cnt);
138        }
139        oldcnt = cnt;
140        reads++;
141    } while((rtc_read_cmos(0xa) & 0x80) == 0);
142
143    printf("Measured %d PIT counts in one RTC second, reads = %d.\n", ticks, reads);
144    return ticks;
145}
146
147#endif
148
149
150/**
151 * \brief Calibrates local APIC timer against PIT.
152 * \return Local APIC timer ticks per PIT second.
153 */
154static uint32_t __attribute__((unused)) calibrate_apic_timer_pit(systime_t *systime_freq)
155{
156    // Set APIC timer to one-shot mode
157    apic_timer_init(true, false);
158    apic_timer_set_divide(xapic_by1);
159
160    // Calibrate against PIT
161    pit_init();
162    pit_timer0_set(0xffff, false, true); // only lsb
163
164    // Start both timers
165    apic_timer_set_count(UINT32_MAX);
166
167    // Wait a second (1,193,182 ticks)
168    uint16_t oldcnt = pit_timer0_read_lsb();
169    uint64_t timestamp = rdtsc();
170    uint32_t ticks = 0;
171    do {
172        uint16_t cnt = pit_timer0_read_lsb();
173        if (cnt <= oldcnt) {
174            ticks += oldcnt - cnt;
175        } else {
176            ticks += oldcnt + 256 - cnt;
177        }
178        oldcnt = cnt;
179
180    } while (ticks < PIT_TIMER0_FREQUENCY);
181
182    uint64_t afreq = UINT32_MAX - apic_timer_get_count();
183    uint64_t sfreq = rdtsc() - timestamp;
184
185    *systime_freq = sfreq;
186    printf("Measured APIC frequency: %ld Hz, systime frequency: %ld Hz\n",
187           afreq, sfreq);
188    return afreq;
189}
190
191
192/// Number of measurement iterations
193#define MAX_ITERATIONS  100
194
195/**
196 * \brief Calibrates TSC against local APIC timer.
197 * \return TSC ticks per local APIC timer tick.
198 */
199static uint64_t __attribute__((unused)) calibrate_tsc_apic_timer(systime_t *systime_freq)
200{
201    // Must tick with higher granularity than a millisecond
202    assert(apic_frequency > 1000);
203    uint32_t ticksperms = apic_frequency / 1000;
204
205    // XXX: Let's hope this fits on the stack (reserve half the stack)
206    /* assert(sizeof(uint64_t) * MAX_ITERATIONS < KERNEL_STACK_SIZE / 2); */
207    uint64_t timestamps[MAX_ITERATIONS];
208    memset(timestamps, 0, sizeof(timestamps));
209
210    // Set APIC timer to periodic mode
211    apic_timer_init(true, true);
212    apic_timer_set_divide(xapic_by1);
213    apic_timer_set_count(UINT32_MAX);
214
215    // Do all measurement iterations
216    uint32_t oldcnt = apic_timer_get_count();
217    for(int i = 0; i < MAX_ITERATIONS; i++) {
218        // Wait a millisecond
219        uint32_t ticks = 0;
220        do {
221            uint32_t cnt = apic_timer_get_count();
222            if(cnt <= oldcnt) {
223                ticks += oldcnt - cnt;
224            } else {
225                ticks += oldcnt + (UINT32_MAX - cnt);
226            }
227            oldcnt = cnt;
228        } while(ticks < ticksperms);
229
230        timestamps[i] = rdtsc();
231    }
232
233    // Calculate average
234    uint64_t tpms = 0;
235    for(int i = 1; i < MAX_ITERATIONS; i++) {
236        assert(timestamps[i - 1] < timestamps[i]);
237        tpms += timestamps[i] - timestamps[i - 1];
238    }
239    tpms /= MAX_ITERATIONS - 1;
240
241    // Check that jitter is not too high.
242    // No more than 1% of values should deviate more than 1%
243    // from the average.
244    unsigned int outliers = 0;
245    uint64_t avgdistance = 0;
246    for(int i = 1; i < MAX_ITERATIONS; i++) {
247        uint64_t diff = timestamps[i] - timestamps[i - 1],
248            distance = diff > tpms ? diff - tpms : tpms - diff;
249
250        if(distance > tpms / 100) {
251            outliers++;
252        }
253
254        avgdistance += distance;
255    }
256    avgdistance /= MAX_ITERATIONS - 1;
257
258    // Always round up
259    if(outliers > ((MAX_ITERATIONS - 1) / 100 + 1)) {
260        printk(LOG_WARN, "Considerable TSC jitter detected! %" PRIu64 " ticks "
261               "on average.\n", avgdistance);
262    }
263
264    *systime_freq = tpms * 1000;
265    printk(LOG_NOTE, "Measured %" PRIu64 " TSC counts per ms, "
266           "%d data points. Average jitter %" PRIu64 " TSC ticks.\n",
267           tpms, MAX_ITERATIONS - 1, avgdistance);
268    return tpms;
269}
270
271void timing_apic_timer_set_ms(unsigned int ms)
272{
273    // Must tick with higher granularity than a millisecond
274    assert(apic_frequency > 1000);
275    assert(ms < UINT32_MAX / (apic_frequency / 1000));
276
277    apic_timer_set_divide(xapic_by1);
278    apic_timer_set_count(ms * (apic_frequency / 1000));
279}
280
281uint32_t timing_get_apic_ticks_per_sec(void)
282{
283    return apic_frequency;
284}
285
286uint64_t timing_get_tsc_per_ms(void)
287{
288    // Has to tick with at least ms granularity
289    assert(tscperms > 0);
290    return tscperms;
291}
292
293void timing_calibrate(void)
294{
295    if (CPU_IS_M5_SIMULATOR) {
296        // Guess -- avoid delay of calibration
297        printk(LOG_WARN, "Warning: using hard-coded timer calibration on M5\n");
298        apic_frequency = 31250000;
299        tscperms = apic_frequency / 1000;
300    } else {
301        if(apic_is_bsp()) {
302#ifdef __k1om__
303            apic_frequency = calibrate_apic_timer_k1om();
304            tscperms = calibrate_tsc_apic_timer(&systime_frequency);
305            systime_frequency = tscperms * 1000;
306#else
307            // apic_frequency = calibrate_apic_timer_pit(&systime_frequency);
308            apic_frequency = calibrate_apic_timer_rtc(&systime_frequency);
309#endif
310            global->apic_frequency = apic_frequency;
311            global->systime_frequency = systime_frequency;
312        } else {
313            apic_frequency = global->apic_frequency;
314            systime_frequency = global->systime_frequency;
315        }
316        tscperms = systime_frequency / 1000;
317        apic_systime_frequency_ratio = ((uint64_t)apic_frequency << 32) / systime_frequency;
318    }
319}
320
321systime_t systime_now(void)
322{
323    return rdtsc();
324}
325