Branch data Line data Source code
1 : : /* calibrate.c: default delay calibration
2 : : *
3 : : * Excised from init/main.c
4 : : * Copyright (C) 1991, 1992 Linus Torvalds
5 : : */
6 : :
7 : : #include <linux/jiffies.h>
8 : : #include <linux/delay.h>
9 : : #include <linux/init.h>
10 : : #include <linux/timex.h>
11 : : #include <linux/smp.h>
12 : : #include <linux/percpu.h>
13 : :
14 : : unsigned long lpj_fine;
15 : : unsigned long preset_lpj;
16 : 0 : static int __init lpj_setup(char *str)
17 : : {
18 : 0 : preset_lpj = simple_strtoul(str,NULL,0);
19 : 0 : return 1;
20 : : }
21 : :
22 : : __setup("lpj=", lpj_setup);
23 : :
24 : : #ifdef ARCH_HAS_READ_CURRENT_TIMER
25 : :
26 : : /* This routine uses the read_current_timer() routine and gets the
27 : : * loops per jiffy directly, instead of guessing it using delay().
28 : : * Also, this code tries to handle non-maskable asynchronous events
29 : : * (like SMIs)
30 : : */
31 : : #define DELAY_CALIBRATION_TICKS ((HZ < 100) ? 1 : (HZ/100))
32 : : #define MAX_DIRECT_CALIBRATION_RETRIES 5
33 : :
34 : 0 : static unsigned long calibrate_delay_direct(void)
35 : : {
36 : : unsigned long pre_start, start, post_start;
37 : : unsigned long pre_end, end, post_end;
38 : : unsigned long start_jiffies;
39 : : unsigned long timer_rate_min, timer_rate_max;
40 : : unsigned long good_timer_sum = 0;
41 : : unsigned long good_timer_count = 0;
42 : : unsigned long measured_times[MAX_DIRECT_CALIBRATION_RETRIES];
43 : : int max = -1; /* index of measured_times with max/min values or not set */
44 : : int min = -1;
45 : : int i;
46 : :
47 [ # # ]: 0 : if (read_current_timer(&pre_start) < 0 )
48 : : return 0;
49 : :
50 : : /*
51 : : * A simple loop like
52 : : * while ( jiffies < start_jiffies+1)
53 : : * start = read_current_timer();
54 : : * will not do. As we don't really know whether jiffy switch
55 : : * happened first or timer_value was read first. And some asynchronous
56 : : * event can happen between these two events introducing errors in lpj.
57 : : *
58 : : * So, we do
59 : : * 1. pre_start <- When we are sure that jiffy switch hasn't happened
60 : : * 2. check jiffy switch
61 : : * 3. start <- timer value before or after jiffy switch
62 : : * 4. post_start <- When we are sure that jiffy switch has happened
63 : : *
64 : : * Note, we don't know anything about order of 2 and 3.
65 : : * Now, by looking at post_start and pre_start difference, we can
66 : : * check whether any asynchronous event happened or not
67 : : */
68 : :
69 [ # # ]: 0 : for (i = 0; i < MAX_DIRECT_CALIBRATION_RETRIES; i++) {
70 : 0 : pre_start = 0;
71 : 0 : read_current_timer(&start);
72 : 0 : start_jiffies = jiffies;
73 [ # # ]: 0 : while (time_before_eq(jiffies, start_jiffies + 1)) {
74 : 0 : pre_start = start;
75 : 0 : read_current_timer(&start);
76 : : }
77 : 0 : read_current_timer(&post_start);
78 : :
79 : : pre_end = 0;
80 : 0 : end = post_start;
81 [ # # ]: 0 : while (time_before_eq(jiffies, start_jiffies + 1 +
82 : : DELAY_CALIBRATION_TICKS)) {
83 : 0 : pre_end = end;
84 : 0 : read_current_timer(&end);
85 : : }
86 : 0 : read_current_timer(&post_end);
87 : :
88 : 0 : timer_rate_max = (post_end - pre_start) /
89 : : DELAY_CALIBRATION_TICKS;
90 : 0 : timer_rate_min = (pre_end - post_start) /
91 : : DELAY_CALIBRATION_TICKS;
92 : :
93 : : /*
94 : : * If the upper limit and lower limit of the timer_rate is
95 : : * >= 12.5% apart, redo calibration.
96 : : */
97 [ # # ]: 0 : if (start >= post_end)
98 : 0 : printk(KERN_NOTICE "calibrate_delay_direct() ignoring "
99 : : "timer_rate as we had a TSC wrap around"
100 : : " start=%lu >=post_end=%lu\n",
101 : : start, post_end);
102 [ # # ][ # # ]: 0 : if (start < post_end && pre_start != 0 && pre_end != 0 &&
[ # # ][ # # ]
103 : 0 : (timer_rate_max - timer_rate_min) < (timer_rate_max >> 3)) {
104 : 0 : good_timer_count++;
105 : 0 : good_timer_sum += timer_rate_max;
106 : 0 : measured_times[i] = timer_rate_max;
107 [ # # ][ # # ]: 0 : if (max < 0 || timer_rate_max > measured_times[max])
108 : : max = i;
109 [ # # ][ # # ]: 0 : if (min < 0 || timer_rate_max < measured_times[min])
110 : : min = i;
111 : : } else
112 : 0 : measured_times[i] = 0;
113 : :
114 : : }
115 : :
116 : : /*
117 : : * Find the maximum & minimum - if they differ too much throw out the
118 : : * one with the largest difference from the mean and try again...
119 : : */
120 [ # # ]: 0 : while (good_timer_count > 1) {
121 : : unsigned long estimate;
122 : : unsigned long maxdiff;
123 : :
124 : : /* compute the estimate */
125 : 0 : estimate = (good_timer_sum/good_timer_count);
126 : 0 : maxdiff = estimate >> 3;
127 : :
128 : : /* if range is within 12% let's take it */
129 [ # # ]: 0 : if ((measured_times[max] - measured_times[min]) < maxdiff)
130 : : return estimate;
131 : :
132 : : /* ok - drop the worse value and try again... */
133 : : good_timer_sum = 0;
134 : : good_timer_count = 0;
135 [ # # ]: 0 : if ((measured_times[max] - estimate) <
136 : 0 : (estimate - measured_times[min])) {
137 : 0 : printk(KERN_NOTICE "calibrate_delay_direct() dropping "
138 : : "min bogoMips estimate %d = %lu\n",
139 : : min, measured_times[min]);
140 : 0 : measured_times[min] = 0;
141 : : min = max;
142 : : } else {
143 : 0 : printk(KERN_NOTICE "calibrate_delay_direct() dropping "
144 : : "max bogoMips estimate %d = %lu\n",
145 : : max, measured_times[max]);
146 : 0 : measured_times[max] = 0;
147 : : max = min;
148 : : }
149 : :
150 [ # # ]: 0 : for (i = 0; i < MAX_DIRECT_CALIBRATION_RETRIES; i++) {
151 [ # # ]: 0 : if (measured_times[i] == 0)
152 : 0 : continue;
153 : 0 : good_timer_count++;
154 : 0 : good_timer_sum += measured_times[i];
155 [ # # ]: 0 : if (measured_times[i] < measured_times[min])
156 : : min = i;
157 [ # # ]: 0 : if (measured_times[i] > measured_times[max])
158 : : max = i;
159 : : }
160 : :
161 : : }
162 : :
163 : 0 : printk(KERN_NOTICE "calibrate_delay_direct() failed to get a good "
164 : : "estimate for loops_per_jiffy.\nProbably due to long platform "
165 : : "interrupts. Consider using \"lpj=\" boot option.\n");
166 : 0 : return 0;
167 : : }
168 : : #else
169 : : static unsigned long calibrate_delay_direct(void)
170 : : {
171 : : return 0;
172 : : }
173 : : #endif
174 : :
175 : : /*
176 : : * This is the number of bits of precision for the loops_per_jiffy. Each
177 : : * time we refine our estimate after the first takes 1.5/HZ seconds, so try
178 : : * to start with a good estimate.
179 : : * For the boot cpu we can skip the delay calibration and assign it a value
180 : : * calculated based on the timer frequency.
181 : : * For the rest of the CPUs we cannot assume that the timer frequency is same as
182 : : * the cpu frequency, hence do the calibration for those.
183 : : */
184 : : #define LPS_PREC 8
185 : :
186 : 0 : static unsigned long calibrate_delay_converge(void)
187 : : {
188 : : /* First stage - slowly accelerate to find initial bounds */
189 : : unsigned long lpj, lpj_base, ticks, loopadd, loopadd_base, chop_limit;
190 : : int trials = 0, band = 0, trial_in_band = 0;
191 : :
192 : : lpj = (1<<12);
193 : :
194 : : /* wait for "start of" clock tick */
195 : 0 : ticks = jiffies;
196 [ # # ]: 0 : while (ticks == jiffies)
197 : : ; /* nothing */
198 : : /* Go .. */
199 : 0 : ticks = jiffies;
200 : : do {
201 [ # # ]: 0 : if (++trial_in_band == (1<<band)) {
202 : 0 : ++band;
203 : : trial_in_band = 0;
204 : : }
205 : 0 : __delay(lpj * band);
206 : 0 : trials += band;
207 [ # # ]: 0 : } while (ticks == jiffies);
208 : : /*
209 : : * We overshot, so retreat to a clear underestimate. Then estimate
210 : : * the largest likely undershoot. This defines our chop bounds.
211 : : */
212 : : trials -= band;
213 : : loopadd_base = lpj * band;
214 : 0 : lpj_base = lpj * trials;
215 : :
216 : : recalibrate:
217 : : lpj = lpj_base;
218 : : loopadd = loopadd_base;
219 : :
220 : : /*
221 : : * Do a binary approximation to get lpj set to
222 : : * equal one clock (up to LPS_PREC bits)
223 : : */
224 : 0 : chop_limit = lpj >> LPS_PREC;
225 [ # # ]: 0 : while (loopadd > chop_limit) {
226 : 0 : lpj += loopadd;
227 : 0 : ticks = jiffies;
228 [ # # ]: 0 : while (ticks == jiffies)
229 : : ; /* nothing */
230 : 0 : ticks = jiffies;
231 : 0 : __delay(lpj);
232 [ # # ]: 0 : if (jiffies != ticks) /* longer than 1 tick */
233 : : lpj -= loopadd;
234 : 0 : loopadd >>= 1;
235 : : }
236 : : /*
237 : : * If we incremented every single time possible, presume we've
238 : : * massively underestimated initially, and retry with a higher
239 : : * start, and larger range. (Only seen on x86_64, due to SMIs)
240 : : */
241 [ # # ]: 0 : if (lpj + loopadd * 2 == lpj_base + loopadd_base * 2) {
242 : : lpj_base = lpj;
243 : 0 : loopadd_base <<= 2;
244 : 0 : goto recalibrate;
245 : : }
246 : :
247 : 0 : return lpj;
248 : : }
249 : :
250 : : static DEFINE_PER_CPU(unsigned long, cpu_loops_per_jiffy) = { 0 };
251 : :
252 : : /*
253 : : * Check if cpu calibration delay is already known. For example,
254 : : * some processors with multi-core sockets may have all cores
255 : : * with the same calibration delay.
256 : : *
257 : : * Architectures should override this function if a faster calibration
258 : : * method is available.
259 : : */
260 : 0 : unsigned long __attribute__((weak)) calibrate_delay_is_known(void)
261 : : {
262 : 0 : return 0;
263 : : }
264 : :
265 : 0 : void calibrate_delay(void)
266 : : {
267 : : unsigned long lpj;
268 : : static bool printed;
269 : 0 : int this_cpu = smp_processor_id();
270 : :
271 [ # # ]: 0 : if (per_cpu(cpu_loops_per_jiffy, this_cpu)) {
272 : 0 : lpj = per_cpu(cpu_loops_per_jiffy, this_cpu);
273 [ # # ]: 0 : if (!printed)
274 : 0 : pr_info("Calibrating delay loop (skipped) "
275 : : "already calibrated this CPU");
276 [ # # ]: 0 : } else if (preset_lpj) {
277 : : lpj = preset_lpj;
278 [ # # ]: 0 : if (!printed)
279 : 0 : pr_info("Calibrating delay loop (skipped) "
280 : : "preset value.. ");
281 [ # # ][ # # ]: 0 : } else if ((!printed) && lpj_fine) {
282 : : lpj = lpj_fine;
283 : 0 : pr_info("Calibrating delay loop (skipped), "
284 : : "value calculated using timer frequency.. ");
285 [ # # ]: 0 : } else if ((lpj = calibrate_delay_is_known())) {
286 : : ;
287 [ # # ]: 0 : } else if ((lpj = calibrate_delay_direct()) != 0) {
288 [ # # ]: 0 : if (!printed)
289 : 0 : pr_info("Calibrating delay using timer "
290 : : "specific routine.. ");
291 : : } else {
292 [ # # ]: 0 : if (!printed)
293 : 0 : pr_info("Calibrating delay loop... ");
294 : 0 : lpj = calibrate_delay_converge();
295 : : }
296 : 0 : per_cpu(cpu_loops_per_jiffy, this_cpu) = lpj;
297 [ # # ]: 0 : if (!printed)
298 : 0 : pr_cont("%lu.%02lu BogoMIPS (lpj=%lu)\n",
299 : : lpj/(500000/HZ),
300 : : (lpj/(5000/HZ)) % 100, lpj);
301 : :
302 : 0 : loops_per_jiffy = lpj;
303 : 0 : printed = true;
304 : 0 : }
|
