1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#include <stdio.h>
28#include <sys/types.h>
29#include <dirent.h>
30#include <stdarg.h>
31#include <stddef.h>
32#include <stdlib.h>
33#include <dlfcn.h>
34#include <door.h>
35#include <errno.h>
36#include <fcntl.h>
37#include <strings.h>
38#include <unistd.h>
39#include <synch.h>
40#include <syslog.h>
41#include <pthread.h>
42#include <thread.h>
43#include <signal.h>
44#include <limits.h>
45#include <locale.h>
46#include <sys/stat.h>
47#include <sys/systeminfo.h>
48#include <sys/wait.h>
49#include <sys/processor.h>
50#include <sys/pset.h>
51#include <ctype.h>
52#include <poll.h>
53#include <sys/wait.h>
54#include <sys/pm.h>
55#include <sys/iso/signal_iso.h>
56#include <sys/procset.h>
57
58#include "fpsapi.h"
59#include "fpsd.h"
60#include "messages.h"
61
62/* Local Functions */
63
64static int
65check_invoke_prog(int   devid, time_t *last,
66unsigned  tstswap, int frequency, int group_no, int fpu_index);
67
68static int identify_fpu_to_run_test(int *freq, int *iteration, int *fpu_index);
69
70void  *test_fpu_thr(void *arg);
71
72#define	CPU_TST_EXEC_FAIL	{		\
73	error = errno;							\
74	fpsd_message(FPSD_EXIT_ERROR,\
75		FPS_WARNING, TST_EXEC_FAIL, testpath, strerror(error)); \
76	}
77
78static int boot_tst_delay = FPS_BOOT_TST_DELAY;
79
80/* Increments failure for the cpu */
81static void
82record_failure(int devid, int index) {
83	if ((index >= 0) &&
84		(index < fpsd.d_conf->m_cpuids_size)) {
85		fpsd.d_conf->m_cpus[index].num_failures++;
86		fpsd_message(FPSD_NO_EXIT, FPS_DEBUG,
87			RECORD_FAILURE_MSG, devid, index);
88	}
89}
90
91/* Returns 1 if testing is diabled for the cpu, else 0 */
92
93static int
94check_if_disabled(int fpu_index) {
95	int is_disabled;
96
97	is_disabled = fpsd.d_conf->m_cpus[fpu_index].disable_test;
98	if (is_disabled) {
99		return (1);
100	} else {
101		return (0);
102	}
103}
104
105/*
106 * Forks and executes "fptest" and waits for an amount of time equal to
107 * the time to schedule next "fptest". Times out if the test does not
108 * complete and unbinds and terminates the test.
109 * Return = 1 implies fptest could be invoked successfully but a non-zero
110 * status would be dealt with by setting the reprobe flag.
111 * Return = -1 implies fptest could not be successfully invoked or test
112 * exited with non-zero exit status that requires the scheduler to retry.
113 */
114static int
115check_invoke_prog(int   devid,		/* cpu-id */
116		time_t *last,	/* Last time it was invoked */
117		unsigned  tstswap, /* Expected swap space required for test */
118		int	frequency, /* Frequency of the processor under test */
119		int	group_no, /* Group no. ==> matrix size to be used */
120		int	fpu_index)
121{
122	int  error;
123	hrtime_t   start_hrtime = 0, end_hrtime = 0, hrmsecs = 0;
124	hrtime_t   hrsecs = 0;
125	pid_t  pid = -1;
126	int  exit_status = 0;
127	char cpuid_c[64];
128	char frequency_c[10];
129	char group_c[10];
130	int ret = -1;
131	int status = 0;
132	char *testpath;
133	char sig_str[32];
134	int elapsed_time;
135	int status_available;
136	int max_timeout;
137	int pb_ret;
138
139	testpath = fpsd.d_conf->m_cpus[fpu_index].fptest_path;
140	if (check_if_disabled(fpu_index)) {
141		return (ret);
142	}
143
144	/* Compare all in seconds.  */
145
146	*last = time(NULL);
147
148	(void) snprintf(cpuid_c, sizeof (cpuid_c), "%d", devid);
149	(void) snprintf(frequency_c, sizeof (frequency_c), "%d", frequency);
150	(void) snprintf(group_c, sizeof (group_c), "%d", group_no);
151
152	/* Check if enough swap space is there; Return -1 if not. */
153
154	if (get_free_swap() < (uint64_t)(tstswap+FPS_SWAP_RESERVE)) {
155		fpsd_message(FPSD_NO_EXIT, FPS_INFO, SWAP_WARN, testpath);
156		return (ret);
157	}
158
159	fpsd_message(FPSD_NO_EXIT, FPS_INFO, START_TEST_MSG,
160	    testpath, frequency_c, group_c, cpuid_c);
161
162	start_hrtime = gethrtime();
163
164	pid = fork1();  /* fork1() duplicates only the calling thread */
165	if (pid == 0) {
166		(void) execl(testpath,   /* Path */
167		    FPS_FPUTST_NAME,	/* Arg 0 */
168		"-f",
169		    frequency_c, /* Frequency */
170		"-p",
171		    group_c,	/* Group no. */
172		"-d",
173		    cpuid_c,	/* CPU ID */
174		    (char *)NULL);
175
176		CPU_TST_EXEC_FAIL	/* Should never reach here */
177	}
178
179	if (pid == (pid_t)-1) {
180		error = errno;
181		if ((error == EAGAIN) || (error == ENOMEM)) {
182			fpsd_message(FPSD_NO_EXIT, FPS_DEBUG,
183			    FORK_FAIL_MSG, testpath, strerror(error));
184		} else {
185			fpsd_message(FPSD_NO_EXIT, FPS_WARNING,
186			    FORK_FAIL_MSG, testpath, strerror(error));
187		}
188		return (-1);
189	}
190
191	/* Synchronously wait here till the child exits */
192
193	elapsed_time = 0;
194	status_available = 0;
195	max_timeout = fpsd.d_interval * 1000;
196	while (elapsed_time < max_timeout) {
197		if (pid == waitpid((pid_t)pid, &status, WNOHANG)) {
198			status_available = 1;
199			break;
200		} else {
201			elapsed_time += 50;
202			(void) poll(NULL, 0, 50);   /* wait 50 milli sec. */
203		}
204	}
205
206	if (!status_available) {
207		exit_status = FPU_TIMED_OUT;
208	} else {
209		exit_status = WEXITSTATUS(status);
210		if (exit_status == 0xFF) {
211			/* As WEXITSTATUS returns 0xFF */
212			exit_status = FPU_UNSUPPORT;
213		}
214	}
215	if (exit_status == FPU_UNSUPPORT) {
216		/* Reprobe */
217		fpsd.d_conf->m_reprobe = 1;
218		ret = 1;
219	} else if (exit_status == FPU_OK) {
220		/* Increment iteration */
221		fpsd.d_iteration++;
222		ret = 1;
223	} else if ((exit_status == FPU_FOROFFLINE) ||
224	    (exit_status == FPU_BIND_FAIL)) {
225		/* Force reprobe */
226		fpsd.d_conf->m_reprobe = 1;
227		ret = 1;
228	} else if (exit_status == FPU_INVALID_ARG) {
229		/* This should not happen; so force exit */
230		fpsd_message(FPSD_EXIT_TEST_USAGE, FPS_WARNING,
231		    FPU_INVALID_ARG_MSG);
232	} else if ((exit_status == FPU_SIG_SEGV) ||
233	    (exit_status == FPU_SIG_BUS)) {
234		fpsd_message(FPSD_NO_EXIT, FPS_INFO, FPU_SIG_RCVD,
235		    devid);
236		record_failure(devid, fpu_index);
237		ret = -1; /* Retry */
238	} else if (exit_status == FPU_SIG_FPE) {
239		fpsd_message(FPSD_NO_EXIT, FPS_INFO, FPU_FPE_MSG,
240		    devid);
241		record_failure(devid, fpu_index);
242		ret = -1;
243	} else if (exit_status == FPU_SIG_ILL) {
244		fpsd_message(FPSD_NO_EXIT, FPS_INFO, FPU_SIG_ILL_MSG,
245		    devid);
246		record_failure(devid, fpu_index);
247		ret = -1;
248	} else if (exit_status == FPU_SYSCALL_FAIL) {
249		fpsd_message(FPSD_NO_EXIT, FPS_INFO, FPU_SYSCALL_FAIL_MSG,
250		    devid);
251		record_failure(devid, fpu_index);
252		fpsd.d_iteration++; /* Iteration skipped */
253		ret = 1; /* Record failure and move on */
254	} else if (exit_status == FPU_EREPORT_INCOM) {
255		fpsd_message(FPSD_NO_EXIT, FPS_INFO, FPU_EREPORT_INCOM_MSG,
256		    devid);
257		fpsd.d_conf->m_reprobe = 1;
258		ret = 1;
259	} else if (exit_status == FPU_SYSCALL_TRYAGAIN) {
260		fpsd_message(FPSD_NO_EXIT, FPS_INFO, FPU_SYSCALL_TRYAGAIN_MSG);
261		ret = -1; /* Retry as it could be some resource issue */
262	} else if (exit_status == FPU_EREPORT_FAIL) {
263		fpsd_message(FPSD_NO_EXIT, FPS_INFO, FPU_EREPORT_FAIL_MSG,
264		    devid);
265		ret = -1;
266	} else if (exit_status == FPU_TIMED_OUT) {
267		pb_ret = processor_bind(P_PID, pid, PBIND_NONE, NULL);
268		if (pb_ret == -1) {
269			fpsd_message(FPSD_NO_EXIT, FPS_INFO,
270			    UNBIND_FAIL_MSG,
271			    strerror(errno));
272		}
273		(void) kill(pid, SIGINT);
274		while (pid != waitpid((pid_t)pid, &status, WUNTRACED)) {
275			(void) poll(NULL, 0, 10);
276			(void) kill(pid, SIGINT);
277		}
278		fpsd_message(FPSD_NO_EXIT, FPS_INFO, FPU_TIMED_OUT_MSG, devid);
279		record_failure(devid, fpu_index);
280		ret = -1;
281	}
282
283	/*
284	 * The following is the case if the test ended due to a
285	 * signal and did not have a handler for the signal.
286	 */
287	if (WIFSIGNALED(status)) {
288		(void) sig2str(WTERMSIG(status), sig_str);
289		fpsd_message(FPSD_NO_EXIT, FPS_INFO,
290		    TST_SIGNALED_MSG, devid,
291		    frequency, sig_str);
292		record_failure(devid, fpu_index);
293		ret = -1; /* Retry */
294	}
295
296	end_hrtime = gethrtime();
297	hrmsecs = ((end_hrtime - start_hrtime)/
298	    ((hrtime_t)1000*1000));
299	hrsecs  = hrmsecs / 1000;
300	fpsd_message(FPSD_NO_EXIT, FPS_INFO, END_TST_MSG,  (int)pid,
301	    (int)(hrsecs/(60*60)),
302	    (int)((hrsecs%3600)/60),
303	    (int)(hrsecs%60),
304	    (int)(hrmsecs%1000),
305	    cpuid_c);
306
307	fpsd_message(FPSD_NO_EXIT, FPS_DEBUG, EXIT_STAT_MSG, exit_status);
308
309	return (ret);
310}
311
312/*
313 *  The test scheduling thread.
314 */
315
316void  *
317test_fpu_thr(/* ARGSUSED */ void *arg)
318{
319	time_t cur = 0,   /* current time in secs */
320	    last = 0;   /* Last time this level testing done in secs */
321	int    ret;
322
323	int	intvl = 0;	/* interval */
324	unsigned	tswap = 0;
325	int	poll_intvl;
326	long	num_cpus;
327	int	idle = 0, remain = 0, max_remain = 0;
328	time_t last_wakeup = 0, wakeup_elapse;
329	int fpuid;
330	int frequency;
331	int group_no;
332
333	int force_skip_test_if_pm_idle = 1;
334	int fpu_index;
335	int max_idle_time_4_tst_run;
336	int j;
337
338	/*
339	 * If enabled, do not run test on idle system, even if test intvl
340	 * explicitly specified.
341	 */
342
343/*
344 * Minimum time to wait before scheduling tests
345 * when the system just wakes up from sleep.
346 */
347#define	MINSLEEP	8
348
349	num_cpus = fpsd.d_conf->m_num_on_fpuids;
350
351	intvl =  poll_intvl = fpsd.d_interval;
352
353	tswap = FPS_LOWTST_SWAP;
354
355	cur  =  time(NULL);
356
357	/*
358	 * Initialize last time test done based on earlier bootup testing.
359	 * This decides when the first time scheduling of the test is
360	 * to be done.
361	 */
362
363	/*
364	 * In systems with less than 3 processors, the initial testing
365	 * has been found to affect the system bootup time.
366	 * Wait for 5 min for those systems before starting any testing.
367	 */
368
369	if (num_cpus < 3)
370		fps_wait_secs(boot_tst_delay);
371
372	/* Soft bind before once before starting test. */
373	if (processor_bind(P_PID, P_MYID, PBIND_SOFT, NULL) != 0) {
374		fpsd_message(FPSD_EXIT_ERROR, FPS_WARNING, SYSTEM_CALL_FAIL,
375		    "processor_bind", strerror(errno));
376	}
377
378	if (pset_bind(PS_SOFT, P_PID, P_MYID, NULL) != 0) {
379		fpsd_message(FPSD_EXIT_ERROR, FPS_WARNING, SYSTEM_CALL_FAIL,
380		    "pset_bind", strerror(errno));
381	}
382
383#define	MAX_IDLE_TIME_FOR_TSTRUN	10
384
385	if (intvl/2 > MAX_IDLE_TIME_FOR_TSTRUN) {
386		max_idle_time_4_tst_run =
387		    MAX_IDLE_TIME_FOR_TSTRUN;
388	} else {
389		max_idle_time_4_tst_run =
390		    (intvl/2) +
391		    MAX_TEST_RUN_TIME;
392	}
393
394	cur  =  time(NULL);
395	last = 0; /* Force the invocation by setting last to zero. */
396
397
398	for (;;) {
399		time_t elapse;
400
401		cur =  time(NULL);
402		elapse = cur - last;
403
404		/*
405		 * Sleep for intvl secs amount of time.
406		 */
407
408		if (elapse >= (time_t)intvl)
409			poll_intvl = 0;
410		else  /* Don't sleep more than 1 min at a time */
411			poll_intvl = (int)((time_t)intvl-elapse);
412
413		/*
414		 * Until poll_intvl becomes zero, sleep.
415		 * If poll gets interrupted for any reason, then also works.
416		 */
417
418		if (poll_intvl > 0) {
419			(void) poll(NULL, 0, poll_intvl*1000);
420			continue;
421		}
422
423#define	INVOKE_PROG	{	\
424	fpuid = identify_fpu_to_run_test(&frequency, &group_no, &fpu_index);\
425	if (intvl != fpsd.d_interval) {	\
426		/*	\
427		 * Interval has changed due to change in	\
428		 * online processors/ config properties.	\
429		 */	\
430		intvl = fpsd.d_interval;	\
431		fpsd_message(FPSD_NO_EXIT, FPS_DEBUG,	\
432			INTVL_CHANGED_MSG, intvl);	\
433	}	\
434	if (fpuid == -1) {\
435		/* Testing could not be done on any cpu */\
436		fpsd_message(FPSD_NO_EXIT, FPS_DEBUG,	\
437		    INVALID_FPU_ID);	\
438		last = time(NULL);	\
439		continue;\
440	}\
441	ret = check_invoke_prog(fpuid, &last, tswap, frequency,	\
442		group_no, fpu_index); \
443	if (ret == -1) {	\
444		for (j = 0; (j < MAX_RETRIES) && (ret != 1); j++) {	\
445			(void) poll(NULL, 0, RETRY_INTVL);	\
446			fpsd_message(FPSD_NO_EXIT, FPS_DEBUG,	\
447				RESCHEDULE_MSG, fpuid);\
448			ret = check_invoke_prog(fpuid, &last, tswap,	\
449				frequency, group_no, fpu_index); \
450		}	\
451		if (ret == -1) {	\
452			/*\
453			 * Tried MAX_RETRIES times. Still seeing failures\
454			 * on this fpu. Skip this iteration and move on.\
455			 */\
456			fpsd.d_iteration++;	\
457		}	\
458	}	\
459}
460
461		/*
462		 * If power management is disabled (or not supported) on the
463		 * system, just go ahead, invoke the program.
464		 */
465		update_pm_state();  /* Update current PM state. */
466		if (sys_pm_state != PM_SYSTEM_PM_ENABLED) {
467		/* autopm disabled. Just go ahead invoke program. */
468			INVOKE_PROG
469			continue;
470		}
471
472		/*
473		 *  Power management is enabled. This system may be CPU PM
474		 * enabled system or just disk(and other) PM enabled.
475		 * If CPU PM not supported, just invoke the program.
476		 */
477		if (!is_estar_system) {
478			INVOKE_PROG
479			continue;
480		}
481
482		/* This system is CPU PM currently supported & enabled. */
483
484		/*
485		 * By deafult, tests are not invoked on E* compliant system.
486		 * However if force_skip_test_if_pm_idle is set to 0, tests
487		 * will be invoked. This is kept for debugging purposes for now.
488		 * Should be removed if no use cases.
489		 */
490
491		if (!force_skip_test_if_pm_idle) {
492			INVOKE_PROG
493			continue;
494		}
495
496		/*
497		 * If the system is in sleep mode, wait until it comes
498		 * to full power mode.
499		 */
500
501		/* If CPUs are not in full power mode, this will return -1 */
502		ret = get_idle_rem_stats(&idle, &remain, &max_remain);
503
504		/*
505		 * Wait until CPU comes to full power mode.
506		 * Call wait for state change function -- the return from the
507		 * function does not guarantee that the system is in full power
508		 * mode. So get the current status later as well.
509		 */
510		if (ret == -1) {
511			while (ret == -1) {
512				/* Avoid busy loop in any case */
513				(void) poll(NULL, 0, 1000);
514				/* Wait until CPU comes to full pwr */
515				wait_for_pm_state_change();
516				ret = get_idle_rem_stats(&idle, &remain,
517				    &max_remain);
518			}
519
520			/* Remember the last time that we woke up. */
521			last_wakeup = time(NULL);
522		}
523
524	/*
525	 * To meet E* requirements, the system should go to
526	 * deep sleep mode in 30 mins on default configs.
527	 * The CPU power management does this by 14.25min+14.25min
528	 * so total 28.5mins. (in sleep mode followed by deep sleep).
529	 * Running the test as the system just becomes active,
530	 * may reset the idle counter and may delay the transition.
531	 * However since we have 1.5 mins cushion to meet E*
532	 * requirements, we are just making use of it.
533	 *
534	 * If system is idle for more than 10 seconds, wait
535	 * until the system idle time is less than 10  seconds.
536	 * Poll in 2 sec interval, so we will catch it as soon
537	 * as the system idle time goes low (as it just becomes busy).
538	 * Basically don't run test on an idle system.
539	 * If the system is continously busy, then this will
540	 * result in continously scheduling the test.
541	 *
542	 * Running test on a system which is just 10 seconds idle,
543	 * may reset the idle counter.
544	 * This will postpone the idle transition to it's lowest power
545	 * by worst case of  10 secs + worst case run time for fptest
546	 * that is approximately 1 sec = 11 sec.
547	 * This is below the 1.5mins cushion CPU PM now has to make
548	 * idle transition.
549	 *
550	 * So if d_interval/2 >= 10 follow the above logic. Else, reduce
551	 * max_idle_time_4_tstrun = d_interval/2 + max_time_taken_by_test
552	 * (which is <= 1s). We want to be conservative in scheduling
553	 * test rather than utilize the cushion to maximum possible
554	 * extent.
555	 * Note: The E* desktop systems have atmost 2 processors, but
556	 * this will work even for more processors in which case the
557	 * interval will be less or if the interval is configured thro'
558	 * SMF.
559	 * As long as atleast any one processor is in full power mode,
560	 * all processors have to be in same power level.
561	 */
562
563		/* Invoke program if system is "busy" */
564
565		if (idle <= max_idle_time_4_tst_run) {
566	/*
567	 * If the system is just waking up from sleep, don't rush into
568	 * testing immediately to avoid hiccups in performance.
569	 *
570	 */
571			wakeup_elapse = time(NULL) - last_wakeup;
572			if (wakeup_elapse < MINSLEEP) {
573				fps_wait_secs((int)(MINSLEEP-wakeup_elapse));
574			}
575			INVOKE_PROG
576			continue;
577		}
578
579	/* The system is "idle". Wait until it becomes "busy" */
580		while (idle > max_idle_time_4_tst_run) {
581
582	/*
583	 * Once in max_idle_time_4_tst_run/2 secs, we are issuing
584	 * ioctl call to catch the system as soon as it becomes
585	 * "busy". Polling is not an efficient way to do this,
586	 * but this is the only way we got right now.
587	 */
588			fps_wait_secs(max_idle_time_4_tst_run / 2);
589			ret = get_idle_rem_stats(&idle, &remain, &max_remain);
590			if (ret == -1) break; /* Incase now in sleep mode */
591		}
592		continue;
593
594	} /* End infinite for loop */
595
596#pragma error_messages(off, E_STATEMENT_NOT_REACHED)
597	/* NOTREACHED */
598	return (NULL);
599}
600
601/*
602 * get_num_onln_cpus(): returns the number of processors that are in
603 * "on-line" state only. This number will be less than the number
604 * returned by sysconf(_SC_NPROCESSORS_ONLN) if there are some
605 * processors in "no-intr" state.
606 */
607
608static int
609get_num_onln_cpus()
610{
611	int i;
612	int num_onln = 0;
613	int total_onln = sysconf(_SC_NPROCESSORS_ONLN);
614
615	for (i = 0; i < fpsd.d_conf->m_max_cpuid; i++) {
616		if (p_online(i, P_STATUS) == P_ONLINE) {
617			num_onln++;
618		}
619		if (num_onln == total_onln) {
620			/* Break after all onln cpuids found */
621			break;
622		}
623	}
624	return (num_onln);
625}
626
627/*
628 * Identifies the fpu on which test will be scheduled next.
629 */
630
631static int
632identify_fpu_to_run_test(int *freq, int *iteration, int *fpu_index)
633{
634	int fpuid = -1;
635	int ascend;
636	int tmp_iter;
637	fps_cpu_t fps_cpu;
638	int i;
639	int num_onln;
640	/* Timestamp at which SIGHUP ts was checked last */
641	static hrtime_t	ts_hup_chkd = 0;
642	hrtime_t tmp_ts;
643	int iter = 0;
644
645	*iteration = *freq = 0;
646	while (fpuid == -1) {
647		/* Check if the number of online processors has changed */
648		num_onln = get_num_onln_cpus();
649		if (num_onln != fpsd.d_conf->m_num_on_fpuids) {
650			fpsd_message(FPSD_NO_EXIT, FPS_DEBUG, REPROBE_MSG);
651			fpsd.d_conf->m_reprobe = 1;
652		}
653
654		tmp_ts = fpsd.d_ts_hup;
655		if (fpsd.d_ts_hup > ts_hup_chkd) {
656			fpsd.d_conf->m_reprobe = 1;
657		}
658		ts_hup_chkd = tmp_ts;
659
660		if (1 == fpsd.d_conf->m_reprobe) {
661			fpsd_read_config();
662		}
663		fpsd_message(FPSD_NO_EXIT, FPS_DEBUG, IDENTIFY_FPU_MSG,
664		    fpsd.d_fpuid_index, fpsd.d_iteration,
665		    fpsd.d_conf->total_iter, fpsd.d_conf->m_cpuids_size);
666		if (iter >=  fpsd.d_conf->m_cpuids_size) {
667			/* Possible infinite loop */
668			fpsd_message(FPSD_EXIT_ERROR, FPS_INFO,
669			    INFINITE_LOOP_MSG);
670		}
671		iter++;
672
673		if (fpsd.d_iteration >= fpsd.d_conf->total_iter) {
674			/* One pass completed */
675			fpsd.d_iteration = 0;
676
677			/* Reinit iterations */
678			for (i = 0; i < fpsd.d_conf->m_cpuids_size; i++) {
679				if (fpsd.d_conf->m_cpus[i].disable_test)
680					continue;
681				ascend = fpsd.d_conf->m_cpus[i].asc;
682				if (ascend) {
683				fpsd.d_conf->m_cpus[i].previous_iteration = 0;
684				} else {
685				fpsd.d_conf->m_cpus[i].previous_iteration =
686				    fpsd.d_conf->m_cpus[i].total_iterations + 1;
687				}
688			}
689		}
690		if (fpsd.d_iteration == 0) { /* Beginning of one pass */
691			fpsd.d_fpuid_index = 0;
692			while (fpsd.d_fpuid_index <
693			    fpsd.d_conf->m_cpuids_size) {
694				if (fpsd.d_conf->m_cpus[fpsd.d_fpuid_index].\
695				    disable_test) {
696					fpsd.d_fpuid_index++;
697				} else {
698					break;
699				}
700			}
701			if (fpsd.d_fpuid_index ==  fpsd.d_conf->m_cpuids_size) {
702				return (-1);
703			}
704		} else {
705			if (fpsd.d_fpuid_index ==
706			    (fpsd.d_conf->m_cpuids_size-1)) {
707				/* One iteration done for all fpus */
708				fpsd.d_fpuid_index = 0;
709			} else {
710				fpsd.d_fpuid_index++;
711			}
712		}
713		fps_cpu = fpsd.d_conf->m_cpus[fpsd.d_fpuid_index];
714		fpuid = fps_cpu.cpuid;
715		if (fps_cpu.disable_test) {
716			fpuid = -1;
717			continue;
718		}
719		*freq = fps_cpu.frequency;
720
721		/* Find the iteration no. */
722		tmp_iter = fps_cpu.previous_iteration;
723		ascend = fpsd.d_conf->m_cpus[fpsd.d_fpuid_index].asc;
724		if (ascend) {
725			if (tmp_iter == fps_cpu.total_iterations) {
726			/*
727			 * 1 pass completed for this fpu;
728			 * skip this fpu and goto the next fpu
729			 */
730				fpuid = -1;
731				continue;
732			} else {
733				fpsd.d_conf->m_cpus[fpsd.d_fpuid_index].\
734				    previous_iteration++;
735			}
736		} else {
737			/* This FPU is tested in descending order of */
738			/* iteration no. ==> matrix size */
739			if (tmp_iter == 1) {
740				/*
741				 * 1 pass completed for this fpu;
742				 * skip this fpu and goto the next fpu
743				 */
744				fpuid = -1;
745				continue;
746			} else {
747				fpsd.d_conf->m_cpus[fpsd.d_fpuid_index].\
748				    previous_iteration--;
749			}
750		}
751		*iteration =
752		    fpsd.d_conf->m_cpus[fpsd.d_fpuid_index].previous_iteration;
753		*fpu_index = fpsd.d_fpuid_index;
754		fpsd_message(FPSD_NO_EXIT, FPS_DEBUG, IDENTIFY_FPU_RTN_MSG,
755		    fpuid, *iteration, *freq,
756		    fpsd.d_conf->m_cpus[fpsd.d_fpuid_index].\
757		    previous_iteration,
758		    fps_cpu.total_iterations);
759	}
760	return (fpuid);
761}
762