domain/pools/Poold.java

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * ident	"%Z%%M%	%I%	%E% SMI"
 */

package com.sun.solaris.domain.pools;

import java.io.*;
import java.util.*;
import java.util.logging.*;
import java.text.DecimalFormat;
import java.util.concurrent.atomic.*;

import com.sun.solaris.service.locality.*;
import com.sun.solaris.service.logging.*;
import com.sun.solaris.service.pools.*;
import com.sun.solaris.service.exception.*;

/*
 * poold overview
 * ----- --------
 *
 * poold manipulates system resources in accordance with administrator
 * specified constraints and objectives. The "goal" of the application
 * is to maximise the efficiency of available resources within these
 * parameters.
 *
 * Constraints are specified as follows:
 *
 * On a resource set:
 *
 *	- min Is the minimum amount of resource a set should
 *	receive. poold will never elect to move resource so that a set
 *	falls below its minimum value. It is possible for a set to
 *	fall below its minimum as a consequence of administrative
 *	intervention, in which case poold will endeavour to bring a
 *	set back to its minimum level at the earliest opportunity.
 *
 *	- max Is the maximum amount of resource a set should
 *	recieve. poold will never elect to move resource so that a set
 *	rises above its maximum value. It is possible for a set to
 *	rise above its maximum as a consequence of administrative
 *	intervention, in which case poold will endeavour to bring a
 *	set back to its maximum level at the earliest opportunity.
 *
 * On a resource component:
 *
 *	- cpu.pinned Is an indication that a CPU should be ignored by
 *	poold for purposes of reallocation. A pinned CPU will never be
 *	moved by poold from one set to another.
 *
 * In addition to constraints, an administrator may also specify
 * objectives. Currently three types of objectives are supported:
 *
 *	- system.wt-load Is an objective set across an entire
 *	configuration. It attempts to ensure that resource is shared
 *	in accordance with current consumption. Those partitions which
 *	are most heavily utilized are give more resource in an attempt
 *	to lower their utilization levels.
 *
 *	- set.locality Is a locality objective which attempts to
 *	minimize or maximize resource locality for a set.
 *
 *	- set.utilization Is a utilization based objective which an
 *	administrator may use to explicitly dictate the utilization
 *	levels which should be achieved for a set.
 *
 * When executing, poold uses information about the current pools
 * configuration; monitored resource utilization and specified
 * constraints and objectives to determine if a move of resources is
 * likely to lead to an increase in objective satisfaction.
 *
 * Candidate moves are generated by observing resource constraints.
 * These moves are evaluated and scored in terms of their contribution
 * to objective satisfaction (note: objectives are weighted according
 * to their importance) and ranked accordingly. If a move can be
 * identified that delivers an improvement, the move is made. Data is
 * collected about past moves and recorded as "Decision History",
 * before the move is made this data is consulted and if the move is
 * expected not to yield an improvement, it may be cancelled. This
 * refinement is designed to improve the quality of decision making by
 * reflecting upon the past performance of decisions as well as the
 * contribution to objective satisfaction.
 *
 * poold structure
 * ----- ---------
 *
 * The common package name begins with:
 *
 * com.sun.solaris
 *
 * The software is divided into two main functional areas:
 *
 *	service
 *
 *	These packages collaborate to provide services to poold
 *	(typically, they use JNI to access exising
 *	functionality). They are not designed to be extended. For more
 *	details on these classes examine the source files in each
 *	directory.
 *
 *	exception	Stack trace formatter
 *	kstat		Interface to Solaris kstat facility
 *	locality	Interface to Solaris lgrp facility
 *	logging		Interface to Solaris syslog facility
 *	pools		Interface to Solaris libpool facility
 *	timer		High resolution timestamps
 *
 *	domain:
 *
 *	These package reflect problem domain concepts and are
 *	responsible for application logic related to these
 *	concepts.
 *
 *	pools		Dynamic Resource Pools specific functionality.
 *
 * This code block will continue to explain in more detail how poold
 * is organized.
 *
 * poold provides the following basic facilities:
 *
 * Monitoring:
 *
 * Basic statistic access is provided by the
 * com.sun.solaris.service.kstat package. The following interfaces and
 * classes collaborate to provide higher level statistic and
 * monitoring facilities to the application:
 *
 *	INTERFACES
 *
 *	AggregateStatistic
 *	Monitor
 *	Statistic
 *	StatisticListener
 *
 *	CLASSES
 *
 *	AbstractStatistic
 *	DoubleStatistic
 *	LongStatistic
 *	ResourceMonitor
 *	StatisticEvent
 *	StatisticList
 *	StatisticOperations
 *	SystemMonitor
 *	UnsignedInt64Statistic
 *
 * Logging:
 *
 * Logging services are provided by the com.sun.solaris.logging
 * package. In addition, the following class implements Poold's
 * specific logging requirements.
 *
 *	CLASSES
 *
 *	Poold
 *
 * Optimization:
 *
 * lgrp service are provided by the com.sun.solaris.service.lgrp
 * package. pools services are provided by the
 * com.sun.solaris.service.pools package. In addition, optimization is
 * implemented in the following Interfaces and Classes:
 *
 *	INTERFACES
 *
 *	Objective
 *	Solver
 *	WorkloadDependentObjective
 *
 *	CLASSES
 *
 *	AbstractObjective
 *	ComponentMove
 *	DecisionHistory
 *	Expression
 *	KExpression
 *	KVEpression
 *	KVOpExpression
 *	LocalityObjective
 *	Move
 *	Poold
 *	QuantityMove
 *	SystemSolver
 *	UtilizationObjective
 *	WeightedLoadObjective
 *
 * Configuration:
 *
 * pools services are provided by the com.sun.solaris.service.pools
 * package, this is used to read poold configuration details from a
 * libpool configuration. In addition, configuration is implemented in
 * the following Classes:
 *
 *	CLASSES
 *
 *	AbstractObjective
 *	Expression
 *	Poold
 *	SystemSolver
 *
 * (NB: Some classes were mentioned in multiple categories where there
 * responsbilities overlap. Inner classes are not listed as their
 * responsibilities can be clearly inferred from their context.)
 *
 * For more details on any of the packages, classes or interfaces
 * mentioned above, look at the documentation associated with each
 * class.
 */

/**
 * The <code>Poold</code> class implements a dynamic resource
 * allocation system for Solaris.
 *
 * <code>Poold</code> is a monitoring daemon, designed to evaluate
 * user specified objectives, monitor workload behaviour and
 * dynamically assign resources in order to satisfy the evaluated
 * objectives. For more details see:
 *
 * <a href="http://sac.eng.sun.com/PSARC/2002/287/">PSARC/2002/287</a>
 */
final class Poold
{
	/**
	 * The configuration which is manipulated.
	 */
	private Configuration conf;

	/**
	 * The monitoring interface.
	 */
	private Monitor monitor;

	/**
	 * The interface to higher level resource managers.
	 */
	private DRM drm;

	/**
	 * The interface to the configuration solver.
	 */
	private Solver solver;

	/**
	 * Default path to the logging properties file.
	 */
	public static final String POOLD_PROPERTIES_PATH =
	    "/usr/lib/pool/poold.properties";

	/**
	 * Logger for records which aren't produced in the Monitoring,
	 * Configuration, or Optimization states.  This logger is the
	 * parent to the loggers used in those states.
	 */
	public static final Logger BASE_LOG = Logger.getLogger(
	    "com.sun.solaris.domain.pools.poold");

	/**
	 * Logger for records produced in the Configuration state.
	 */
	public static final Logger CONF_LOG = Logger.getLogger(
	    "com.sun.solaris.domain.pools.poold.Configuration");

	/**
	 * Logger for records produced in the Monitoring state.
	 */
	public static final Logger MON_LOG = Logger.getLogger(
	    "com.sun.solaris.domain.pools.poold.Monitoring");

	/**
	 * Logger for records produced in the Optimization state.
	 */
	public static final Logger OPT_LOG = Logger.getLogger(
	    "com.sun.solaris.domain.pools.poold.Optimization");

	/**
	 * Singleton instance of Poold.
	 */
	private static Poold instance;

	/**
	 * The main sampling and solving thread.
	 */
	private Thread mainThread;

	/**
	 * Process exit code indicating a failure.
	 */
	private static final int E_PO_FAILURE = 2;

	/**
	 * Keep track of whether initialize() has been invoked, to
	 * output the "starting" message on the first.
	 */
	private AtomicBoolean firstInitialization = new AtomicBoolean(true);

	/**
	 * Flags whether poold should run or exit.
	 */
	private AtomicBoolean shouldRun = new AtomicBoolean(true);

	private static class logHelper {
		/**
		 * Default logfile location
		 */
		public static final String DEF_LOG_LOC = "/var/log/pool/poold";

		/**
		 * Log location indicating <code>syslog</code>, as
		 * opposed to a file, should be used.
		 */
		public static final String SYSLOG_LOG_LOC = "SYSLOG";

		/**
		 * Default Log severity (if not overridden)
		 */
		public static final Severity DEF_SEVERITY = Severity.INFO;

		/**
		 * Name of configuration property, log location.
		 */
		public static final String PROPERTY_NAME_LOG_LOC =
		    "system.poold.log-location";

		/**
		 * Name of configuration property, log level.
		 */
		public static final String PROPERTY_NAME_LOG_LEVEL =
		    "system.poold.log-level";

		/**
		 * Location of logfile -- an absolute filename, or
		 * "SYSLOG".
		 */
		private static String location;

		/**
		 * Logfile handler, responsible for taking log messages
		 * and exporting them.
		 */
		private static Handler handler;

		/**
		 * Logfile severity, log messages below this severity are
		 * ignored.
		 */
		private static Severity severity;

		/**
		 * Flag recording whether preinitialization has occurred.
		 */
		private static boolean preinitialized = false;

		/**
		 * Flag recording whether the logging Severity has been
		 * overridden with the -l command-line option, which
		 * means the console is the only thing being logged to,
		 * and the configuration's logging properties are
		 * ignored.
		 */
		private static boolean usingConsole;

		/**
		 * Indicates whether logging semantics should be changed
		 * to facilitate debugging.
		 */
		private static final boolean loggingDebugging = false;

		/**
		 * Do the pre-initialization initialization:  install
		 * loggers for reporting errors during initialization.
		 *
		 * @param consoleSeverity If non-null, indicates that
		 * the configuration property-controlled logging behavior
		 * is to be overridden (the <code>-l</code> option was
		 * specified), and messages are to be logged only to the
		 * console, with (at most) the given maximum severity.
		 */
		private static void preinitialize(Severity consoleSeverity) {
			if (preinitialized)
				return;

			/*
			 * Read logging properties affecting the
			 * FileHandler and ConsoleHandler from
			 * <code>poold.properties</code>.
			 */
			Properties props = new Properties();
			ByteArrayOutputStream bos = new ByteArrayOutputStream();
			try {
				props.load(
				    new FileInputStream(POOLD_PROPERTIES_PATH));
				props.store(bos, "");
				LogManager.getLogManager().readConfiguration(
				    new ByteArrayInputStream(
				    bos.toByteArray()));
			} catch (IOException ioe) {
				Poold.MON_LOG.log(Severity.WARNING, "could not "
				    + "read logging properties from "
				    + "poold.properties: " + ioe);
			}

			if (consoleSeverity == null || loggingDebugging) {
				/*
				 * Log messages to /var/log/pool/poold,
				 * the default location, until the pools
				 * configuration properties are read and
				 * applied, which may change the logging
				 * file and severity.
				 *
				 * Under normal circumstances, it's
				 * expected that NO INFO-level messages
				 * will be emitted until that time; this
				 * is only a measure to ensure that
				 * unanticipated errors are reported in
				 * some log.
				 */
				location = SYSLOG_LOG_LOC;
				handler = SyslogHandler.getInstance("poold",
				    Facility.DAEMON);
				severity = DEF_SEVERITY;
				handler.setLevel(severity);
				BASE_LOG.addHandler(handler);
			}

			if (consoleSeverity != null) {
				/*
				 * If -l is specified, log to the
				 * console.  Unless loggingDebug is
				 * true, this will also mean that the
				 * logging properties are ignored.
				 *
				 * Determine if the user has specified
				 * the use of a ConsoleHandler through
				 * poold.properties.
				 */
				Logger root = Logger.getLogger("");
				Handler[] handler = root.getHandlers();
				ConsoleHandler ch = null;
				for (int i = 0; i < handler.length && ch ==
				    null; i++)
					if (handler[i]
					    instanceof ConsoleHandler)
						ch = (ConsoleHandler)handler[i];
				/*
				 * If none was previously, install a
				 * ConsoleHandler.
				 */
				if (ch == null) {
					ch = new ConsoleHandler();

					ch.setFormatter(
					    new SysloglikeFormatter());
					ch.setLevel(consoleSeverity);
					root.addHandler(ch);
				}
				severity = consoleSeverity;
				BASE_LOG.log(Severity.DEBUG,
				    "logging with level " + severity);

				/**
				 * Allow logging properties to be
				 * effective if loggingDebugging is not
				 * set.
				 */
				if (!loggingDebugging)
					usingConsole = true;
			}
			preinitialized = true;
		}

		/**
		 * Configure loggers based on the logging-related
		 * configuration properties.  Outputs a description of
		 * any changes to the configuration logger.
		 *
		 * @throws ConfigurationException if there is an error
		 * applying libpool configuration properties to
		 * <code>poold</code>
		 */
		public static void initializeWithConfiguration(
		    Configuration conf) throws ConfigurationException
		{
			String newLogLocation;
			Severity newLogSeverity;
			String newLogSeverityName = null;

			/*
			 * Set the log location as specified by the
			 * configuration's system properties.
			 */
			try {
				newLogLocation = conf.getStringProperty(
				    PROPERTY_NAME_LOG_LOC);
			} catch (PoolsException e) {
				newLogLocation = DEF_LOG_LOC;
			}

			try {
				newLogSeverityName = conf.getStringProperty(
				    PROPERTY_NAME_LOG_LEVEL);
				newLogSeverity = Severity.getSeverityWithName(
				    newLogSeverityName);
				assert(newLogSeverity != null);
			} catch (PoolsException e) {
				newLogSeverity = DEF_SEVERITY;
			} catch (IllegalArgumentException e) {
				throw(ConfigurationException)
				    (new ConfigurationException(
				    "invalid " + PROPERTY_NAME_LOG_LEVEL +
				    "value: " + newLogSeverityName)
				    .initCause(e));
			}

			Handler newLogHandler = null;

			/*
			 * (Re)install the logger for the poold class
			 * hierarchy.  This means that only poold
			 * messages are controlled by the pools
			 * configuration properties/command-line
			 * options.
			 */

			/*
			 * The logfile is always re-opened, in case the
			 * cause for reinitialization is due to SIGHUP
			 * following a log rotation.
			 */
			if (handler != null) {
				BASE_LOG.removeHandler(handler);
				handler.close();
				handler = null;
			}

			if (newLogLocation.toUpperCase().equals(
			    SYSLOG_LOG_LOC.toUpperCase()))
				newLogHandler =
				    SyslogHandler.getInstance("poold",
				    Facility.DAEMON);
			else {
				if (!newLogLocation.startsWith("/"))
					throw
					    new ConfigurationException(
						PROPERTY_NAME_LOG_LOC +
						" value is not an" +
						" absolute path");
				try {
					newLogHandler =
					    new FileHandler(
					    newLogLocation, 0, 1, true);
					newLogHandler.setFormatter(
					    new SysloglikeFormatter());
				} catch (java.io.IOException ioe) {
					Poold.utility.die(
					    Poold.CONF_LOG,
					    new PooldException(
					    newLogLocation +
					    ": can't write")
					    .initCause(ioe), false);
				}
			}

			if (!severity.equals(newLogSeverity) ||
			    !location.equals(newLogLocation))
				CONF_LOG.log(Severity.DEBUG,
				    "logging with level " + severity);

			severity = newLogSeverity;
			handler = newLogHandler;
			location = newLogLocation;

			handler.setLevel(severity);
			BASE_LOG.addHandler(handler);
		}

		/**
		 * Configure the loggers based on the pool's logging
		 * properties, or, if the -l option was specified on the
		 * command line, continue to use the console.
		 */
		public static void initialize(Configuration conf)
		    throws ConfigurationException
		{
			if (usingConsole)
				return;
			else
				initializeWithConfiguration(conf);
		}

		/**
		 * Return the current logging level.
		 */
		public static Severity getSeverity()
		{
			return (severity);
		}

		public static void close()
		{
			if (handler != null) {
				BASE_LOG.removeHandler(handler);
				handler.close();
			}
		}
	}

	/**
	 * Constructor
	 *
	 * Only one poold instance should be running per system.
	 *
	 * @param consoleSeverity If non-null, indicates that the
	 * configuration property-controlled logging behavior is to be
	 * overridden (the <code>-l</code> option was specified), and
	 * messages are to be logged only to the console, with (at most)
	 * the given maximum severity.
	 */
	private Poold(Severity consoleSeverity)
	{
		/*
		 * Establish loggers for recording errors during
		 * initialization.  Under normal circumstances, no
		 * messages will be emitted; this is only a measure to
		 * make sure that unanticipated errors are reported in
		 * some log, or the console, if the -l option is used.
		 */
		logHelper.preinitialize(consoleSeverity);

		/*
		 * Try opening the configuration read-write in hopes the
		 * ability will be possessed henceforth.
		 */
		try {
			conf = new Configuration(PoolInternal.
			    pool_dynamic_location(), PoolInternal.PO_RDWR);
			conf.close();
		} catch (PoolsException pe) {
			Poold.utility.die(CONF_LOG, new PooldException(
			    "cannot open dynamic pools configuration " +
			     "read-write (" + pe.getMessage() + ")")
			     .initCause(pe), false);
		}

		try {
			conf = new Configuration(PoolInternal.
			    pool_dynamic_location(), PoolInternal.PO_RDONLY);
		} catch (PoolsException pe) {
			Poold.utility.die(CONF_LOG, pe);
		}

		/*
		 * Create the required sub-components:
		 * - a monitoring object
		 * - a DRM implementer
		 * - a solver
		 */
		monitor = new SystemMonitor();
		drm = new LogDRM();
		solver = new SystemSolver(monitor);
	}

	/**
	 * Returns a reference to the singleton <code>Poold</code>,
	 * constructing one if necessary.
	 *
	 * @param consoleSeverity If non-null, indicates that the
	 * configuration property-controlled logging behavior is to be
	 * overridden (the <code>-l</code> option was specified), and
	 * messages are to be logged only to the console, with (at most)
	 * the given maximum severity.
	 * @throws IllegalArgumentException if the given console
	 * severity doesn't match that of an existing instance.
	 */
	public static Poold getInstanceWithConsoleLogging(
	    Severity consoleSeverity)
	{
		if (instance == null)
			return (instance = new Poold(consoleSeverity));
		else
			if (logHelper.usingConsole == false &&
			    consoleSeverity == null || consoleSeverity !=
			    null && logHelper.getSeverity().equals(
			    consoleSeverity))
				return (instance);
			else
				throw new IllegalArgumentException();
	}

	/**
	 * Initializes <code>Poold</code> for operation at startup or
	 * in response to a detected libpool configuration change.
	 */
	private void initialize()
	{
		try {
			logHelper.initialize(conf);
			if (firstInitialization.get())
				CONF_LOG.log(Severity.INFO, "starting");
			/*
			 * When a system is extremely busy, it may
			 * prove difficult to initialize poold. Just
			 * keep trying until we succeed.
			 */
			boolean busy = true;
			while (busy && shouldRun.get()) {
				busy = false;
				try {
					monitor.initialize(conf);
					CONF_LOG.log(Severity.DEBUG,
					    "configuring solver...");
					solver.initialize(conf);
					CONF_LOG.log(Severity.INFO,
					    "configuration complete");
				} catch (PoolsException pe) {
					CONF_LOG.log(Severity.INFO,
					    "The system is too busy to " +
					    "initialize, attempting " +
					    "initialization again");
					/*
					 * pause for a while before
					 * re-attempting the
					 * initialization.
					 */
					try {
						Thread.sleep(50);
					} catch (InterruptedException ie) {
						/*
						 * Safe to ignore this
						 * exception as we
						 * will simply try
						 * again sooner.
						 */
					}
					busy = true;
				} catch (StaleMonitorException sme) {
					CONF_LOG.log(Severity.INFO,
					    "The system is too busy to " +
					    "initialize, attempting " +
					    "initialization again");
					/*
					 * pause for a while before
					 * re-attempting the
					 * initialization.
					 */
					try {
						Thread.sleep(50);
					} catch (InterruptedException ie) {
						/*
						 * Safe to ignore this
						 * exception as we
						 * will simply try
						 * again sooner.
						 */
					}
					busy = true;
				}
			}
			if (firstInitialization.get())
				firstInitialization.set(false);
		} catch (ConfigurationException ce) {
			Poold.utility.die(CONF_LOG, ce);
		}
	}

	/**
	 * Execute <code>Poold</code> indefinitely. This method is
	 * invoked after <code>Poold</code> completes
	 * configuration. It will continue to execute until
	 * <code>Poold</code> is terminated.
	 *
	 * @throws Exception If an there is an error in execution.
	 */
	private void execute() throws Exception
	{
		int changed = 0;
		boolean confRequired = false;

		while (shouldRun.get()) {
			try {
				changed = conf.update();
				assert(!confRequired || confRequired &&
				    changed != 0);
				if (changed != 0) {
					CONF_LOG.log(Severity.DEBUG,
					    "configuration change detected");
					if (!confRequired)
						CONF_LOG.log(Severity.INFO,
						    "configuration changed " +
						    "externally");
					CONF_LOG.log(Severity.INFO,
					    "reconfiguring...");
				}
				confRequired = false;
			} catch (PoolsException pe) {
				Poold.utility.die(CONF_LOG, pe);
			}
			if (changed != 0)
				initialize();

			boolean gotNext = false;
			while (shouldRun.get() && !gotNext) {
				try {
					monitor.getNext();
					gotNext = true;

					/*
					 * All workload-dependent
					 * objectives must now be
					 * checked for violations.  The
					 * solver holds all objectives
					 * and it makes the decision
					 * about whether a
					 * reconfiguration is required.
					 */
					if (solver.examine(monitor)) {
						MON_LOG.log(Severity.INFO,
						    "reconfiguration required");
						confRequired = solver.solve();
					} else {
						MON_LOG.log(Severity.INFO,
						    "all evaluated objectives"
						    + " satisfied");
					}
				} catch (StaleMonitorException e) {
					/*
					 * Later, assert that every
					 * cause of the
					 * StaleMonitorException is
					 * handled by the above
					 * conf.update().
					 */
					confRequired = true;
				} catch (InterruptedException ie) {
					Poold.MON_LOG.log(Severity.INFO,
					    "interrupted");
					break;
				}
			}
			if (!shouldRun.get())
				break;
			System.runFinalization();
		}
		Poold.BASE_LOG.log(Severity.NOTICE, "exiting");
	}

	/**
	 * Cleanup any resources when the application terminates.
	 */
	private void cleanup()
	{
		conf.close();
		logHelper.close();
                instance = null;
	}

	/**
	 * Invoke <code>Poold</code>. This main function is provided
	 * so that <code>poold</code> can be executed.  Execution will
	 * continue indefinitely unless there is an error, in which case
	 * when execute() terminates.
	 */
	public void run()
	{
		mainThread = Thread.currentThread();

		Runtime.getRuntime().addShutdownHook(new Thread() {
			public void run() {
				try {
					cleanup();
				} catch (Throwable t) {
				}
			}
		});

		try {
			initialize();
			execute();
		} catch (Throwable t) {
			Poold.utility.die(BASE_LOG, t);
		}
	}

	/**
	 * Stops <code>Poold</code>.  Sets a flag indicating that run()
	 * should break out of the monitor/solve loop and clean up.
	 */
	public void shutdown() {
		/*
		 * Flag that the main thread should break out of the
		 * sample/solve loop as soon as possible.
		 */
		shouldRun.set(false);

		/*
		 * Interrupt the main thread, which will cause the
		 * monitor to break out of its sleep if it's waiting for
		 * the sample time to arrive, and cause the sample/solve
		 * loop condition to be evaluated sooner.  But make some
		 * effort not to cause an InterruptedIOException if
		 * we're not even through initialization yet; we'll get
		 * around to shutting down soon enough.
		 */
		if (!firstInitialization.get() && mainThread != null)
			mainThread.interrupt();
	}

	public static void main(String args[]) throws IllegalArgumentException
	{
		Severity severity = null;

		if (args.length > 0) {
			if (args[0].compareTo("-l") == 0 && args.length == 2) {
				severity = (Severity) Severity.parse(args[1]);
			} else
				throw new IllegalArgumentException(
				    "usage: poold [-l level]");
		}
		Poold p = getInstanceWithConsoleLogging(severity);
		p.run();
	}

	/**
	 * The <code>utility</code> class provides various
	 * <code>Poold</code> related static utility methods that
	 * don't naturally reside on any class.
	 */
	static class utility {
		/**
		 * Outputs a near-final message corresponding
		 * to an exception (or other throwable) to the named
		 * logger before causing the VM to exit.  The message
		 * will have ERROR severity unless an instance of
		 * PoolsException is given, in which case the message
		 * will adopt the PoolsException's severity.  Similarly,
		 * if the PoolsException specifies an exit code, it will
		 * be used; otherwise the default of
		 * <code>E_PO_FAILURE</code> will be used.
		 *
		 * @param logger Logger used to log the message
		 * @param t The cause.  A stack trace will be affixed to
		 * the message.
		 */
		public static void die(Logger logger, Throwable t)
		{
			die(logger, t, true);
		}

		/**
		 * Outputs a near-final message corresponding
		 * to an exception (or other throwable) to the named
		 * logger before causing the VM to exit.  The message
		 * will have ERROR severity unless an instance of
		 * PoolsException is given, in which case the message
		 * will adopt the PoolsException's severity.  Similarly,
		 * if the PoolsException specifies an exit code, it will
		 * be used; otherwise the default of
		 * <code>E_PO_FAILURE</code> will be used.
		 *
		 * @param logger Logger used to log the message
		 * @param t The cause.
		 * @param showStackTrace If true, a stack trace will be
		 * affixed to the message.
		 */
		public static void die(Logger logger, Throwable t,
		    boolean showStackTrace)
		{
			try {
				Severity severity;
				/*
				 * Configure the message's exception and
				 * severity.
				 */
				LogRecord record;
				if (t instanceof PooldException)
					record = new LogRecord(
					    ((PooldException)t).getSeverity(),
					    t.getMessage());
				else
					record = new LogRecord(Severity.ERR,
					    t.getMessage());
				if (record.getMessage() == null)
					record.setMessage("exception " +
					    t.getClass().getName());
				if (showStackTrace)
					record.setThrown(t);

				record.setLoggerName(logger.getName());
				logger.log(record);

				if (logHelper.handler != null)
					logHelper.handler.flush();
				if (t instanceof PooldException)
					System.exit(((PooldException)t)
					    .getExitCode());
				else
					System.exit(E_PO_FAILURE);
			} catch (Exception e) {
				SuccinctStackTraceFormatter.printStackTrace(e);
				System.exit(-1);
			}
		}

		/**
		 * Outputs a warning-level message to the named logger.
		 *
		 * @param logger Logger used to log the message
		 * @param t The cause.
		 * @param showStackTrace If true, a stack trace will be
		 * affixed to the message.
		 */
		public static void warn(Logger logger, Throwable t,
		    boolean showStackTrace)
		{
			try {
				Severity severity;
				/*
				 * Configure the message's exception and
				 * severity.
				 */
				LogRecord record;
				if (t instanceof PooldException)
					record = new LogRecord(
					    ((PooldException)t).getSeverity(),
					    t.getMessage());
				else
					record = new LogRecord(Severity.WARNING,
					    t.getMessage());
				if (record.getMessage() == null)
					record.setMessage("exception " +
					    t.getClass().getName());
				if (showStackTrace)
					record.setThrown(t);

				record.setLoggerName(logger.getName());
				logger.log(record);

				if (logHelper.handler != null)
					logHelper.handler.flush();
			} catch (Exception e) {
				SuccinctStackTraceFormatter.printStackTrace(e);
				System.exit(-1);
			}
		}
	}
}

class ConfigurationException extends Exception
{
	public ConfigurationException(String message)
	{
		super(message);
	}
}

class IllegalOFValueException extends RuntimeException
{
	public IllegalOFValueException(String message)
	{
		super(message);
	}
}

class PooldException extends Exception {
	/**
	 * The exit code which the virtual machine should exit at if
	 * this exception cannot be handled.
	 */
	private int exitCode;

	/**
	 * The severity of this message.  See <code>Severity</code>.
	 */
	private Severity severity;

	/**
	 * Constructs a message with default exit code and
	 * <code>System.ERR</code> severity.
	 */
	public PooldException(String message)
	{
		this(message, 1, Severity.ERR);
	}

	/**
	 * Constructs a message with given exit code and
	 * <code>System.ERR</code> severity.
	 */
	public PooldException(String message, int exitCode)
	{
		this(message, exitCode, Severity.ERR);
	}

	/**
	 * Constructs a message with a given exit code and severity.
	 */
	public PooldException(String message, int exitCode, Severity severity)
	{
		super(message);
		this.exitCode = exitCode;
		this.severity = severity;
	}

	/**
	 * The exit code which the virtual machine should exit at if
	 * this exception cannot be handled.
	 */
	public int getExitCode()
	{
		return (exitCode);
	}

	public Severity getSeverity()
	{
		return (severity);
	}
}