gsg_txn/JAVA/filemanagement.html

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>
    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
    <title>Chapter 5. Managing DB Files</title>
    <link rel="stylesheet" href="gettingStarted.css" type="text/css" />
    <meta name="generator" content="DocBook XSL Stylesheets V1.73.2" />
    <link rel="start" href="index.html" title="Getting Started with Berkeley DB Transaction Processing" />
    <link rel="up" href="index.html" title="Getting Started with Berkeley DB Transaction Processing" />
    <link rel="prev" href="reversesplit.html" title="Reverse BTree Splits" />
    <link rel="next" href="backuprestore.html" title="Backup Procedures" />
  </head>
  <body>
    <div class="navheader">
      <table width="100%" summary="Navigation header">
        <tr>
          <th colspan="3" align="center">Chapter 5. Managing DB Files</th>
        </tr>
        <tr>
          <td width="20%" align="left"><a accesskey="p" href="reversesplit.html">Prev</a> </td>
          <th width="60%" align="center"> </th>
          <td width="20%" align="right"> <a accesskey="n" href="backuprestore.html">Next</a></td>
        </tr>
      </table>
      <hr />
    </div>
    <div class="chapter" lang="en" xml:lang="en">
      <div class="titlepage">
        <div>
          <div>
            <h2 class="title"><a id="filemanagement"></a>Chapter 5. Managing DB Files</h2>
          </div>
        </div>
      </div>
      <div class="toc">
        <p>
          <b>Table of Contents</b>
        </p>
        <dl>
          <dt>
            <span class="sect1">
              <a href="filemanagement.html#checkpoints">Checkpoints</a>
            </span>
          </dt>
          <dt>
            <span class="sect1">
              <a href="backuprestore.html">Backup Procedures</a>
            </span>
          </dt>
          <dd>
            <dl>
              <dt>
                <span class="sect2">
                  <a href="backuprestore.html#copyutilities">About Unix Copy Utilities</a>
                </span>
              </dt>
              <dt>
                <span class="sect2">
                  <a href="backuprestore.html#standardbackup">Offline Backups</a>
                </span>
              </dt>
              <dt>
                <span class="sect2">
                  <a href="backuprestore.html#hotbackup">Hot Backup</a>
                </span>
              </dt>
              <dt>
                <span class="sect2">
                  <a href="backuprestore.html#incrementalbackups">Incremental Backups</a>
                </span>
              </dt>
            </dl>
          </dd>
          <dt>
            <span class="sect1">
              <a href="recovery.html">Recovery Procedures</a>
            </span>
          </dt>
          <dd>
            <dl>
              <dt>
                <span class="sect2">
                  <a href="recovery.html#normalrecovery">Normal Recovery</a>
                </span>
              </dt>
              <dt>
                <span class="sect2">
                  <a href="recovery.html#catastrophicrecovery">Catastrophic Recovery</a>
                </span>
              </dt>
            </dl>
          </dd>
          <dt>
            <span class="sect1">
              <a href="architectrecovery.html">Designing Your Application for Recovery</a>
            </span>
          </dt>
          <dd>
            <dl>
              <dt>
                <span class="sect2">
                  <a href="architectrecovery.html#multithreadrecovery">Recovery for Multi-Threaded Applications</a>
                </span>
              </dt>
              <dt>
                <span class="sect2">
                  <a href="architectrecovery.html#multiprocessrecovery">Recovery in Multi-Process Applications</a>
                </span>
              </dt>
            </dl>
          </dd>
          <dt>
            <span class="sect1">
              <a href="hotfailover.html">Using Hot Failovers</a>
            </span>
          </dt>
          <dt>
            <span class="sect1">
              <a href="logfileremoval.html">Removing Log Files</a>
            </span>
          </dt>
          <dt>
            <span class="sect1">
              <a href="logconfig.html">Configuring the Logging Subsystem</a>
            </span>
          </dt>
          <dd>
            <dl>
              <dt>
                <span class="sect2">
                  <a href="logconfig.html#logfilesize">Setting the Log File Size</a>
                </span>
              </dt>
              <dt>
                <span class="sect2">
                  <a href="logconfig.html#logregionsize">Configuring the Logging Region Size</a>
                </span>
              </dt>
              <dt>
                <span class="sect2">
                  <a href="logconfig.html#inmemorylogging">Configuring In-Memory Logging</a>
                </span>
              </dt>
              <dt>
                <span class="sect2">
                  <a href="logconfig.html#logbuffer">Setting the In-Memory Log Buffer Size</a>
                </span>
              </dt>
            </dl>
          </dd>
        </dl>
      </div>
      <p>
        DB is capable of storing several types of files on disk:
    </p>
      <div class="itemizedlist">
        <ul type="disc">
          <li>
            <p>
                Data files, which contain the actual data in your database.
            </p>
          </li>
          <li>
            <p>
                Log files, which contain information required to recover your
                database in the event of a system or application failure.
            </p>
          </li>
          <li>
            <p>
                Region files, which contain information necessary for the
                overall operation of your application.
            </p>
          </li>
          <li>
            <p>
                Temporary files, which are created only under certain special circumstances. These files never need to
                be backed up or otherwise managed and so they are not a consideration for the topics described in this
                chapter. See <a class="xref" href="enabletxn.html#security" title="Security Considerations">Security Considerations</a>
                for more information on temporary files.
            </p>
          </li>
        </ul>
      </div>
      <p>
        Of these, you must manage your data and log files by ensuring that they
        are backed up. You should also pay attention to the amount of disk space
        your log files are consuming, and periodically remove any unneeded
        files. Finally, you can optionally tune your logging subsystem to best
        suit your application's needs and requirements.
        These topics are discussed in this chapter.
    </p>
      <div class="sect1" lang="en" xml:lang="en">
        <div class="titlepage">
          <div>
            <div>
              <h2 class="title" style="clear: both"><a id="checkpoints"></a>Checkpoints</h2>
            </div>
          </div>
        </div>
        <p>
            Before we can discuss DB file management, we need to
            describe checkpoints. When databases are modified (that is, a
            transaction is committed), the modifications are recorded in
            DB's logs, but they are <span class="emphasis"><em>not</em></span>
            necessarily reflected in the actual database files on disk.
        </p>
        <p>
            This means that as time goes on, increasingly
            more data is contained in your log files that is not
            contained in your data files. As a result, you must keep more
            log files around than you might actually need. Also, any
            recovery run from your log files will take increasingly longer
            amounts of time, because there is more data in the log files
            that must be reflected back into the data files during the
            recovery process.
        </p>
        <p>
            You can reduce these problems by periodically
            running a checkpoint against your environment. The checkpoint:
        </p>
        <div class="itemizedlist">
          <ul type="disc">
            <li>
              <p>
                    Flushes dirty pages from the in-memory cache. This means that data modifications found in your
                    in-memory cache are written to the database files on disk. Note that a checkpoint also causes data
                    dirtied by an uncommitted transaction to also be written to your database files on disk. In this latter
                    case, DB's normal recovery is used to remove any such modifications that were subsequently
                    abandoned by your application using a transaction abort.
                </p>
              <p>
                    Normal recovery is describe in <a class="xref" href="recovery.html" title="Recovery Procedures">Recovery Procedures</a>.
                </p>
            </li>
            <li>
              <p>
                    Writes a checkpoint record.
                </p>
            </li>
            <li>
              <p>
                    Flushes the log. This causes all log data that has not yet been written to disk to be written.
                </p>
            </li>
            <li>
              <p>
                    Writes a list of open databases.
                </p>
            </li>
          </ul>
        </div>
        <p>
            There are several ways to run a checkpoint. One way is to use
            the <span class="command"><strong>db_checkpoint</strong></span> command line utility. (Note, however, that this command line utility
            cannot be used if your environment was opened using

                <span>
                    <code class="methodname">EnvironmentConfig.setPrivate()</code>.)
                </span>
         </p>
        <p>
            You can also run a thread that periodically checkpoints your
            environment for you by calling the


                <code class="methodname">Environment.checkpoint()</code>
            method.
        </p>
        <p>
            Note that you can prevent a checkpoint from occurring unless more
            than a specified amount of log data has been written since the
            last checkpoint. You can also prevent the checkpoint from
            running unless more than a specified amount of time has
            occurred since the last checkpoint. These conditions are
            particularly interesting if you have multiple threads
                <span>or processes</span>
            running checkpoints.
        </p>
        <p>
            For configuration information, see the


                <a class="ulink" href="http://www.oracle.com/technology/documentation/berkeley-db/db/java/com/sleepycat/db/CheckpointConfig.html" target="_top">
                    CheckpointConfig Javadoc page.
                </a>
        </p>
        <p>
        Note that running checkpoints can be quite expensive. DB must
        flush every dirty page to the backing database files. On the
        other hand, if you do not run checkpoints often enough, your
        recovery time can be unnecessarily long and you may be using more
        disk space than you really need. Also, you cannot remove log files
        until a checkpoint is run. Therefore, deciding how frequently
        to run a checkpoint is one of the most
        common tuning activity for DB applications.
    </p>
        <p>
    For example, the following class performs a checkpoint every 60 seconds, so long as 500 kb of logging data has been
    written since the last checkpoint:
</p>
        <pre class="programlisting"><span>package db.txn;</span>

import com.sleepycat.db.CheckpointConfig;
import com.sleepycat.db.DatabaseException;
import com.sleepycat.db.Environment;

public class CheckPointer extends Thread
{
    private CheckpointConfig cpc = new CheckpointConfig();
    private Environment myEnv = null;
    private static boolean canRun = true;


    // Constructor.
    CheckPointer(Environment env) {
        myEnv = env;
        // Run a checkpoint only if 500 kbytes of log data has been
        // written.
        cpc.setKBytes(500);
    }

    // Thread method that performs a checkpoint every
    // 60 seconds
    public void run () {
        while (canRun) {
            try {
                myEnv.checkpoint(cpc);
                sleep(60000);
            } catch (DatabaseException de) {
                System.err.println("Checkpoint error: " +
                    de.toString());
            } catch (InterruptedException e) {
                // Should never get here
                System.err.println("got interrupted exception");
            }
        }
    }

   public static void stopRunning() {
        canRun = false;
   }
} </pre>
        <p>
    And you use this class as follows. Note that we add the call to shutdown the checkpoint thread in our application's
    shutdown code:
</p>
        <pre class="programlisting">package db.txn;

import com.sleepycat.db.DatabaseException;
import com.sleepycat.db.Environment;
import com.sleepycat.db.EnvironmentConfig;

import java.io.File;
import java.io.FileNotFoundException;


public class TryCheckPoint {

    private static String myEnvPath = "./";

    private static Environment myEnv = null;


    private static void usage() {
        System.out.println("TxnGuide [-h &lt;env directory&gt;]");
        System.exit(-1);
    }

    public static void main(String args[]) {
        try {
            // Parse the arguments list
            parseArgs(args);
            // Open the environment and databases
            openEnv();

            // Start the checkpoint thread
            CheckPointer cp = new CheckPointer(myEnv);
            cp.start();

            //////////////////////////////////
            // Do database work here as normal
            //////////////////////////////////

            // Once all database work is completed, stop the checkpoint
            // thread.
            CheckPointer.stopRunning();

            // Join the checkpoint thread in case it needs some time to
            // cleanly shutdown.
            cp.join();

        } catch (Exception e) {
            System.err.println("TryCheckPoint: " + e.toString());
            e.printStackTrace();
        } finally {
            closeEnv();
        }
        System.out.println("All done.");
    }

    // Open an environment and databases
    private static void openEnv() throws DatabaseException {
        System.out.println("opening env");

        // Set up the environment.
        EnvironmentConfig myEnvConfig = new EnvironmentConfig();
        myEnvConfig.setAllowCreate(true);
        myEnvConfig.setInitializeCache(true);
        myEnvConfig.setInitializeLocking(true);
        myEnvConfig.setInitializeLogging(true);
        myEnvConfig.setTransactional(true);
        // EnvironmentConfig.setThreaded(true) is the default behavior
        // in Java, so we do not have to do anything to cause the
        // environment handle to be free-threaded.

        try {
            // Open the environment
            myEnv = new Environment(new File(myEnvPath),    // Env home
                                    myEnvConfig);

            // Skipping the database opens and closes for brevity

        } catch (FileNotFoundException fnfe) {
            System.err.println("openEnv: " + fnfe.toString());
            System.exit(-1);
        }
    }

    // Close the environment and databases
    private static void closeEnv() {
        System.out.println("Closing env");
        if (myEnv != null ) {
            try {
                myEnv.close();
            } catch (DatabaseException e) {
                System.err.println("closeEnv: " + e.toString());
                e.printStackTrace();
            }
        }
    }

    private TryCheckPoint() {}

    private static void parseArgs(String args[]) {
        for(int i = 0; i &lt; args.length; ++i) {
            if (args[i].startsWith("-")) {
                switch(args[i].charAt(1)) {
                    case 'h':
                        myEnvPath = new String(args[++i]);
                    break;
                    default:
                        usage();
                }
            }
        }
    }
} </pre>
      </div>
    </div>
    <div class="navfooter">
      <hr />
      <table width="100%" summary="Navigation footer">
        <tr>
          <td width="40%" align="left"><a accesskey="p" href="reversesplit.html">Prev</a> </td>
          <td width="20%" align="center"> </td>
          <td width="40%" align="right"> <a accesskey="n" href="backuprestore.html">Next</a></td>
        </tr>
        <tr>
          <td width="40%" align="left" valign="top">Reverse BTree Splits </td>
          <td width="20%" align="center">
            <a accesskey="h" href="index.html">Home</a>
          </td>
          <td width="40%" align="right" valign="top"> Backup Procedures</td>
        </tr>
      </table>
    </div>
  </body>
</html>