1/*-
2 * See the file LICENSE for redistribution information.
3 *
4 * Copyright (c) 2002,2008 Oracle.  All rights reserved.
5 *
6 * $Id: Store.java,v 1.5 2008/02/18 14:48:11 mark Exp $
7 */
8
9package com.sleepycat.persist.impl;
10
11import java.io.FileNotFoundException;
12import java.util.ArrayList;
13import java.util.Comparator;
14import java.util.HashMap;
15import java.util.HashSet;
16import java.util.IdentityHashMap;
17import java.util.List;
18import java.util.Map;
19import java.util.Set;
20import java.util.WeakHashMap;
21
22import com.sleepycat.bind.EntityBinding;
23import com.sleepycat.bind.tuple.StringBinding;
24import com.sleepycat.compat.DbCompat;
25import com.sleepycat.db.Cursor;
26import com.sleepycat.db.CursorConfig;
27import com.sleepycat.db.Database;
28import com.sleepycat.db.DatabaseConfig;
29import com.sleepycat.db.DatabaseEntry;
30import com.sleepycat.db.DatabaseException;
31import com.sleepycat.db.Environment;
32import com.sleepycat.db.ForeignKeyDeleteAction;
33import com.sleepycat.db.OperationStatus;
34import com.sleepycat.db.SecondaryConfig;
35import com.sleepycat.db.SecondaryDatabase;
36import com.sleepycat.db.Sequence;
37import com.sleepycat.db.SequenceConfig;
38import com.sleepycat.db.Transaction;
39import com.sleepycat.persist.DatabaseNamer;
40import com.sleepycat.persist.PrimaryIndex;
41import com.sleepycat.persist.SecondaryIndex;
42import com.sleepycat.persist.StoreConfig;
43import com.sleepycat.persist.evolve.Converter;
44import com.sleepycat.persist.evolve.EvolveConfig;
45import com.sleepycat.persist.evolve.EvolveEvent;
46import com.sleepycat.persist.evolve.EvolveInternal;
47import com.sleepycat.persist.evolve.EvolveListener;
48import com.sleepycat.persist.evolve.EvolveStats;
49import com.sleepycat.persist.evolve.Mutations;
50import com.sleepycat.persist.model.ClassMetadata;
51import com.sleepycat.persist.model.DeleteAction;
52import com.sleepycat.persist.model.EntityMetadata;
53import com.sleepycat.persist.model.EntityModel;
54import com.sleepycat.persist.model.FieldMetadata;
55import com.sleepycat.persist.model.ModelInternal;
56import com.sleepycat.persist.model.PrimaryKeyMetadata;
57import com.sleepycat.persist.model.Relationship;
58import com.sleepycat.persist.model.SecondaryKeyMetadata;
59import com.sleepycat.persist.raw.RawObject;
60import com.sleepycat.util.keyrange.KeyRange;
61
62/**
63 * Base implementation for EntityStore and  RawStore.  The methods here
64 * correspond directly to those in EntityStore; see EntityStore documentation
65 * for details.
66 *
67 * @author Mark Hayes
68 */
69public class Store {
70
71    public static final String NAME_SEPARATOR = "#";
72    private static final String NAME_PREFIX = "persist" + NAME_SEPARATOR;
73    private static final String DB_NAME_PREFIX = "com.sleepycat.persist.";
74    private static final String CATALOG_DB = DB_NAME_PREFIX + "formats";
75    private static final String SEQUENCE_DB = DB_NAME_PREFIX + "sequences";
76
77    private static Map<Environment,Map<String,PersistCatalog>> catalogPool =
78        new WeakHashMap<Environment,Map<String,PersistCatalog>>();
79
80    /* For unit testing. */
81    private static SyncHook syncHook;
82
83    private Environment env;
84    private boolean locking;
85    private boolean rawAccess;
86    private PersistCatalog catalog;
87    private EntityModel model;
88    private Mutations mutations;
89    private StoreConfig storeConfig;
90    private String storeName;
91    private String storePrefix;
92    private Map<String,PrimaryIndex> priIndexMap;
93    private Map<String,SecondaryIndex> secIndexMap;
94    private Map<String,DatabaseConfig> priConfigMap;
95    private Map<String,SecondaryConfig> secConfigMap;
96    private Map<String,PersistKeyBinding> keyBindingMap;
97    private Map<String,Sequence> sequenceMap;
98    private Map<String,SequenceConfig> sequenceConfigMap;
99    private Database sequenceDb;
100    private IdentityHashMap<Database,Object> deferredWriteDatabases;
101    private Map<String,Set<String>> inverseRelatedEntityMap;
102
103    public Store(Environment env,
104                 String storeName,
105                 StoreConfig config,
106                 boolean rawAccess)
107        throws DatabaseException {
108
109        this.env = env;
110        this.storeName = storeName;
111        this.rawAccess = rawAccess;
112
113        if (env == null || storeName == null) {
114            throw new NullPointerException
115                ("env and storeName parameters must not be null");
116        }
117        if (config != null) {
118            model = config.getModel();
119            mutations = config.getMutations();
120        }
121        if (config == null) {
122            storeConfig = StoreConfig.DEFAULT;
123        } else {
124            storeConfig = config.cloneConfig();
125        }
126
127        locking = DbCompat.getInitializeLocking(env.getConfig());
128
129        storePrefix = NAME_PREFIX + storeName + NAME_SEPARATOR;
130        priIndexMap = new HashMap<String,PrimaryIndex>();
131        secIndexMap = new HashMap<String,SecondaryIndex>();
132        priConfigMap = new HashMap<String,DatabaseConfig>();
133        secConfigMap = new HashMap<String,SecondaryConfig>();
134        keyBindingMap = new HashMap<String,PersistKeyBinding>();
135        sequenceMap = new HashMap<String,Sequence>();
136        sequenceConfigMap = new HashMap<String,SequenceConfig>();
137        deferredWriteDatabases = new IdentityHashMap<Database,Object>();
138
139        if (rawAccess) {
140            /* Open a read-only catalog that uses the stored model. */
141            if (model != null) {
142                throw new IllegalArgumentException
143                    ("A model may not be specified when opening a RawStore");
144            }
145            DatabaseConfig dbConfig = new DatabaseConfig();
146            dbConfig.setReadOnly(true);
147            dbConfig.setTransactional
148                (storeConfig.getTransactional());
149            catalog = new PersistCatalog
150                (null, env, storePrefix, storePrefix + CATALOG_DB, dbConfig,
151                 model, mutations, rawAccess, this);
152        } else {
153            /* Open the shared catalog that uses the current model. */
154            synchronized (catalogPool) {
155                Map<String,PersistCatalog> catalogMap = catalogPool.get(env);
156                if (catalogMap == null) {
157                    catalogMap = new HashMap<String,PersistCatalog>();
158                    catalogPool.put(env, catalogMap);
159                }
160                catalog = catalogMap.get(storeName);
161                if (catalog != null) {
162                    catalog.openExisting();
163                } else {
164                    Transaction txn = null;
165                    if (storeConfig.getTransactional() &&
166			DbCompat.getThreadTransaction(env) == null) {
167                        txn = env.beginTransaction(null, null);
168                    }
169                    boolean success = false;
170                    try {
171                        DatabaseConfig dbConfig = new DatabaseConfig();
172                        dbConfig.setAllowCreate(storeConfig.getAllowCreate());
173                        dbConfig.setReadOnly(storeConfig.getReadOnly());
174                        dbConfig.setTransactional
175                            (storeConfig.getTransactional());
176                        DbCompat.setTypeBtree(dbConfig);
177                        catalog = new PersistCatalog
178                            (txn, env, storePrefix, storePrefix + CATALOG_DB,
179                             dbConfig, model, mutations, rawAccess, this);
180                        catalogMap.put(storeName, catalog);
181                        success = true;
182                    } finally {
183                        if (txn != null) {
184                            if (success) {
185                                txn.commit();
186                            } else {
187                                txn.abort();
188                            }
189                        }
190                    }
191                }
192            }
193        }
194
195        /* Get the merged mutations from the catalog. */
196        mutations = catalog.getMutations();
197
198        /*
199         * If there is no model parameter, use the default or stored model
200         * obtained from the catalog.
201         */
202        model = catalog.getResolvedModel();
203
204        /*
205         * Give the model a reference to the catalog to fully initialize the
206         * model.  Only then may we initialize the Converter mutations, which
207         * themselves may call model methods and expect the model to be fully
208         * initialized.
209         */
210        ModelInternal.setCatalog(model, catalog);
211        for (Converter converter : mutations.getConverters()) {
212            converter.getConversion().initialize(model);
213        }
214
215        /*
216         * For each existing entity with a relatedEntity reference, create an
217         * inverse map (back pointer) from the class named in the relatedEntity
218         * to the class containing the secondary key.  This is used to open the
219         * class containing the secondary key whenever we open the
220         * relatedEntity class, to configure foreign key constraints. Note that
221         * we do not need to update this map as new primary indexes are
222         * created, because opening the new index will setup the foreign key
223         * constraints. [#15358]
224         */
225        inverseRelatedEntityMap = new HashMap<String,Set<String>>();
226        List<Format> entityFormats = new ArrayList<Format>();
227        catalog.getEntityFormats(entityFormats);
228        for (Format entityFormat : entityFormats) {
229            EntityMetadata entityMeta = entityFormat.getEntityMetadata();
230            for (SecondaryKeyMetadata secKeyMeta :
231                 entityMeta.getSecondaryKeys().values()) {
232                String relatedClsName = secKeyMeta.getRelatedEntity();
233                if (relatedClsName != null) {
234                    Set<String> inverseClassNames =
235                        inverseRelatedEntityMap.get(relatedClsName);
236                    if (inverseClassNames == null) {
237                        inverseClassNames = new HashSet<String>();
238                        inverseRelatedEntityMap.put
239                            (relatedClsName, inverseClassNames);
240                    }
241                    inverseClassNames.add(entityMeta.getClassName());
242                }
243            }
244        }
245    }
246
247    public Environment getEnvironment() {
248        return env;
249    }
250
251    public StoreConfig getConfig() {
252        return storeConfig.cloneConfig();
253    }
254
255    public String getStoreName() {
256        return storeName;
257    }
258
259    public void dumpCatalog() {
260        catalog.dump();
261    }
262
263
264    public EntityModel getModel() {
265        return model;
266    }
267
268    public Mutations getMutations() {
269        return mutations;
270    }
271
272    /**
273     * A getPrimaryIndex with extra parameters for opening a raw store.
274     * primaryKeyClass and entityClass are used for generic typing; for a raw
275     * store, these should always be Object.class and RawObject.class.
276     * primaryKeyClassName is used for consistency checking and should be null
277     * for a raw store only.  entityClassName is used to identify the store and
278     * may not be null.
279     */
280    public synchronized <PK,E> PrimaryIndex<PK,E>
281        getPrimaryIndex(Class<PK> primaryKeyClass,
282                        String primaryKeyClassName,
283                        Class<E> entityClass,
284                        String entityClassName)
285        throws DatabaseException {
286
287        assert (rawAccess && entityClass == RawObject.class) ||
288              (!rawAccess && entityClass != RawObject.class);
289        assert (rawAccess && primaryKeyClassName == null) ||
290              (!rawAccess && primaryKeyClassName != null);
291
292        checkOpen();
293
294        PrimaryIndex<PK,E> priIndex = priIndexMap.get(entityClassName);
295        if (priIndex == null) {
296
297            /* Check metadata. */
298            EntityMetadata entityMeta = checkEntityClass(entityClassName);
299            PrimaryKeyMetadata priKeyMeta = entityMeta.getPrimaryKey();
300            if (primaryKeyClassName == null) {
301                primaryKeyClassName = priKeyMeta.getClassName();
302            } else {
303                String expectClsName =
304                    SimpleCatalog.keyClassName(priKeyMeta.getClassName());
305                if (!primaryKeyClassName.equals(expectClsName)) {
306                    throw new IllegalArgumentException
307                        ("Wrong primary key class: " + primaryKeyClassName +
308                         " Correct class is: " + expectClsName);
309                }
310            }
311
312            /* Create bindings. */
313            PersistEntityBinding entityBinding =
314                new PersistEntityBinding(catalog, entityClassName, rawAccess);
315            PersistKeyBinding keyBinding = getKeyBinding(primaryKeyClassName);
316
317            /* If not read-only, get the primary key sequence. */
318            String seqName = priKeyMeta.getSequenceName();
319            if (!storeConfig.getReadOnly() && seqName != null) {
320                entityBinding.keyAssigner = new PersistKeyAssigner
321                    (keyBinding, entityBinding, getSequence(seqName));
322            }
323
324            /*
325             * Use a single transaction for opening the primary DB and its
326             * secondaries.  If opening any secondary fails, abort the
327             * transaction and undo the changes to the state of the store.
328             * Also support undo if the store is non-transactional.
329             */
330            Transaction txn = null;
331            DatabaseConfig dbConfig = getPrimaryConfig(entityMeta);
332            if (dbConfig.getTransactional() &&
333		DbCompat.getThreadTransaction(env) == null) {
334                txn = env.beginTransaction(null, null);
335            }
336            PrimaryOpenState priOpenState =
337                new PrimaryOpenState(entityClassName);
338            boolean success = false;
339            try {
340
341                /* Open the primary database. */
342                String[] fileAndDbNames =
343                    parseDbName(storePrefix + entityClassName);
344                Database db;
345                try {
346                    db = DbCompat.openDatabase
347                        (env, txn, fileAndDbNames[0], fileAndDbNames[1],
348                         dbConfig);
349                } catch (FileNotFoundException e) {
350                    throw new DatabaseException(e);
351                }
352                priOpenState.addDatabase(db);
353
354                /* Create index object. */
355                priIndex = new PrimaryIndex
356                    (db, primaryKeyClass, keyBinding, entityClass,
357                     entityBinding);
358
359                /* Update index and database maps. */
360                priIndexMap.put(entityClassName, priIndex);
361                if (DbCompat.getDeferredWrite(dbConfig)) {
362                    deferredWriteDatabases.put(db, null);
363                }
364
365                /* If not read-only, open all associated secondaries. */
366                if (!dbConfig.getReadOnly()) {
367                    openSecondaryIndexes(txn, entityMeta, priOpenState);
368
369                    /*
370                     * To enable foreign key contratints, also open all primary
371                     * indexes referring to this class via a relatedEntity
372                     * property in another entity. [#15358]
373                     */
374                    Set<String> inverseClassNames =
375                        inverseRelatedEntityMap.get(entityClassName);
376                    if (inverseClassNames != null) {
377                        for (String relatedClsName : inverseClassNames) {
378                            getRelatedIndex(relatedClsName);
379                        }
380                    }
381                }
382                success = true;
383            } finally {
384                if (success) {
385                    if (txn != null) {
386                        txn.commit();
387                    }
388                } else {
389                    if (txn != null) {
390                        txn.abort();
391                    } else {
392                        priOpenState.closeDatabases();
393                    }
394                    priOpenState.undoState();
395                }
396            }
397        }
398        return priIndex;
399    }
400
401    /**
402     * Holds state information about opening a primary index and its secondary
403     * indexes.  Used to undo the state of this object if the transaction
404     * opening the primary and secondaries aborts.  Also used to close all
405     * databases opened during this process for a non-transactional store.
406     */
407    private class PrimaryOpenState {
408
409        private String entityClassName;
410        private IdentityHashMap<Database,Object> databases;
411        private Set<String> secNames;
412
413        PrimaryOpenState(String entityClassName) {
414            this.entityClassName = entityClassName;
415            databases = new IdentityHashMap<Database,Object>();
416            secNames = new HashSet<String>();
417        }
418
419        /**
420         * Save a database that was opening during this operation.
421         */
422        void addDatabase(Database db) {
423            databases.put(db, null);
424        }
425
426        /**
427         * Save the name of a secondary index that was opening during this
428         * operation.
429         */
430        void addSecondaryName(String secName) {
431            secNames.add(secName);
432        }
433
434        /**
435         * Close any databases opened during this operation when it fails.
436         * This method should be called if a non-transactional operation fails,
437         * since we cannot rely on the transaction abort to cleanup any
438         * databases that were opened.
439         */
440        void closeDatabases() {
441            for (Database db : databases.keySet()) {
442                try {
443                    db.close();
444                } catch (Exception ignored) {
445                }
446            }
447        }
448
449        /**
450         * Reset all state information when this operation fails.  This method
451         * should be called for both transactional and non-transsactional
452         * operation.
453         */
454        void undoState() {
455            priIndexMap.remove(entityClassName);
456            for (String secName : secNames) {
457                secIndexMap.remove(secName);
458            }
459            for (Database db : databases.keySet()) {
460                deferredWriteDatabases.remove(db);
461            }
462        }
463    }
464
465    /**
466     * Opens a primary index related via a foreign key (relatedEntity).
467     * Related indexes are not opened in the same transaction used by the
468     * caller to open a primary or secondary.  It is OK to leave the related
469     * index open when the caller's transaction aborts.  It is only important
470     * to open a primary and its secondaries atomically.
471     */
472    private PrimaryIndex getRelatedIndex(String relatedClsName)
473        throws DatabaseException {
474
475        PrimaryIndex relatedIndex = priIndexMap.get(relatedClsName);
476        if (relatedIndex == null) {
477            EntityMetadata relatedEntityMeta =
478                checkEntityClass(relatedClsName);
479            Class relatedKeyCls;
480            String relatedKeyClsName;
481            Class relatedCls;
482            if (rawAccess) {
483                relatedCls = RawObject.class;
484                relatedKeyCls = Object.class;
485                relatedKeyClsName = null;
486            } else {
487                try {
488                    relatedCls = EntityModel.classForName(relatedClsName);
489                } catch (ClassNotFoundException e) {
490                    throw new IllegalArgumentException
491                        ("Related entity class not found: " +
492                         relatedClsName);
493                }
494                relatedKeyClsName = SimpleCatalog.keyClassName
495                    (relatedEntityMeta.getPrimaryKey().getClassName());
496                relatedKeyCls =
497                    SimpleCatalog.keyClassForName(relatedKeyClsName);
498            }
499
500            /*
501             * Cycles are prevented here by adding primary indexes to the
502             * priIndexMap as soon as they are created, before opening related
503             * indexes.
504             */
505            relatedIndex = getPrimaryIndex
506                (relatedKeyCls, relatedKeyClsName,
507                 relatedCls, relatedClsName);
508        }
509        return relatedIndex;
510    }
511
512    /**
513     * A getSecondaryIndex with extra parameters for opening a raw store.
514     * keyClassName is used for consistency checking and should be null for a
515     * raw store only.
516     */
517    public synchronized <SK,PK,E1,E2 extends E1> SecondaryIndex<SK,PK,E2>
518        getSecondaryIndex(PrimaryIndex<PK,E1> primaryIndex,
519                          Class<E2> entityClass,
520                          String entityClassName,
521                          Class<SK> keyClass,
522                          String keyClassName,
523                          String keyName)
524        throws DatabaseException {
525
526        assert (rawAccess && keyClassName == null) ||
527              (!rawAccess && keyClassName != null);
528
529        checkOpen();
530
531        EntityMetadata entityMeta = null;
532        SecondaryKeyMetadata secKeyMeta = null;
533
534        /* Validate the subclass for a subclass index. */
535        if (entityClass != primaryIndex.getEntityClass()) {
536            entityMeta = model.getEntityMetadata(entityClassName);
537            assert entityMeta != null;
538            secKeyMeta = checkSecKey(entityMeta, keyName);
539            String subclassName = entityClass.getName();
540            String declaringClassName = secKeyMeta.getDeclaringClassName();
541            if (!subclassName.equals(declaringClassName)) {
542                throw new IllegalArgumentException
543                    ("Key for subclass " + subclassName +
544                     " is declared in a different class: " +
545                     makeSecName(declaringClassName, keyName));
546            }
547        }
548
549        /*
550         * Even though the primary is already open, we can't assume the
551         * secondary is open because we don't automatically open all
552         * secondaries when the primary is read-only.  Use auto-commit (a null
553         * transaction) since we're opening only one database.
554         */
555        String secName = makeSecName(entityClassName, keyName);
556        SecondaryIndex<SK,PK,E2> secIndex = secIndexMap.get(secName);
557        if (secIndex == null) {
558            if (entityMeta == null) {
559                entityMeta = model.getEntityMetadata(entityClassName);
560                assert entityMeta != null;
561            }
562            if (secKeyMeta == null) {
563                secKeyMeta = checkSecKey(entityMeta, keyName);
564            }
565
566            /* Check metadata. */
567            if (keyClassName == null) {
568                keyClassName = getSecKeyClass(secKeyMeta);
569            } else {
570                String expectClsName = getSecKeyClass(secKeyMeta);
571                if (!keyClassName.equals(expectClsName)) {
572                    throw new IllegalArgumentException
573                        ("Wrong secondary key class: " + keyClassName +
574                         " Correct class is: " + expectClsName);
575                }
576            }
577
578            secIndex = openSecondaryIndex
579                (null, primaryIndex, entityClass, entityMeta,
580                 keyClass, keyClassName, secKeyMeta, secName,
581                 false /*doNotCreate*/, null /*priOpenState*/);
582        }
583        return secIndex;
584    }
585
586    /**
587     * Opens any secondary indexes defined in the given entity metadata that
588     * are not already open.  This method is called when a new entity subclass
589     * is encountered when an instance of that class is stored, and the
590     * EntityStore.getSubclassIndex has not been previously called for that
591     * class. [#15247]
592     */
593    synchronized void openSecondaryIndexes(Transaction txn,
594                                           EntityMetadata entityMeta,
595                                           PrimaryOpenState priOpenState)
596        throws DatabaseException {
597
598        String entityClassName = entityMeta.getClassName();
599        PrimaryIndex<Object,Object> priIndex =
600            priIndexMap.get(entityClassName);
601        assert priIndex != null;
602        Class<Object> entityClass = priIndex.getEntityClass();
603
604        for (SecondaryKeyMetadata secKeyMeta :
605             entityMeta.getSecondaryKeys().values()) {
606            String keyName = secKeyMeta.getKeyName();
607            String secName = makeSecName(entityClassName, keyName);
608            SecondaryIndex<Object,Object,Object> secIndex =
609                secIndexMap.get(secName);
610            if (secIndex == null) {
611                String keyClassName = getSecKeyClass(secKeyMeta);
612                /* RawMode: should not require class. */
613                Class keyClass =
614                    SimpleCatalog.keyClassForName(keyClassName);
615                openSecondaryIndex
616                    (txn, priIndex, entityClass, entityMeta,
617                     keyClass, keyClassName, secKeyMeta,
618                     makeSecName
619                        (entityClassName, secKeyMeta.getKeyName()),
620                     storeConfig.getSecondaryBulkLoad() /*doNotCreate*/,
621                     priOpenState);
622            }
623        }
624    }
625
626    /**
627     * Opens a secondary index with a given transaction and adds it to the
628     * secIndexMap.  We assume that the index is not already open.
629     */
630    private <SK,PK,E1,E2 extends E1> SecondaryIndex<SK,PK,E2>
631        openSecondaryIndex(Transaction txn,
632                           PrimaryIndex<PK,E1> primaryIndex,
633                           Class<E2> entityClass,
634                           EntityMetadata entityMeta,
635                           Class<SK> keyClass,
636                           String keyClassName,
637                           SecondaryKeyMetadata secKeyMeta,
638                           String secName,
639                           boolean doNotCreate,
640                           PrimaryOpenState priOpenState)
641        throws DatabaseException {
642
643        assert !secIndexMap.containsKey(secName);
644        String[] fileAndDbNames = parseDbName(storePrefix + secName);
645        SecondaryConfig config =
646            getSecondaryConfig(secName, entityMeta, keyClassName, secKeyMeta);
647        Database priDb = primaryIndex.getDatabase();
648        DatabaseConfig priConfig = priDb.getConfig();
649
650        String relatedClsName = secKeyMeta.getRelatedEntity();
651        if (relatedClsName != null) {
652            PrimaryIndex relatedIndex = getRelatedIndex(relatedClsName);
653            config.setForeignKeyDatabase(relatedIndex.getDatabase());
654        }
655
656        if (config.getTransactional() != priConfig.getTransactional() ||
657            DbCompat.getDeferredWrite(config) !=
658            DbCompat.getDeferredWrite(priConfig) ||
659            config.getReadOnly() != priConfig.getReadOnly()) {
660            throw new IllegalArgumentException
661                ("One of these properties was changed to be inconsistent" +
662                 " with the associated primary database: " +
663                 " Transactional, DeferredWrite, ReadOnly");
664        }
665
666        PersistKeyBinding keyBinding = getKeyBinding(keyClassName);
667
668        /*
669         * doNotCreate is true when StoreConfig.getSecondaryBulkLoad is true
670         * and we are opening a secondary as a side effect of opening a
671         * primary, i.e., getSecondaryIndex is not being called.  If
672         * doNotCreate is true and the database does not exist, we silently
673         * ignore the DatabaseNotFoundException and return null.  When
674         * getSecondaryIndex is subsequently called, the secondary database
675         * will be created and populated from the primary -- a bulk load.
676         */
677        SecondaryDatabase db;
678        boolean saveAllowCreate = config.getAllowCreate();
679        try {
680            if (doNotCreate) {
681                config.setAllowCreate(false);
682            }
683            db = DbCompat.openSecondaryDatabase
684                (env, txn, fileAndDbNames[0], fileAndDbNames[1], priDb,
685                 config);
686        } catch (FileNotFoundException e) {
687            if (doNotCreate) {
688                return null;
689            } else {
690                throw new DatabaseException(e);
691            }
692        } finally {
693            if (doNotCreate) {
694                config.setAllowCreate(saveAllowCreate);
695            }
696        }
697        SecondaryIndex<SK,PK,E2> secIndex = new SecondaryIndex
698            (db, null, primaryIndex, keyClass, keyBinding);
699
700        /* Update index and database maps. */
701        secIndexMap.put(secName, secIndex);
702        if (DbCompat.getDeferredWrite(config)) {
703            deferredWriteDatabases.put(db, null);
704        }
705        if (priOpenState != null) {
706            priOpenState.addDatabase(db);
707            priOpenState.addSecondaryName(secName);
708        }
709        return secIndex;
710    }
711
712
713    public void truncateClass(Class entityClass)
714        throws DatabaseException {
715
716        truncateClass(null, entityClass);
717    }
718
719    public synchronized void truncateClass(Transaction txn, Class entityClass)
720        throws DatabaseException {
721
722        checkOpen();
723
724        /* Close primary and secondary databases. */
725        closeClass(entityClass);
726
727        String clsName = entityClass.getName();
728        EntityMetadata entityMeta = checkEntityClass(clsName);
729
730        /*
731         * Truncate the primary first and let any exceptions propogate
732         * upwards.  Then truncate each secondary, only throwing the first
733         * exception.
734         */
735        boolean primaryExists = truncateIfExists(txn, storePrefix + clsName);
736        if (primaryExists) {
737            DatabaseException firstException = null;
738            for (SecondaryKeyMetadata keyMeta :
739                 entityMeta.getSecondaryKeys().values()) {
740                try {
741                    truncateIfExists
742                        (txn,
743                         storePrefix +
744                         makeSecName(clsName, keyMeta.getKeyName()));
745                    /* Ignore secondaries that do not exist. */
746                } catch (DatabaseException e) {
747                    if (firstException == null) {
748                        firstException = e;
749                    }
750                }
751            }
752            if (firstException != null) {
753                throw firstException;
754            }
755        }
756    }
757
758    private boolean truncateIfExists(Transaction txn, String dbName)
759        throws DatabaseException {
760
761        try {
762            String[] fileAndDbNames = parseDbName(dbName);
763            DbCompat.truncateDatabase
764                (env, txn, fileAndDbNames[0], fileAndDbNames[1],
765                 false/*returnCount*/);
766            return true;
767        } catch (FileNotFoundException e) {
768            return false;
769        }
770    }
771
772    public synchronized void closeClass(Class entityClass)
773        throws DatabaseException {
774
775        checkOpen();
776        String clsName = entityClass.getName();
777        EntityMetadata entityMeta = checkEntityClass(clsName);
778
779        PrimaryIndex priIndex = priIndexMap.get(clsName);
780        if (priIndex != null) {
781            /* Close the secondaries first. */
782            DatabaseException firstException = null;
783            for (SecondaryKeyMetadata keyMeta :
784                 entityMeta.getSecondaryKeys().values()) {
785
786                String secName = makeSecName(clsName, keyMeta.getKeyName());
787                SecondaryIndex secIndex = secIndexMap.get(secName);
788                if (secIndex != null) {
789                    Database db = secIndex.getDatabase();
790                    firstException = closeDb(db, firstException);
791                    firstException =
792                        closeDb(secIndex.getKeysDatabase(), firstException);
793                    secIndexMap.remove(secName);
794                    deferredWriteDatabases.remove(db);
795                }
796            }
797            /* Close the primary last. */
798            Database db = priIndex.getDatabase();
799            firstException = closeDb(db, firstException);
800            priIndexMap.remove(clsName);
801            deferredWriteDatabases.remove(db);
802
803            /* Throw the first exception encountered. */
804            if (firstException != null) {
805                throw firstException;
806            }
807        }
808    }
809
810    public synchronized void close()
811        throws DatabaseException {
812
813        checkOpen();
814        DatabaseException firstException = null;
815        try {
816            if (rawAccess) {
817                boolean allClosed = catalog.close();
818                assert allClosed;
819            } else {
820                synchronized (catalogPool) {
821                    Map<String,PersistCatalog> catalogMap =
822                        catalogPool.get(env);
823                    assert catalogMap != null;
824                    if (catalog.close()) {
825                        /* Remove when the reference count goes to zero. */
826                        catalogMap.remove(storeName);
827                    }
828                }
829            }
830            catalog = null;
831        } catch (DatabaseException e) {
832            if (firstException == null) {
833                firstException = e;
834            }
835        }
836        firstException = closeDb(sequenceDb, firstException);
837        for (SecondaryIndex index : secIndexMap.values()) {
838            firstException = closeDb(index.getDatabase(), firstException);
839            firstException = closeDb(index.getKeysDatabase(), firstException);
840        }
841        for (PrimaryIndex index : priIndexMap.values()) {
842            firstException = closeDb(index.getDatabase(), firstException);
843        }
844        if (firstException != null) {
845            throw firstException;
846        }
847    }
848
849    public synchronized Sequence getSequence(String name)
850        throws DatabaseException {
851
852        checkOpen();
853
854        if (storeConfig.getReadOnly()) {
855            throw new IllegalStateException("Store is read-only");
856        }
857
858        Sequence seq = sequenceMap.get(name);
859        if (seq == null) {
860            if (sequenceDb == null) {
861                String[] fileAndDbNames =
862                    parseDbName(storePrefix + SEQUENCE_DB);
863                DatabaseConfig dbConfig = new DatabaseConfig();
864                dbConfig.setTransactional(storeConfig.getTransactional());
865                dbConfig.setAllowCreate(true);
866                DbCompat.setTypeBtree(dbConfig);
867                try {
868                    sequenceDb = DbCompat.openDatabase
869                        (env, null/*txn*/, fileAndDbNames[0],
870                         fileAndDbNames[1], dbConfig);
871                } catch (FileNotFoundException e) {
872                    throw new DatabaseException(e);
873                }
874            }
875            DatabaseEntry entry = new DatabaseEntry();
876            StringBinding.stringToEntry(name, entry);
877            seq = sequenceDb.openSequence(null, entry, getSequenceConfig(name));
878            sequenceMap.put(name, seq);
879        }
880        return seq;
881    }
882
883    public synchronized SequenceConfig getSequenceConfig(String name) {
884        checkOpen();
885        SequenceConfig config = sequenceConfigMap.get(name);
886        if (config == null) {
887            config = new SequenceConfig();
888            config.setInitialValue(1);
889            config.setRange(1, Long.MAX_VALUE);
890            config.setCacheSize(100);
891            config.setAutoCommitNoSync(true);
892            config.setAllowCreate(!storeConfig.getReadOnly());
893            sequenceConfigMap.put(name, config);
894        }
895        return config;
896    }
897
898    public synchronized void setSequenceConfig(String name,
899                                               SequenceConfig config) {
900        checkOpen();
901        sequenceConfigMap.put(name, config);
902    }
903
904    public synchronized DatabaseConfig getPrimaryConfig(Class entityClass) {
905        checkOpen();
906        String clsName = entityClass.getName();
907        EntityMetadata meta = checkEntityClass(clsName);
908        return getPrimaryConfig(meta).cloneConfig();
909    }
910
911    private synchronized DatabaseConfig getPrimaryConfig(EntityMetadata meta) {
912        String clsName = meta.getClassName();
913        DatabaseConfig config = priConfigMap.get(clsName);
914        if (config == null) {
915            config = new DatabaseConfig();
916            config.setTransactional(storeConfig.getTransactional());
917            config.setAllowCreate(!storeConfig.getReadOnly());
918            config.setReadOnly(storeConfig.getReadOnly());
919            DbCompat.setTypeBtree(config);
920            setBtreeComparator(config, meta.getPrimaryKey().getClassName());
921            priConfigMap.put(clsName, config);
922        }
923        return config;
924    }
925
926    public synchronized void setPrimaryConfig(Class entityClass,
927                                              DatabaseConfig config) {
928        checkOpen();
929        String clsName = entityClass.getName();
930        if (priIndexMap.containsKey(clsName)) {
931            throw new IllegalStateException
932                ("Cannot set config after DB is open");
933        }
934        EntityMetadata meta = checkEntityClass(clsName);
935        DatabaseConfig dbConfig = getPrimaryConfig(meta);
936        if (config.getSortedDuplicates() ||
937            config.getBtreeComparator() != dbConfig.getBtreeComparator()) {
938            throw new IllegalArgumentException
939                ("One of these properties was illegally changed: " +
940                 " SortedDuplicates or BtreeComparator");
941        }
942        if (!DbCompat.isTypeBtree(config)) {
943            throw new IllegalArgumentException("Only type BTREE allowed");
944        }
945        priConfigMap.put(clsName, config);
946    }
947
948    public synchronized SecondaryConfig getSecondaryConfig(Class entityClass,
949                                                           String keyName) {
950        checkOpen();
951        String entityClsName = entityClass.getName();
952        EntityMetadata entityMeta = checkEntityClass(entityClsName);
953        SecondaryKeyMetadata secKeyMeta = checkSecKey(entityMeta, keyName);
954        String keyClassName = getSecKeyClass(secKeyMeta);
955        String secName = makeSecName(entityClass.getName(), keyName);
956        return (SecondaryConfig) getSecondaryConfig
957            (secName, entityMeta, keyClassName, secKeyMeta).cloneConfig();
958    }
959
960    private SecondaryConfig getSecondaryConfig(String secName,
961                                               EntityMetadata entityMeta,
962                                               String keyClassName,
963                                               SecondaryKeyMetadata
964                                               secKeyMeta) {
965        SecondaryConfig config = secConfigMap.get(secName);
966        if (config == null) {
967            /* Set common properties to match the primary DB. */
968            DatabaseConfig priConfig = getPrimaryConfig(entityMeta);
969            config = new SecondaryConfig();
970            config.setTransactional(priConfig.getTransactional());
971            config.setAllowCreate(!priConfig.getReadOnly());
972            config.setReadOnly(priConfig.getReadOnly());
973            DbCompat.setTypeBtree(config);
974            DbCompat.setDeferredWrite
975                (config, DbCompat.getDeferredWrite(priConfig));
976            /* Set secondary properties based on metadata. */
977            config.setAllowPopulate(true);
978            Relationship rel = secKeyMeta.getRelationship();
979            config.setSortedDuplicates(rel == Relationship.MANY_TO_ONE ||
980                                       rel == Relationship.MANY_TO_MANY);
981            setBtreeComparator(config, secKeyMeta.getClassName());
982            PersistKeyCreator keyCreator = new PersistKeyCreator
983                (catalog, entityMeta, keyClassName, secKeyMeta);
984            if (rel == Relationship.ONE_TO_MANY ||
985                rel == Relationship.MANY_TO_MANY) {
986                config.setMultiKeyCreator(keyCreator);
987            } else {
988                config.setKeyCreator(keyCreator);
989            }
990            DeleteAction deleteAction = secKeyMeta.getDeleteAction();
991            if (deleteAction != null) {
992                ForeignKeyDeleteAction baseDeleteAction;
993                switch (deleteAction) {
994                case ABORT:
995                    baseDeleteAction = ForeignKeyDeleteAction.ABORT;
996                    break;
997                case CASCADE:
998                    baseDeleteAction = ForeignKeyDeleteAction.CASCADE;
999                    break;
1000                case NULLIFY:
1001                    baseDeleteAction = ForeignKeyDeleteAction.NULLIFY;
1002                    break;
1003                default:
1004                    throw new IllegalStateException(deleteAction.toString());
1005                }
1006                config.setForeignKeyDeleteAction(baseDeleteAction);
1007                if (deleteAction == DeleteAction.NULLIFY) {
1008                    config.setForeignMultiKeyNullifier(keyCreator);
1009                }
1010            }
1011            secConfigMap.put(secName, config);
1012        }
1013        return config;
1014    }
1015
1016    public synchronized void setSecondaryConfig(Class entityClass,
1017                                                String keyName,
1018                                                SecondaryConfig config) {
1019        checkOpen();
1020        String entityClsName = entityClass.getName();
1021        EntityMetadata entityMeta = checkEntityClass(entityClsName);
1022        SecondaryKeyMetadata secKeyMeta = checkSecKey(entityMeta, keyName);
1023        String keyClassName = getSecKeyClass(secKeyMeta);
1024        String secName = makeSecName(entityClass.getName(), keyName);
1025        if (secIndexMap.containsKey(secName)) {
1026            throw new IllegalStateException
1027                ("Cannot set config after DB is open");
1028        }
1029        SecondaryConfig dbConfig =
1030            getSecondaryConfig(secName, entityMeta, keyClassName, secKeyMeta);
1031        if (config.getSortedDuplicates() != dbConfig.getSortedDuplicates() ||
1032            config.getBtreeComparator() != dbConfig.getBtreeComparator() ||
1033            config.getDuplicateComparator() != null ||
1034            config.getAllowPopulate() != dbConfig.getAllowPopulate() ||
1035            config.getKeyCreator() != dbConfig.getKeyCreator() ||
1036            config.getMultiKeyCreator() != dbConfig.getMultiKeyCreator() ||
1037            config.getForeignKeyNullifier() !=
1038                dbConfig.getForeignKeyNullifier() ||
1039            config.getForeignMultiKeyNullifier() !=
1040                dbConfig.getForeignMultiKeyNullifier() ||
1041            config.getForeignKeyDeleteAction() !=
1042                dbConfig.getForeignKeyDeleteAction() ||
1043            config.getForeignKeyDatabase() != null) {
1044            throw new IllegalArgumentException
1045                ("One of these properties was illegally changed: " +
1046                 " SortedDuplicates, BtreeComparator, DuplicateComparator," +
1047                 " AllowPopulate, KeyCreator, MultiKeyCreator," +
1048                 " ForeignKeyNullifer, ForeignMultiKeyNullifier," +
1049                 " ForeignKeyDeleteAction, ForeignKeyDatabase");
1050        }
1051        if (!DbCompat.isTypeBtree(config)) {
1052            throw new IllegalArgumentException("Only type BTREE allowed");
1053        }
1054        secConfigMap.put(secName, config);
1055    }
1056
1057    private static String makeSecName(String entityClsName, String keyName) {
1058         return entityClsName + NAME_SEPARATOR + keyName;
1059    }
1060
1061    static String makePriDbName(String storePrefix, String entityClsName) {
1062        return storePrefix + entityClsName;
1063    }
1064
1065    static String makeSecDbName(String storePrefix,
1066                                String entityClsName,
1067                                String keyName) {
1068        return storePrefix + makeSecName(entityClsName, keyName);
1069    }
1070
1071    /**
1072     * Parses a whole DB name and returns an array of 2 strings where element 0
1073     * is the file name (always null for JE, always non-null for DB core) and
1074     * element 1 is the logical DB name (always non-null for JE, may be null
1075     * for DB core).
1076     */
1077    public String[] parseDbName(String wholeName) {
1078        return parseDbName(wholeName, storeConfig.getDatabaseNamer());
1079    }
1080
1081    /**
1082     * Allows passing a namer to a static method for testing.
1083     */
1084    public static String[] parseDbName(String wholeName, DatabaseNamer namer) {
1085        String[] result = new String[2];
1086        if (DbCompat.SEPARATE_DATABASE_FILES) {
1087            String[] splitName = wholeName.split(NAME_SEPARATOR);
1088            assert splitName.length == 3 || splitName.length == 4 : wholeName;
1089            assert splitName[0].equals("persist") : wholeName;
1090            String storeName = splitName[1];
1091            String clsName = splitName[2];
1092            String keyName = (splitName.length > 3) ? splitName[3] : null;
1093            result[0] = namer.getFileName(storeName, clsName, keyName);
1094            result[1] = null;
1095        } else {
1096            result[0] = null;
1097            result[1] = wholeName;
1098        }
1099        return result;
1100    }
1101
1102    private void checkOpen() {
1103        if (catalog == null) {
1104            throw new IllegalStateException("Store has been closed");
1105        }
1106    }
1107
1108    private EntityMetadata checkEntityClass(String clsName) {
1109        EntityMetadata meta = model.getEntityMetadata(clsName);
1110        if (meta == null) {
1111            throw new IllegalArgumentException
1112                ("Class could not be loaded or is not an entity class: " +
1113                 clsName);
1114        }
1115        return meta;
1116    }
1117
1118    private SecondaryKeyMetadata checkSecKey(EntityMetadata entityMeta,
1119                                             String keyName) {
1120        SecondaryKeyMetadata secKeyMeta =
1121            entityMeta.getSecondaryKeys().get(keyName);
1122        if (secKeyMeta == null) {
1123            throw new IllegalArgumentException
1124                ("Not a secondary key: " +
1125                 makeSecName(entityMeta.getClassName(), keyName));
1126        }
1127        return secKeyMeta;
1128    }
1129
1130    private String getSecKeyClass(SecondaryKeyMetadata secKeyMeta) {
1131        String clsName = secKeyMeta.getElementClassName();
1132        if (clsName == null) {
1133            clsName = secKeyMeta.getClassName();
1134        }
1135        return SimpleCatalog.keyClassName(clsName);
1136    }
1137
1138    private PersistKeyBinding getKeyBinding(String keyClassName) {
1139        PersistKeyBinding binding = keyBindingMap.get(keyClassName);
1140        if (binding == null) {
1141            binding = new PersistKeyBinding(catalog, keyClassName, rawAccess);
1142            keyBindingMap.put(keyClassName, binding);
1143        }
1144        return binding;
1145    }
1146
1147    private void setBtreeComparator(DatabaseConfig config, String clsName) {
1148        if (!rawAccess) {
1149            ClassMetadata meta = model.getClassMetadata(clsName);
1150            if (meta != null) {
1151                List<FieldMetadata> compositeKeyFields =
1152                    meta.getCompositeKeyFields();
1153                if (compositeKeyFields != null) {
1154                    Class keyClass = SimpleCatalog.keyClassForName(clsName);
1155                    if (Comparable.class.isAssignableFrom(keyClass)) {
1156                        Comparator<Object> cmp = new PersistComparator
1157                            (clsName, compositeKeyFields,
1158                             getKeyBinding(clsName));
1159                        config.setBtreeComparator(cmp);
1160                    }
1161                }
1162            }
1163        }
1164    }
1165
1166    private DatabaseException closeDb(Database db,
1167                                      DatabaseException firstException) {
1168        if (db != null) {
1169            try {
1170                db.close();
1171            } catch (DatabaseException e) {
1172                if (firstException == null) {
1173                    firstException = e;
1174                }
1175            }
1176        }
1177        return firstException;
1178    }
1179
1180    public EvolveStats evolve(EvolveConfig config)
1181        throws DatabaseException {
1182
1183        checkOpen();
1184        List<Format> toEvolve = new ArrayList<Format>();
1185        Set<String> configToEvolve = config.getClassesToEvolve();
1186        if (configToEvolve.isEmpty()) {
1187            catalog.getEntityFormats(toEvolve);
1188        } else {
1189            for (String name : configToEvolve) {
1190                Format format = catalog.getFormat(name);
1191                if (format == null) {
1192                    throw new IllegalArgumentException
1193                        ("Class to evolve is not persistent: " + name);
1194                }
1195                if (!format.isEntity()) {
1196                    throw new IllegalArgumentException
1197                        ("Class to evolve is not an entity class: " + name);
1198                }
1199                toEvolve.add(format);
1200            }
1201        }
1202
1203        EvolveEvent event = EvolveInternal.newEvent();
1204        for (Format format : toEvolve) {
1205            if (format.getEvolveNeeded()) {
1206                evolveIndex(format, event, config.getEvolveListener());
1207                format.setEvolveNeeded(false);
1208                catalog.flush();
1209            }
1210        }
1211
1212        return event.getStats();
1213    }
1214
1215    private void evolveIndex(Format format,
1216                             EvolveEvent event,
1217                             EvolveListener listener)
1218        throws DatabaseException {
1219
1220        /* We may make this configurable later. */
1221        final int WRITES_PER_TXN = 1;
1222
1223        Class entityClass = format.getType();
1224        String entityClassName = format.getClassName();
1225        EntityMetadata meta = model.getEntityMetadata(entityClassName);
1226        String keyClassName = meta.getPrimaryKey().getClassName();
1227        keyClassName = SimpleCatalog.keyClassName(keyClassName);
1228        DatabaseConfig dbConfig = getPrimaryConfig(meta);
1229
1230        PrimaryIndex<Object,Object> index = getPrimaryIndex
1231            (Object.class, keyClassName, entityClass, entityClassName);
1232        Database db = index.getDatabase();
1233
1234        EntityBinding binding = index.getEntityBinding();
1235        DatabaseEntry key = new DatabaseEntry();
1236        DatabaseEntry data = new DatabaseEntry();
1237
1238        CursorConfig cursorConfig = null;
1239        Transaction txn = null;
1240        if (dbConfig.getTransactional()) {
1241            txn = env.beginTransaction(null, null);
1242            cursorConfig = CursorConfig.READ_COMMITTED;
1243        }
1244
1245        Cursor cursor = null;
1246        int nWritten = 0;
1247        try {
1248            cursor = db.openCursor(txn, cursorConfig);
1249            OperationStatus status = cursor.getFirst(key, data, null);
1250            while (status == OperationStatus.SUCCESS) {
1251                boolean oneWritten = false;
1252                if (evolveNeeded(key, data, binding)) {
1253                    cursor.putCurrent(data);
1254                    oneWritten = true;
1255                    nWritten += 1;
1256                }
1257                if (listener != null) {
1258                    EvolveInternal.updateEvent
1259                        (event, entityClassName, 1, oneWritten ? 1 : 0);
1260                    if (!listener.evolveProgress(event)) {
1261                        break;
1262                    }
1263                }
1264                if (txn != null && nWritten >= WRITES_PER_TXN) {
1265                    cursor.close();
1266                    cursor = null;
1267                    txn.commit();
1268                    txn = null;
1269                    txn = env.beginTransaction(null, null);
1270                    cursor = db.openCursor(txn, cursorConfig);
1271                    DatabaseEntry saveKey = KeyRange.copy(key);
1272                    status = cursor.getSearchKeyRange(key, data, null);
1273                    if (status == OperationStatus.SUCCESS &&
1274                        KeyRange.equalBytes(key, saveKey)) {
1275                        status = cursor.getNext(key, data, null);
1276                    }
1277                } else {
1278                    status = cursor.getNext(key, data, null);
1279                }
1280            }
1281        } finally {
1282            if (cursor != null) {
1283                cursor.close();
1284            }
1285            if (txn != null) {
1286                if (nWritten > 0) {
1287                    txn.commit();
1288                } else {
1289                    txn.abort();
1290                }
1291            }
1292        }
1293    }
1294
1295    /**
1296     * Checks whether the given data is in the current format by translating it
1297     * to/from an object.  If true is returned, data is updated.
1298     */
1299    private boolean evolveNeeded(DatabaseEntry key,
1300                                 DatabaseEntry data,
1301                                 EntityBinding binding) {
1302        Object entity = binding.entryToObject(key, data);
1303        DatabaseEntry newData = new DatabaseEntry();
1304        binding.objectToData(entity, newData);
1305        if (data.equals(newData)) {
1306            return false;
1307        } else {
1308            byte[] bytes = newData.getData();
1309            int off = newData.getOffset();
1310            int size = newData.getSize();
1311            data.setData(bytes, off, size);
1312            return true;
1313        }
1314    }
1315
1316    /**
1317     * For unit testing.
1318     */
1319    public static void setSyncHook(SyncHook hook) {
1320        syncHook = hook;
1321    }
1322
1323    /**
1324     * For unit testing.
1325     */
1326    public interface SyncHook {
1327        void onSync(Database db, boolean flushLog);
1328    }
1329}
1330