IndexUpgrader.java

  1. /*
  2.  *  IndexUpgrader.java
  3.  *
  4.  *  Copyright (c) 2007-2014, The University of Sheffield.
  5.  *
  6.  *  This file is part of GATE Mímir (see http://gate.ac.uk/family/mimir.html), 
  7.  *  and is free software, licenced under the GNU Lesser General Public License,
  8.  *  Version 3, June 2007 (also included with this distribution as file
  9.  *  LICENCE-LGPL3.html).
  10.  *
  11.  *  Valentin Tablan, 27 Feb 2014
  12.  *
  13.  *  $Id: IndexUpgrader.java 20281 2017-12-05 00:35:07Z ian_roberts $
  14.  */
  15. package gate.mimir.util;

  17. import gate.Gate;
  18. import gate.mimir.IndexConfig;
  19. import gate.mimir.IndexConfig.SemanticIndexerConfig;
  20. import gate.mimir.MimirIndex;
  21. import gate.mimir.SemanticAnnotationHelper;
  22. import gate.mimir.index.AtomicIndex;
  23. import gate.mimir.index.DocumentCollection;
  24. import gate.mimir.index.IndexException;
  25. import it.unimi.di.big.mg4j.index.DiskBasedIndex;
  26. import it.unimi.di.big.mg4j.index.cluster.DocumentalCluster;
  27. import it.unimi.di.big.mg4j.io.IOFactory;
  28. import it.unimi.di.big.mg4j.io.IOFactories;

  30. import it.unimi.dsi.util.Properties;


  33. import java.io.File;
  34. import java.io.FilenameFilter;
  35. import java.io.IOException;
  36. import java.nio.file.Files;
  37. import java.nio.file.StandardCopyOption;
  38. import java.util.Arrays;
  39. import java.util.HashSet;
  40. import java.util.Set;

  42. import org.slf4j.Logger;
  43. import org.slf4j.LoggerFactory;

  45. /**
  46.  * Implementation of an algorithm to upgrade a 4.x or 5.0 Mímir
  47.  * index to the format used by the current version.
  48.  */
  49. public class IndexUpgrader {
  50.   
  51.   protected static Logger logger = LoggerFactory.getLogger(IndexUpgrader.class);
  52.   
  53.   /**
  54.    * A minimal set of files required for a valid index.
  55.    */
  56.   protected static final String[] REQUIRED_INDEX_FILE_EXTENSIONS = new String[] {
  57.     DiskBasedIndex.INDEX_EXTENSION,
  58.     DiskBasedIndex.POSITIONS_EXTENSION,
  59.     DiskBasedIndex.TERMS_EXTENSION,
  60.     DiskBasedIndex.OFFSETS_EXTENSION
  61.   };

  63.   /**
  64.    * A minimal set of files required for a valid quasi-succinct index.
  65.    */
  66.   protected static final String[] REQUIRED_QS_INDEX_FILE_EXTENSIONS = new String[] {
  67.     DiskBasedIndex.COUNTS_EXTENSION,
  68.     DiskBasedIndex.COUNTS_EXTENSION + "offsets",
  69.     DiskBasedIndex.POINTERS_EXTENSIONS,
  70.     DiskBasedIndex.POINTERS_EXTENSIONS + "offsets",
  71.     DiskBasedIndex.POSITIONS_EXTENSION,
  72.     DiskBasedIndex.POSITIONS_EXTENSION + "offsets",
  73.     DiskBasedIndex.TERMS_EXTENSION,
  74.   };
  75.    
  76.   /**
  77.    * A minimal set of files required for a valid direct index.
  78.    */
  79.   protected static final String[] REQUIRED_DIRECT_INDEX_FILE_EXTENSIONS = new String[] {
  80.     DiskBasedIndex.INDEX_EXTENSION,
  81.     DiskBasedIndex.TERMS_EXTENSION,
  82.     DiskBasedIndex.OFFSETS_EXTENSION
  83.   };

  85.   /**
  86.    * A minimal set of files required for a valid quasi-succinct direct index.
  87.    */
  88.   protected static final String[] REQUIRED_QS_DIRECT_INDEX_FILE_EXTENSIONS = new String[] {
  89.     DiskBasedIndex.COUNTS_EXTENSION,
  90.     DiskBasedIndex.COUNTS_EXTENSION + "offsets",
  91.     DiskBasedIndex.POINTERS_EXTENSIONS,
  92.     DiskBasedIndex.POINTERS_EXTENSIONS + "offsets",
  93.     DiskBasedIndex.TERMS_EXTENSION,
  94.   };
  95.   
  96.   public static void upgradeIndex(File indexDirectory) throws IOException, 
  97.       IndexException {
  98.     File indexConfigFile = new File(indexDirectory, 
  99.         MimirIndex.INDEX_CONFIG_FILENAME);
  100.     IndexConfig indexConfig = IndexConfig.readConfigFromFile(indexConfigFile);
  101.     //test the version
  102.     if(indexConfig.getFormatVersion() == 7) {
  103.       upgradeFromV7(indexDirectory);
  104.     } else {
  105.       if(indexConfig.getFormatVersion() > 6 || indexConfig.getFormatVersion() < 4){
  106.         throw new IndexException(
  107.             "Unsupported index version: " + indexConfig.getFormatVersion());
  108.       }
  109.       
  110.       //check that none of the files to be created exist already
  111.       for(int i = 0 ; i < indexConfig.getTokenIndexers().length; i++) {
  112.         File tokenDir = new File(indexDirectory, "token-" + i);
  113.         if(tokenDir.exists()) {
  114.           throw new IndexException(
  115.               "Location required by upgraded index already exists:" + 
  116.               tokenDir.getAbsolutePath());
  117.         }
  118.       }
  119.       for(int i = 0 ; i < indexConfig.getSemanticIndexers().length; i++) {
  120.         File tokenDir = new File(indexDirectory, "mention-" + i);
  121.         if(tokenDir.exists()) {
  122.           throw new IndexException(
  123.               "Location required by upgraded index already exists:" + 
  124.               tokenDir.getAbsolutePath());
  125.         }
  126.       }
  127.       
  128.       // check access
  129.       File sourceDir = new File(indexDirectory, "mg4j");
  130.       if(!sourceDir.isDirectory()) throw new IndexException(
  131.           "Invalid index: could not find source directory at" + 
  132.           sourceDir.getAbsolutePath());
  133.       if(!sourceDir.canRead()) throw new IndexException(
  134.           "Could not read source directory at" + sourceDir.getAbsolutePath());
  135.       // check that we know how to deal with the S-A-H implementations
  136.       Class<? extends SemanticAnnotationHelper> dbSahClass = null;
  137.       try {
  138.         dbSahClass = Class.forName(
  139.             "gate.mimir.db.DBSemanticAnnotationHelper", 
  140.             true, Gate.getClassLoader()).asSubclass(
  141.                 SemanticAnnotationHelper.class);
  142.       } catch(ClassNotFoundException e) {
  143.         throw new IndexException("Could not find the DB S-A-H class. "
  144.             + "Is the 'db-h2' plugin loaded?", e);
  145.       }
  146.       for(int subIndexIdx = 0 ; 
  147.           subIndexIdx < indexConfig.getSemanticIndexers().length; 
  148.           subIndexIdx++) {
  149.         SemanticIndexerConfig sic = indexConfig.getSemanticIndexers()[subIndexIdx];
  150.         for(SemanticAnnotationHelper sah : sic.getHelpers()) {
  151.           while(sah instanceof DelegatingSemanticAnnotationHelper) {
  152.             sah = ((DelegatingSemanticAnnotationHelper)sah).getDelegate();
  153.           }
  154.           if(!dbSahClass.isAssignableFrom(sah.getClass())) {
  155.             throw new IndexException("Cannot convert mentions index mentions-" +
  156.                 subIndexIdx + " because it does not use the DB H2 " + 
  157.                 "Annotation Helper, which is the only one supported by " + 
  158.                 "this automatic upgrade process");
  159.           }
  160.         }
  161.       }
  162.       // move files
  163.       //collection files
  164.       File[] collectionFiles = sourceDir.listFiles(
  165.           DocumentCollection.CollectionFile.FILENAME_FILTER);
  166.       for(File aColFile : collectionFiles) {
  167.         File dest = new File(indexDirectory, aColFile.getName());
  168.         if(! aColFile.renameTo(dest)) {
  169.           throw new IndexException("Could not rename " + 
  170.               aColFile.getAbsolutePath() + " to " + dest.getAbsolutePath());
  171.         }
  172.       }
  173.       //token indexes
  174.       for(int subIndexIdx = 0 ; 
  175.           subIndexIdx < indexConfig.getTokenIndexers().length; 
  176.           subIndexIdx++) {
  177.         upgradeSubIndex(indexDirectory, subIndexIdx, 
  178.             indexConfig.getTokenIndexers()[subIndexIdx].isDirectIndexEnabled(), 
  179.             null);
  180.       }
  181.       // mention indexes
  182.       for(int subIndexIdx = 0 ; 
  183.           subIndexIdx < indexConfig.getSemanticIndexers().length; 
  184.           subIndexIdx++) {
  185.         SemanticIndexerConfig sic = indexConfig.getSemanticIndexers()[subIndexIdx]; 
  186.         upgradeSubIndex(indexDirectory, subIndexIdx, sic.isDirectIndexEnabled(),
  187.             sic);
  188.       }
  189.       // cleanup old dirs (only if empty)
  190.       if(sourceDir.listFiles().length == 0) {
  191.         if(!sourceDir.delete()) {
  192.           logger.info("Could not delete old MG4J directory " + sourceDir + 
  193.               " even though it appears empty.");
  194.         }
  195.       }
  196.       File sourceDBDir = new File(indexDirectory, "db");
  197.       if(sourceDBDir.listFiles().length == 0) {
  198.         if(!sourceDBDir.delete()) {
  199.           logger.info("Could not delete old DB directory " + sourceDBDir + 
  200.               " even though it appears empty.");
  201.         }
  202.       }
  203.     }
  204.     //update the version number in the index config
  205.     indexConfig.setFormatVersion(IndexConfig.FORMAT_VERSION);
  206.     IndexConfig.writeConfigToFile(indexConfig, indexConfigFile);
  207.   }
  208.   
  209.   /**
  210.    * Moves the file belonging to one sub-index. 
  211.    * @param indexDirectory the top level index directory for the M&iacute;mir 
  212.    * index being upgraded. 
  213.    * @param subIndexIdx the index (position) of the sub-index
  214.    * @param mentionsConfig if this is a mentions index, then this parameter
  215.    *  contains the mentions indexer config, null otherwise.
  216.    * @param direct doe this sub-index have a direct index also?
  217.    * @throws IndexException
  218.    * @throws IOException 
  219.    */
  220.   protected static void upgradeSubIndex(File indexDirectory, int subIndexIdx, 
  221.         final boolean direct, SemanticIndexerConfig mentionsConfig) throws IndexException, IOException {
  222.     File sourceDir = new File(indexDirectory, "mg4j");
  223.     // sanity checks
  224.     final String inputFilePrefix = 
  225.         (mentionsConfig != null ? "mimir-mentions-" : "mimir-token-") + 
  226.         subIndexIdx;
  227.     
  228.     File[] atomicIndexFiles = sourceDir.listFiles(new FilenameFilter() {
  229.       @Override
  230.       public boolean accept(File dir, String name) {
  231.         return name.startsWith(inputFilePrefix + ".") || 
  232.             (direct && name.startsWith(inputFilePrefix + 
  233.              AtomicIndex.DIRECT_INDEX_NAME_SUFFIX + "."));
  234.       }
  235.     });

  237.     Set<String> requiredExtensions = new HashSet<String>();
  238.     Set<String> requiredDirectExtensions = new HashSet<String>();
  239.     
  240.     String indexPropertiesFile = new File(sourceDir, inputFilePrefix + DiskBasedIndex.PROPERTIES_EXTENSION).getAbsolutePath();
  241.     try {
  242.       Properties indexProperties = IOFactories.loadProperties(IOFactory.FILESYSTEM_FACTORY,
  243.           indexPropertiesFile);
  244.       if("it.unimi.di.big.mg4j.index.QuasiSuccinctIndex".equals(indexProperties.getString("indexclass"))) {
  245.         requiredExtensions.addAll(Arrays.asList(REQUIRED_QS_INDEX_FILE_EXTENSIONS));
  246.       } else {
  247.         requiredExtensions.addAll(Arrays.asList(REQUIRED_INDEX_FILE_EXTENSIONS));
  248.       }
  249.     } catch(Exception e) {
  250.       throw new IndexException("Error reading " + indexPropertiesFile, e);
  251.     }

  253.     if(direct) {
  254.       String directPropertiesFile = new File(sourceDir, inputFilePrefix + AtomicIndex.DIRECT_INDEX_NAME_SUFFIX
  255.           + DiskBasedIndex.PROPERTIES_EXTENSION).getAbsolutePath();
  256.       try {
  257.         Properties directProperties = IOFactories.loadProperties(IOFactory.FILESYSTEM_FACTORY,
  258.             directPropertiesFile);
  259.         if("it.unimi.di.big.mg4j.index.QuasiSuccinctIndex".equals(directProperties.getString("indexclass"))) {
  260.           requiredDirectExtensions.addAll(Arrays.asList(REQUIRED_QS_DIRECT_INDEX_FILE_EXTENSIONS));
  261.         } else {
  262.           requiredDirectExtensions.addAll(Arrays.asList(REQUIRED_DIRECT_INDEX_FILE_EXTENSIONS));
  263.         }
  264.       } catch(Exception e) {
  265.         throw new IndexException("Error reading " + directPropertiesFile, e);
  266.       }
  267.     }

  269.     for(File aFile : atomicIndexFiles) {
  270.       String extension = aFile.getName().substring(inputFilePrefix.length());
  271.       if(direct && extension.startsWith(AtomicIndex.DIRECT_INDEX_NAME_SUFFIX)) {
  272.         extension = extension.substring(AtomicIndex.DIRECT_INDEX_NAME_SUFFIX.length());
  273.         requiredDirectExtensions.remove(extension);
  274.       } else {
  275.         requiredExtensions.remove(extension);  
  276.       }
  277.     }
  278.     // check that we've seen all files we wanted
  279.     if(!requiredExtensions.isEmpty() ||
  280.         (direct && ! requiredDirectExtensions.isEmpty())) {
  281.       //not all required files were found
  282.       StringBuilder str = new StringBuilder(
  283.           "Some required files were not found for index '");
  284.       str.append(inputFilePrefix).append("': ");
  285.       for(String extension : requiredExtensions) {
  286.         str.append(new File(sourceDir, 
  287.             inputFilePrefix + extension).getAbsolutePath());
  288.         str.append("\n");
  289.       }
  290.       if(direct) {
  291.         for(String extension : requiredDirectExtensions) {
  292.           str.append(new File(sourceDir, 
  293.               inputFilePrefix + extension).getAbsolutePath());
  294.           str.append("\n");
  295.         } 
  296.       }
  297.       throw new IndexException(str.toString());
  298.     }
  299.     
  300.     // all tests passed - start creating the new directories
  301.     String outputFilePrefix = (mentionsConfig != null ? "mention-" : "token-") + 
  302.         subIndexIdx;
  303.     File atomicIndexDir = new File(indexDirectory, outputFilePrefix);
  304.     File headDir = new File(atomicIndexDir, AtomicIndex.HEAD_FILE_NAME);
  305.     if(!headDir.mkdirs()) {
  306.       throw new IndexException(
  307.           "Location required by upgraded index could not be created:" + 
  308.           headDir.getAbsolutePath());
  309.     }
  310.     for(File sourceFile : atomicIndexFiles) {
  311.       String extension = sourceFile.getName().substring(inputFilePrefix.length());
  312.       File destinationFile = new File(headDir, outputFilePrefix + extension);
  313.       if(!sourceFile.renameTo(destinationFile)) {
  314.         throw new IndexException("Could not rename " + 
  315.             sourceFile.getAbsolutePath() + " to " + 
  316.             destinationFile.getAbsolutePath());
  317.       }
  318.     }
  319.     // create Bloom filter, and regenerate the term map
  320.     File termsFile = new File(headDir, outputFilePrefix + 
  321.         DiskBasedIndex.TERMS_EXTENSION); // guaranteed to exist, as tested already
  322.     File termMapFile = new File(headDir, outputFilePrefix + 
  323.         DiskBasedIndex.TERMMAP_EXTENSION); // may not exist but that's OK
  324.     File bloomFile = new File(headDir, outputFilePrefix + 
  325.         DocumentalCluster.BLOOM_EXTENSION); // guaranteed to exist, as tested already
  326.     if(termMapFile.exists()) {
  327.       if(!termMapFile.renameTo(new File(headDir, outputFilePrefix + 
  328.           DiskBasedIndex.TERMMAP_EXTENSION + ".old"))) {
  329.         logger.warn("Unable to back up old termmap for " + outputFilePrefix + "/" + headDir.getName());
  330.       }
  331.     }
  332.     AtomicIndex.generateTermMap(termsFile, termMapFile, bloomFile);
  333.     
  334.     if(direct) {
  335.       // create the direct.terms file by copying the terms file from 
  336.       // the **inverted** index in head
  337.       File dest = new File(atomicIndexDir, AtomicIndex.DIRECT_TERMS_FILENAME);
  338.       Files.copy(termsFile.toPath(), dest.toPath(), 
  339.           StandardCopyOption.COPY_ATTRIBUTES);
  340.       // create direct Bloom filter and regenerate term map
  341.       File dirTermsFile = new File(headDir, outputFilePrefix + 
  342.           AtomicIndex.DIRECT_INDEX_NAME_SUFFIX + 
  343.           DiskBasedIndex.TERMS_EXTENSION); // guaranteed to exist, as tested already
  344.       File dirTermMapFile = new File(headDir, outputFilePrefix + 
  345.           AtomicIndex.DIRECT_INDEX_NAME_SUFFIX +
  346.           DiskBasedIndex.TERMMAP_EXTENSION); // may not exist but that's OK
  347.       File dirBloomFile = new File(headDir, outputFilePrefix + 
  348.           AtomicIndex.DIRECT_INDEX_NAME_SUFFIX +
  349.           DocumentalCluster.BLOOM_EXTENSION); // guaranteed to exist, as tested already
  350.       if(dirTermMapFile.exists()) {
  351.         if(!dirTermMapFile.renameTo(new File(headDir, outputFilePrefix + 
  352.             AtomicIndex.DIRECT_INDEX_NAME_SUFFIX +
  353.             DiskBasedIndex.TERMMAP_EXTENSION + ".old"))) {
  354.           logger.warn("Unable to back up old direct termmap for " + outputFilePrefix + "/" + headDir.getName());
  355.         }
  356.       }
  357.       AtomicIndex.generateTermMap(dirTermsFile, dirTermMapFile, dirBloomFile);
  358.     }
  359.     
  360.     // move the DB files
  361.     if(mentionsConfig != null) {
  362.       // We know that the DB-H2 S-A-H was used, as we've already tested for that
  363.       File sourceDBDir = new File(indexDirectory, "db");
  364.       File destDBDir = new File(atomicIndexDir, "db");
  365.       if(!destDBDir.mkdirs()) {
  366.         throw new IndexException(
  367.             "Location required by upgraded index could not be created:" + 
  368.             destDBDir.getAbsolutePath());
  369.       }
  370.       for(String annType : mentionsConfig.getAnnotationTypes()) {
  371.         String tableBaseName = annType.replaceAll("[^\\p{Alnum}_]", "_");
  372.         File source = new File(sourceDBDir, tableBaseName + ".h2.db");
  373.         File dest = new File(destDBDir, tableBaseName + ".h2.db");
  374.         if(!source.renameTo(dest)) {
  375.           throw new IndexException("Could not rename " +  
  376.               source.getAbsolutePath() + " to " + dest.getAbsolutePath());
  377.         }
  378.       }
  379.     }
  380.   }
  381.   
  382.   protected static void upgradeFromV7(File indexDirectory) throws IndexException, IOException {
  383.     // just need to regenerate the term maps
  384.     for(File subIndexDirectory : indexDirectory.listFiles((File parent, String name) -> name.matches("(?:token|mention)-\\d+"))) {
  385.       String outputFilePrefix = subIndexDirectory.getName();
  386.       for(File batchDir : subIndexDirectory.listFiles((File parent, String name) -> name.matches("head|tail-\\d+"))) {
  387.         // create termmap and Bloom filter
  388.         File termsFile = new File(batchDir, outputFilePrefix + 
  389.             DiskBasedIndex.TERMS_EXTENSION);
  390.         File termMapFile = new File(batchDir, outputFilePrefix + 
  391.             DiskBasedIndex.TERMMAP_EXTENSION);
  392.         File bloomFile = new File(batchDir, outputFilePrefix + 
  393.             DocumentalCluster.BLOOM_EXTENSION);
  394.         if(!termsFile.exists()) {
  395.           throw new IndexException("No terms file found for " + outputFilePrefix + "/" + batchDir.getName());
  396.         }
  397.         if(termMapFile.exists()) {
  398.           if(!termMapFile.renameTo(new File(batchDir, outputFilePrefix + 
  399.               DiskBasedIndex.TERMMAP_EXTENSION + ".old"))) {
  400.             logger.warn("Unable to back up old termmap for " + outputFilePrefix + "/" + batchDir.getName());
  401.           }
  402.         }
  403.         if(bloomFile.exists()) {
  404.           if(!bloomFile.renameTo(new File(batchDir, outputFilePrefix + 
  405.               DocumentalCluster.BLOOM_EXTENSION + ".old"))) {
  406.             logger.warn("Unable to back up old bloom filter for " + outputFilePrefix + "/" + batchDir.getName());
  407.           }
  408.         }
  409.         AtomicIndex.generateTermMap(termsFile, termMapFile, bloomFile);
  410.         
  411.         // create direct termmap and Bloom filter
  412.         File dirTermsFile = new File(batchDir, outputFilePrefix + 
  413.             AtomicIndex.DIRECT_INDEX_NAME_SUFFIX + 
  414.             DiskBasedIndex.TERMS_EXTENSION);
  415.         if(dirTermsFile.exists()) {
  416.           File dirTermMapFile = new File(batchDir, outputFilePrefix + 
  417.               AtomicIndex.DIRECT_INDEX_NAME_SUFFIX +
  418.               DiskBasedIndex.TERMMAP_EXTENSION);
  419.           File dirBloomFile = new File(batchDir, outputFilePrefix + 
  420.               AtomicIndex.DIRECT_INDEX_NAME_SUFFIX +
  421.               DocumentalCluster.BLOOM_EXTENSION);
  422.           if(dirTermMapFile.exists()) {
  423.             if(!dirTermMapFile.renameTo(new File(batchDir, outputFilePrefix + 
  424.                 AtomicIndex.DIRECT_INDEX_NAME_SUFFIX +
  425.                 DiskBasedIndex.TERMMAP_EXTENSION + ".old"))) {
  426.               logger.warn("Unable to back up old direct termmap for " + outputFilePrefix + "/" + batchDir.getName());
  427.             }
  428.           }
  429.           if(dirBloomFile.exists()) {
  430.             if(!dirBloomFile.renameTo(new File(batchDir, outputFilePrefix + 
  431.                 AtomicIndex.DIRECT_INDEX_NAME_SUFFIX + 
  432.                 DocumentalCluster.BLOOM_EXTENSION + ".old"))) {
  433.               logger.warn("Unable to back up old direct bloom filter for " + outputFilePrefix + "/" + batchDir.getName());
  434.             }
  435.           }

  437.           AtomicIndex.generateTermMap(dirTermsFile, dirTermMapFile, dirBloomFile);
  438.         }

  440.       }
  441.     }
  442.   }

  444. }
Created with JaCoCo 0.8.5.201910111838