AbstractIndexTermsQuery.java

  1. /*
  2.  * AbstractIndexTermsQuery.java
  3.  * 
  4.  * Copyright (c) 2007-2011, The University of Sheffield.
  5.  * 
  6.  * This file is part of GATE Mímir (see http://gate.ac.uk/family/mimir.html),
  7.  * and is free software, licenced under the GNU Lesser General Public License,
  8.  * Version 3, June 2007 (also included with this distribution as file
  9.  * LICENCE-LGPL3.html).
  10.  * 
  11.  * Valentin Tablan, 17 Jul 2012
  12.  * 
  13.  * $Id: AbstractIndexTermsQuery.java 17255 2014-01-29 15:29:10Z valyt $
  14.  */
  15. package gate.mimir.search.terms;

  17. import gate.mimir.SemanticAnnotationHelper;
  18. import gate.mimir.index.AtomicIndex;
  19. import gate.mimir.search.IndexReaderPool;
  20. import gate.mimir.search.QueryEngine;
  21. import gate.mimir.search.QueryEngine.IndexType;
  22. import it.unimi.di.big.mg4j.search.DocumentIterator;
  23. import it.unimi.di.big.mg4j.search.visitor.CounterCollectionVisitor;
  24. import it.unimi.di.big.mg4j.search.visitor.CounterSetupVisitor;
  25. import it.unimi.di.big.mg4j.search.visitor.TermCollectionVisitor;
  26. import it.unimi.dsi.fastutil.ints.IntArrayList;
  27. import it.unimi.dsi.fastutil.objects.ObjectArrayList;

  29. import java.io.IOException;
  30. import java.util.HashSet;
  31. import java.util.Set;

  33. import org.slf4j.Logger;
  34. import org.slf4j.LoggerFactory;

  36. /**
  37.  * Base class for terms queries that use an MG4J direct index for their search.
  38.  */
  39. public abstract class AbstractIndexTermsQuery extends
  40.   AbstractDocumentsBasedTermsQuery {
  41.   /**
  42.    * Serialization ID.
  43.    */
  44.   private static final long serialVersionUID = 8382919427152317859L;

  46.   private static final Logger logger = LoggerFactory
  47.     .getLogger(AbstractIndexTermsQuery.class);

  49.   /**
  50.    * The name of the subindex in which the terms are sought. Each Mímir index
  51.    * includes multiple sub-indexes (some storing tokens, other storing
  52.    * annotations), identified by a name. For token indexes, the index name is
  53.    * the name of the token feature being indexed; for annotation indexes, the
  54.    * index name is the annotation type.
  55.    */
  56.   protected final String indexName;

  58.   /**
  59.    * The type of index being searched (tokens or annotations).
  60.    */
  61.   protected final IndexType indexType;

  63.   /**
  64.    * The atomic index used for executing the query. This includes both the 
  65.    * inverted and the direct index (if configured).
  66.    */
  67.   protected transient AtomicIndex atomicIndex;

  69.   /**
  70.    * The semantic annotation helper for the correct annotation type (as given by
  71.    * {@link #indexName}), if {@link #indexType} is {@link IndexType#ANNOTATIONS}
  72.    * , <code>null</code> otherwise.
  73.    */
  74.   protected transient SemanticAnnotationHelper annotationHelper;

  76.   /**
  77.    * Should stop words be filtered out of the results?
  78.    */
  79.   protected boolean stopWordsBlocked = false;

  81.   /**
  82.    * Stop words set used for filtering out stop words. See
  83.    * {@link #stopWordsBlocked}.
  84.    */
  85.   protected Set<String> stopWords = null;

  87.   /**
  88.    * If set to true, term strings for annotation mentions are replaced with
  89.    * their description (see
  90.    * {@link SemanticAnnotationHelper#describeMention(String)}.
  91.    */
  92.   protected final boolean describeAnnotations;

  94.   /**
  95.    * The query engine used to execute this query.
  96.    */
  97.   protected transient QueryEngine engine;

  99.   protected final boolean countsEnabled;

  101.   /**
  102.    * @return the countsEnabled
  103.    */
  104.   public boolean isCountsEnabled() {
  105.     return countsEnabled;
  106.   }

  108.   /**
  109.    * The default set of stop words.
  110.    */
  111.   public static final String[] DEFAULT_STOP_WORDS = new String[]{",", ".", "?",
  112.     "!", ":", ";", "#", "~", "^", "@", "%", "&", "(", ")", "[", "]", "{", "}",
  113.     "|", "\\", "<", ">", "-", "+", "*", "/", "=", "'", "\"", "'s", "1", "2",
  114.     "3", "4", "5", "6", "7", "8", "9", "0", "a", "about", "above", "above",
  115.     "across", "after", "afterwards", "again", "against", "all", "almost",
  116.     "alone", "along", "already", "also", "although", "always", "am", "among",
  117.     "amongst", "amoungst", "amount", "an", "and", "another", "any", "anyhow",
  118.     "anyone", "anything", "anyway", "anywhere", "are", "around", "as", "at",
  119.     "b", "back", "be", "became", "because", "become", "becomes", "becoming",
  120.     "been", "before", "beforehand", "behind", "being", "below", "beside",
  121.     "besides", "between", "beyond", "bill", "both", "bottom", "but", "by", "c",
  122.     "call", "can", "cannot", "cant", "co", "con", "could", "couldnt", "cry",
  123.     "d", "de", "describe", "detail", "do", "done", "down", "due", "during",
  124.     "e", "each", "eg", "eight", "either", "eleven", "else", "elsewhere",
  125.     "empty", "enough", "etc", "even", "ever", "every", "everyone",
  126.     "everything", "everywhere", "except", "f", "few", "fifteen", "fify",
  127.     "fill", "find", "fire", "first", "five", "for", "former", "formerly",
  128.     "forty", "found", "four", "from", "front", "full", "further", "g", "get",
  129.     "give", "go", "h", "had", "has", "hasnt", "have", "he", "hence", "her",
  130.     "here", "hereafter", "hereby", "herein", "hereupon", "hers", "herself",
  131.     "him", "himself", "his", "how", "however", "hundred", "i", "ie", "if",
  132.     "in", "inc", "indeed", "interest", "into", "is", "it", "its", "itself",
  133.     "j", "k", "keep", "l", "last", "latter", "latterly", "least", "less",
  134.     "ltd", "m", "made", "many", "may", "me", "meanwhile", "might", "mill",
  135.     "mine", "more", "moreover", "most", "mostly", "move", "much", "must", "my",
  136.     "myself", "n", "name", "namely", "neither", "never", "nevertheless",
  137.     "next", "nine", "no", "nobody", "none", "noone", "nor", "not", "nothing",
  138.     "now", "nowhere", "o", "of", "off", "often", "on", "once", "one", "only",
  139.     "onto", "or", "other", "others", "otherwise", "our", "ours", "ourselves",
  140.     "out", "over", "own", "p", "part", "per", "perhaps", "please", "put", "q",
  141.     "r", "rather", "re", "s", "same", "see", "seem", "seemed", "seeming",
  142.     "seems", "serious", "several", "she", "should", "show", "side", "since",
  143.     "sincere", "six", "sixty", "so", "some", "somehow", "someone", "something",
  144.     "sometime", "sometimes", "somewhere", "still", "such", "system", "t",
  145.     "take", "ten", "than", "that", "the", "their", "them", "themselves",
  146.     "then", "thence", "there", "thereafter", "thereby", "therefore", "therein",
  147.     "thereupon", "these", "they", "thickv", "thin", "third", "this", "those",
  148.     "though", "three", "through", "throughout", "thru", "thus", "to",
  149.     "together", "too", "top", "toward", "towards", "twelve", "twenty", "two",
  150.     "u", "un", "under", "until", "up", "upon", "us", "v", "very", "via", "w",
  151.     "was", "we", "well", "were", "what", "whatever", "when", "whence",
  152.     "whenever", "where", "whereafter", "whereas", "whereby", "wherein",
  153.     "whereupon", "wherever", "whether", "which", "while", "whither", "who",
  154.     "whoever", "whole", "whom", "whose", "why", "will", "with", "within",
  155.     "without", "would", "x", "y", "yet", "you", "your", "yours", "yourself",
  156.     "yourselves", "z"};

  158.   /**
  159.    * @param indexName
  160.    *          The name of the subindex in which the terms are sought. Each Mímir
  161.    *          index includes multiple sub-indexes (some storing tokens, other
  162.    *          storing annotations), identified by a name. For token indexes, the
  163.    *          index name is the name of the token feature being indexed; for
  164.    *          annotation indexes, the index name is the annotation type.
  165.    * @param indexType
  166.    *          The type of index to be searched (tokens or annotations).
  167.    * @param countsEnabled
  168.    *          should term counts be obtained?
  169.    * @param describeAnnotations
  170.    *          If the index being interrogated is of type
  171.    *          {@link IndexType#ANNOTATIONS} then the indexed term strings are
  172.    *          URIs whose format depends on the actual implementation of the
  173.    *          index. These strings make little sense outside of the index. If
  174.    *          this is set to <code>true</code>, then term descriptions are also
  175.    *          included in the results set. See
  176.    *          {@link TermsResultSet#termDescriptions} and
  177.    *          {@link SemanticAnnotationHelper#describeMention(String)}. Setting
  178.    *          this to <code>true</code> has no effect if the index being
  179.    *          interrogated is a {@link IndexType#TOKENS} index.
  180.    */
  181.   public AbstractIndexTermsQuery(String indexName, IndexType indexType,
  182.                                  boolean countsEnabled,
  183.                                  boolean describeAnnotations,
  184.                                  long... documentIDs) {
  185.     super(documentIDs);
  186.     this.indexName = indexName;
  187.     this.indexType = indexType;
  188.     this.countsEnabled = countsEnabled;
  189.     this.describeAnnotations =
  190.       describeAnnotations && (indexType == IndexType.ANNOTATIONS);
  191.   }

  193.   /**
  194.    * Populates the internal state by obtaining references to the direct and
  195.    * indirect indexes from the {@link QueryEngine}.
  196.    * 
  197.    * @param engine
  198.    *          the {@link QueryEngine} used to execute this query.
  199.    * @throws IllegalArgumentException
  200.    *           if the index represented by the provided query engine does not
  201.    *           have a direct index for the given sub-index (as specified by
  202.    *           {@link #indexType} and {@link #indexName}).
  203.    */
  204.   protected void prepare(QueryEngine engine) {
  205.     this.engine = engine;
  206.     switch(indexType){
  207.       case ANNOTATIONS:
  208.         atomicIndex = engine.getAnnotationIndex(indexName);
  209.         annotationHelper = engine.getAnnotationHelper(indexName);
  210.         break;
  211.       case TOKENS:
  212.         atomicIndex = engine.getTokenIndex(indexName);
  213.         break;
  214.       default:
  215.         throw new IllegalArgumentException("Invalid index type: " +
  216.           indexType.toString());
  217.     }
  218.     if(!atomicIndex.hasDirectIndex()) { throw new IllegalArgumentException(
  219.       "This type of query requires a " +
  220.         "direct index, but one was not found for (" +
  221.         indexType.toString().toLowerCase() + ") sub-index \"" + indexName +
  222.         "\""); }
  223.   }

  225.   protected TermsResultSet buildResultSet(DocumentIterator documentIterator)
  226.     throws IOException {
  227.     // prepare local data
  228.     ObjectArrayList<String> termStrings = new ObjectArrayList<String>();
  229.     ObjectArrayList<String> termDescriptions =
  230.       describeAnnotations ? new ObjectArrayList<String>() : null;
  231.     IntArrayList termCounts = countsEnabled ? new IntArrayList() : null;
  232.     TermCollectionVisitor termCollectionVisitor = null;
  233.     CounterSetupVisitor counterSetupVisitor = null;
  234.     CounterCollectionVisitor counterCollectionVisitor = null;
  235.     if(countsEnabled) {
  236.       termCollectionVisitor = new TermCollectionVisitor();
  237.       counterSetupVisitor = new CounterSetupVisitor(termCollectionVisitor);
  238.       counterCollectionVisitor =
  239.         new CounterCollectionVisitor(counterSetupVisitor);
  240.       termCollectionVisitor.prepare();
  241.       documentIterator.accept(termCollectionVisitor);
  242.       counterSetupVisitor.prepare();
  243.       documentIterator.accept(counterSetupVisitor);
  244.     }
  245.     if(stopWordsBlocked) {
  246.       // use the default list if no custom one was set
  247.       if(stopWords == null) setStopWords(DEFAULT_STOP_WORDS);
  248.     }
  249.     long termId = documentIterator.nextDocument();
  250.     terms: while(termId != DocumentIterator.END_OF_LIST && termId != -1) {
  251.       int termCount = -1;
  252.       if(countsEnabled) {
  253.         counterSetupVisitor.clear();
  254.         documentIterator.acceptOnTruePaths(counterCollectionVisitor);
  255.         termCount = 0;
  256.         for(int aCount : counterSetupVisitor.count)
  257.           termCount += aCount;
  258.       }
  259.       String termString = null;
  260.       // get the term string
  261.       try {
  262.         termString = atomicIndex.getDirectTerm(termId).toString();
  263.       } catch(Exception e) {
  264.         System.err.println("Error reading indirect index term with ID " +
  265.           termId);
  266.         e.printStackTrace();
  267.         termId = documentIterator.nextDocument();
  268.         continue terms;
  269.       }
  270.       if(stopWordsBlocked && stopWords.contains(termString)) {
  271.         // skip this term
  272.         termId = documentIterator.nextDocument();
  273.         continue terms;
  274.       }
  275.       if(indexType == IndexType.ANNOTATIONS) {
  276.         if(!annotationHelper.isMentionUri(termString)) {
  277.           // skip this term (not produced by our helper)
  278.           termId = documentIterator.nextDocument();
  279.           continue terms;
  280.         }
  281.         if(describeAnnotations) {
  282.           termDescriptions.add(annotationHelper.describeMention(termString));
  283.         }
  284.       }
  285.       termStrings.add(termString);
  286.       if(countsEnabled) {
  287.         termCounts.add(termCount);
  288.       }
  289.       termId = documentIterator.nextDocument();
  290.     }
  291.     // construct the result
  292.     TermsResultSet res =
  293.       new TermsResultSet(termStrings.toArray(new String[termStrings.size()]),
  294.         null, countsEnabled ? termCounts.toIntArray() : null,
  295.         describeAnnotations
  296.           ? termDescriptions.toArray(new String[termDescriptions.size()])
  297.           : null);
  298.     if(describeAnnotations) res = TermsResultSet.groupByDescription(res);
  299.     return res;
  300.   }

  302.   /**
  303.    * Should stop words be filtered out from the results? Defaults to
  304.    * <code>false</code>.
  305.    * 
  306.    * @return the stopWordsBlocked
  307.    */
  308.   public boolean isStopWordsBlocked() {
  309.     return stopWordsBlocked;
  310.   }

  312.   /**
  313.    * Enables or disables the filtering of stop words from the results. If a
  314.    * custom list of stop words has been set (by calling
  315.    * {@link #setStopWords(String[])}) then it is used, otherwise the
  316.    * {@link #DEFAULT_STOP_WORDS} list is used.
  317.    * 
  318.    * @param stopWordsBlocked
  319.    *          the stopWordsBlocked to set
  320.    */
  321.   public void setStopWordsBlocked(boolean stopWordsBlocked) {
  322.     this.stopWordsBlocked = stopWordsBlocked;
  323.   }

  325.   /**
  326.    * Gets the current custom list of stop words.
  327.    * 
  328.    * @return the stopWords
  329.    */
  330.   public Set<String> getStopWords() {
  331.     return stopWords;
  332.   }

  334.   public void setStopWords(Set<String> stopWords) {
  335.     this.stopWords = new HashSet<String>(stopWords);
  336.   }

  338.   /**
  339.    * Sets the custom list of stop words that should be blocked from query
  340.    * results. The actual blocking also needs to be enabled by calling
  341.    * {@link #setStopWordsBlocked(boolean)}. If this array is set to
  342.    * <code>null<code>, then the 
  343.    * {@link #DEFAULT_STOP_WORDS} are used.
  344.    * 
  345.    * @param stopWords
  346.    *          the stopWords to set
  347.    */
  348.   public void setStopWords(String[] stopWords) {
  349.     this.stopWords = new HashSet<String>(stopWords.length);
  350.     for(String sw : stopWords)
  351.       this.stopWords.add(sw);
  352.   }
  353. }
Created with JaCoCo 0.8.5.201910111838