AtomicTokenIndex.java

  1. /*
  2.  *  AtomicTokenIndex.java
  3.  *
  4.  *  Copyright (c) 2007-2013, The University of Sheffield.
  5.  *
  6.  *  This file is part of GATE Mímir (see http://gate.ac.uk/family/mimir.html), 
  7.  *  and is free software, licenced under the GNU Lesser General Public License,
  8.  *  Version 3, June 2007 (also included with this distribution as file
  9.  *  LICENCE-LGPL3.html).
  10.  *
  11.  *  Valentin Tablan, 19 Dec 2013
  12.  *
  13.  *  $Id: AtomicTokenIndex.java 17371 2014-02-20 15:45:05Z valyt $
  14.  */
  15. package gate.mimir.index;

  17. import gate.Annotation;
  18. import gate.FeatureMap;
  19. import gate.mimir.DocumentMetadataHelper;
  20. import gate.mimir.IndexConfig.TokenIndexerConfig;
  21. import gate.mimir.MimirIndex;
  22. import it.unimi.di.big.mg4j.index.Index;
  23. import it.unimi.dsi.lang.ObjectParser;

  25. import java.io.File;
  26. import java.io.IOException;
  27. import java.io.UnsupportedEncodingException;
  28. import java.nio.ByteBuffer;
  29. import java.nio.CharBuffer;
  30. import java.nio.charset.CharacterCodingException;
  31. import java.nio.charset.Charset;
  32. import java.nio.charset.CharsetDecoder;
  33. import java.nio.charset.CharsetEncoder;
  34. import java.nio.charset.CodingErrorAction;
  35. import java.util.LinkedList;
  36. import java.util.List;
  37. import java.util.concurrent.BlockingQueue;

  39. import org.slf4j.Logger;
  40. import org.slf4j.LoggerFactory;

  42. /**
  43.  * An {@link AtomicIndex} implementation for indexing tokens.
  44.  */
  45. public class AtomicTokenIndex extends AtomicIndex {
  46.   
  47.   private final static Logger logger = LoggerFactory.getLogger(AtomicTokenIndex.class);
  48.   
  49.   /**
  50.    * A constant (empty String array) used for filtering terms from indexing.
  51.    * @see #calculateTermStringForAnnotation(Annotation, GATEDocument)
  52.    * implementation.
  53.    */
  54.   private static final String[] DO_NOT_INDEX = new String[]{};
  55.   
  56.   
  57.   protected final CharsetEncoder UTF8_CHARSET_ENCODER = Charset.forName("UTF-8").newEncoder();
  58.   
  59.   protected final CharsetDecoder UTF8_CHARSET_DECODER = Charset.forName("UTF-8").newDecoder();
  60.   
  61.   /**
  62.    * Is this token index responsible for writing the zip collection?
  63.    */
  64.   protected boolean zipCollectionEnabled = false;
  65.   
  66.   /**
  67.    * Stores the document tokens for writing to the zip collection;
  68.    */
  69.   protected List<String> documentTokens;
  70.   
  71.   /**
  72.    * Stores the document non-tokens for writing to the zip collection;
  73.    */
  74.   protected List<String> documentNonTokens;
  75.   
  76.   /**
  77.    * An array of helpers for creating document metadata. 
  78.    */
  79.   protected DocumentMetadataHelper[] docMetadataHelpers;
  80.   
  81.   /**
  82.    * GATE document factory used by the zip builder, and also to
  83.    * translate field indexes to field names.
  84.    */
  85.   protected GATEDocumentFactory factory;
  86.   
  87.   
  88.   /**
  89.    * The feature name corresponding to the field.
  90.    */
  91.   protected String featureName;
  92.   

  94.   
  95.   /**
  96.    * Creates a new atomic index for indexing tokens. 
  97.    * @param parent the top level {@link MimirIndex} to which this new atomic 
  98.    * index belongs.
  99.    * @param name the name for the new atomic index. This will be used as the
  100.    * name of the top level directory for this atomic index (which is a 
  101.    * sub-directory of the parent) and as a base name for all the files of this 
  102.    * atomic index.
  103.    * @param hasDirectIndex should a direct index be created as well.
  104.    * @param inputQueue the queue where documents are submitted for indexing;
  105.    * @param outputQueue the queue where indexed documents are returned to;
  106.    * @throws IndexException 
  107.    * @throws IOException 
  108.    */
  109.   public AtomicTokenIndex(MimirIndex parent, String name,
  110.       boolean hasDirectIndex, BlockingQueue<GATEDocument> inputQueue,
  111.       BlockingQueue<GATEDocument> outputQueue, TokenIndexerConfig config,
  112.       boolean zipCollection) throws IOException, IndexException {
  113.     super(parent, name, hasDirectIndex, 
  114.         config.getTermProcessor(), inputQueue, outputQueue);
  115.     this.featureName = config.getFeatureName();
  116.     this.zipCollectionEnabled = zipCollection;
  117.     if(zipCollectionEnabled) {
  118.       documentTokens = new LinkedList<String>();
  119.       documentNonTokens = new LinkedList<String>();
  120.       docMetadataHelpers = parent.getIndexConfig().getDocMetadataHelpers();
  121.     }
  122.     
  123.     // save the term processor
  124.     additionalProperties.setProperty(Index.PropertyKeys.TERMPROCESSOR, 
  125.         ObjectParser.toSpec(termProcessor));
  126.     
  127.     try {
  128.       UTF8_CHARSET_ENCODER.replaceWith("[?]".getBytes("UTF-8"));
  129.       UTF8_CHARSET_ENCODER.onMalformedInput(CodingErrorAction.REPLACE);
  130.       UTF8_CHARSET_ENCODER.onUnmappableCharacter(CodingErrorAction.REPLACE);
  131.     } catch(UnsupportedEncodingException e) {
  132.       // this should never happen
  133.       throw new RuntimeException("UTF-8 not supported");
  134.     }
  135.     
  136.     indexingThread = new Thread(this, "Mimir-" + name + " indexing thread");
  137.     indexingThread.start();
  138.   }

  140.   
  141.   /**
  142.    * If zipping, inform the collection builder that a new document
  143.    * is about to start.
  144.    */
  145.   protected void documentStarting(GATEDocument gateDocument) throws IndexException {
  146.     if(zipCollectionEnabled) {
  147.       // notify the metadata helpers
  148.       if(docMetadataHelpers != null){
  149.         for(DocumentMetadataHelper aHelper : docMetadataHelpers){
  150.           aHelper.documentStart(gateDocument);
  151.         }
  152.       }
  153.     }
  154.     // set lastTokenIndex to -1 so we don't have to special-case the first
  155.     // token in the document in calculateStartPosition
  156.     tokenPosition = -1;
  157.   }

  159.   /**
  160.    * If zipping, inform the collection builder that we finished
  161.    * the current document.
  162.    */
  163.   protected void documentEnding(GATEDocument gateDocument) throws IndexException {
  164.     if(zipCollectionEnabled) {
  165.       DocumentData docData = new DocumentData(
  166.           gateDocument.uri().toString(), 
  167.           gateDocument.title().toString(),
  168.           documentTokens.toArray(new String[documentTokens.size()]),
  169.           documentNonTokens.toArray(new String[documentNonTokens.size()])); 
  170.       if(docMetadataHelpers != null){
  171.         for(DocumentMetadataHelper aHelper : docMetadataHelpers){
  172.           aHelper.documentEnd(gateDocument, docData);
  173.         }
  174.       }
  175.       parent.writeZipDocumentData(docData);
  176.       documentTokens.clear();
  177.       documentNonTokens.clear();
  178.     }
  179.   }

  181.   /**
  182.    * Get the token annotations from this document, in increasing
  183.    * order of offset.
  184.    */
  185.   protected Annotation[] getAnnotsToProcess(GATEDocument gateDocument) {
  186.     Annotation[] tokens = gateDocument.getTokenAnnots(); 
  187.     return tokens;
  188.   }

  190.   /**
  191.    * This indexer always adds one posting per token, so the start
  192.    * position for the next annotation is always one more than the
  193.    * previous one.
  194.    * 
  195.    * @param ann
  196.    * @param gateDocument
  197.    */
  198.   protected void calculateStartPositionForAnnotation(Annotation ann,
  199.           GATEDocument gateDocument) {
  200.     tokenPosition++;
  201.   }

  203.   /**
  204.    * For a token annotation, the "string" we index is the feature value
  205.    * corresponding to the name of the field to index.  As well as
  206.    * calculating the string, this method writes an entry to the zip
  207.    * collection builder if it exists.
  208.    * 
  209.    * @param ann
  210.    * @param gateDocument
  211.    */
  212.   protected String[] calculateTermStringForAnnotation(Annotation ann,
  213.           GATEDocument gateDocument) throws IndexException {
  214.     FeatureMap tokenFeatures = ann.getFeatures();
  215.     String value = (String)tokenFeatures.get(featureName);
  216.     // make sure we get valid UTF-8 content
  217.    // illegal strings will simply be rendered as "[UNMAPPED]"
  218.     if(value != null) {
  219.       try {
  220.         CharBuffer cb = CharBuffer.wrap(value);
  221.         ByteBuffer bb = UTF8_CHARSET_ENCODER.encode(cb);
  222.         cb = UTF8_CHARSET_DECODER.decode(bb);
  223.         value  = cb.toString();
  224.       } catch(CharacterCodingException e) {
  225.         // this should not happen
  226.         value = null;
  227.         logger.error("Error while normalizing input", e);
  228.       }      
  229.     }

  231.     
  232.     currentTerm.replace(value == null ? "" : value);
  233.     //save the *unprocessed* term to the collection, if required.
  234.     if(zipCollectionEnabled) {
  235.       documentTokens.add(currentTerm.toString());
  236.       documentNonTokens.add(gateDocument.getNonTokens()[tokenPosition]);
  237.     }
  238.     if(termProcessor.processTerm(currentTerm)){
  239.       //the processor has changed the term, and allowed us to index it
  240.       return null;  
  241.     }else{
  242.       //the processor has filtered the term -> don't index it.
  243.       return DO_NOT_INDEX;
  244.     }
  245.   }

  247.   /**
  248.    * Overridden to close the zip collection builder.
  249.    */
  250.   @Override
  251.   protected void flush() throws IOException {
  252.   }
  253. }
Created with JaCoCo 0.8.5.201910111838