<commit_before>// BSD License (http://lemurproject.org/galago-license)
package org.lemurproject.galago.core.index.disk;

import java.util.logging.Level;
import org.lemurproject.galago.core.retrieval.iterator.NullExtentIterator;
import org.lemurproject.galago.core.retrieval.iterator.ModifiableIterator;
import org.lemurproject.galago.core.retrieval.query.Node;
import org.lemurproject.galago.core.retrieval.query.NodeType;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Logger;
import org.lemurproject.galago.core.index.AggregateReader.CollectionStatistics;
import org.lemurproject.galago.core.index.BTreeFactory;
import org.lemurproject.galago.core.index.BTreeReader;
import org.lemurproject.galago.core.index.Index;
import org.lemurproject.galago.core.index.KeyListReader;
import org.lemurproject.galago.core.index.NamesReader;
import org.lemurproject.galago.core.index.IndexPartModifier;
import org.lemurproject.galago.core.index.IndexPartReader;
import org.lemurproject.galago.core.index.LengthsReader;
import org.lemurproject.galago.core.index.ValueIterator;
import org.lemurproject.galago.core.index.corpus.CorpusReader;
import org.lemurproject.galago.core.index.corpus.SplitBTreeReader;
import org.lemurproject.galago.core.parse.Document;
import org.lemurproject.galago.core.retrieval.query.NodeParameters;
import org.lemurproject.galago.tupleflow.Parameters;

/**
 * This is the main class for a disk based index structure
 * 
 * A index is a set of parts and modifiers
 * Each part is a index file that offers one or more iterators
 * Each modifier alters or extends the data provided by the corresponding part
 * Queries are be processed in this class using a tree of iterators
 * 
 * See the Index interface for the list of public functions.
 * 
 * @author trevor, sjh, irmarc
 */
public class DiskIndex implements Index {

  protected File location;
  protected Parameters manifest = new Parameters();
  protected LengthsReader lengthsReader = null;
  protected NamesReader namesReader = null;
  
  protected Map<String, IndexPartReader> parts = new HashMap<String, IndexPartReader>();
  protected Map<String, CollectionStatistics> partStatistics = new HashMap<String, CollectionStatistics>();
  protected Map<String, HashMap<String, IndexPartModifier>> modifiers = new HashMap<String, HashMap<String, IndexPartModifier>>();
  protected HashMap<String, String> defaultIndexOperators = new HashMap<String, String>();
  protected HashSet<String> knownIndexOperators = new HashSet<String>();

  // useful to assemble an index from odd pieces
  public DiskIndex(Collection<String> indexParts) throws IOException{
    location = null;

    for(String indexPart : indexParts){
      File part = new File(indexPart);
      IndexComponentReader component = openIndexComponent(part.getAbsolutePath());
      initializeComponent(part.getName(), component);
    }
    // Initialize these now b/c they're so common
    if (parts.containsKey("lengths")) {
      lengthsReader = (DiskLengthsReader) parts.get("lengths");
    }
    if (parts.containsKey("names")) {
      namesReader = (DiskNameReader) parts.get("names");
    }

    initializeIndexOperators();
  }


  public DiskIndex(String indexPath) throws IOException {
    // Make sure it's a valid location    
    location = new File(indexPath);
    if (!location.isDirectory()) {
      throw new IOException(String.format("%s is not a directory.", indexPath));
    }

    // Load all parts
    openDiskParts("", location);

    // Initialize these now b/c they're so common
    if (parts.containsKey("lengths")) {
      lengthsReader = (DiskLengthsReader) parts.get("lengths");
    }
    if (parts.containsKey("names")) {
      namesReader = (DiskNameReader) parts.get("names");
    }

    initializeIndexOperators();
  }

  /**
   * recursively open index parts
   *  + infer if the file/folder is a part or a modifier
   * 
   *  prefix should be empty string or a path ending with a slash
   */
  protected void openDiskParts(String name, File directory) throws IOException {
    // check if the directory is a split index folder: (e.g. corpus)
    if (SplitBTreeReader.isBTree(directory)) {
      IndexComponentReader component = openIndexComponent(directory.getAbsolutePath());
      if (component != null) {
        initializeComponent(name, component);
      }
      return;
    }

    // otherwise the directory might contain stand-alone index files
    for (File part : directory.listFiles()) {
      String partName = (name.length() == 0) ? part.getName() : name + "/" + part.getName();
      if (part.isDirectory()) {
        openDiskParts(partName, part);
      } else {
        IndexComponentReader component = openIndexComponent(part.getAbsolutePath());
        if (component != null) {
          initializeComponent(partName, component);
        }
      }
    }
  }

  protected void initializeComponent(String name, IndexComponentReader component) {
    if (IndexPartReader.class.isAssignableFrom(component.getClass())) {
      parts.put(name, (IndexPartReader) component);
      partStatistics.put(name, new CollectionStatistics(name, component.getManifest()));

    } else if (IndexPartModifier.class.isAssignableFrom(component.getClass())) {
      // need to pop off the dirname : (e.g. mod/)
      String[] nameParts = name.split("\\.");

      if (nameParts.length >= 2) {
        if (!modifiers.containsKey(nameParts[0])) {
          modifiers.put(nameParts[0], new HashMap<String, IndexPartModifier>());
        }
        modifiers.get(nameParts[0]).put(nameParts[1], (IndexPartModifier) component);
      }
    }
  }

  public File getIndexLocation() {
    return location;
  }

  // I'd really prefer to get this from the manifest, but writing the manifest reliably seems
  // difficult right now, so just use it if available, otherwise use well-known defaults.
  @Override
  public String getDefaultPart() {
    if (manifest.containsKey("defaultPart")) {
      String part = manifest.getString("defaultPart");
      if (parts.containsKey(part)) {
        return part;
      }
    }

    // otherwise, try to default
    if (parts.containsKey("postings.porter")) {
      return "postings.porter";
    }
    if (parts.containsKey("postings.krovetz")) {
      return "postings.porter";
    }
    if (parts.containsKey("postings")) {
      return "postings";
    }
    // otherwise - anything will do.
    return parts.keySet().iterator().next();<commit_msg>Fixed defaulty part assignment - postings.krovetz<commit_after>// BSD License (http://lemurproject.org/galago-license)
package org.lemurproject.galago.core.index.disk;

import java.util.logging.Level;
import org.lemurproject.galago.core.retrieval.iterator.NullExtentIterator;
import org.lemurproject.galago.core.retrieval.iterator.ModifiableIterator;
import org.lemurproject.galago.core.retrieval.query.Node;
import org.lemurproject.galago.core.retrieval.query.NodeType;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Logger;
import org.lemurproject.galago.core.index.AggregateReader.CollectionStatistics;
import org.lemurproject.galago.core.index.BTreeFactory;
import org.lemurproject.galago.core.index.BTreeReader;
import org.lemurproject.galago.core.index.Index;
import org.lemurproject.galago.core.index.KeyListReader;
import org.lemurproject.galago.core.index.NamesReader;
import org.lemurproject.galago.core.index.IndexPartModifier;
import org.lemurproject.galago.core.index.IndexPartReader;
import org.lemurproject.galago.core.index.LengthsReader;
import org.lemurproject.galago.core.index.ValueIterator;
import org.lemurproject.galago.core.index.corpus.CorpusReader;
import org.lemurproject.galago.core.index.corpus.SplitBTreeReader;
import org.lemurproject.galago.core.parse.Document;
import org.lemurproject.galago.core.retrieval.query.NodeParameters;
import org.lemurproject.galago.tupleflow.Parameters;

/**
 * This is the main class for a disk based index structure
 * 
 * A index is a set of parts and modifiers
 * Each part is a index file that offers one or more iterators
 * Each modifier alters or extends the data provided by the corresponding part
 * Queries are be processed in this class using a tree of iterators
 * 
 * See the Index interface for the list of public functions.
 * 
 * @author trevor, sjh, irmarc
 */
public class DiskIndex implements Index {

  protected File location;
  protected Parameters manifest = new Parameters();
  protected LengthsReader lengthsReader = null;
  protected NamesReader namesReader = null;
  
  protected Map<String, IndexPartReader> parts = new HashMap<String, IndexPartReader>();
  protected Map<String, CollectionStatistics> partStatistics = new HashMap<String, CollectionStatistics>();
  protected Map<String, HashMap<String, IndexPartModifier>> modifiers = new HashMap<String, HashMap<String, IndexPartModifier>>();
  protected HashMap<String, String> defaultIndexOperators = new HashMap<String, String>();
  protected HashSet<String> knownIndexOperators = new HashSet<String>();

  // useful to assemble an index from odd pieces
  public DiskIndex(Collection<String> indexParts) throws IOException{
    location = null;

    for(String indexPart : indexParts){
      File part = new File(indexPart);
      IndexComponentReader component = openIndexComponent(part.getAbsolutePath());
      initializeComponent(part.getName(), component);
    }
    // Initialize these now b/c they're so common
    if (parts.containsKey("lengths")) {
      lengthsReader = (DiskLengthsReader) parts.get("lengths");
    }
    if (parts.containsKey("names")) {
      namesReader = (DiskNameReader) parts.get("names");
    }

    initializeIndexOperators();
  }


  public DiskIndex(String indexPath) throws IOException {
    // Make sure it's a valid location    
    location = new File(indexPath);
    if (!location.isDirectory()) {
      throw new IOException(String.format("%s is not a directory.", indexPath));
    }

    // Load all parts
    openDiskParts("", location);

    // Initialize these now b/c they're so common
    if (parts.containsKey("lengths")) {
      lengthsReader = (DiskLengthsReader) parts.get("lengths");
    }
    if (parts.containsKey("names")) {
      namesReader = (DiskNameReader) parts.get("names");
    }

    initializeIndexOperators();
  }

  /**
   * recursively open index parts
   *  + infer if the file/folder is a part or a modifier
   * 
   *  prefix should be empty string or a path ending with a slash
   */
  protected void openDiskParts(String name, File directory) throws IOException {
    // check if the directory is a split index folder: (e.g. corpus)
    if (SplitBTreeReader.isBTree(directory)) {
      IndexComponentReader component = openIndexComponent(directory.getAbsolutePath());
      if (component != null) {
        initializeComponent(name, component);
      }
      return;
    }

    // otherwise the directory might contain stand-alone index files
    for (File part : directory.listFiles()) {
      String partName = (name.length() == 0) ? part.getName() : name + "/" + part.getName();
      if (part.isDirectory()) {
        openDiskParts(partName, part);
      } else {
        IndexComponentReader component = openIndexComponent(part.getAbsolutePath());
        if (component != null) {
          initializeComponent(partName, component);
        }
      }
    }
  }

  protected void initializeComponent(String name, IndexComponentReader component) {
    if (IndexPartReader.class.isAssignableFrom(component.getClass())) {
      parts.put(name, (IndexPartReader) component);
      partStatistics.put(name, new CollectionStatistics(name, component.getManifest()));

    } else if (IndexPartModifier.class.isAssignableFrom(component.getClass())) {
      // need to pop off the dirname : (e.g. mod/)
      String[] nameParts = name.split("\\.");

      if (nameParts.length >= 2) {
        if (!modifiers.containsKey(nameParts[0])) {
          modifiers.put(nameParts[0], new HashMap<String, IndexPartModifier>());
        }
        modifiers.get(nameParts[0]).put(nameParts[1], (IndexPartModifier) component);
      }
    }
  }

  public File getIndexLocation() {
    return location;
  }

  // I'd really prefer to get this from the manifest, but writing the manifest reliably seems
  // difficult right now, so just use it if available, otherwise use well-known defaults.
  @Override
  public String getDefaultPart() {
    if (manifest.containsKey("defaultPart")) {
      String part = manifest.getString("defaultPart");
      if (parts.containsKey(part)) {
        return part;
      }
    }

    // otherwise, try to default
    if (parts.containsKey("postings.porter")) {
      return "postings.porter";
    }
    if (parts.containsKey("postings.krovetz")) {
      return "postings.krovetz";
    }
    if (parts.containsKey("postings")) {
      return "postings";
    }
    // otherwise - anything will do.
    return parts.keySet().iterator().next();
{'fraction_non_alphanumeric': 0.11050548794146196, 'fraction_numerical': 0.0008959904427686105, 'mean_word_length': 3.8670058139534884, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6204560', 'n_tokens_mistral': 3802, 'n_tokens_neox': 3751, 'n_words': 969}