<commit_before>package pl.edu.icm.coansys.citations

import pl.edu.icm.coansys.commons.scala.strings.rotations
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.hadoop.io.{Writable, Text, MapFile}
import collection.mutable.ListBuffer
import com.nicta.scoobi.core.DList
import com.nicta.scoobi.io.sequence.SeqSchema
import com.nicta.scoobi.Persist.persist
import com.nicta.scoobi.InputsOutputs.convertToSequenceFile
import com.nicta.scoobi.application.ScoobiConfiguration

/**
 * A class helping in approximate index saved in MapFile usage.
 *
 * @author <NAME> (<EMAIL>)
 */
class ApproximateIndex[V <: Writable : Manifest](val indexFileUri: String) {
  val conf = new Configuration()
  val reader = new MapFile.Reader(new Path(indexFileUri), conf)

  def get(query: String): Iterable[V] = {
    def isTooBig(query: String, key: String): Boolean =
      !key.startsWith(query.substring(0, query.length - 1))

    def isMatching(query: String, key: String): Boolean =
      (key.startsWith(query.substring(0, query.length - 1)) && key.length <= query.length) ||
        (key.startsWith(query) && key.length <= query.length + 1)
<commit_msg>Use local index file (for distributed cache)<commit_after>package pl.edu.icm.coansys.citations

import pl.edu.icm.coansys.commons.scala.strings.rotations
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.LocalFileSystem
import org.apache.hadoop.io.{Writable, Text, MapFile}
import collection.mutable.ListBuffer
import com.nicta.scoobi.core.DList
import com.nicta.scoobi.io.sequence.SeqSchema
import com.nicta.scoobi.Persist.persist
import com.nicta.scoobi.InputsOutputs.convertToSequenceFile
import com.nicta.scoobi.application.ScoobiConfiguration

/**
 * A class helping in approximate index saved in MapFile usage.
 *
 * @author <NAME> (<EMAIL>)
 */
class ApproximateIndex[V <: Writable : Manifest](val indexFileUri: String) {
  val conf = new Configuration()
  val reader = new MapFile.Reader(new LocalFileSystem(), indexFileUri, conf)

  def get(query: String): Iterable[V] = {
    def isTooBig(query: String, key: String): Boolean =
      !key.startsWith(query.substring(0, query.length - 1))

    def isMatching(query: String, key: String): Boolean =
      (key.startsWith(query.substring(0, query.length - 1)) && key.length <= query.length) ||
        (key.startsWith(query) && key.length <= query.length + 1)

{'fraction_non_alphanumeric': 0.12587701196863393, 'fraction_numerical': 0.004127115146512587, 'mean_word_length': 4.716981132075472, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '141947', 'n_tokens_mistral': 803, 'n_tokens_neox': 793, 'n_words': 169}