<commit_before>package leo.datastructures.internal

import leo.datastructures.internal.terms._
import scala.Some
import scala.Some
import scala.Some
import scala.Some

/**
 * Created by lex on 16.10.14.
 */
object TermIndex {


  var terms: Set[Term] = Set.empty
  var symbol_of: Map[Signature#Key, Set[Term]] = Map.empty
  var headsymbol_of: Map[Term, Set[Term]] = Map.empty
  var occurs_in: Map[Term, Set[(Term, Position)]] = Map.empty
  var occurs_at: Map[Term, Map[Position, Set[Term]]] = Map.empty

  def insert(term: Term): Term = {
    val t = term.betaNormalize
    val t2 = t.makeGlobal
    // Force computation of lazy values
    t2.headSymbol
    t2.freeVars
    t2.occurrences

    // insert to data structures
    for (s <- t2.symbols) {
      symbol_of.get(s) match {
        case None => symbol_of += ((s,Set(t2)))
        case Some(set) => symbol_of += ((s, set + t2))
      }
    }
    val hs = t2.headSymbol
    headsymbol_of.get(hs) match {
      case None => headsymbol_of += ((hs, Set(t2)))
      case Some(set) => headsymbol_of += ((hs, set + t2))
    }
    insertSubterms(t2, t2, Position.root)
    terms += t2

    t2
  }

  def byHeadsymbol(head: Term): Set[Term] = headsymbol_of.getOrElse(head, Set())
  def bySymbol(sym: Signature#Key): Set[Term] = symbol_of.getOrElse(sym, Set())

  def bySubterm(subterm: Term): Set[(Term, Position)] = occurs_in.getOrElse(subterm, Set())
  def bySubtermAtPos(subterm: Term, pos: Position): Set[(Term)] = occurs_at.get(subterm) match {
    case None => Set()
    case Some(inner) => inner.getOrElse(pos, Set())
  }

  protected def insertSubterms(term: Term, subterm: Term, position: Position): Unit = {
    occurs_in.get(subterm) match {
      case None => occurs_in += ((subterm, Set((term, position))))
      case Some(set) => occurs_in += ((subterm, set + ((term, position))))
    }

    occurs_at.get(subterm) match {
      case None => occurs_at += ((subterm, Map((position, term))))
      case Some(inner) => inner.get(position) match {
        case None => occurs_at += ((subterm, inner + ((position, Set(term)))))
        case Some(set) => occurs_at += ((subterm, inner + ((position, set + term))))
      }
    }

    subterm match {
      case Bound(t,scope) => ()
      case Symbol(id)     => ()
      case s @@@ t        => () // not implemented for curried terms
      case f ∙ args       => insertSubterms(term, f, position.headPos); var i = 1
                              for(arg <- args) {
                                arg match {
                                  case Left(t) => insertSubterms(term, t, position.spinePos.argPos(i)); i = i+1
                                  case Right(_) => ()
                                }
                              }
      case s @@@@ ty      => () // not implemented for curried terms
      case ty :::> s      => insertSubterms(term, s, position.abstrPos)
      case TypeLambda(t)  => insertSubterms(term, t, position.abstrPos)
    }
  }
}<commit_msg>fixed typo in term index insertion<commit_after>package leo.datastructures.internal

import leo.datastructures.internal.terms._
import scala.Some
import scala.Some
import scala.Some
import scala.Some

/**
 * Created by lex on 16.10.14.
 */
object TermIndex {


  var terms: Set[Term] = Set.empty
  var symbol_of: Map[Signature#Key, Set[Term]] = Map.empty
  var headsymbol_of: Map[Term, Set[Term]] = Map.empty
  var occurs_in: Map[Term, Set[(Term, Position)]] = Map.empty
  var occurs_at: Map[Term, Map[Position, Set[Term]]] = Map.empty

  def insert(term: Term): Term = {
    val t = term.betaNormalize
    val t2 = t.makeGlobal
    // Force computation of lazy values
    t2.headSymbol
    t2.freeVars
    t2.occurrences

    // insert to data structures
    for (s <- t2.symbols) {
      symbol_of.get(s) match {
        case None => symbol_of += ((s,Set(t2)))
        case Some(set) => symbol_of += ((s, set + t2))
      }
    }
    val hs = t2.headSymbol
    headsymbol_of.get(hs) match {
      case None => headsymbol_of += ((hs, Set(t2)))
      case Some(set) => headsymbol_of += ((hs, set + t2))
    }
    insertSubterms(t2, t2, Position.root)
    terms += t2

    t2
  }

  def byHeadsymbol(head: Term): Set[Term] = headsymbol_of.getOrElse(head, Set())
  def bySymbol(sym: Signature#Key): Set[Term] = symbol_of.getOrElse(sym, Set())

  def bySubterm(subterm: Term): Set[(Term, Position)] = occurs_in.getOrElse(subterm, Set())
  def bySubtermAtPos(subterm: Term, pos: Position): Set[(Term)] = occurs_at.get(subterm) match {
    case None => Set()
    case Some(inner) => inner.getOrElse(pos, Set())
  }

  protected def insertSubterms(term: Term, subterm: Term, position: Position): Unit = {
    occurs_in.get(subterm) match {
      case None => occurs_in += ((subterm, Set((term, position))))
      case Some(set) => occurs_in += ((subterm, set + ((term, position))))
    }

    occurs_at.get(subterm) match {
      case None => occurs_at += ((subterm, Map((position, Set(term)))))
      case Some(inner) => inner.get(position) match {
        case None => occurs_at += ((subterm, inner + ((position, Set(term)))))
        case Some(set) => occurs_at += ((subterm, inner + ((position, set + term))))
      }
    }

    subterm match {
      case Bound(t,scope) => ()
      case Symbol(id)     => ()
      case s @@@ t        => () // not implemented for curried terms
      case f ∙ args       => insertSubterms(term, f, position.headPos); var i = 1
                              for(arg <- args) {
                                arg match {
                                  case Left(t) => insertSubterms(term, t, position.spinePos.argPos(i)); i = i+1
                                  case Right(_) => ()
                                }
                              }
      case s @@@@ ty      => () // not implemented for curried terms
      case ty :::> s      => insertSubterms(term, s, position.abstrPos)
      case TypeLambda(t)  => insertSubterms(term, t, position.abstrPos)
    }
  }
}
{'fraction_non_alphanumeric': 0.14529058116232466, 'fraction_numerical': 0.007348029392117568, 'mean_word_length': 2.4105922551252847, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2889560', 'n_tokens_mistral': 1997, 'n_tokens_neox': 1922, 'n_words': 495}