<commit_before>/* Copyright (C) 2008-2014 University of Massachusetts Amherst.
   This file is part of "FACTORIE" (Factor graphs, Imperative, Extensible)
   http://factorie.cs.umass.edu, http://github.com/factorie
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License. */
package cc.factorie.app.nlp.pos

import cc.factorie.app.nlp._
import cc.factorie.app.chain.ChainModel
import cc.factorie.app.chain.Observations._
import java.io._
import java.util.{HashMap, HashSet}
import cc.factorie.util.{HyperparameterMain, ClasspathURL, BinarySerializer}
import cc.factorie.optimize.Trainer
import cc.factorie.variable.{HammingObjective, BinaryFeatureVectorVariable, CategoricalVectorDomain, CategoricalVariable, LabeledVar, LabeledMutableDiscreteVar}
import scala.reflect.ClassTag

/** A linear-chain CRF part-of-speech tagger, doing inference by Viterbi.
    @author <NAME>, <NAME>
 */
abstract class ChainPosTagger[A<:PosTag](val tagConstructor:(Token)=>A)(implicit ct:ClassTag[A]) extends DocumentAnnotator {
  def this(tagConstructor:(Token)=>A, url:java.net.URL)(implicit ct:ClassTag[A]) = { this(tagConstructor); deserialize(url.openConnection().getInputStream) }
  def process(document: Document) = {
    document.sentences.foreach(s => {
      if (s.nonEmpty) {
        s.tokens.foreach(t => if (!t.attr.contains[A]) t.attr += tagConstructor(t))
        initPOSFeatures(s)
        model.maximize(s.tokens.map(_.attr[A]))(null)
      }
    })
    document
  }

  def prereqAttrs = Seq(classOf[Token], classOf[Sentence])
  def postAttrs = Seq(ct.runtimeClass)
  def tokenAnnotationString(token: Token) = { val label = token.attr[A with LabeledVar]; if (label ne null) label.categoryValue else "(null)" }

  def serialize(stream: OutputStream) {
    import cc.factorie.util.CubbieConversions._
    val dstream = new DataOutputStream(new BufferedOutputStream(stream))
    BinarySerializer.serialize(PosFeaturesDomain.dimensionDomain, dstream)
    BinarySerializer.serialize(model, dstream)
    dstream.close()
  }
  def deserialize(stream: InputStream) {
    import cc.factorie.util.CubbieConversions._
    val dstream = new DataInputStream(new BufferedInputStream(stream))
    BinarySerializer.deserialize(PosFeaturesDomain.dimensionDomain, dstream)
    BinarySerializer.deserialize(model, dstream)
    dstream.close()
  }

  def train(trainSentences:Seq[Sentence],
            testSentences:Seq[Sentence],
            lrate:Double = 0.1,
            decay:Double = 0.01,
            cutoff:Int = 2,
            doBootstrap:Boolean = true,
            useHingeLoss:Boolean = false,
            numIterations: Int = 5,
            l1Factor:Double = 0.000001,
            l2Factor:Double = 0.000001)(implicit random: scala.util.Random) {
    // TODO Accomplish this TokenNormalization instead by calling POS3.preProcess
    println("Initializing POS features for training sentences")
    trainSentences.foreach(initPOSFeatures)
    println("Finished initializing POS features for training sentences")
    PosFeaturesDomain.freeze()
    println("Initializing POS features for testing sentences")
    testSentences.foreach(initPOSFeatures)
    println("Finished initializing POS features for testing sentences")
    def evaluate() {
      println("Evaluating")
      (trainSentences ++ testSentences).foreach(s => model.maximize(s.tokens.map(_.attr[A with LabeledVar]))(null) )
      println("Train accuracy: "+ HammingObjective.accuracy(trainSentences.flatMap(s => s.tokens.map(_.attr[A with LabeledVar]))))
      println("Test accuracy: "+ HammingObjective.accuracy(testSentences.flatMap(s => s.tokens.map(_.attr[A with LabeledVar]))))
    }
    val examples =
    if(useHingeLoss)
      trainSentences.map(sentence => new model.ChainStructuredSVMExample(sentence.tokens.map(_.attr[A with LabeledMutableDiscreteVar]))).toSeq
    else
      trainSentences.map(sentence => new model.ChainLikelihoodExample(sentence.tokens.map(_.attr[A with LabeledMutableDiscreteVar])))
    //val optimizer = new cc.factorie.optimize.AdaGrad(rate=lrate)
    val optimizer = new cc.factorie.optimize.AdaGradRDA(rate=lrate, l1=l1Factor/examples.length, l2=l2Factor/examples.length)
    println("Running Parameter Optimization")
    Trainer.onlineTrain(model.parameters, examples, maxIterations=numIterations, optimizer=optimizer, evaluate=evaluate, useParallelTrainer = false)
  }


  object PosFeaturesDomain extends CategoricalVectorDomain[String]
  class PosFeatures(val token:Token) extends BinaryFeatureVectorVariable[String] { def domain = PosFeaturesDomain; override def skipNonCategories = true }

  val posDomain = tagConstructor(null).domain
  val model = new ChainModel[A, PosFeatures, Token](posDomain,
    PosFeaturesDomain,
    l => l.token.attr[PosFeatures],
    l => l.token,
    t => t.attr[A]){
    useObsMarkov = false
  }

  def initPOSFeatures(sentence: Sentence): Unit
}

class WSJChainPosTagger extends ChainPosTagger((t:Token) => new PennPosTag(t, 0)) {
  def this(url: java.net.URL) = {
    this()  
    deserialize(url.openConnection().getInputStream)
  }

  def initPOSFeatures(sentence: Sentence): Unit = {
    import cc.factorie.app.strings.simplifyDigits
    for (token <- sentence.tokens) {
      if(token.attr[PosFeatures] ne null)
        token.attr.remove[PosFeatures]

      val features = token.attr += new PosFeatures(token)
      val rawWord = token.string
      val word = simplifyDigits(rawWord).toLowerCase
      features += "W="+word
      features += "STEM=" + cc.factorie.app.strings.porterStem(word)
      features += "SHAPE2=" + cc.factorie.app.strings.stringShape(rawWord, 2)
      features += "SHAPE3=" + cc.factorie.app.strings.stringShape(rawWord, 3)
      // pre/suf of length 1..9
      //for (i <- 1 to 9) {
      val i = 3
      features += "SUFFIX" + i + "=" + word.takeRight(i)
      features += "PREFIX" + i + "=" + word.take(i)
      //}
      if (token.isCapitalized) features += "CAPITALIZED"
      if (token.string.matches("[A-Z]")) features += "CONTAINS_CAPITAL"
      if (token.string.matches("-")) features += "CONTAINS_DASH"
      if (token.containsDigit) features += "NUMERIC"
      if (token.isPunctuation) features += "PUNCTUATION"
    }
    addNeighboringFeatureConjunctions(sentence.tokens, (t: Token) => t.attr[PosFeatures], "W=[^@]*$", List(-2), List(-1), List(1), List(-2,-1), List(-1,0))
  }
}
object WSJChainPosTagger extends WSJChainPosTagger(ClasspathURL[WSJChainPosTagger](".factorie"))

class OntonotesChainPosTagger extends ChainPosTagger((t:Token) => new PennPosTag(t, 0)) {
  def this(url: java.net.URL) = {
    this()  
    deserialize(url.openConnection().getInputStream)
  }

  def initPOSFeatures(sentence: Sentence): Unit = {
    import cc.factorie.app.strings.simplifyDigits
    for (token <- sentence.tokens) {
      if(token.attr[PosFeatures] ne null)
        token.attr.remove[PosFeatures]

      val features = token.attr += new PosFeatures(token)
      val rawWord = token.string
      val word = simplifyDigits(rawWord).toLowerCase
      features += "W="+word
      features += "STEM=" + cc.factorie.app.strings.porterStem(word)
      features += "SHAPE2=" + cc.factorie.app.strings.stringShape(rawWord, 2)
      features += "SHAPE3=" + cc.factorie.app.strings.stringShape(rawWord, 3)
      // pre/suf of length 1..9
      //for (i <- 1 to 9) {
      val i = 3
      features += "SUFFIX" + i + "=" + word.takeRight(i)
      features += "PREFIX" + i + "=" + word.take(i)
      //}
      if (token.isCapitalized) features += "CAPITALIZED"
      if (token.string.matches("[A-Z]")) features += "CONTAINS_CAPITAL"
      if (token.string.matches("-")) features += "CONTAINS_DASH"
      if (token.containsDigit) features += "NUMERIC"
      if (token.isPunctuation) features += "PUNCTUATION"
    }
    addNeighboringFeatureConjunctions(sentence.tokens, (t: Token) => t.attr[PosFeatures], "W=[^@]*$", List(-2), List(-1), List(1), List(-2,-1), List(-1,0))
  }
}
object OntonotesChainPosTagger extends OntonotesChainPosTagger(ClasspathURL[OntonotesChainPosTagger](".factorie"))

class CtbChainPosTagger extends ChainPosTagger((t:Token) => new CtbPosTag(t, 0)) {

  private var prefixMap = new HashMap[Char, HashSet[String]]
  private var suffixMap = new HashMap[Char, HashSet[String]]

  def this(url: java.net.URL) = {
    this()  
    deserialize(url.openConnection().getInputStream)
  }

  override def train(trainSentences:Seq[Sentence],
                     testSentences:Seq[Sentence],
                     lrate:Double = 0.1,
                     decay:Double = 0.01,
                     cutoff:Int = 2,
                     doBootstrap:Boolean = true,
                     useHingeLoss:Boolean = false,
                     numIterations: Int = 5,
                     l1Factor:Double = 0.000001,
                     l2Factor:Double = 0.000001)(implicit random: scala.util.Random): Unit = {
    initPrefixAndSuffixMaps(trainSentences.flatMap(_.tokens))
    super.train(trainSentences, testSentences, lrate, decay, cutoff, doBootstrap, useHingeLoss, numIterations, l1Factor, l2Factor)
  }

  def initPOSFeatures(sentence: Sentence): Unit = {
    import cc.factorie.app.chineseStrings._

    for (token <- sentence.tokens) {
      if(token.attr[PosFeatures] ne null)
        token.attr.remove[PosFeatures]

      val features = token.attr += new PosFeatures(token)
      val rawWord = token.string
      val prefix = rawWord(0)
      val suffix = rawWord(rawWord.size - 1)

      features += "W="+rawWord

      val i = 3

      features += "SUFFIX" + i + "=" + rawWord.takeRight(i)
      features += "PREFIX" + i + "=" + rawWord.take(i)

      if(prefixMap.containsKey(prefix)) {
        val prefixLabelSet = prefixMap.get(prefix)
        val prefixCTBMorph = posDomain.categories.map{
          category =>

            val hasCategory = {
              if(prefixLabelSet.contains(category))
                "TRUE"
              else
                "FALSE"
            }

            "PRE_" + category + "_" + hasCategory
        }

        features ++= prefixCTBMorph
      }

      if(suffixMap.containsKey(suffix)) {
        val suffixLabelSet = suffixMap.get(suffix)
        val suffixCTBMorph = posDomain.categories.map{
          category =>

            val hasCategory = {
              if(suffixLabelSet.contains(category))
                "TRUE"
              else
                "FALSE"
            }

            "SUF_" + category + "_" + hasCategory
        }

        features ++= suffixCTBMorph
      }

      if (hasPunctuation(rawWord)) features += "PUNCTUATION"
      if (hasNumeric(rawWord)) features += "NUMERIC"
      if (hasChineseNumeric(rawWord)) features += "CHINESE_NUMERIC"
      if (hasAlpha(rawWord)) features += "ALPHA"
    }

    addNeighboringFeatureConjunctions(sentence.tokens,
                                      (t: Token) => t.attr[PosFeatures],
                                      "W=[^@]*$",
                                      List(-2),
                                      List(-1),
                                      List(1),
                                      List(-2,-1),
                                      List(-1,0))
  }

  def initPrefixAndSuffixMaps(tokens: Seq[Token]): Unit = {
    prefixMap.clear()
    suffixMap.clear()

    tokens.map(
      token => (token.string, token.attr[LabeledCtbPosTag].categoryValue)
    ).foreach{
      case (word, label) =>
<commit_msg>Removed some features<commit_after>/* Copyright (C) 2008-2014 University of Massachusetts Amherst.
   This file is part of "FACTORIE" (Factor graphs, Imperative, Extensible)
   http://factorie.cs.umass.edu, http://github.com/factorie
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License. */
package cc.factorie.app.nlp.pos

import cc.factorie.app.nlp._
import cc.factorie.app.chain.ChainModel
import cc.factorie.app.chain.Observations._
import java.io._
import java.util.{HashMap, HashSet}
import cc.factorie.util.{HyperparameterMain, ClasspathURL, BinarySerializer}
import cc.factorie.optimize.Trainer
import cc.factorie.variable.{HammingObjective, BinaryFeatureVectorVariable, CategoricalVectorDomain, CategoricalVariable, LabeledVar, LabeledMutableDiscreteVar}
import scala.reflect.ClassTag

/** A linear-chain CRF part-of-speech tagger, doing inference by Viterbi.
    @author <NAME>, <NAME>
 */
abstract class ChainPosTagger[A<:PosTag](val tagConstructor:(Token)=>A)(implicit ct:ClassTag[A]) extends DocumentAnnotator {
  def this(tagConstructor:(Token)=>A, url:java.net.URL)(implicit ct:ClassTag[A]) = { this(tagConstructor); deserialize(url.openConnection().getInputStream) }
  def process(document: Document) = {
    document.sentences.foreach(s => {
      if (s.nonEmpty) {
        s.tokens.foreach(t => if (!t.attr.contains[A]) t.attr += tagConstructor(t))
        initPOSFeatures(s)
        model.maximize(s.tokens.map(_.attr[A]))(null)
      }
    })
    document
  }

  def prereqAttrs = Seq(classOf[Token], classOf[Sentence])
  def postAttrs = Seq(ct.runtimeClass)
  def tokenAnnotationString(token: Token) = { val label = token.attr[A with LabeledVar]; if (label ne null) label.categoryValue else "(null)" }

  def serialize(stream: OutputStream) {
    import cc.factorie.util.CubbieConversions._
    val dstream = new DataOutputStream(new BufferedOutputStream(stream))
    BinarySerializer.serialize(PosFeaturesDomain.dimensionDomain, dstream)
    BinarySerializer.serialize(model, dstream)
    dstream.close()
  }
  def deserialize(stream: InputStream) {
    import cc.factorie.util.CubbieConversions._
    val dstream = new DataInputStream(new BufferedInputStream(stream))
    BinarySerializer.deserialize(PosFeaturesDomain.dimensionDomain, dstream)
    BinarySerializer.deserialize(model, dstream)
    dstream.close()
  }

  def train(trainSentences:Seq[Sentence],
            testSentences:Seq[Sentence],
            lrate:Double = 0.1,
            decay:Double = 0.01,
            cutoff:Int = 2,
            doBootstrap:Boolean = true,
            useHingeLoss:Boolean = false,
            numIterations: Int = 5,
            l1Factor:Double = 0.000001,
            l2Factor:Double = 0.000001)(implicit random: scala.util.Random) {
    // TODO Accomplish this TokenNormalization instead by calling POS3.preProcess
    println("Initializing POS features for training sentences")
    trainSentences.foreach(initPOSFeatures)
    println("Finished initializing POS features for training sentences")
    PosFeaturesDomain.freeze()
    println("Initializing POS features for testing sentences")
    testSentences.foreach(initPOSFeatures)
    println("Finished initializing POS features for testing sentences")
    def evaluate() {
      println("Evaluating")
      (trainSentences ++ testSentences).foreach(s => model.maximize(s.tokens.map(_.attr[A with LabeledVar]))(null) )
      println("Train accuracy: "+ HammingObjective.accuracy(trainSentences.flatMap(s => s.tokens.map(_.attr[A with LabeledVar]))))
      println("Test accuracy: "+ HammingObjective.accuracy(testSentences.flatMap(s => s.tokens.map(_.attr[A with LabeledVar]))))
    }
    val examples =
    if(useHingeLoss)
      trainSentences.map(sentence => new model.ChainStructuredSVMExample(sentence.tokens.map(_.attr[A with LabeledMutableDiscreteVar]))).toSeq
    else
      trainSentences.map(sentence => new model.ChainLikelihoodExample(sentence.tokens.map(_.attr[A with LabeledMutableDiscreteVar])))
    //val optimizer = new cc.factorie.optimize.AdaGrad(rate=lrate)
    val optimizer = new cc.factorie.optimize.AdaGradRDA(rate=lrate, l1=l1Factor/examples.length, l2=l2Factor/examples.length)
    println("Running Parameter Optimization")
    Trainer.onlineTrain(model.parameters, examples, maxIterations=numIterations, optimizer=optimizer, evaluate=evaluate, useParallelTrainer = false)
  }


  object PosFeaturesDomain extends CategoricalVectorDomain[String]
  class PosFeatures(val token:Token) extends BinaryFeatureVectorVariable[String] { def domain = PosFeaturesDomain; override def skipNonCategories = true }

  val posDomain = tagConstructor(null).domain
  val model = new ChainModel[A, PosFeatures, Token](posDomain,
    PosFeaturesDomain,
    l => l.token.attr[PosFeatures],
    l => l.token,
    t => t.attr[A]){
    useObsMarkov = false
  }

  def initPOSFeatures(sentence: Sentence): Unit
}

class WSJChainPosTagger extends ChainPosTagger((t:Token) => new PennPosTag(t, 0)) {
  def this(url: java.net.URL) = {
    this()  
    deserialize(url.openConnection().getInputStream)
  }

  def initPOSFeatures(sentence: Sentence): Unit = {
    import cc.factorie.app.strings.simplifyDigits
    for (token <- sentence.tokens) {
      if(token.attr[PosFeatures] ne null)
        token.attr.remove[PosFeatures]

      val features = token.attr += new PosFeatures(token)
      val rawWord = token.string
      val word = simplifyDigits(rawWord).toLowerCase
      features += "W="+word
      features += "STEM=" + cc.factorie.app.strings.porterStem(word)
      features += "SHAPE2=" + cc.factorie.app.strings.stringShape(rawWord, 2)
      features += "SHAPE3=" + cc.factorie.app.strings.stringShape(rawWord, 3)
      // pre/suf of length 1..9
      //for (i <- 1 to 9) {
      val i = 3
      features += "SUFFIX" + i + "=" + word.takeRight(i)
      features += "PREFIX" + i + "=" + word.take(i)
      //}
      if (token.isCapitalized) features += "CAPITALIZED"
      if (token.string.matches("[A-Z]")) features += "CONTAINS_CAPITAL"
      if (token.string.matches("-")) features += "CONTAINS_DASH"
      if (token.containsDigit) features += "NUMERIC"
      if (token.isPunctuation) features += "PUNCTUATION"
    }
    addNeighboringFeatureConjunctions(sentence.tokens, (t: Token) => t.attr[PosFeatures], "W=[^@]*$", List(-2), List(-1), List(1), List(-2,-1), List(-1,0))
  }
}
object WSJChainPosTagger extends WSJChainPosTagger(ClasspathURL[WSJChainPosTagger](".factorie"))

class OntonotesChainPosTagger extends ChainPosTagger((t:Token) => new PennPosTag(t, 0)) {
  def this(url: java.net.URL) = {
    this()  
    deserialize(url.openConnection().getInputStream)
  }

  def initPOSFeatures(sentence: Sentence): Unit = {
    import cc.factorie.app.strings.simplifyDigits
    for (token <- sentence.tokens) {
      if(token.attr[PosFeatures] ne null)
        token.attr.remove[PosFeatures]

      val features = token.attr += new PosFeatures(token)
      val rawWord = token.string
      val word = simplifyDigits(rawWord).toLowerCase
      features += "W="+word
      features += "STEM=" + cc.factorie.app.strings.porterStem(word)
      features += "SHAPE2=" + cc.factorie.app.strings.stringShape(rawWord, 2)
      features += "SHAPE3=" + cc.factorie.app.strings.stringShape(rawWord, 3)
      // pre/suf of length 1..9
      //for (i <- 1 to 9) {
      val i = 3
      features += "SUFFIX" + i + "=" + word.takeRight(i)
      features += "PREFIX" + i + "=" + word.take(i)
      //}
      if (token.isCapitalized) features += "CAPITALIZED"
      if (token.string.matches("[A-Z]")) features += "CONTAINS_CAPITAL"
      if (token.string.matches("-")) features += "CONTAINS_DASH"
      if (token.containsDigit) features += "NUMERIC"
      if (token.isPunctuation) features += "PUNCTUATION"
    }
    addNeighboringFeatureConjunctions(sentence.tokens, (t: Token) => t.attr[PosFeatures], "W=[^@]*$", List(-2), List(-1), List(1), List(-2,-1), List(-1,0))
  }
}
object OntonotesChainPosTagger extends OntonotesChainPosTagger(ClasspathURL[OntonotesChainPosTagger](".factorie"))

class CtbChainPosTagger extends ChainPosTagger((t:Token) => new CtbPosTag(t, 0)) {

  private var prefixMap = new HashMap[Char, HashSet[String]]
  private var suffixMap = new HashMap[Char, HashSet[String]]

  def this(url: java.net.URL) = {
    this()  
    deserialize(url.openConnection().getInputStream)
  }

  override def train(trainSentences:Seq[Sentence],
                     testSentences:Seq[Sentence],
                     lrate:Double = 0.1,
                     decay:Double = 0.01,
                     cutoff:Int = 2,
                     doBootstrap:Boolean = true,
                     useHingeLoss:Boolean = false,
                     numIterations: Int = 5,
                     l1Factor:Double = 0.000001,
                     l2Factor:Double = 0.000001)(implicit random: scala.util.Random): Unit = {
    initPrefixAndSuffixMaps(trainSentences.flatMap(_.tokens))
    super.train(trainSentences, testSentences, lrate, decay, cutoff, doBootstrap, useHingeLoss, numIterations, l1Factor, l2Factor)
  }

  def initPOSFeatures(sentence: Sentence): Unit = {
    import cc.factorie.app.chineseStrings._

    for (token <- sentence.tokens) {
      if(token.attr[PosFeatures] ne null)
        token.attr.remove[PosFeatures]

      val features = token.attr += new PosFeatures(token)
      val rawWord = token.string
      val prefix = rawWord(0)
      val suffix = rawWord(rawWord.size - 1)

      features += "W="+rawWord

      (0 to 4).foreach {
        i =>
          features += "SUFFIX" + i + "=" + rawWord.takeRight(i)
          features += "PREFIX" + i + "=" + rawWord.take(i)
      }

      if(prefixMap.containsKey(prefix)) {
        val prefixLabelSet = prefixMap.get(prefix)
        val prefixCTBMorph = posDomain.categories.map{
          category =>

            val hasCategory = {
              if(prefixLabelSet.contains(category))
                "TRUE"
              else
                "FALSE"
            }

            "PRE_" + category + "_" + hasCategory
        }

        features ++= prefixCTBMorph
      }

      if(suffixMap.containsKey(suffix)) {
        val suffixLabelSet = suffixMap.get(suffix)
        val suffixCTBMorph = posDomain.categories.map{
          category =>

            val hasCategory = {
              if(suffixLabelSet.contains(category))
                "TRUE"
              else
                "FALSE"
            }

            "SUF_" + category + "_" + hasCategory
        }

        features ++= suffixCTBMorph
      }
/*
      if (hasPunctuation(rawWord)) features += "PUNCTUATION"
      if (hasNumeric(rawWord)) features += "NUMERIC"
      if (hasChineseNumeric(rawWord)) features += "CHINESE_NUMERIC"
      if (hasAlpha(rawWord)) features += "ALPHA"
*/
    }

    addNeighboringFeatureConjunctions(sentence.tokens,
                                      (t: Token) => t.attr[PosFeatures],
                                      "W=[^@]*$",
                                      List(-2),
                                      List(-1),
                                      List(1),
                                      List(-2,-1),
                                      List(-1,0))
  }

  def initPrefixAndSuffixMaps(tokens: Seq[Token]): Unit = {
    prefixMap.clear()
    suffixMap.clear()

    tokens.map(
      token => (token.string, token.attr[LabeledCtbPosTag].categoryValue)
    ).foreach{
      case (word, label) =>

{'fraction_non_alphanumeric': 0.12113077761932724, 'fraction_numerical': 0.009269356597600873, 'mean_word_length': 2.990460251046025, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '529836', 'n_tokens_mistral': 7506, 'n_tokens_neox': 7139, 'n_words': 1507}