<commit_before>package opennlp.textgrounder.tr.app

import java.io._
import java.util.zip._

import opennlp.textgrounder.tr.util._
import opennlp.textgrounder.tr.topo._
import opennlp.textgrounder.tr.topo.gaz._
import opennlp.textgrounder.tr.text._
import opennlp.textgrounder.tr.text.prep._
import opennlp.textgrounder.tr.text.io._

import scala.collection.JavaConversions._

import org.apache.tools.bzip2._
import org.clapper.argot._
import ArgotConverters._

import opennlp.maxent._
import opennlp.maxent.io._
import opennlp.model._

object SupervisedTRMaxentModelTrainer extends App {
  val parser = new ArgotParser("textgrounder run opennlp.textgrounder.tr.app.SupervisedTRMaxentModelTrainer", preUsage = Some("Textgrounder"))

  val wikiCorpusInputFile = parser.option[String](List("c", "corpus"), "corpus", "wiki training corpus input file")
  val wikiTextInputFile = parser.option[String](List("w", "wiki"), "wiki", "wiki text input file")
  val trInputFile = parser.option[String](List("i", "tr-input"), "tr-input", "TR-CoNLL input path")
  val gazInputFile = parser.option[String](List("g", "gaz"), "gaz", "serialized gazetteer input file")
  val stoplistInputFile = parser.option[String](List("s", "stoplist"), "stoplist", "stopwords input file")
  val modelsOutputDir = parser.option[String](List("d", "models-dir"), "models-dir", "models output directory")

  val windowSize = 20
  val dpc = 1.0
  val threshold = 1.0
  val iterations = 10
  val cutoff = 5<commit_msg>Added parameter for distance threshold.<commit_after>package opennlp.textgrounder.tr.app

import java.io._
import java.util.zip._

import opennlp.textgrounder.tr.util._
import opennlp.textgrounder.tr.topo._
import opennlp.textgrounder.tr.topo.gaz._
import opennlp.textgrounder.tr.text._
import opennlp.textgrounder.tr.text.prep._
import opennlp.textgrounder.tr.text.io._

import scala.collection.JavaConversions._

import org.apache.tools.bzip2._
import org.clapper.argot._
import ArgotConverters._

import opennlp.maxent._
import opennlp.maxent.io._
import opennlp.model._

object SupervisedTRMaxentModelTrainer extends App {
  val parser = new ArgotParser("textgrounder run opennlp.textgrounder.tr.app.SupervisedTRMaxentModelTrainer", preUsage = Some("Textgrounder"))

  val wikiCorpusInputFile = parser.option[String](List("c", "corpus"), "corpus", "wiki training corpus input file")
  val wikiTextInputFile = parser.option[String](List("w", "wiki"), "wiki", "wiki text input file")
  val trInputFile = parser.option[String](List("i", "tr-input"), "tr-input", "TR-CoNLL input path")
  val gazInputFile = parser.option[String](List("g", "gaz"), "gaz", "serialized gazetteer input file")
  val stoplistInputFile = parser.option[String](List("s", "stoplist"), "stoplist", "stopwords input file")
  val modelsOutputDir = parser.option[String](List("d", "models-dir"), "models-dir", "models output directory")
  val thresholdParam = parser.option[Double](List("t", "threshold"), "threshold", "maximum distance threshold")

  val windowSize = 20
  val dpc = 1.0
  val threshold = if(thresholdParam.value != None) thresholdParam.value.get else 1.0
  val iterations = 10
  val cutoff = 5
{'fraction_non_alphanumeric': 0.13373379702813784, 'fraction_numerical': 0.006323110970597534, 'mean_word_length': 5.619246861924686, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '682554', 'n_tokens_mistral': 1067, 'n_tokens_neox': 1033, 'n_words': 237}