<commit_before>package org.allenai.semparse.pipeline.science_data

import org.json4s._
import org.json4s.JsonDSL._

import com.mattg.util.FileUtil

import org.allenai.semparse.pipeline.base._

object ScienceQuestionPipeline {
  val fileUtil = new FileUtil

  //////////////////////////////////////////////////////////
  // Step 1: Take sentences and generating training data
  //////////////////////////////////////////////////////////

  val sentenceProcessorParams: JValue =
    ("max word count per sentence" -> 10) ~
    ("logical forms" -> ("nested" -> true)) ~
    ("output format" -> "debug") ~
    ("data name" -> "petert_sentences") ~
    ("data directory" -> "/home/mattg/data/petert_science_sentences")
  val sentenceProcessorType: JValue = ("type" -> "science sentence processor")
  val sentenceProcessorParamsWithType: JValue = sentenceProcessorParams merge sentenceProcessorType

  //////////////////////////////////////////////////////////
  // Step 2: Create a KB graph
  //////////////////////////////////////////////////////////

  val animalTensor = "/home/mattg/data/aristo_kb/animal_tensor.tsv"
  val biomeTensor = "/home/mattg/data/aristo_kb/biome_tensor.tsv"
  val kbGraphCreatorParams: JValue =
    ("graph name" -> "animal_and_biome") ~
    ("relation sets" -> List(animalTensor, biomeTensor)) ~
    ("type" -> "kb graph creator")

  ////////////////////////////////////////////////////////////////
  // Step 3: Processing the training data into Jayant's lisp files
  ////////////////////////////////////////////////////////////////

  val trainingDataParams: JValue =
    ("training data creator" -> sentenceProcessorParamsWithType) ~
    ("data name" -> "science/petert_science_sentences") ~
    ("lines to use" -> 700000) ~
    ("word count threshold" -> 5)

  ////////////////////////////////////////////////////////////////
  // Step 4: Select features for each word
  ////////////////////////////////////////////////////////////////

  val SFE_SPEC_FILE = "src/main/resources/science_sfe_spec.json"

  val trainingDataFeatureParams: JValue =
    ("training data" -> trainingDataParams) ~
    ("sfe spec file" -> SFE_SPEC_FILE) ~
    ("graph creator" -> kbGraphCreatorParams)

  val trainingDataPmiParams: JValue =
    ("training data features" -> trainingDataFeatureParams)

  ////////////////////////////////////////////////////////////////
  // Step 5: Train a model
  ////////////////////////////////////////////////////////////////

  val modelParams: JValue =
    ("model type" -> "combined") ~
    ("feature computer" -> trainingDataPmiParams)

  ////////////////////////////////////////////////////////////////
  // Step 6: Process the questions into logical forms
  ////////////////////////////////////////////////////////////////

  val questionProcessorParams: JValue =
    ("question file" -> "data/science/monarch_questions/raw_questions.tsv") ~
    ("output format" -> "debug") ~
    ("logical forms" -> ("nested" -> true)) ~
    ("data name" -> "monarch_questions")

  /////////////////////////////////////////////////////////////////////
  // Step 7: Score the answer options for each question using the model
  /////////////////////////////////////////////////////////////////////

  val questionScorerParams: JValue =
    ("questions" -> questionProcessorParams) ~
    ("model" -> modelParams)

  def main(args: Array[String]) {
    //new Trainer(modelParams, fileUtil).runPipeline()
    new ScienceSentenceProcessor(sentenceProcessorParams, fileUtil).runPipeline()
  }
}<commit_msg>Added the sentence corruptor to the pipeline<commit_after>package org.allenai.semparse.pipeline.science_data

import org.json4s._
import org.json4s.JsonDSL._

import com.mattg.util.FileUtil

import org.allenai.semparse.pipeline.base._

object ScienceQuestionPipeline {
  val fileUtil = new FileUtil

  //////////////////////////////////////////////////////////
  // Step 1: Take sentences and generating training data
  //////////////////////////////////////////////////////////

  val sentenceProcessorParams: JValue =
    ("max word count per sentence" -> 10) ~
    ("logical forms" -> ("nested" -> true)) ~
    ("output format" -> "debug") ~
    ("data name" -> "petert_sentences") ~
    ("data directory" -> "/home/mattg/data/petert_science_sentences")
  val sentenceProcessorType: JValue = ("type" -> "science sentence processor")
  val sentenceProcessorParamsWithType: JValue = sentenceProcessorParams merge sentenceProcessorType

  //////////////////////////////////////////////////////////////////
  // Step 1.5: Use noise contrastive estimation to get negative data
  //////////////////////////////////////////////////////////////////

  val sentenceCorruptorParams: JValue =
    ("positive data" -> sentenceProcessorParams)

  //////////////////////////////////////////////////////////
  // Step 2: Create a KB graph
  //////////////////////////////////////////////////////////

  val animalTensor = "/home/mattg/data/aristo_kb/animal_tensor.tsv"
  val biomeTensor = "/home/mattg/data/aristo_kb/biome_tensor.tsv"
  val kbGraphCreatorParams: JValue =
    ("graph name" -> "animal_and_biome") ~
    ("relation sets" -> List(animalTensor, biomeTensor)) ~
    ("type" -> "kb graph creator")

  ////////////////////////////////////////////////////////////////
  // Step 3: Processing the training data into Jayant's lisp files
  ////////////////////////////////////////////////////////////////

  val trainingDataParams: JValue =
    ("training data creator" -> sentenceProcessorParamsWithType) ~
    ("data name" -> "science/petert_science_sentences") ~
    ("lines to use" -> 700000) ~
    ("word count threshold" -> 5)

  ////////////////////////////////////////////////////////////////
  // Step 4: Select features for each word
  ////////////////////////////////////////////////////////////////

  val SFE_SPEC_FILE = "src/main/resources/science_sfe_spec.json"

  val trainingDataFeatureParams: JValue =
    ("training data" -> trainingDataParams) ~
    ("sfe spec file" -> SFE_SPEC_FILE) ~
    ("graph creator" -> kbGraphCreatorParams)

  val trainingDataPmiParams: JValue =
    ("training data features" -> trainingDataFeatureParams)

  ////////////////////////////////////////////////////////////////
  // Step 5: Train a model
  ////////////////////////////////////////////////////////////////

  val modelParams: JValue =
    ("model type" -> "combined") ~
    ("feature computer" -> trainingDataPmiParams)

  ////////////////////////////////////////////////////////////////
  // Step 6: Process the questions into logical forms
  ////////////////////////////////////////////////////////////////

  val questionProcessorParams: JValue =
    ("question file" -> "data/science/monarch_questions/raw_questions.tsv") ~
    ("output format" -> "debug") ~
    ("logical forms" -> ("nested" -> true)) ~
    ("data name" -> "monarch_questions")

  /////////////////////////////////////////////////////////////////////
  // Step 7: Score the answer options for each question using the model
  /////////////////////////////////////////////////////////////////////

  val questionScorerParams: JValue =
    ("questions" -> questionProcessorParams) ~
    ("model" -> modelParams)

  def main(args: Array[String]) {
    //new Trainer(modelParams, fileUtil).runPipeline()
    //new ScienceSentenceProcessor(sentenceProcessorParams, fileUtil).runPipeline()
    new SentenceCorruptor(sentenceCorruptorParams, fileUtil).runPipeline()
  }
}
{'fraction_non_alphanumeric': 0.3430637386181039, 'fraction_numerical': 0.005088377075522228, 'mean_word_length': 1.40625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1120681', 'n_tokens_mistral': 1804, 'n_tokens_neox': 1710, 'n_words': 443}