<commit_before>package org.grammaticalframework.parser

import org.grammaticalframework.reader.PGF
import org.grammaticalframework.Trees.PrettyPrinter
import org.grammaticalframework.linearizer.Linearizer

import java.util.logging._;

object BasicParserTest {
  def main(args: Array[String]) {
    println("Current dir is : " + System.getProperty("user.dir"));

    // Setting up logging
    val logger = Logger.getLogger("org.grammaticalframework.parser")
    logger.setLevel(Level.INFO)
    val hdlr = new ConsoleHandler()
    hdlr.setLevel(Level.FINEST)
    logger.addHandler(hdlr)

    // Testing with the food grammar
    val foodGrammar = new TestGrammar("pgf/Foods.pgf", "FoodsEng")
    foodGrammar.parseAndPrint("this fresh pizza is Italian")
    foodGrammar.parseAndPrint("those boring fish are expensive")
    val foodGrammarSwe = new TestGrammar("pgf/Foods.pgf", "FoodsSwe")
    foodGrammarSwe.parseAndPrint("den här läckra pizzan är färsk")
    val foodGrammarIta = new TestGrammar("pgf/Foods.pgf", "FoodsIta")
    foodGrammarIta.parseAndPrint("questa pizza deliziosa è fresca")
    // testing whith the phrasebook
    val engPhrasebook = new TestGrammar("pgf/Phrasebook.pgf", "PhrasebookFre")
    engPhrasebook.parseAndPrint("le restaurant est ouvert le lundi .")
  }
}


class TestGrammar(pgfFile:String, langName:String) {
  val grammar:PGF = PGF.readFromFile(pgfFile)
  val parser = new Parser(grammar.concrete(langName))

  def parseAndPrint(txt:String):Unit = {
    println("Parsing string \"" + txt + "\" with grammar " +
            pgfFile + ":" + langName)
    val tokens = txt.split(" ")
    parser.parse(tokens)
    val trees = parser.getTrees
    trees.map(PrettyPrinter.print).foreach(println)
    trees.foreach( t => {
      val line = new Linearizer(grammar, grammar.concrete(langName))
      println(line.renderLin(line.linearize(t)))
    })
    //parser.printState
  }
}<commit_msg>Updating the test script for more efficiency<commit_after>package org.grammaticalframework.parser

import org.grammaticalframework.reader.PGF
import org.grammaticalframework.Trees.PrettyPrinter
import org.grammaticalframework.linearizer.Linearizer

import java.util.logging._;

object BasicParserTest {
  def main(args: Array[String]) {
    println("Current dir is : " + System.getProperty("user.dir"));

    // Setting up logging
    val logger = Logger.getLogger("org.grammaticalframework.parser")
    logger.setLevel(Level.INFO)
    val hdlr = new ConsoleHandler()
    hdlr.setLevel(Level.FINEST)
    logger.addHandler(hdlr)

    // Testing with the food grammar
    var grammar = new TestGrammar("pgf/Foods.pgf", "FoodsEng")
    grammar.parseAndPrint("this fresh pizza is Italian")
    grammar.parseAndPrint("those boring fish are expensive")
    grammar = new TestGrammar("pgf/Foods.pgf", "FoodsSwe")
    grammar.parseAndPrint("den här läckra pizzan är färsk")
    grammar = new TestGrammar("pgf/Foods.pgf", "FoodsIta")
    grammar.parseAndPrint("questa pizza deliziosa è fresca")
    // Testing whith the phrasebook
    grammar = new TestGrammar("pgf/Phrasebook.pgf", "PhrasebookFre")
    grammar.parseAndPrint("le restaurant est ouvert le lundi")
  }
}


class TestGrammar(pgfFile:String, langName:String) {
  val grammar:PGF = PGF.readFromFile(pgfFile)
  val parser = new Parser(grammar.concrete(langName))
  val linearizer = new Linearizer(grammar, grammar.concrete(langName))

  def parseAndPrint(txt:String):Unit = {
    println("Parsing string \"" + txt + "\" with grammar " +
            pgfFile + ":" + langName)
    val tokens = txt.split(" ")
    parser.parse(tokens)
    val trees = parser.getTrees
    trees.foreach( t => {
      println(PrettyPrinter.print(t))
      println(linearizer.renderLin(linearizer.linearize(t)))
      println()
    })
  }
}
{'fraction_non_alphanumeric': 0.11531340915101825, 'fraction_numerical': 0.0, 'mean_word_length': 4.10391363022942, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '112769', 'n_tokens_mistral': 1273, 'n_tokens_neox': 1241, 'n_words': 251}