<commit_before>package epic.parser
/*
 Copyright 2010 <NAME>

 Licensed under the Apache License, Version 2.0 (the "License")
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

import epic.parser.projections.GrammarRefinements

import epic.trees.annotations._
import breeze.linalg.Counter2
import breeze.config.Help
import epic.parser.ParserParams.XbarGrammar
import epic.lexicon.{SimpleTagScorer, Lexicon, SimpleLexicon}
import java.io.{FileWriter, BufferedWriter, File}
import epic.trees._

/**
 * Contains codes to read off parsers and grammars from
 * a treebank.
 */
object GenerativeParser {
  /**
   * Extracts a [[epic.parser.RuleTopology]] and [[epic.lexicon.Lexicon]]
   * from a treebank
   * @param data the treebank
   * @return
   */
  def extractLexiconAndGrammar(data: Traversable[TreeInstance[AnnotatedLabel, String]]): (Lexicon[AnnotatedLabel, String], RuleTopology[AnnotatedLabel]) = {
    val root = data.head.tree.label
    val (words, binary, unary) = extractCounts(data)
    val grammar = RuleTopology(root,
      binary.keysIterator.map(_._2) ++ unary.keysIterator.map(_._2)
    )

    val lexicon = new SimpleLexicon(grammar.labelIndex, words)
    (lexicon, grammar)
  }

  /**
   * Makes a basic
   * @param data
   * @tparam L
   * @tparam W
   * @return
   */
  def fromTrees[L, W](data: Traversable[TreeInstance[L, W]])(implicit deb: Debinarizer[L]):Parser[L, W] = {
    val grammar = extractGrammar(data.head.tree.label, data)
    Parser(grammar)<commit_msg>thinking about switching lexicons<commit_after>package epic.parser
/*
 Copyright 2010 <NAME>

 Licensed under the Apache License, Version 2.0 (the "License")
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

import java.io.{BufferedWriter, File, FileWriter}

import breeze.config.Help
import breeze.linalg.Counter2
import epic.features.WordShapeGenerator
import epic.lexicon.{Lexicon, SignatureLexicon, SimpleTagScorer}
import epic.parser.ParserParams.XbarGrammar
import epic.parser.projections.GrammarRefinements
import epic.trees._
import epic.trees.annotations._

/**
 * Contains codes to read off parsers and grammars from
 * a treebank.
 */
object GenerativeParser {
  /**
   * Extracts a [[epic.parser.RuleTopology]] and [[epic.lexicon.Lexicon]]
   * from a treebank
   * @param data the treebank
   * @return
   */
  def extractLexiconAndGrammar(data: Traversable[TreeInstance[AnnotatedLabel, String]]): (Lexicon[AnnotatedLabel, String], RuleTopology[AnnotatedLabel]) = {
    val root = data.head.tree.label
    val (words, binary, unary) = extractCounts(data)
    val grammar = RuleTopology(root,
      binary.keysIterator.map(_._2) ++ unary.keysIterator.map(_._2)
    )

    val sigs = words.activeIterator.collect { case ((label, word), count) if count < 2 => (label, "SHAPE-" + WordShapeGenerator(word))}
    val multimap = (words.activeKeysIterator ++ sigs).toSet[(AnnotatedLabel, String)].groupBy(_._2).mapValues(_.map(pair => grammar.labelIndex(pair._1))).toMap

    val lexicon = new SignatureLexicon(grammar.labelIndex, multimap, (w: String) => "SHAPE-" + WordShapeGenerator(w))
    //val lexicon = new SimpleLexicon(grammar.labelIndex, words)
    (lexicon, grammar)
  }

  /**
   * Makes a basic
   * @param data
   * @tparam L
   * @tparam W
   * @return
   */
  def fromTrees[L, W](data: Traversable[TreeInstance[L, W]])(implicit deb: Debinarizer[L]):Parser[L, W] = {
    val grammar = extractGrammar(data.head.tree.label, data)
    Parser(grammar)
{'fraction_non_alphanumeric': 0.10652680652680653, 'fraction_numerical': 0.005827505827505828, 'mean_word_length': 4.258578431372549, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5894116', 'n_tokens_mistral': 1364, 'n_tokens_neox': 1296, 'n_words': 390}