<commit_before>package jigg.pipeline

/*
 Copyright 2013-2015 <NAME> and <NAME>

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

import jigg.nlp.ccg.JapaneseShiftReduceParsing
import jigg.nlp.ccg.{InputOptions, TaggerOptions, ParserOptions}
import jigg.nlp.ccg.lexicon.{PoSTaggedSentence, Derivation, Point}
import jigg.nlp.ccg.tagger.{MaxEntMultiTagger}
import jigg.nlp.ccg.parser.{TransitionBasedParser, KBestDecoder}
import jigg.util.PropertiesUtil
import jigg.util.XMLUtil

import java.util.Properties
import scala.xml._
import scala.collection.mutable.HashMap
import scala.collection.mutable.ArrayBuffer

/** Currently this class is ugly; it largely depends on global variables defined in jigg.nlp.ccg.Options.
  * TODO: revise this class and ShiftReduceParsing class.
  */
class CCGParseAnnotator(override val name: String, override val props: Properties) extends SentencesAnnotator {

  val parsing = new JapaneseShiftReduceParsing
  var tagger: MaxEntMultiTagger = _
  var decoder: TransitionBasedParser = _

  @Prop(gloss = "Path to the trained model (you can omit this if you load a jar which packs models)") var model = ""
  @Prop(gloss = "Pruning parameter in supertagging") var beta = 0.001
  @Prop(gloss = "Maximum category candidates for each by supertagging, -1 for infinity") var maxK = -1
  @Prop(gloss = "Beam size (usually 64 is ok); Recommend to use the same beam size at training the model. This is automatically done if the model is loaded from a model jar.") var beam = 64
  @Prop(gloss = "Outputs k-best derivations if this value > 1") var kBest = 1
  @Prop(gloss = "Beam size (usually 64 is ok); Recommend to use the same beam size at training the model. This is automatically done if the model is loaded from a model jar.") var preferConnected = false
  readProps()

  if (model != "") InputOptions.loadModelPath = model
  TaggerOptions.beta = beta
  TaggerOptions.maxK = maxK
  ParserOptions.beam = beam

  override def init = {
    configParsing
    tagger = parsing.tagging.getTagger
    decoder = parsing.getPredDecoder
  }

  def configParsing = {
    System.err.println("The path of CCG parser model: " + InputOptions.loadModelPath)
    try {
      parsing.load
    }
    catch{
      case e: Exception =>
        val errorMsg = s"""Failed to start CCG parser. Make sure the model file of CCG is already installed. If not, execute the following command in jigg directory:
  ./script/download_ccg_model.sh
"""
        argumentError("command", errorMsg)
    }
  }
<commit_msg>Small fix bug in error msg<commit_after>package jigg.pipeline

/*
 Copyright 2013-2015 <NAME> and <NAME>

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

import jigg.nlp.ccg.JapaneseShiftReduceParsing
import jigg.nlp.ccg.{InputOptions, TaggerOptions, ParserOptions}
import jigg.nlp.ccg.lexicon.{PoSTaggedSentence, Derivation, Point}
import jigg.nlp.ccg.tagger.{MaxEntMultiTagger}
import jigg.nlp.ccg.parser.{TransitionBasedParser, KBestDecoder}
import jigg.util.PropertiesUtil
import jigg.util.XMLUtil

import java.util.Properties
import scala.xml._
import scala.collection.mutable.HashMap
import scala.collection.mutable.ArrayBuffer

/** Currently this class is ugly; it largely depends on global variables defined in jigg.nlp.ccg.Options.
  * TODO: revise this class and ShiftReduceParsing class.
  */
class CCGParseAnnotator(override val name: String, override val props: Properties) extends SentencesAnnotator {

  val parsing = new JapaneseShiftReduceParsing
  var tagger: MaxEntMultiTagger = _
  var decoder: TransitionBasedParser = _

  @Prop(gloss = "Path to the trained model (you can omit this if you load a jar which packs models)") var model = ""
  @Prop(gloss = "Pruning parameter in supertagging") var beta = 0.001
  @Prop(gloss = "Maximum category candidates for each by supertagging, -1 for infinity") var maxK = -1
  @Prop(gloss = "Beam size (usually 64 is ok); Recommend to use the same beam size at training the model. This is automatically done if the model is loaded from a model jar.") var beam = 64
  @Prop(gloss = "Outputs k-best derivations if this value > 1") var kBest = 1
  @Prop(gloss = "Beam size (usually 64 is ok); Recommend to use the same beam size at training the model. This is automatically done if the model is loaded from a model jar.") var preferConnected = false
  readProps()

  if (model != "") InputOptions.loadModelPath = model
  TaggerOptions.beta = beta
  TaggerOptions.maxK = maxK
  ParserOptions.beam = beam

  override def init = {
    configParsing
    tagger = parsing.tagging.getTagger
    decoder = parsing.getPredDecoder
  }

  def configParsing = {
    System.err.println("The path of CCG parser model: " + InputOptions.loadModelPath)
    try {
      parsing.load
    }
    catch{
      case e: Exception =>
        val errorMsg = s"""Failed to start CCG parser. Make sure the model file of CCG is already installed. If not, execute the following command in jigg directory:
  ./script/download_ccg_model.sh
"""
        argumentError("model", errorMsg)
    }
  }

{'fraction_non_alphanumeric': 0.07729468599033816, 'fraction_numerical': 0.008662335498917208, 'mean_word_length': 4.0369127516778525, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6075641', 'n_tokens_mistral': 1800, 'n_tokens_neox': 1701, 'n_words': 681}