<commit_before>/**
 * @file    Tagger.cc                                                        
 * @Author  <NAME>                                               
 * @brief   Class that can label a Sentence.                                  
 */

///////////////////////////////////////////////////////////////////////////////
//                                                                           //
// (C) Copyright 2014, University of Helsinki                                //
// Licensed under the Apache License, Version 2.0 (the "License");           //
// you may not use this file except in compliance with the License.          //
// You may obtain a copy of the License at                                   //
// http://www.apache.org/licenses/LICENSE-2.0                                //
// Unless required by applicable law or agreed to in writing, software       //
// distributed under the License is distributed on an "AS IS" BASIS,         //
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  //
// See the License for the specific language governing permissions and       //
// limitations under the License.                                            //
//                                                                           //
///////////////////////////////////////////////////////////////////////////////

#include "Tagger.hh"

#ifndef TEST_Tagger_cc

#include <sstream>

#include "PerceptronTrainer.hh"
#include "SGDTrainer.hh"
#include "Trellis.hh"
#include "process_aux.hh"

#define FINN_POS_ID_STRING "FinnPosModel"
const int ENDIANNESS_MARKER=1;

using finnposaux::StringPairVector;

Tagger::Tagger(std::ostream &msg_out):
  msg_out(msg_out)
{}

Tagger::Tagger(const TaggerOptions &tagger_options, 
	       std::ostream &msg_out):
  line_counter(0),
  tagger_options(tagger_options),
  label_extractor(tagger_options),
  msg_out(msg_out)
{
  if (tagger_options.estimator == AVG_PERC)
    {
      if (tagger_options.regularization != NONE or 
	  tagger_options.delta != -1            or 
	  tagger_options.sigma != -1)
	{ 
	  msg_out << "Warning! Averaged perceptron doesn't utilize delta," 
		  << std::endl
		  << "sigma or regularization. Options will be discarded." 
		  << std::endl 
		  << std::endl;
	}
    }
}

Tagger::Tagger(std::istream &tagger_opt_in, std::ostream &msg_out):
  line_counter(0),
  tagger_options(tagger_opt_in, line_counter),
  label_extractor(tagger_options),
  msg_out(msg_out)
{
  if (tagger_options.estimator == AVG_PERC)
    {
      if (tagger_options.regularization != NONE or 
	  tagger_options.delta != -1            or 
	  tagger_options.sigma != -1)
	{ 
	  msg_out << "Warning! Averaged perceptron doesn't utilize delta," 
		  << std::endl
		  << "sigma or regularization. Options will be discarded." 
		  << std::endl 
		  << std::endl;
	}
    }

  line_counter = 0;
}

void Tagger::set_param_filter(const TaggerOptions &options)
{ param_table.set_param_filter(options); }

void Tagger::set_options(const TaggerOptions &tagger_options)
{ this->tagger_options = tagger_options; }

#include <cassert>

void Tagger::train(std::istream &train_in,
		   std::istream &dev_in)
{
  msg_out << "Reading training data." << std::endl;
  Data train_data(train_in, 1, label_extractor, param_table, 
		  tagger_options.degree);
  train_data.clear_label_guesses(); 
  train_data.randomize();

  msg_out << "Training label guesser." << std::endl;
  label_extractor.train(train_data);
  param_table.set_label_extractor(label_extractor);
  
  msg_out << "Reading development data." << std::endl;
  Data dev_data(dev_in, 1, label_extractor, param_table, tagger_options.degree);
  dev_data.clear_label_guesses(); 

  if (tagger_options.max_train_passes != 0)
    {

      msg_out << "Setting label guesses." << std::endl;
      train_data.set_label_guesses(label_extractor, 0, 
				   tagger_options.guess_mass, 
				   tagger_options.guesses);
      std::cerr << "Average number of label candidates: "
		<< LabelExtractor::all_time_guess_count * 1.0 / 
	LabelExtractor::all_time_word_count << std::endl;
      
      dev_data.set_label_guesses(label_extractor, 0, 
				 tagger_options.guess_mass,
				 tagger_options.guesses);
      
      msg_out << "Estimating lemmatizer parameters." << std::endl;
    }
  
  lemma_extractor.set_max_passes(tagger_options.max_lemmatizer_passes);
  lemma_extractor.train(train_data, dev_data, label_extractor, msg_out,
			tagger_options);

  msg_out << "Estimating tagger parameters." << std::endl;
  if (tagger_options.estimator == AVG_PERC)
    {
      PerceptronTrainer trainer(tagger_options.max_train_passes, 
				tagger_options.max_useless_passes, 
				param_table, 
				label_extractor, 
				lemma_extractor,
				msg_out,
				tagger_options);
      
      trainer.train(train_data, dev_data, tagger_options.beam, 		    
		    tagger_options.beam_mass);
    }
  else
    { throw NotImplemented(); }

  param_table.set_label_extractor(label_extractor);
}

void Tagger::evaluate(std::istream &in)
{
  msg_out << "Evaluating." << std::endl;<commit_msg>Don't set label guesses if no tagger is going tobe trained.<commit_after>/**
 * @file    Tagger.cc                                                        
 * @Author  <NAME>                                               
 * @brief   Class that can label a Sentence.                                  
 */

///////////////////////////////////////////////////////////////////////////////
//                                                                           //
// (C) Copyright 2014, University of Helsinki                                //
// Licensed under the Apache License, Version 2.0 (the "License");           //
// you may not use this file except in compliance with the License.          //
// You may obtain a copy of the License at                                   //
// http://www.apache.org/licenses/LICENSE-2.0                                //
// Unless required by applicable law or agreed to in writing, software       //
// distributed under the License is distributed on an "AS IS" BASIS,         //
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  //
// See the License for the specific language governing permissions and       //
// limitations under the License.                                            //
//                                                                           //
///////////////////////////////////////////////////////////////////////////////

#include "Tagger.hh"

#ifndef TEST_Tagger_cc

#include <sstream>

#include "PerceptronTrainer.hh"
#include "SGDTrainer.hh"
#include "Trellis.hh"
#include "process_aux.hh"

#define FINN_POS_ID_STRING "FinnPosModel"
const int ENDIANNESS_MARKER=1;

using finnposaux::StringPairVector;

Tagger::Tagger(std::ostream &msg_out):
  msg_out(msg_out)
{}

Tagger::Tagger(const TaggerOptions &tagger_options, 
	       std::ostream &msg_out):
  line_counter(0),
  tagger_options(tagger_options),
  label_extractor(tagger_options),
  msg_out(msg_out)
{
  if (tagger_options.estimator == AVG_PERC)
    {
      if (tagger_options.regularization != NONE or 
	  tagger_options.delta != -1            or 
	  tagger_options.sigma != -1)
	{ 
	  msg_out << "Warning! Averaged perceptron doesn't utilize delta," 
		  << std::endl
		  << "sigma or regularization. Options will be discarded." 
		  << std::endl 
		  << std::endl;
	}
    }
}

Tagger::Tagger(std::istream &tagger_opt_in, std::ostream &msg_out):
  line_counter(0),
  tagger_options(tagger_opt_in, line_counter),
  label_extractor(tagger_options),
  msg_out(msg_out)
{
  if (tagger_options.estimator == AVG_PERC)
    {
      if (tagger_options.regularization != NONE or 
	  tagger_options.delta != -1            or 
	  tagger_options.sigma != -1)
	{ 
	  msg_out << "Warning! Averaged perceptron doesn't utilize delta," 
		  << std::endl
		  << "sigma or regularization. Options will be discarded." 
		  << std::endl 
		  << std::endl;
	}
    }

  line_counter = 0;
}

void Tagger::set_param_filter(const TaggerOptions &options)
{ param_table.set_param_filter(options); }

void Tagger::set_options(const TaggerOptions &tagger_options)
{ this->tagger_options = tagger_options; }

#include <cassert>

void Tagger::train(std::istream &train_in,
		   std::istream &dev_in)
{
  msg_out << "Reading training data." << std::endl;
  Data train_data(train_in, 1, label_extractor, param_table, 
		  tagger_options.degree);
  train_data.clear_label_guesses(); 
  train_data.randomize();

  msg_out << "Training label guesser." << std::endl;
  label_extractor.train(train_data);
  param_table.set_label_extractor(label_extractor);
  
  msg_out << "Reading development data." << std::endl;
  Data dev_data(dev_in, 1, label_extractor, param_table, tagger_options.degree);
  dev_data.clear_label_guesses(); 

  if (tagger_options.max_train_passes != 0)
    {

      msg_out << "Setting label guesses." << std::endl;
      train_data.set_label_guesses(label_extractor, 0, 
				   tagger_options.guess_mass, 
				   tagger_options.guesses);
      std::cerr << "Average number of label candidates: "
		<< LabelExtractor::all_time_guess_count * 1.0 / 
	LabelExtractor::all_time_word_count << std::endl;
      
      dev_data.set_label_guesses(label_extractor, 0, 
				 tagger_options.guess_mass,
				 tagger_options.guesses);
      
    }

  msg_out << "Estimating lemmatizer parameters." << std::endl;
  lemma_extractor.set_max_passes(tagger_options.max_lemmatizer_passes);
  lemma_extractor.train(train_data, dev_data, label_extractor, msg_out,
			tagger_options);

  msg_out << "Estimating tagger parameters." << std::endl;
  if (tagger_options.estimator == AVG_PERC)
    {
      PerceptronTrainer trainer(tagger_options.max_train_passes, 
				tagger_options.max_useless_passes, 
				param_table, 
				label_extractor, 
				lemma_extractor,
				msg_out,
				tagger_options);
      
      trainer.train(train_data, dev_data, tagger_options.beam, 		    
		    tagger_options.beam_mass);
    }
  else
    { throw NotImplemented(); }

  param_table.set_label_extractor(label_extractor);
}

void Tagger::evaluate(std::istream &in)
{
  msg_out << "Evaluating." << std::endl;
{'fraction_non_alphanumeric': 0.13542688910696762, 'fraction_numerical': 0.004514229636898921, 'mean_word_length': 2.28741935483871, 'pattern_counts': {'":': 0, '<': 117, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1316855', 'n_tokens_mistral': 3355, 'n_tokens_neox': 3083, 'n_words': 641}