<commit_before>// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

#include <algorithm>
#include <sstream>

#include "morphodita/morpho/generic_morpho_encoder.h"
#include "morphodita/morpho/morpho.h"
#include "morphodita/morpho/morpho_ids.h"
#include "morphodita/morpho/morpho_statistical_guesser_trainer.h"
#include "morphodita/tagger/conllu_elementary_features.h"
#include "morphodita/tagger/feature_sequences.h"
#include "morphodita/tagger/perceptron_tagger_trainer.h"
#include "morphodita/tagger/tagger_ids.h"
#include "morphodita/tagger/tagger_trainer.h"
#include "morphodita/tokenizer/generic_tokenizer.h"
#include "morphodita/tokenizer/generic_tokenizer_factory_encoder.h"
#include "morphodita/tokenizer/gru_tokenizer_trainer.h"
#include "morphodita/tokenizer/tokenizer_ids.h"
#include "parsito/parser/parser_nn_trainer.h"
#include "sentence/input_format.h"
#include "sentence/sentence.h"
#include "tokenizer/detokenizer.h"
#include "tokenizer/multiword_splitter_trainer.h"
#include "trainer.h"
#include "trainer_morphodita_parsito.h"
#include "unilib/unicode.h"
#include "unilib/utf8.h"
#include "utils/options.h"
#include "utils/parse_double.h"
#include "utils/parse_int.h"
#include "utils/split.h"
#include "version/version.h"

namespace ufal {
namespace udpipe {

bool trainer_morphodita_parsito::train(const vector<sentence>& training, const vector<sentence>& heldout,
                                       const string& tokenizer, const string& tagger, const string& parser, ostream& os, string& error) {
  error.clear();

  // Save model version info
  os.put(model_morphodita_parsito::VERSION_LATEST);
  // Add sentinel required since version 2
  os.put(0x7F).put(0x7F);

  // Check input data
  for (auto&& sentence : training)
    for (size_t i = 1; i < sentence.words.size(); i++)
      if (!can_combine_tag(sentence.words[i], error))
        return false;
  for (auto&& sentence : heldout)
    for (size_t i = 1; i < sentence.words.size(); i++)
      if (!can_combine_tag(sentence.words[i], error))
        return false;

  if (!train_tokenizer(training, heldout, tokenizer, os, error)) return false;
  string tagger_model;
  {
    ostringstream os_tagger;
    if (!train_tagger(training, heldout, tagger, os_tagger, error)) return false;
    tagger_model.assign(os_tagger.str());
    os.write(tagger_model.data(), tagger_model.size());
  }
  if (!train_parser(training, heldout, parser, tagger_model, os, error)) return false;

  return true;
}

bool trainer_morphodita_parsito::train_tokenizer(const vector<sentence>& training, const vector<sentence>& heldout,
                                                 const string& options, ostream& os, string& error) {
  if (options == NONE) {
    os.put(0);
  } else {
    // Tokenizer options
    named_values::map tokenizer;
    if (!named_values::parse(options, tokenizer, error)) return false;
    int run = 0; if (!option_int(tokenizer, "run", run, error)) return false;

    if (tokenizer.count("from_model")) {
      // Use specified tokenizer model
      string_piece tokenizer_data;
      if (!load_model(tokenizer["from_model"], TOKENIZER_MODEL, tokenizer_data))
        return error.assign("Cannot load model from which the tokenizer should be used!"), false;

      cerr << "Using tokenizer from given model." << endl;
      os.write(tokenizer_data.str, tokenizer_data.len);
    } else {
      os.put(1);
      const string& model = option_str(tokenizer, "model");

      // Tokenizer itself
      if (model == "generic") {
        os.put(morphodita::tokenizer_id::GENERIC);
        morphodita::generic_tokenizer_factory_encoder::encode(morphodita::generic_tokenizer::LATEST, os);
      } else if (model.empty() || model == "gru") {
        // Create a detokenizator if required
        unique_ptr<detokenizer> detokenizer;
        if (tokenizer.count("detokenize")) {
          detokenizer.reset(new udpipe::detokenizer(tokenizer["detokenize"]));
          if (!detokenizer) return error.assign("Cannot create detokenizer!"), false;
        }

        // Prepare training data for the gru_tokenizer
        vector<morphodita::tokenized_sentence> sentences;
        bool spaces_in_training = false;
        for (size_t training_sentence = 0; training_sentence < training.size(); training_sentence++) {
          sentence s = training[training_sentence];
          if (detokenizer) detokenizer->detokenize(s);

          auto& sentence = (sentences.emplace_back(), sentences.back());

          for (size_t i = 1, j = 0; i < s.words.size(); i++) {
            const string& form = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? s.multiword_tokens[j].form : s.words[i].form;

            sentence.tokens.emplace_back(sentence.sentence.size(), 0);
            for (auto&& chr : unilib::utf8::decoder(form)) {
              sentence.sentence.push_back(chr);
              if (unilib::unicode::category(chr) & unilib::unicode::Zs) spaces_in_training = true;
            }
            sentence.tokens.back().length = sentence.sentence.size() - sentence.tokens.back().start;

            const string& misc = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? s.multiword_tokens[j].misc : s.words[i].misc;
            if (misc.find(space_after_no) == string::npos) sentence.sentence.push_back(' ');

            if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
              i = s.multiword_tokens[j++].id_last;
          }
          string doc_par_id;
          if (training_sentence + 1 < training.size() &&
              (training[training_sentence + 1].get_new_doc(doc_par_id) || training[training_sentence + 1].get_new_par(doc_par_id)))
            sentence.sentence.append(2, '\n');
        }

        // Heldout data
        vector<morphodita::tokenized_sentence> heldout_sentences;

        bool detokenize_handout = true; if (!option_bool(tokenizer, "detokenize_handout", detokenize_handout, error)) return false;
        for (size_t heldout_sentence = 0; heldout_sentence < heldout.size(); heldout_sentence++) {
          sentence s = heldout[heldout_sentence];
          if (detokenizer && detokenize_handout) detokenizer->detokenize(s);

          auto& sentence = (heldout_sentences.emplace_back(), heldout_sentences.back());

          for (size_t i = 1, j = 0; i < s.words.size(); i++) {
            const string& form = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? s.multiword_tokens[j].form : s.words[i].form;

            sentence.tokens.emplace_back(sentence.sentence.size(), 0);
            for (auto&& chr : unilib::utf8::decoder(form))
              sentence.sentence.push_back(chr);
            sentence.tokens.back().length = sentence.sentence.size() - sentence.tokens.back().start;

            const string& misc = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? s.multiword_tokens[j].misc : s.words[i].misc;
            if (misc.find(space_after_no) == string::npos) sentence.sentence.push_back(' ');

            if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
              i = s.multiword_tokens[j++].id_last;
          }
          string doc_par_id;
          if (heldout_sentence + 1 < heldout.size() &&
              (heldout[heldout_sentence + 1].get_new_doc(doc_par_id) || heldout[heldout_sentence + 1].get_new_par(doc_par_id)))
            sentence.sentence.append(2, '\n');
        }

        // Options
        bool tokenize_url = true; if (!option_bool(tokenizer, "tokenize_url", tokenize_url, error)) return false;
        int segment_size = 50; // if (!option_int(tokenizer, "segment_size", segment_size, error)) return false;
        bool allow_spaces = spaces_in_training; if (!option_bool(tokenizer, "allow_spaces", allow_spaces, error)) return false;
        int dimension = 24; // if (!option_int(tokenizer, "dimension", dimension, error)) return false;
        int epochs = 100; if (!option_int(tokenizer, "epochs", epochs, error)) return false;
        int batch_size = run <= 1 ? 50 : 50 + 50 * hyperparameter_integer(run, 1, 0, 1);
        if (!option_int(tokenizer, "batch_size", batch_size, error)) return false;
        double learning_rate = run <= 1 ? 0.005 : hyperparameter_logarithmic(run, 2, 0.0005, 0.01);
        if (!option_double(tokenizer, "learning_rate", learning_rate, error)) return false;
        double learning_rate_final = 0.0; // if (!option_double(tokenizer, "learning_rate_final", learning_rate_final, error)) return false;
        double dropout = 0.1; if (!option_double(tokenizer, "dropout", dropout, error)) return false;
        double initialization_range = 0.5; if (!option_double(tokenizer, "initialization_range", initialization_range, error)) return false;
        bool early_stopping = !heldout_sentences.empty(); if (!option_bool(tokenizer, "early_stopping", early_stopping, error)) return false;

        if (run >= 1) cerr << "Random search run " << run << ", batch_size=" << batch_size
                           << ", learning_rate=" << fixed << setprecision(8) << learning_rate << endl;
        if (allow_spaces) cerr << "Allowing tokenizer to return spaces in tokens." << endl;


        // Train and encode gru_tokenizer
        os.put(morphodita::tokenizer_ids::GRU);
        if (!morphodita::gru_tokenizer_trainer::train(tokenize_url ? morphodita::gru_tokenizer_trainer::URL_EMAIL_LATEST : 0,
                                                      segment_size, allow_spaces, dimension, epochs, batch_size, learning_rate,
                                                      learning_rate_final, dropout, initialization_range, early_stopping,
                                                      sentences, heldout_sentences, os, error))
          return false;
      } else {
        return error.assign("Unknown tokenizer model '").append(model).append("'!"), false;
      }

      // Multiword splitter
      if (!multiword_splitter_trainer::train(training, os, error)) return false;
    }
  }

  return true;
}

bool trainer_morphodita_parsito::train_tagger(const vector<sentence>& training, const vector<sentence>& heldout,
                                              const string& options, ostream& os, string& error) {
  if (options == NONE) {
    os.put(0);
  } else {
    // Parse options
    named_values::map tagger;
    if (!named_values::parse(options, tagger, error)) return false;

    if (tagger.count("from_model")) {
      // Use specified tokenizer model(s)
      int model_index = 1, taggers_total = 0;
      string model_name = "from_model";
      vector<string_piece> taggers_data;
      do {
        taggers_data.emplace_back();
        if (!load_model(tagger[model_name], TAGGER_MODEL, taggers_data.back()))
          return error.assign("Cannot load model from which the tagger should be used!"), false;
        if (taggers_data.back().str[0])
          taggers_total += taggers_data.back().str[0];
        else
          taggers_data.pop_back();
        model_name = "from_model_" + to_string(++model_index);
      } while (tagger.count(model_name));
      if (taggers_total < 0 || taggers_total > 4) return error.assign("Cannot create more than four tagger models!"), false;

      cerr << "Using tagger from given model(s)." << endl;
      os.put(taggers_total);
      for (auto&& tagger_data : taggers_data)
        os.write(tagger_data.str + 1, tagger_data.len - 1);
    } else {
      // Create MorphoDiTa model(s)
      int models = 1; if (!option_int(tagger, "models", models, error)) return false;
      if (models <= 0) return error.assign("Number of tagger models cannot be negative or zero!"), false;
      if (models > 4) return error.assign("Cannot create more than four tagger models!"), false;

      os.put(models);
      for (int model = 0; model < models; model++)
        if (!train_tagger_model(training, heldout, model, models, tagger, os, error))
          return false;
    }
  }

  return true;
}

bool trainer_morphodita_parsito::train_parser(const vector<sentence>& training, const vector<sentence>& heldout,
                                              const string& options, const string& tagger_model, ostream& os, string& error) {
  if (options == NONE) {
    os.put(0);
  } else {
    // Create Parsito model
    named_values::map parser;
    if (!named_values::parse(options, parser, error)) return false;
    int run = 0; if (!option_int(parser, "run", run, error)) return false;

    if (parser.count("from_model")) {
      // Use specified parser model
      string_piece parser_data;
      if (!load_model(parser["from_model"], PARSER_MODEL, parser_data))
        return error.assign("Cannot load model from which the parser should be used!"), false;

      cerr << "Using parser from given model." << endl;
      os.write(parser_data.str, parser_data.len);
    } else {
      os.put(1);

      // Parsito options
      string transition_system = parser.count("transition_system") ? parser["transition_system"] : "projective";
      string transition_oracle = parser.count("transition_oracle") ? parser["transition_oracle"] :
          transition_system == "projective" ? "dynamic" :
          transition_system == "swap" ? "static_lazy" :
          "static";

      int embedding_upostag = 20; if (!option_int(parser, "embedding_upostag", embedding_upostag, error)) return false;
      int embedding_feats = 20; if (!option_int(parser, "embedding_feats", embedding_feats, error)) return false;
      int embedding_xpostag = 0; if (!option_int(parser, "embedding_xpostag", embedding_xpostag, error)) return false;
      int embedding_form = 50; if (!option_int(parser, "embedding_form", embedding_form, error)) return false;
      int embedding_lemma = 0; if (!option_int(parser, "embedding_lemma", embedding_lemma, error)) return false;
      int embedding_deprel = 20; if (!option_int(parser, "embedding_deprel", embedding_deprel, error)) return false;
      string embeddings;
      if (embedding_upostag) embeddings.append("universal_tag ").append(to_string(embedding_upostag)).append(" 1\n");
      if (embedding_feats) embeddings.append("feats ").append(to_string(embedding_feats)).append(" 1\n");
      if (embedding_xpostag) embeddings.append("tag ").append(to_string(embedding_xpostag)).append(" 1\n");
      if (embedding_form) {
        embeddings.append("form ").append(to_string(embedding_form)).append(" 2");
        if (!option_str(parser, "embedding_form_file").empty()) embeddings.append(" ").append(option_str(parser, "embedding_form_file"));
        embeddings.push_back('\n');
      }
      if (embedding_lemma) {
        embeddings.append("lemma ").append(to_string(embedding_lemma)).append(" 2");
        if (!option_str(parser, "embedding_lemma_file").empty()) embeddings.append(" ").append(option_str(parser, "embedding_lemma_file"));
        embeddings.push_back('\n');
      }
      if (embedding_deprel) embeddings.append("deprel ").append(to_string(embedding_deprel)).append(" 1\n");

      int iterations = 10; if (!option_int(parser, "iterations", iterations, error)) return false;
      int hidden_layer = 200; if (!option_int(parser, "hidden_layer", hidden_layer, error)) return false;
      int batch_size = 10; if (!option_int(parser, "batch_size", batch_size, error)) return false;
      int structured_interval = run <= 1 ? 8 : hyperparameter_integer(run,1,0,2) == 2 ? 0 : 8 + 2*hyperparameter_integer(run,1,0,2);
      if (!option_int(parser, "structured_interval", structured_interval, error)) return false;
      double learning_rate = run <= 1 ? 0.02 : hyperparameter_logarithmic(run, 2, 0.005, 0.04);
      if (!option_double(parser, "learning_rate", learning_rate, error)) return false;
      double learning_rate_final = 0.001; if (!option_double(parser, "learning_rate_final", learning_rate_final, error)) return false;
      double l2 = run <= 1 ? 0.5 : hyperparameter_uniform(run, 3, 0.2, 0.6);
      if (!option_double(parser, "l2", l2, error)) return false;

      if (run >= 1) cerr << "Random search run " << run << ", structured_interval=" << structured_interval
                         << ", learning_rate=" << fixed << setprecision(8) << learning_rate
                         << ", l2=" << l2 << endl;

      // Prepare data in the correct format
      parsito::network_parameters parameters;
      parameters.iterations = iterations;
      parameters.structured_interval = structured_interval;
      parameters.hidden_layer = hidden_layer;
      parameters.hidden_layer_type = parsito::activation_function::TANH;
      parameters.trainer.algorithm = parsito::network_trainer::SGD;
      parameters.trainer.learning_rate = learning_rate;
      parameters.trainer.learning_rate_final = learning_rate_final;
      parameters.trainer.momentum = 0;
      parameters.trainer.epsilon = 0;
      parameters.batch_size = batch_size;
      parameters.initialization_range = 0.1f;
      parameters.l1_regularization = 0;
      parameters.l2_regularization = l2;
      parameters.maxnorm_regularization = 0;
      parameters.dropout_hidden = 0;
      parameters.dropout_input = 0;

      // Tag the input if required
      unique_ptr<model> tagger;
      bool use_gold_tags = false; if (!option_bool(parser, "use_gold_tags", use_gold_tags, error)) return false;
      if (!use_gold_tags && !tagger_model.empty() && tagger_model[0]) {
        stringstream tagger_description;
        tagger_description.put(model_morphodita_parsito::VERSION_LATEST).put(0x7F).put(0x7F).put(0).write(tagger_model.data(), tagger_model.size()).put(0);
        tagger.reset(model_morphodita_parsito::load(tagger_description));
        if (!tagger) return error.assign("Cannot load the tagger model for parser training data generation!"), false;
      }

      // Training data
      sentence tagged;
      vector<parsito::tree> train_trees;
      for (auto&& sentence : training) {
        tagged = sentence;
        if (tagger && !tagger->tag(tagged, DEFAULT, error)) return false;

        train_trees.emplace_back();
        for (size_t i = 1; i < tagged.words.size(); i++) {
          train_trees.back().add_node(string());
          model_normalize_form(tagged.words[i].form, train_trees.back().nodes.back().form);
          train_trees.back().nodes.back().lemma.assign(tagged.words[i].lemma);
          train_trees.back().nodes.back().upostag.assign(tagged.words[i].upostag);
          train_trees.back().nodes.back().xpostag.assign(tagged.words[i].xpostag);
          train_trees.back().nodes.back().feats.assign(tagged.words[i].feats);
        }
        for (size_t i = 1; i < tagged.words.size(); i++)
          train_trees.back().set_head(tagged.words[i].id, tagged.words[i].head, tagged.words[i].deprel);
      }

      // Heldout data
      vector<parsito::tree> heldout_trees;
      for (auto&& sentence : heldout) {
        tagged = sentence;
        if (tagger && !tagger->tag(tagged, DEFAULT, error)) return false;

        heldout_trees.emplace_back();
        for (size_t i = 1; i < tagged.words.size(); i++) {
          heldout_trees.back().add_node(string());
          model_normalize_form(tagged.words[i].form, heldout_trees.back().nodes.back().form);
          heldout_trees.back().nodes.back().lemma.assign(tagged.words[i].lemma);
          heldout_trees.back().nodes.back().upostag.assign(tagged.words[i].upostag);
          heldout_trees.back().nodes.back().xpostag.assign(tagged.words[i].xpostag);
          heldout_trees.back().nodes.back().feats.assign(tagged.words[i].feats);
        }
        for (size_t i = 1; i < tagged.words.size(); i++)
          heldout_trees.back().set_head(tagged.words[i].id, tagged.words[i].head, tagged.words[i].deprel);
      }

      // Train the parser
      binary_encoder enc;
      enc.add_str("nn");
      parsito::parser_nn_trainer::train(transition_system, transition_oracle, embeddings, parser_nodes,
                                        parameters, 1, train_trees, heldout_trees, enc);
      compressor::save(os, enc);
    }
  }

  return true;
}

bool trainer_morphodita_parsito::load_model(const string& data, model_type model, string_piece& range) {
  istringstream is(data);

  // Check that it is morphodita_parsito model.
  char len;
  if (!is.get(len)) return false;
  string name(len, ' ');
  if (!is.read(&name[0], len)) return false;
  if (name != "morphodita_parsito") return false;

  char version;
  if (!is.get(version)) return false;

  // Tokenizer
  {
    if (model == TOKENIZER_MODEL) range.str = data.data() + is.tellg();
    char tokenizer; if (!is.get(tokenizer)) return false;
    unique_ptr<morphodita::tokenizer_factory> tokenizer_factory(tokenizer ? morphodita::tokenizer_factory::load(is) : nullptr);
    if (tokenizer && !tokenizer_factory) return false;
    if (model == TOKENIZER_MODEL) return range.len = is.tellg() - streampos(range.str - data.data()), true;
  }

  // Tagger
  {
    if (model == TAGGER_MODEL) range.str = data.data() + is.tellg();
    char taggers; if (!is.get(taggers)) return false;
    for (char i = 0; i < taggers; i++) {<commit_msg>Update model_morphodita_parsito loading in the trainer.<commit_after>// This file is part of UDPipe <http://github.com/ufal/udpipe/>.
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

#include <algorithm>
#include <sstream>

#include "morphodita/morpho/generic_morpho_encoder.h"
#include "morphodita/morpho/morpho.h"
#include "morphodita/morpho/morpho_ids.h"
#include "morphodita/morpho/morpho_statistical_guesser_trainer.h"
#include "morphodita/tagger/conllu_elementary_features.h"
#include "morphodita/tagger/feature_sequences.h"
#include "morphodita/tagger/perceptron_tagger_trainer.h"
#include "morphodita/tagger/tagger_ids.h"
#include "morphodita/tagger/tagger_trainer.h"
#include "morphodita/tokenizer/generic_tokenizer.h"
#include "morphodita/tokenizer/generic_tokenizer_factory_encoder.h"
#include "morphodita/tokenizer/gru_tokenizer_trainer.h"
#include "morphodita/tokenizer/tokenizer_ids.h"
#include "parsito/parser/parser_nn_trainer.h"
#include "sentence/input_format.h"
#include "sentence/sentence.h"
#include "tokenizer/detokenizer.h"
#include "tokenizer/multiword_splitter_trainer.h"
#include "trainer.h"
#include "trainer_morphodita_parsito.h"
#include "unilib/unicode.h"
#include "unilib/utf8.h"
#include "utils/options.h"
#include "utils/parse_double.h"
#include "utils/parse_int.h"
#include "utils/split.h"
#include "version/version.h"

namespace ufal {
namespace udpipe {

bool trainer_morphodita_parsito::train(const vector<sentence>& training, const vector<sentence>& heldout,
                                       const string& tokenizer, const string& tagger, const string& parser, ostream& os, string& error) {
  error.clear();

  // Save model version info
  os.put(model_morphodita_parsito::VERSION_LATEST);
  // Add sentinel required since version 2
  os.put(0x7F).put(0x7F);

  // Check input data
  for (auto&& sentence : training)
    for (size_t i = 1; i < sentence.words.size(); i++)
      if (!can_combine_tag(sentence.words[i], error))
        return false;
  for (auto&& sentence : heldout)
    for (size_t i = 1; i < sentence.words.size(); i++)
      if (!can_combine_tag(sentence.words[i], error))
        return false;

  if (!train_tokenizer(training, heldout, tokenizer, os, error)) return false;
  string tagger_model;
  {
    ostringstream os_tagger;
    if (!train_tagger(training, heldout, tagger, os_tagger, error)) return false;
    tagger_model.assign(os_tagger.str());
    os.write(tagger_model.data(), tagger_model.size());
  }
  if (!train_parser(training, heldout, parser, tagger_model, os, error)) return false;

  return true;
}

bool trainer_morphodita_parsito::train_tokenizer(const vector<sentence>& training, const vector<sentence>& heldout,
                                                 const string& options, ostream& os, string& error) {
  if (options == NONE) {
    os.put(0);
  } else {
    // Tokenizer options
    named_values::map tokenizer;
    if (!named_values::parse(options, tokenizer, error)) return false;
    int run = 0; if (!option_int(tokenizer, "run", run, error)) return false;

    if (tokenizer.count("from_model")) {
      // Use specified tokenizer model
      string_piece tokenizer_data;
      if (!load_model(tokenizer["from_model"], TOKENIZER_MODEL, tokenizer_data))
        return error.assign("Cannot load model from which the tokenizer should be used!"), false;

      cerr << "Using tokenizer from given model." << endl;
      os.write(tokenizer_data.str, tokenizer_data.len);
    } else {
      os.put(1);
      const string& model = option_str(tokenizer, "model");

      // Tokenizer itself
      if (model == "generic") {
        os.put(morphodita::tokenizer_id::GENERIC);
        morphodita::generic_tokenizer_factory_encoder::encode(morphodita::generic_tokenizer::LATEST, os);
      } else if (model.empty() || model == "gru") {
        // Create a detokenizator if required
        unique_ptr<detokenizer> detokenizer;
        if (tokenizer.count("detokenize")) {
          detokenizer.reset(new udpipe::detokenizer(tokenizer["detokenize"]));
          if (!detokenizer) return error.assign("Cannot create detokenizer!"), false;
        }

        // Prepare training data for the gru_tokenizer
        vector<morphodita::tokenized_sentence> sentences;
        bool spaces_in_training = false;
        for (size_t training_sentence = 0; training_sentence < training.size(); training_sentence++) {
          sentence s = training[training_sentence];
          if (detokenizer) detokenizer->detokenize(s);

          auto& sentence = (sentences.emplace_back(), sentences.back());

          for (size_t i = 1, j = 0; i < s.words.size(); i++) {
            const string& form = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? s.multiword_tokens[j].form : s.words[i].form;

            sentence.tokens.emplace_back(sentence.sentence.size(), 0);
            for (auto&& chr : unilib::utf8::decoder(form)) {
              sentence.sentence.push_back(chr);
              if (unilib::unicode::category(chr) & unilib::unicode::Zs) spaces_in_training = true;
            }
            sentence.tokens.back().length = sentence.sentence.size() - sentence.tokens.back().start;

            const string& misc = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? s.multiword_tokens[j].misc : s.words[i].misc;
            if (misc.find(space_after_no) == string::npos) sentence.sentence.push_back(' ');

            if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
              i = s.multiword_tokens[j++].id_last;
          }
          string doc_par_id;
          if (training_sentence + 1 < training.size() &&
              (training[training_sentence + 1].get_new_doc(doc_par_id) || training[training_sentence + 1].get_new_par(doc_par_id)))
            sentence.sentence.append(2, '\n');
        }

        // Heldout data
        vector<morphodita::tokenized_sentence> heldout_sentences;

        bool detokenize_handout = true; if (!option_bool(tokenizer, "detokenize_handout", detokenize_handout, error)) return false;
        for (size_t heldout_sentence = 0; heldout_sentence < heldout.size(); heldout_sentence++) {
          sentence s = heldout[heldout_sentence];
          if (detokenizer && detokenize_handout) detokenizer->detokenize(s);

          auto& sentence = (heldout_sentences.emplace_back(), heldout_sentences.back());

          for (size_t i = 1, j = 0; i < s.words.size(); i++) {
            const string& form = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? s.multiword_tokens[j].form : s.words[i].form;

            sentence.tokens.emplace_back(sentence.sentence.size(), 0);
            for (auto&& chr : unilib::utf8::decoder(form))
              sentence.sentence.push_back(chr);
            sentence.tokens.back().length = sentence.sentence.size() - sentence.tokens.back().start;

            const string& misc = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? s.multiword_tokens[j].misc : s.words[i].misc;
            if (misc.find(space_after_no) == string::npos) sentence.sentence.push_back(' ');

            if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
              i = s.multiword_tokens[j++].id_last;
          }
          string doc_par_id;
          if (heldout_sentence + 1 < heldout.size() &&
              (heldout[heldout_sentence + 1].get_new_doc(doc_par_id) || heldout[heldout_sentence + 1].get_new_par(doc_par_id)))
            sentence.sentence.append(2, '\n');
        }

        // Options
        bool tokenize_url = true; if (!option_bool(tokenizer, "tokenize_url", tokenize_url, error)) return false;
        int segment_size = 50; // if (!option_int(tokenizer, "segment_size", segment_size, error)) return false;
        bool allow_spaces = spaces_in_training; if (!option_bool(tokenizer, "allow_spaces", allow_spaces, error)) return false;
        int dimension = 24; // if (!option_int(tokenizer, "dimension", dimension, error)) return false;
        int epochs = 100; if (!option_int(tokenizer, "epochs", epochs, error)) return false;
        int batch_size = run <= 1 ? 50 : 50 + 50 * hyperparameter_integer(run, 1, 0, 1);
        if (!option_int(tokenizer, "batch_size", batch_size, error)) return false;
        double learning_rate = run <= 1 ? 0.005 : hyperparameter_logarithmic(run, 2, 0.0005, 0.01);
        if (!option_double(tokenizer, "learning_rate", learning_rate, error)) return false;
        double learning_rate_final = 0.0; // if (!option_double(tokenizer, "learning_rate_final", learning_rate_final, error)) return false;
        double dropout = 0.1; if (!option_double(tokenizer, "dropout", dropout, error)) return false;
        double initialization_range = 0.5; if (!option_double(tokenizer, "initialization_range", initialization_range, error)) return false;
        bool early_stopping = !heldout_sentences.empty(); if (!option_bool(tokenizer, "early_stopping", early_stopping, error)) return false;

        if (run >= 1) cerr << "Random search run " << run << ", batch_size=" << batch_size
                           << ", learning_rate=" << fixed << setprecision(8) << learning_rate << endl;
        if (allow_spaces) cerr << "Allowing tokenizer to return spaces in tokens." << endl;


        // Train and encode gru_tokenizer
        os.put(morphodita::tokenizer_ids::GRU);
        if (!morphodita::gru_tokenizer_trainer::train(tokenize_url ? morphodita::gru_tokenizer_trainer::URL_EMAIL_LATEST : 0,
                                                      segment_size, allow_spaces, dimension, epochs, batch_size, learning_rate,
                                                      learning_rate_final, dropout, initialization_range, early_stopping,
                                                      sentences, heldout_sentences, os, error))
          return false;
      } else {
        return error.assign("Unknown tokenizer model '").append(model).append("'!"), false;
      }

      // Multiword splitter
      if (!multiword_splitter_trainer::train(training, os, error)) return false;
    }
  }

  return true;
}

bool trainer_morphodita_parsito::train_tagger(const vector<sentence>& training, const vector<sentence>& heldout,
                                              const string& options, ostream& os, string& error) {
  if (options == NONE) {
    os.put(0);
  } else {
    // Parse options
    named_values::map tagger;
    if (!named_values::parse(options, tagger, error)) return false;

    if (tagger.count("from_model")) {
      // Use specified tokenizer model(s)
      int model_index = 1, taggers_total = 0;
      string model_name = "from_model";
      vector<string_piece> taggers_data;
      do {
        taggers_data.emplace_back();
        if (!load_model(tagger[model_name], TAGGER_MODEL, taggers_data.back()))
          return error.assign("Cannot load model from which the tagger should be used!"), false;
        if (taggers_data.back().str[0])
          taggers_total += taggers_data.back().str[0];
        else
          taggers_data.pop_back();
        model_name = "from_model_" + to_string(++model_index);
      } while (tagger.count(model_name));
      if (taggers_total < 0 || taggers_total > 4) return error.assign("Cannot create more than four tagger models!"), false;

      cerr << "Using tagger from given model(s)." << endl;
      os.put(taggers_total);
      for (auto&& tagger_data : taggers_data)
        os.write(tagger_data.str + 1, tagger_data.len - 1);
    } else {
      // Create MorphoDiTa model(s)
      int models = 1; if (!option_int(tagger, "models", models, error)) return false;
      if (models <= 0) return error.assign("Number of tagger models cannot be negative or zero!"), false;
      if (models > 4) return error.assign("Cannot create more than four tagger models!"), false;

      os.put(models);
      for (int model = 0; model < models; model++)
        if (!train_tagger_model(training, heldout, model, models, tagger, os, error))
          return false;
    }
  }

  return true;
}

bool trainer_morphodita_parsito::train_parser(const vector<sentence>& training, const vector<sentence>& heldout,
                                              const string& options, const string& tagger_model, ostream& os, string& error) {
  if (options == NONE) {
    os.put(0);
  } else {
    // Create Parsito model
    named_values::map parser;
    if (!named_values::parse(options, parser, error)) return false;
    int run = 0; if (!option_int(parser, "run", run, error)) return false;

    if (parser.count("from_model")) {
      // Use specified parser model
      string_piece parser_data;
      if (!load_model(parser["from_model"], PARSER_MODEL, parser_data))
        return error.assign("Cannot load model from which the parser should be used!"), false;

      cerr << "Using parser from given model." << endl;
      os.write(parser_data.str, parser_data.len);
    } else {
      os.put(1);

      // Parsito options
      string transition_system = parser.count("transition_system") ? parser["transition_system"] : "projective";
      string transition_oracle = parser.count("transition_oracle") ? parser["transition_oracle"] :
          transition_system == "projective" ? "dynamic" :
          transition_system == "swap" ? "static_lazy" :
          "static";

      int embedding_upostag = 20; if (!option_int(parser, "embedding_upostag", embedding_upostag, error)) return false;
      int embedding_feats = 20; if (!option_int(parser, "embedding_feats", embedding_feats, error)) return false;
      int embedding_xpostag = 0; if (!option_int(parser, "embedding_xpostag", embedding_xpostag, error)) return false;
      int embedding_form = 50; if (!option_int(parser, "embedding_form", embedding_form, error)) return false;
      int embedding_lemma = 0; if (!option_int(parser, "embedding_lemma", embedding_lemma, error)) return false;
      int embedding_deprel = 20; if (!option_int(parser, "embedding_deprel", embedding_deprel, error)) return false;
      string embeddings;
      if (embedding_upostag) embeddings.append("universal_tag ").append(to_string(embedding_upostag)).append(" 1\n");
      if (embedding_feats) embeddings.append("feats ").append(to_string(embedding_feats)).append(" 1\n");
      if (embedding_xpostag) embeddings.append("tag ").append(to_string(embedding_xpostag)).append(" 1\n");
      if (embedding_form) {
        embeddings.append("form ").append(to_string(embedding_form)).append(" 2");
        if (!option_str(parser, "embedding_form_file").empty()) embeddings.append(" ").append(option_str(parser, "embedding_form_file"));
        embeddings.push_back('\n');
      }
      if (embedding_lemma) {
        embeddings.append("lemma ").append(to_string(embedding_lemma)).append(" 2");
        if (!option_str(parser, "embedding_lemma_file").empty()) embeddings.append(" ").append(option_str(parser, "embedding_lemma_file"));
        embeddings.push_back('\n');
      }
      if (embedding_deprel) embeddings.append("deprel ").append(to_string(embedding_deprel)).append(" 1\n");

      int iterations = 10; if (!option_int(parser, "iterations", iterations, error)) return false;
      int hidden_layer = 200; if (!option_int(parser, "hidden_layer", hidden_layer, error)) return false;
      int batch_size = 10; if (!option_int(parser, "batch_size", batch_size, error)) return false;
      int structured_interval = run <= 1 ? 8 : hyperparameter_integer(run,1,0,2) == 2 ? 0 : 8 + 2*hyperparameter_integer(run,1,0,2);
      if (!option_int(parser, "structured_interval", structured_interval, error)) return false;
      double learning_rate = run <= 1 ? 0.02 : hyperparameter_logarithmic(run, 2, 0.005, 0.04);
      if (!option_double(parser, "learning_rate", learning_rate, error)) return false;
      double learning_rate_final = 0.001; if (!option_double(parser, "learning_rate_final", learning_rate_final, error)) return false;
      double l2 = run <= 1 ? 0.5 : hyperparameter_uniform(run, 3, 0.2, 0.6);
      if (!option_double(parser, "l2", l2, error)) return false;

      if (run >= 1) cerr << "Random search run " << run << ", structured_interval=" << structured_interval
                         << ", learning_rate=" << fixed << setprecision(8) << learning_rate
                         << ", l2=" << l2 << endl;

      // Prepare data in the correct format
      parsito::network_parameters parameters;
      parameters.iterations = iterations;
      parameters.structured_interval = structured_interval;
      parameters.hidden_layer = hidden_layer;
      parameters.hidden_layer_type = parsito::activation_function::TANH;
      parameters.trainer.algorithm = parsito::network_trainer::SGD;
      parameters.trainer.learning_rate = learning_rate;
      parameters.trainer.learning_rate_final = learning_rate_final;
      parameters.trainer.momentum = 0;
      parameters.trainer.epsilon = 0;
      parameters.batch_size = batch_size;
      parameters.initialization_range = 0.1f;
      parameters.l1_regularization = 0;
      parameters.l2_regularization = l2;
      parameters.maxnorm_regularization = 0;
      parameters.dropout_hidden = 0;
      parameters.dropout_input = 0;

      // Tag the input if required
      unique_ptr<model> tagger;
      bool use_gold_tags = false; if (!option_bool(parser, "use_gold_tags", use_gold_tags, error)) return false;
      if (!use_gold_tags && !tagger_model.empty() && tagger_model[0]) {
        stringstream tagger_description;
        tagger_description.put(model_morphodita_parsito::VERSION_LATEST).put(0x7F).put(0x7F).put(0).write(tagger_model.data(), tagger_model.size()).put(0);
        tagger.reset(model_morphodita_parsito::load(tagger_description));
        if (!tagger) return error.assign("Cannot load the tagger model for parser training data generation!"), false;
      }

      // Training data
      sentence tagged;
      vector<parsito::tree> train_trees;
      for (auto&& sentence : training) {
        tagged = sentence;
        if (tagger && !tagger->tag(tagged, DEFAULT, error)) return false;

        train_trees.emplace_back();
        for (size_t i = 1; i < tagged.words.size(); i++) {
          train_trees.back().add_node(string());
          model_normalize_form(tagged.words[i].form, train_trees.back().nodes.back().form);
          train_trees.back().nodes.back().lemma.assign(tagged.words[i].lemma);
          train_trees.back().nodes.back().upostag.assign(tagged.words[i].upostag);
          train_trees.back().nodes.back().xpostag.assign(tagged.words[i].xpostag);
          train_trees.back().nodes.back().feats.assign(tagged.words[i].feats);
        }
        for (size_t i = 1; i < tagged.words.size(); i++)
          train_trees.back().set_head(tagged.words[i].id, tagged.words[i].head, tagged.words[i].deprel);
      }

      // Heldout data
      vector<parsito::tree> heldout_trees;
      for (auto&& sentence : heldout) {
        tagged = sentence;
        if (tagger && !tagger->tag(tagged, DEFAULT, error)) return false;

        heldout_trees.emplace_back();
        for (size_t i = 1; i < tagged.words.size(); i++) {
          heldout_trees.back().add_node(string());
          model_normalize_form(tagged.words[i].form, heldout_trees.back().nodes.back().form);
          heldout_trees.back().nodes.back().lemma.assign(tagged.words[i].lemma);
          heldout_trees.back().nodes.back().upostag.assign(tagged.words[i].upostag);
          heldout_trees.back().nodes.back().xpostag.assign(tagged.words[i].xpostag);
          heldout_trees.back().nodes.back().feats.assign(tagged.words[i].feats);
        }
        for (size_t i = 1; i < tagged.words.size(); i++)
          heldout_trees.back().set_head(tagged.words[i].id, tagged.words[i].head, tagged.words[i].deprel);
      }

      // Train the parser
      binary_encoder enc;
      enc.add_str("nn");
      parsito::parser_nn_trainer::train(transition_system, transition_oracle, embeddings, parser_nodes,
                                        parameters, 1, train_trees, heldout_trees, enc);
      compressor::save(os, enc);
    }
  }

  return true;
}

bool trainer_morphodita_parsito::load_model(const string& data, model_type model, string_piece& range) {
  istringstream is(data);

  // Check that it is morphodita_parsito model.
  char len;
  if (!is.get(len)) return false;
  string name(len, ' ');
  if (!is.read(&name[0], len)) return false;
  if (name != "morphodita_parsito") return false;

  char version;
  if (!is.get(version)) return false;
  if (!(version >= 1 && version <= model_morphodita_parsito::VERSION_LATEST)) return false;

  // Because UDPipe 1.0 does not check the model version,
  // a specific sentinel was added since version 2 so that
  // loading of such model fail on UDPipe 1.0
  if (version >= 2) {
    char sentinel;
    if (!is.get(sentinel) || sentinel != 0x7F) return false;
    if (!is.get(sentinel) || sentinel != 0x7F) return false;
  }

  // Tokenizer
  {
    if (model == TOKENIZER_MODEL) range.str = data.data() + is.tellg();
    char tokenizer; if (!is.get(tokenizer)) return false;
    unique_ptr<morphodita::tokenizer_factory> tokenizer_factory(tokenizer ? morphodita::tokenizer_factory::load(is) : nullptr);
    if (tokenizer && !tokenizer_factory) return false;
    if (model == TOKENIZER_MODEL) return range.len = is.tellg() - streampos(range.str - data.data()), true;
  }

  // Tagger
  {
    if (model == TAGGER_MODEL) range.str = data.data() + is.tellg();
    char taggers; if (!is.get(taggers)) return false;
    for (char i = 0; i < taggers; i++) {
{'fraction_non_alphanumeric': 0.1333994144902146, 'fraction_numerical': 0.008736543648140891, 'mean_word_length': 3.3364654138344663, 'pattern_counts': {'":': 0, '<': 208, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '401958', 'n_tokens_mistral': 14065, 'n_tokens_neox': 13362, 'n_words': 2980}