<commit_before>/***********************************************************************
 Moses - factored phrase-based language decoder
 Copyright (C) 2010 University of Edinburgh

 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
 version 2.1 of the License, or (at your option) any later version.

 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 Lesser General Public License for more details.

 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 ***********************************************************************/

#include <algorithm>
#include <cstdlib>
#include <ctime>
#include <string>
#include <vector>
#include <map>

#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>

#ifdef MPI_ENABLE
#include <boost/mpi.hpp>
namespace mpi = boost::mpi;
#endif

#include "Main.h"
#include "FeatureVector.h"
#include "StaticData.h"
#include "ChartTrellisPathList.h"
#include "ChartTrellisPath.h"
#include "ScoreComponentCollection.h"
#include "Decoder.h"
#include "Optimiser.h"
#include "Hildreth.h"
#include "ThreadPool.h"

using namespace Mira;
using namespace std;
using namespace Moses;
namespace po = boost::program_options;

int main(int argc, char** argv) {
	size_t rank = 0;
	size_t size = 1;
#ifdef MPI_ENABLE
	mpi::environment env(argc,argv);
	mpi::communicator world;
	rank = world.rank();
	size = world.size();
#endif
	cerr << "Rank: " << rank << " Size: " << size << endl;

	bool help;
	int verbosity;
	string mosesConfigFile;
	string inputFile;
	vector<string> referenceFiles;
	vector<string> mosesConfigFilesFolds, inputFilesFolds, referenceFilesFolds;
	string coreWeightFile;
	size_t epochs;
	string learner;
	bool shuffle;
	size_t mixingFrequency;
	size_t weightDumpFrequency;
	string weightDumpStem;
	float min_learning_rate;
	bool scale_margin, scale_margin_precision;
	bool scale_update, scale_update_precision;
	size_t n;
	size_t batchSize;
	bool distinctNbest;
	bool onlyViolatedConstraints;
	bool accumulateWeights;
	float historySmoothing;
	bool scaleByInputLength, scaleByAvgInputLength;
	bool scaleByInverseLength, scaleByAvgInverseLength;
	float scaleByX;
	float slack, dummy;
	float slack_step;
	float slack_min;
	bool averageWeights;
	bool weightConvergence;
	float learning_rate;
	float mira_learning_rate;
	float perceptron_learning_rate;
	bool logFeatureValues;
	size_t baseOfLog;
	string decoder_settings;
	float min_weight_change;
	float decrease_learning_rate;
	bool normaliseWeights, normaliseMargin;
	bool print_feature_values;
	bool historyOf1best;
	bool historyOfOracles;
	bool sentenceLevelBleu;
	float bleuScoreWeight, bleuScoreWeight_hope, bleuScoreWeight_fear;
	float margin_slack;
	float margin_slack_incr;
	bool perceptron_update;
	bool hope_fear, hope_fear_rank, hope_model;
	bool model_hope_fear, rank_only;
	int hope_n, fear_n, rank_n;
	int threadcount;
	size_t adapt_after_epoch;
	size_t bleu_smoothing_scheme;
	float max_length_dev_all;
	float max_length_dev_hypos;
	float max_length_dev_hope_ref;
	float max_length_dev_fear_ref;
	float relax_BP;
	float min_oracle_bleu;
	float minBleuRatio, maxBleuRatio;
	bool boost;
	bool decode_hope, decode_fear, decode_model;
	string decode_filename;
	size_t update_scheme;
	bool separateUpdates, batchEqualsShard;
	bool sparseAverage, dumpMixedWeights, sparseNoAverage;
	bool useSourceLengthHistory;
	po::options_description desc("Allowed options");
	desc.add_options()
		("slack", po::value<float>(&slack)->default_value(0.01), "Use slack in optimiser")
		("dummy", po::value<float>(&dummy)->default_value(-1), "Dummy variable for slack")
		("accumulate-weights", po::value<bool>(&accumulateWeights)->default_value(false), "Accumulate and average weights over all epochs")
		("adapt-after-epoch", po::value<size_t>(&adapt_after_epoch)->default_value(0), "Index of epoch after which adaptive parameters will be adapted")
		("average-weights", po::value<bool>(&averageWeights)->default_value(false), "Set decoder weights to average weights after each update")
		("base-of-log", po::value<size_t>(&baseOfLog)->default_value(10), "Base for taking logs of feature values")
		("batch-equals-shard", po::value<bool>(&batchEqualsShard)->default_value(false), "Batch size is equal to shard size (purely batch)")
		("batch-size,b", po::value<size_t>(&batchSize)->default_value(1), "Size of batch that is send to optimiser for weight adjustments")
		("bleu-score-weight", po::value<float>(&bleuScoreWeight)->default_value(1.0), "Bleu score weight used in the decoder objective function (on top of the Bleu objective weight)")
		("bleu-score-weight-hope", po::value<float>(&bleuScoreWeight_hope)->default_value(-1), "Bleu score weight used in the decoder objective function for hope translations")
	    ("bleu-score-weight-fear", po::value<float>(&bleuScoreWeight_fear)->default_value(-1), "Bleu score weight used in the decoder objective function for fear translations")
		("bleu-smoothing-scheme", po::value<size_t>(&bleu_smoothing_scheme)->default_value(1), "Set a smoothing scheme for sentence-Bleu: +1 (1), +0.1 (2), papineni (3) (default:1)")
		("boost", po::value<bool>(&boost)->default_value(false), "Apply boosting factor to updates on misranked candidates")
		("config,f", po::value<string>(&mosesConfigFile), "Moses ini-file")
		("configs-folds", po::value<vector<string> >(&mosesConfigFilesFolds), "Moses ini-files, one for each fold")
		("core-weights", po::value<string>(&coreWeightFile), "Weight file containing the core weights (already tuned, have to be non-zero)")
		("decode-hope", po::value<bool>(&decode_hope)->default_value(false), "Decode dev input set according to hope objective")
		("decode-fear", po::value<bool>(&decode_fear)->default_value(false), "Decode dev input set according to fear objective")
		("decode-model", po::value<bool>(&decode_model)->default_value(false), "Decode dev input set according to normal objective")
		("decode-filename", po::value<string>(&decode_filename), "Filename for Bleu objective translations")
		("decoder-settings", po::value<string>(&decoder_settings)->default_value(""), "Decoder settings for tuning runs")
		("decr-learning-rate", po::value<float>(&decrease_learning_rate)->default_value(0),"Decrease learning rate by the given value after every epoch")
		("distinct-nbest", po::value<bool>(&distinctNbest)->default_value(true), "Use n-best list with distinct translations in inference step")
		("dump-mixed-weights", po::value<bool>(&dumpMixedWeights)->default_value(false), "Dump mixed weights instead of averaged weights")
		("epochs,e", po::value<size_t>(&epochs)->default_value(10), "Number of epochs")
		("fear-n", po::value<int>(&fear_n)->default_value(-1), "Number of fear translations used")
		("help", po::value(&help)->zero_tokens()->default_value(false), "Print this help message and exit")
		("history-of-1best", po::value<bool>(&historyOf1best)->default_value(false), "Use 1best translations to update the history")
		("history-of-oracles", po::value<bool>(&historyOfOracles)->default_value(false), "Use oracle translations to update the history")
		("history-smoothing", po::value<float>(&historySmoothing)->default_value(0.9), "Adjust the factor for history smoothing")
		("hope-fear", po::value<bool>(&hope_fear)->default_value(true), "Use only hope and fear translations for optimisation (not model)")
		("hope-fear-rank", po::value<bool>(&hope_fear_rank)->default_value(false), "Use hope and fear translations for optimisation, use model for ranking")
		("hope-model", po::value<bool>(&hope_model)->default_value(false), "Use only hope and model translations for optimisation (use --fear-n to set number of model translations)")
		("hope-n", po::value<int>(&hope_n)->default_value(-1), "Number of hope translations used")
		("input-file,i", po::value<string>(&inputFile), "Input file containing tokenised source")
		("input-files-folds", po::value<vector<string> >(&inputFilesFolds), "Input files containing tokenised source, one for each fold")
		("learner,l", po::value<string>(&learner)->default_value("mira"), "Learning algorithm")
		("log-feature-values", po::value<bool>(&logFeatureValues)->default_value(false), "Take log of feature values according to the given base.")
		("margin-incr", po::value<float>(&margin_slack_incr)->default_value(0), "Increment margin slack after every epoch by this amount")
		("margin-slack", po::value<float>(&margin_slack)->default_value(0), "Slack when comparing left and right hand side of constraints")
		("max-length-dev-all", po::value<float>(&max_length_dev_all)->default_value(-1), "Make use of all 3 following options")
		("max-length-dev-hypos", po::value<float>(&max_length_dev_hypos)->default_value(-1), "Number between 0 and 1 specifying the percentage of admissible length deviation between hope and fear translations")
		("max-length-dev-hope-ref", po::value<float>(&max_length_dev_hope_ref)->default_value(-1), "Number between 0 and 1 specifying the percentage of admissible length deviation between hope and reference translations")
		("max-length-dev-fear-ref", po::value<float>(&max_length_dev_fear_ref)->default_value(-1), "Number between 0 and 1 specifying the percentage of admissible length deviation between fear and reference translations")
		("min-bleu-ratio", po::value<float>(&minBleuRatio)->default_value(-1), "Set a minimum BLEU ratio between hope and fear")
		("max-bleu-ratio", po::value<float>(&maxBleuRatio)->default_value(-1), "Set a maximum BLEU ratio between hope and fear")
		("min-learning-rate", po::value<float>(&min_learning_rate)->default_value(0), "Set a minimum learning rate")
		("min-oracle-bleu", po::value<float>(&min_oracle_bleu)->default_value(0), "Set a minimum oracle BLEU score")
		("min-weight-change", po::value<float>(&min_weight_change)->default_value(0.01), "Set minimum weight change for stopping criterion")
		("mira-learning-rate", po::value<float>(&mira_learning_rate)->default_value(1), "Learning rate for MIRA (fixed or flexible)")
		("mixing-frequency", po::value<size_t>(&mixingFrequency)->default_value(1), "How often per epoch to mix weights, when using mpi")
		("model-hope-fear", po::value<bool>(&model_hope_fear)->default_value(false), "Use model, hope and fear translations for optimisation")
		("nbest,n", po::value<size_t>(&n)->default_value(1), "Number of translations in n-best list")
		("normalise", po::value<bool>(&normaliseWeights)->default_value(false), "Whether to normalise the updated weights before passing them to the decoder")
		("normalise-margin", po::value<bool>(&normaliseMargin)->default_value(false), "Normalise the margin: squash between 0 and 1")
		("only-violated-constraints", po::value<bool>(&onlyViolatedConstraints)->default_value(false), "Add only violated constraints to the optimisation problem")
		("perceptron-learning-rate", po::value<float>(&perceptron_learning_rate)->default_value(0.01), "Perceptron learning rate")
		("print-feature-values", po::value<bool>(&print_feature_values)->default_value(false), "Print out feature values")
		("rank-n", po::value<int>(&rank_n)->default_value(-1), "Number of translations used for ranking")
		("rank-only", po::value<bool>(&rank_only)->default_value(false), "Use only model translations for optimisation")
		("reference-files,r", po::value<vector<string> >(&referenceFiles), "Reference translation files for training")
		("reference-files-folds", po::value<vector<string> >(&referenceFilesFolds), "Reference translation files for training, one for each fold")		
		("relax-BP", po::value<float>(&relax_BP)->default_value(1), "Relax the BP by setting this value between 0 and 1")
		("scale-by-inverse-length", po::value<bool>(&scaleByInverseLength)->default_value(false), "Scale the BLEU score by (a history of) the inverse input length")
		("scale-by-input-length", po::value<bool>(&scaleByInputLength)->default_value(true), "Scale the BLEU score by (a history of) the input length")
		("scale-by-avg-input-length", po::value<bool>(&scaleByAvgInputLength)->default_value(false), "Scale BLEU by an average of the input length")
		("scale-by-avg-inverse-length", po::value<bool>(&scaleByAvgInverseLength)->default_value(false), "Scale BLEU by an average of the inverse input length")
		("scale-by-x", po::value<float>(&scaleByX)->default_value(1), "Scale the BLEU score by value x")
		("scale-margin", po::value<bool>(&scale_margin)->default_value(0), "Scale the margin by the Bleu score of the oracle translation")
		("scale-margin-precision", po::value<bool>(&scale_margin_precision)->default_value(0), "Scale the margin by the precision of the oracle translation")
		("scale-update", po::value<bool>(&scale_update)->default_value(0), "Scale the update by the Bleu score of the oracle translation")       
		("scale-update-precision", po::value<bool>(&scale_update_precision)->default_value(0), "Scale the update by the precision of the oracle translation")	
		("sentence-level-bleu", po::value<bool>(&sentenceLevelBleu)->default_value(true), "Use a sentences level Bleu scoring function")
		("separate-updates", po::value<bool>(&separateUpdates)->default_value(false), "Compute separate updates for each sentence in a batch")
		("shuffle", po::value<bool>(&shuffle)->default_value(false), "Shuffle input sentences before processing")
		("slack-min", po::value<float>(&slack_min)->default_value(0.01), "Minimum slack used")
		("slack-step", po::value<float>(&slack_step)->default_value(0), "Increase slack from epoch to epoch by the value provided")
		("sparse-average", po::value<bool>(&sparseAverage)->default_value(false), "Average weights by the number of processes")
		("sparse-no-average", po::value<bool>(&sparseNoAverage)->default_value(false), "Don't average sparse weights, just sum")
		("stop-weights", po::value<bool>(&weightConvergence)->default_value(true), "Stop when weights converge")
		("threads", po::value<int>(&threadcount)->default_value(1), "Number of threads used")
		("update-scheme", po::value<size_t>(&update_scheme)->default_value(1), "Update scheme, default: 1")
		("use-source-length-history", po::value<bool>(&useSourceLengthHistory)->default_value(false), "Use history of source length instead of target length for history Bleu")
		("verbosity,v", po::value<int>(&verbosity)->default_value(0), "Verbosity level")
		("weight-dump-frequency", po::value<size_t>(&weightDumpFrequency)->default_value(1), "How often per epoch to dump weights, when using mpi")
		("weight-dump-stem", po::value<string>(&weightDumpStem)->default_value("weights"), "Stem of filename to use for dumping weights");

	po::options_description cmdline_options;
	cmdline_options.add(desc);
	po::variables_map vm;
	po::store(po::command_line_parser(argc, argv). options(cmdline_options).run(), vm);
	po::notify(vm);

	if (help) {
		std::cout << "Usage: " + string(argv[0])
		    + " -f mosesini-file -i input-file -r reference-file(s) [options]"
		    << std::endl;
		std::cout << desc << std::endl;
		return 0;
	}

  // create threadpool, if using multi-threaded decoding
  // note: multi-threading is done on sentence-level,
  // each thread translates one sentence
/*#ifdef WITH_THREADS
  if (threadcount < 1) {
    cerr << "Error: Need to specify a positive number of threads" << endl;
    exit(1);
  }
  ThreadPool pool(threadcount);
#else
  if (threadcount > 1) {
    cerr << "Error: Thread count of " << threadcount << " but moses not built with thread support" << endl;
    exit(1);
  }
#endif*/

  bool trainWithMultipleFolds = false; 
  if (mosesConfigFilesFolds.size() > 0 || inputFilesFolds.size() > 0 || referenceFilesFolds.size() > 0)
	  trainWithMultipleFolds = true;
  
  if (dummy != -1)
    slack = dummy;

  if (dumpMixedWeights && (mixingFrequency != weightDumpFrequency)) {
	  cerr << "Set mixing frequency = weight dump frequency for dumping mixed weights!" << endl;
	  exit(1);
  }

  if ((sparseAverage || sparseNoAverage) && averageWeights) {
	  cerr << "Parameters --sparse-average 1/--sparse-no-average 1 and --average-weights 1 are incompatible (not implemented)" << endl;
	  exit(1);
  }

  if (trainWithMultipleFolds) {
	  if (!mosesConfigFilesFolds.size()) {
		  cerr << "Error: No moses ini files specified for training with folds" << endl;
		  exit(1);
	  }
	  
	  if (!inputFilesFolds.size()) {
		  cerr << "Error: No input files specified for training with folds" << endl;
		  exit(1);
	  }

	  if (!referenceFilesFolds.size()) {
		  cerr << "Error: No reference files specified for training with folds" << endl;
		  exit(1);
	  }
  }
  else {
	  if (mosesConfigFile.empty()) {
		  cerr << "Error: No moses ini file specified" << endl;
		  return 1;
	  }
	  
	  if (inputFile.empty()) {
		  cerr << "Error: No input file specified" << endl;
		  return 1;
	  }

	  if (!referenceFiles.size()) {
		  cerr << "Error: No reference files specified" << endl;
		  return 1;
	  }
  }

	// load input and references
  	vector<string> inputSentences;
  	size_t inputSize = trainWithMultipleFolds? inputFilesFolds.size(): 0;
  	size_t refSize = trainWithMultipleFolds? referenceFilesFolds.size(): referenceFiles.size(); 
  	vector<vector<string> > inputSentencesFolds(inputSize);
  	vector<vector<string> > referenceSentences(refSize);
  	
	// number of cores for each fold
  	size_t coresPerFold = 0, myFold = 0;
  	if (trainWithMultipleFolds) {
  		coresPerFold = size/mosesConfigFilesFolds.size();
  		if (size % coresPerFold > 0) {
  			cerr << "Number of cores has to be a multiple of the number of folds" << endl;
  			exit(1);
  		} 
		
  		if (rank == 0)
  			cerr << "Number of cores per fold: " << coresPerFold << endl;		
  		myFold = rank/coresPerFold;
  		cerr << "Rank " << rank << ", my fold: " << myFold << endl;
  	}
  	
  	if (trainWithMultipleFolds) {
  		if (!loadSentences(inputFilesFolds[myFold], inputSentencesFolds[myFold])) {
  			cerr << "Error: Failed to load input sentences from " << inputFilesFolds[myFold] << endl;
  			exit(1);
  		}
  		cerr << "Rank " << rank << " reading inputs from " << inputFilesFolds[myFold] << endl;
  		
  		if (!loadSentences(referenceFilesFolds[myFold], referenceSentences[myFold])) {
  			cerr << "Error: Failed to load reference sentences from " << referenceFilesFolds[myFold] << endl;
  			exit(1);
  		}
  		if (referenceSentences[myFold].size() != inputSentencesFolds[myFold].size()) {
  			cerr << "Error: Input file length (" << inputSentencesFolds[myFold].size() << ") != ("
  				<< referenceSentences[myFold].size() << ") reference file length (rank " << rank << ")" << endl;
  			exit(1);
  		}
  		cerr << "Rank " << rank << " reading references from " << referenceFilesFolds[myFold] << endl;  		
  	}
  	else {
  		if (!loadSentences(inputFile, inputSentences)) {
  			cerr << "Error: Failed to load input sentences from " << inputFile << endl;
  			return 1;
  		}
  		
  		for (size_t i = 0; i < referenceFiles.size(); ++i) {
  			if (!loadSentences(referenceFiles[i], referenceSentences[i])) {
  				cerr << "Error: Failed to load reference sentences from "
  						<< referenceFiles[i] << endl;
  				return 1;
  			}
  			if (referenceSentences[i].size() != inputSentences.size()) {
  				cerr << "Error: Input file length (" << inputSentences.size() << ") != ("
  						<< referenceSentences[i].size() << ") length of reference file " << i
  						<< endl;
  				return 1;
  			}
  		}
  	}

	if (scaleByAvgInputLength ||  scaleByInverseLength || scaleByAvgInverseLength)
		scaleByInputLength = false;

	if (historyOf1best || historyOfOracles)
		sentenceLevelBleu = false;
	if (!sentenceLevelBleu) {
		if (!historyOf1best && !historyOfOracles) {
			historyOf1best = true;
		}
	}

	// initialise Moses
	// add initial Bleu weight and references to initialize Bleu feature
	decoder_settings += " -weight-bl 1 -references";
	if (trainWithMultipleFolds) {
		decoder_settings += " ";
		decoder_settings += referenceFilesFolds[myFold];
	}
	else {
		for (size_t i=0; i < referenceFiles.size(); ++i) {
			decoder_settings += " ";
			decoder_settings += referenceFiles[i];
		}
	}
	
	vector<string> decoder_params;
	boost::split(decoder_params, decoder_settings, boost::is_any_of("\t "));
	
	string configFile = trainWithMultipleFolds? mosesConfigFilesFolds[myFold] : mosesConfigFile;
	cerr << "Rank " << rank << " reading config file from " << configFile << endl;
	MosesDecoder* decoder = new MosesDecoder(configFile, verbosity, decoder_params.size(), decoder_params);
	decoder->setBleuParameters(sentenceLevelBleu, scaleByInputLength, scaleByAvgInputLength,
			scaleByInverseLength, scaleByAvgInverseLength,
			scaleByX, historySmoothing, bleu_smoothing_scheme, relax_BP, useSourceLengthHistory);
	if (normaliseWeights) {
		ScoreComponentCollection startWeights = decoder->getWeights();
		startWeights.L1Normalise();
		decoder->setWeights(startWeights);
	}

	if (decode_hope || decode_fear || decode_model) {
		size_t decode = 1;
		if (decode_fear) decode = 2;
		if (decode_model) decode = 3;
		decodeHopeOrFear(rank, size, decode, decode_filename, inputSentences, decoder, n);
	}

	// Optionally shuffle the sentences
	vector<size_t> order;
	if (trainWithMultipleFolds) {  	
		for (size_t i = 0; i < inputSentencesFolds[myFold].size(); ++i) {
			order.push_back(i);
		}
		
		if (shuffle) {
			cerr << "Shuffling input sentences.." << endl;
			RandomIndex rindex;
			random_shuffle(order.begin(), order.end(), rindex);
		}
	}
	else {
		if (rank == 0) {
			for (size_t i = 0; i < inputSentences.size(); ++i) {
				order.push_back(i);
			}
			
			if (shuffle) {
				cerr << "Shuffling input sentences.." << endl;
				RandomIndex rindex;
				random_shuffle(order.begin(), order.end(), rindex);
			}
		}
	}

	// initialise optimizer
	Optimiser* optimiser = NULL;
	if (learner == "mira") {
		if (rank == 0) {
			cerr << "Optimising using Mira" << endl;
			cerr << "slack: " << slack << ", learning rate: " << mira_learning_rate << endl;
		}
		optimiser = new MiraOptimiser(onlyViolatedConstraints, slack, scale_margin, scale_margin_precision,
				scale_update, scale_update_precision, margin_slack, boost, update_scheme, normaliseMargin);
		learning_rate = mira_learning_rate;
		perceptron_update = false;
	} else if (learner == "perceptron") {
		if (rank == 0) {
			cerr << "Optimising using Perceptron" << endl;
		}
		optimiser = new Perceptron();
		learning_rate = perceptron_learning_rate;
		perceptron_update = true;
		model_hope_fear = false; // mira only
		rank_only = false; // mira only
		hope_fear = false; // mira only
		hope_fear_rank = false; // mira only
		hope_model = false; // mira only
		n = 1;
		hope_n = 1;
		fear_n = 1;
	} else {
		cerr << "Error: Unknown optimiser: " << learner << endl;
		return 1;
	}

	// resolve parameter dependencies
	if (batchSize > 1 && perceptron_update) {
		batchSize = 1;
		cerr << "Info: Setting batch size to 1 for perceptron update" << endl;
	}

	if (hope_n == -1)
		hope_n = n;
	if (fear_n == -1)
		fear_n = n;
	if (rank_n == -1)
		rank_n = n;

	if (model_hope_fear && hope_fear)
		hope_fear = false; // is true by default
	if (rank_only && hope_fear)
		hope_fear = false; // is true by default
	if (hope_fear_rank && hope_fear)
		hope_fear = false; // is true by default
	if (hope_model && hope_fear)
		hope_fear = false; // is true by default
	if (learner == "mira" && !(hope_fear || hope_model || model_hope_fear || rank_only || hope_fear_rank)) {
		cerr << "Error: Need to select an one of parameters --hope-fear/--model-hope-fear for mira update." << endl;
		return 1;
	}

	if (bleuScoreWeight_hope == -1) {
		bleuScoreWeight_hope = bleuScoreWeight;
	}
	if (bleuScoreWeight_fear == -1) {
		bleuScoreWeight_fear = bleuScoreWeight;
	}

	if (max_length_dev_all != -1) {
		max_length_dev_hypos = max_length_dev_all;
		max_length_dev_hope_ref = max_length_dev_all;
		max_length_dev_fear_ref = max_length_dev_all;
	}

#ifdef MPI_ENABLE
	mpi::broadcast(world, order, 0);
#endif

	// Create shards according to the number of processes used
	vector<size_t> shard;
	if (trainWithMultipleFolds) {			
		float shardSize = (float) (order.size())/coresPerFold;
		size_t shardStart = (size_t) (shardSize * (rank % coresPerFold));
		size_t shardEnd = shardStart + shardSize;
		if (rank % coresPerFold == coresPerFold - 1) { // last rank of each fold 
			shardEnd = order.size();
			shardSize = shardEnd - shardStart;
		}		
		cerr << "Rank: " << rank << ", shard size: " << shardSize << endl;
		cerr << "Rank: " << rank << ", shard start: " << shardStart << " shard end: " << shardEnd << endl;
		shard.resize(shardSize);
		copy(order.begin() + shardStart, order.begin() + shardEnd, shard.begin());
		batchSize = 1;
	}
	else {
		float shardSize = (float) (order.size()) / size;
		size_t shardStart = (size_t) (shardSize * rank);
		size_t shardEnd = (size_t) (shardSize * (rank + 1));
		if (rank == size - 1) {
			shardEnd = order.size();
			shardSize = shardEnd - shardStart;
		}
		VERBOSE(1, "Shard size: " << shardSize << endl);
		VERBOSE(1, "Rank: " << rank << " Shard start: " << shardStart << " Shard end: " << shardEnd << endl);<commit_msg>enable multiple cores per fold, 2<commit_after>/***********************************************************************
 Moses - factored phrase-based language decoder
 Copyright (C) 2010 University of Edinburgh

 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
 version 2.1 of the License, or (at your option) any later version.

 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 Lesser General Public License for more details.

 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 ***********************************************************************/

#include <algorithm>
#include <cstdlib>
#include <ctime>
#include <string>
#include <vector>
#include <map>

#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>

#ifdef MPI_ENABLE
#include <boost/mpi.hpp>
namespace mpi = boost::mpi;
#endif

#include "Main.h"
#include "FeatureVector.h"
#include "StaticData.h"
#include "ChartTrellisPathList.h"
#include "ChartTrellisPath.h"
#include "ScoreComponentCollection.h"
#include "Decoder.h"
#include "Optimiser.h"
#include "Hildreth.h"
#include "ThreadPool.h"

using namespace Mira;
using namespace std;
using namespace Moses;
namespace po = boost::program_options;

int main(int argc, char** argv) {
	size_t rank = 0;
	size_t size = 1;
#ifdef MPI_ENABLE
	mpi::environment env(argc,argv);
	mpi::communicator world;
	rank = world.rank();
	size = world.size();
#endif
	cerr << "Rank: " << rank << " Size: " << size << endl;

	bool help;
	int verbosity;
	string mosesConfigFile;
	string inputFile;
	vector<string> referenceFiles;
	vector<string> mosesConfigFilesFolds, inputFilesFolds, referenceFilesFolds;
	string coreWeightFile;
	size_t epochs;
	string learner;
	bool shuffle;
	size_t mixingFrequency;
	size_t weightDumpFrequency;
	string weightDumpStem;
	float min_learning_rate;
	bool scale_margin, scale_margin_precision;
	bool scale_update, scale_update_precision;
	size_t n;
	size_t batchSize;
	bool distinctNbest;
	bool onlyViolatedConstraints;
	bool accumulateWeights;
	float historySmoothing;
	bool scaleByInputLength, scaleByAvgInputLength;
	bool scaleByInverseLength, scaleByAvgInverseLength;
	float scaleByX;
	float slack, dummy;
	float slack_step;
	float slack_min;
	bool averageWeights;
	bool weightConvergence;
	float learning_rate;
	float mira_learning_rate;
	float perceptron_learning_rate;
	bool logFeatureValues;
	size_t baseOfLog;
	string decoder_settings;
	float min_weight_change;
	float decrease_learning_rate;
	bool normaliseWeights, normaliseMargin;
	bool print_feature_values;
	bool historyOf1best;
	bool historyOfOracles;
	bool sentenceLevelBleu;
	float bleuScoreWeight, bleuScoreWeight_hope, bleuScoreWeight_fear;
	float margin_slack;
	float margin_slack_incr;
	bool perceptron_update;
	bool hope_fear, hope_fear_rank, hope_model;
	bool model_hope_fear, rank_only;
	int hope_n, fear_n, rank_n;
	int threadcount;
	size_t adapt_after_epoch;
	size_t bleu_smoothing_scheme;
	float max_length_dev_all;
	float max_length_dev_hypos;
	float max_length_dev_hope_ref;
	float max_length_dev_fear_ref;
	float relax_BP;
	float min_oracle_bleu;
	float minBleuRatio, maxBleuRatio;
	bool boost;
	bool decode_hope, decode_fear, decode_model;
	string decode_filename;
	size_t update_scheme;
	bool separateUpdates, batchEqualsShard;
	bool sparseAverage, dumpMixedWeights, sparseNoAverage;
	bool useSourceLengthHistory;
	po::options_description desc("Allowed options");
	desc.add_options()
		("slack", po::value<float>(&slack)->default_value(0.01), "Use slack in optimiser")
		("dummy", po::value<float>(&dummy)->default_value(-1), "Dummy variable for slack")
		("accumulate-weights", po::value<bool>(&accumulateWeights)->default_value(false), "Accumulate and average weights over all epochs")
		("adapt-after-epoch", po::value<size_t>(&adapt_after_epoch)->default_value(0), "Index of epoch after which adaptive parameters will be adapted")
		("average-weights", po::value<bool>(&averageWeights)->default_value(false), "Set decoder weights to average weights after each update")
		("base-of-log", po::value<size_t>(&baseOfLog)->default_value(10), "Base for taking logs of feature values")
		("batch-equals-shard", po::value<bool>(&batchEqualsShard)->default_value(false), "Batch size is equal to shard size (purely batch)")
		("batch-size,b", po::value<size_t>(&batchSize)->default_value(1), "Size of batch that is send to optimiser for weight adjustments")
		("bleu-score-weight", po::value<float>(&bleuScoreWeight)->default_value(1.0), "Bleu score weight used in the decoder objective function (on top of the Bleu objective weight)")
		("bleu-score-weight-hope", po::value<float>(&bleuScoreWeight_hope)->default_value(-1), "Bleu score weight used in the decoder objective function for hope translations")
	    ("bleu-score-weight-fear", po::value<float>(&bleuScoreWeight_fear)->default_value(-1), "Bleu score weight used in the decoder objective function for fear translations")
		("bleu-smoothing-scheme", po::value<size_t>(&bleu_smoothing_scheme)->default_value(1), "Set a smoothing scheme for sentence-Bleu: +1 (1), +0.1 (2), papineni (3) (default:1)")
		("boost", po::value<bool>(&boost)->default_value(false), "Apply boosting factor to updates on misranked candidates")
		("config,f", po::value<string>(&mosesConfigFile), "Moses ini-file")
		("configs-folds", po::value<vector<string> >(&mosesConfigFilesFolds), "Moses ini-files, one for each fold")
		("core-weights", po::value<string>(&coreWeightFile), "Weight file containing the core weights (already tuned, have to be non-zero)")
		("decode-hope", po::value<bool>(&decode_hope)->default_value(false), "Decode dev input set according to hope objective")
		("decode-fear", po::value<bool>(&decode_fear)->default_value(false), "Decode dev input set according to fear objective")
		("decode-model", po::value<bool>(&decode_model)->default_value(false), "Decode dev input set according to normal objective")
		("decode-filename", po::value<string>(&decode_filename), "Filename for Bleu objective translations")
		("decoder-settings", po::value<string>(&decoder_settings)->default_value(""), "Decoder settings for tuning runs")
		("decr-learning-rate", po::value<float>(&decrease_learning_rate)->default_value(0),"Decrease learning rate by the given value after every epoch")
		("distinct-nbest", po::value<bool>(&distinctNbest)->default_value(true), "Use n-best list with distinct translations in inference step")
		("dump-mixed-weights", po::value<bool>(&dumpMixedWeights)->default_value(false), "Dump mixed weights instead of averaged weights")
		("epochs,e", po::value<size_t>(&epochs)->default_value(10), "Number of epochs")
		("fear-n", po::value<int>(&fear_n)->default_value(-1), "Number of fear translations used")
		("help", po::value(&help)->zero_tokens()->default_value(false), "Print this help message and exit")
		("history-of-1best", po::value<bool>(&historyOf1best)->default_value(false), "Use 1best translations to update the history")
		("history-of-oracles", po::value<bool>(&historyOfOracles)->default_value(false), "Use oracle translations to update the history")
		("history-smoothing", po::value<float>(&historySmoothing)->default_value(0.9), "Adjust the factor for history smoothing")
		("hope-fear", po::value<bool>(&hope_fear)->default_value(true), "Use only hope and fear translations for optimisation (not model)")
		("hope-fear-rank", po::value<bool>(&hope_fear_rank)->default_value(false), "Use hope and fear translations for optimisation, use model for ranking")
		("hope-model", po::value<bool>(&hope_model)->default_value(false), "Use only hope and model translations for optimisation (use --fear-n to set number of model translations)")
		("hope-n", po::value<int>(&hope_n)->default_value(-1), "Number of hope translations used")
		("input-file,i", po::value<string>(&inputFile), "Input file containing tokenised source")
		("input-files-folds", po::value<vector<string> >(&inputFilesFolds), "Input files containing tokenised source, one for each fold")
		("learner,l", po::value<string>(&learner)->default_value("mira"), "Learning algorithm")
		("log-feature-values", po::value<bool>(&logFeatureValues)->default_value(false), "Take log of feature values according to the given base.")
		("margin-incr", po::value<float>(&margin_slack_incr)->default_value(0), "Increment margin slack after every epoch by this amount")
		("margin-slack", po::value<float>(&margin_slack)->default_value(0), "Slack when comparing left and right hand side of constraints")
		("max-length-dev-all", po::value<float>(&max_length_dev_all)->default_value(-1), "Make use of all 3 following options")
		("max-length-dev-hypos", po::value<float>(&max_length_dev_hypos)->default_value(-1), "Number between 0 and 1 specifying the percentage of admissible length deviation between hope and fear translations")
		("max-length-dev-hope-ref", po::value<float>(&max_length_dev_hope_ref)->default_value(-1), "Number between 0 and 1 specifying the percentage of admissible length deviation between hope and reference translations")
		("max-length-dev-fear-ref", po::value<float>(&max_length_dev_fear_ref)->default_value(-1), "Number between 0 and 1 specifying the percentage of admissible length deviation between fear and reference translations")
		("min-bleu-ratio", po::value<float>(&minBleuRatio)->default_value(-1), "Set a minimum BLEU ratio between hope and fear")
		("max-bleu-ratio", po::value<float>(&maxBleuRatio)->default_value(-1), "Set a maximum BLEU ratio between hope and fear")
		("min-learning-rate", po::value<float>(&min_learning_rate)->default_value(0), "Set a minimum learning rate")
		("min-oracle-bleu", po::value<float>(&min_oracle_bleu)->default_value(0), "Set a minimum oracle BLEU score")
		("min-weight-change", po::value<float>(&min_weight_change)->default_value(0.01), "Set minimum weight change for stopping criterion")
		("mira-learning-rate", po::value<float>(&mira_learning_rate)->default_value(1), "Learning rate for MIRA (fixed or flexible)")
		("mixing-frequency", po::value<size_t>(&mixingFrequency)->default_value(1), "How often per epoch to mix weights, when using mpi")
		("model-hope-fear", po::value<bool>(&model_hope_fear)->default_value(false), "Use model, hope and fear translations for optimisation")
		("nbest,n", po::value<size_t>(&n)->default_value(1), "Number of translations in n-best list")
		("normalise", po::value<bool>(&normaliseWeights)->default_value(false), "Whether to normalise the updated weights before passing them to the decoder")
		("normalise-margin", po::value<bool>(&normaliseMargin)->default_value(false), "Normalise the margin: squash between 0 and 1")
		("only-violated-constraints", po::value<bool>(&onlyViolatedConstraints)->default_value(false), "Add only violated constraints to the optimisation problem")
		("perceptron-learning-rate", po::value<float>(&perceptron_learning_rate)->default_value(0.01), "Perceptron learning rate")
		("print-feature-values", po::value<bool>(&print_feature_values)->default_value(false), "Print out feature values")
		("rank-n", po::value<int>(&rank_n)->default_value(-1), "Number of translations used for ranking")
		("rank-only", po::value<bool>(&rank_only)->default_value(false), "Use only model translations for optimisation")
		("reference-files,r", po::value<vector<string> >(&referenceFiles), "Reference translation files for training")
		("reference-files-folds", po::value<vector<string> >(&referenceFilesFolds), "Reference translation files for training, one for each fold")		
		("relax-BP", po::value<float>(&relax_BP)->default_value(1), "Relax the BP by setting this value between 0 and 1")
		("scale-by-inverse-length", po::value<bool>(&scaleByInverseLength)->default_value(false), "Scale the BLEU score by (a history of) the inverse input length")
		("scale-by-input-length", po::value<bool>(&scaleByInputLength)->default_value(true), "Scale the BLEU score by (a history of) the input length")
		("scale-by-avg-input-length", po::value<bool>(&scaleByAvgInputLength)->default_value(false), "Scale BLEU by an average of the input length")
		("scale-by-avg-inverse-length", po::value<bool>(&scaleByAvgInverseLength)->default_value(false), "Scale BLEU by an average of the inverse input length")
		("scale-by-x", po::value<float>(&scaleByX)->default_value(1), "Scale the BLEU score by value x")
		("scale-margin", po::value<bool>(&scale_margin)->default_value(0), "Scale the margin by the Bleu score of the oracle translation")
		("scale-margin-precision", po::value<bool>(&scale_margin_precision)->default_value(0), "Scale the margin by the precision of the oracle translation")
		("scale-update", po::value<bool>(&scale_update)->default_value(0), "Scale the update by the Bleu score of the oracle translation")       
		("scale-update-precision", po::value<bool>(&scale_update_precision)->default_value(0), "Scale the update by the precision of the oracle translation")	
		("sentence-level-bleu", po::value<bool>(&sentenceLevelBleu)->default_value(true), "Use a sentences level Bleu scoring function")
		("separate-updates", po::value<bool>(&separateUpdates)->default_value(false), "Compute separate updates for each sentence in a batch")
		("shuffle", po::value<bool>(&shuffle)->default_value(false), "Shuffle input sentences before processing")
		("slack-min", po::value<float>(&slack_min)->default_value(0.01), "Minimum slack used")
		("slack-step", po::value<float>(&slack_step)->default_value(0), "Increase slack from epoch to epoch by the value provided")
		("sparse-average", po::value<bool>(&sparseAverage)->default_value(false), "Average weights by the number of processes")
		("sparse-no-average", po::value<bool>(&sparseNoAverage)->default_value(false), "Don't average sparse weights, just sum")
		("stop-weights", po::value<bool>(&weightConvergence)->default_value(true), "Stop when weights converge")
		("threads", po::value<int>(&threadcount)->default_value(1), "Number of threads used")
		("update-scheme", po::value<size_t>(&update_scheme)->default_value(1), "Update scheme, default: 1")
		("use-source-length-history", po::value<bool>(&useSourceLengthHistory)->default_value(false), "Use history of source length instead of target length for history Bleu")
		("verbosity,v", po::value<int>(&verbosity)->default_value(0), "Verbosity level")
		("weight-dump-frequency", po::value<size_t>(&weightDumpFrequency)->default_value(1), "How often per epoch to dump weights, when using mpi")
		("weight-dump-stem", po::value<string>(&weightDumpStem)->default_value("weights"), "Stem of filename to use for dumping weights");

	po::options_description cmdline_options;
	cmdline_options.add(desc);
	po::variables_map vm;
	po::store(po::command_line_parser(argc, argv). options(cmdline_options).run(), vm);
	po::notify(vm);

	if (help) {
		std::cout << "Usage: " + string(argv[0])
		    + " -f mosesini-file -i input-file -r reference-file(s) [options]"
		    << std::endl;
		std::cout << desc << std::endl;
		return 0;
	}

  // create threadpool, if using multi-threaded decoding
  // note: multi-threading is done on sentence-level,
  // each thread translates one sentence
/*#ifdef WITH_THREADS
  if (threadcount < 1) {
    cerr << "Error: Need to specify a positive number of threads" << endl;
    exit(1);
  }
  ThreadPool pool(threadcount);
#else
  if (threadcount > 1) {
    cerr << "Error: Thread count of " << threadcount << " but moses not built with thread support" << endl;
    exit(1);
  }
#endif*/

  bool trainWithMultipleFolds = false; 
  if (mosesConfigFilesFolds.size() > 0 || inputFilesFolds.size() > 0 || referenceFilesFolds.size() > 0)
	  trainWithMultipleFolds = true;
  
  if (dummy != -1)
    slack = dummy;

  if (dumpMixedWeights && (mixingFrequency != weightDumpFrequency)) {
	  cerr << "Set mixing frequency = weight dump frequency for dumping mixed weights!" << endl;
	  exit(1);
  }

  if ((sparseAverage || sparseNoAverage) && averageWeights) {
	  cerr << "Parameters --sparse-average 1/--sparse-no-average 1 and --average-weights 1 are incompatible (not implemented)" << endl;
	  exit(1);
  }

  if (trainWithMultipleFolds) {
	  if (!mosesConfigFilesFolds.size()) {
		  cerr << "Error: No moses ini files specified for training with folds" << endl;
		  exit(1);
	  }
	  
	  if (!inputFilesFolds.size()) {
		  cerr << "Error: No input files specified for training with folds" << endl;
		  exit(1);
	  }

	  if (!referenceFilesFolds.size()) {
		  cerr << "Error: No reference files specified for training with folds" << endl;
		  exit(1);
	  }
  }
  else {
	  if (mosesConfigFile.empty()) {
		  cerr << "Error: No moses ini file specified" << endl;
		  return 1;
	  }
	  
	  if (inputFile.empty()) {
		  cerr << "Error: No input file specified" << endl;
		  return 1;
	  }

	  if (!referenceFiles.size()) {
		  cerr << "Error: No reference files specified" << endl;
		  return 1;
	  }
  }

	// load input and references
  	vector<string> inputSentences;
  	size_t inputSize = trainWithMultipleFolds? inputFilesFolds.size(): 0;
  	size_t refSize = trainWithMultipleFolds? referenceFilesFolds.size(): referenceFiles.size(); 
  	vector<vector<string> > inputSentencesFolds(inputSize);
  	vector<vector<string> > referenceSentences(refSize);
  	
	// number of cores for each fold
  	size_t coresPerFold = 0, myFold = 0;
  	if (trainWithMultipleFolds) {
  		coresPerFold = size/mosesConfigFilesFolds.size();
  		if (size % coresPerFold > 0) {
  			cerr << "Number of cores has to be a multiple of the number of folds" << endl;
  			exit(1);
  		} 
		
  		if (rank == 0)
  			cerr << "Number of cores per fold: " << coresPerFold << endl;		
  		myFold = rank/coresPerFold;
  		cerr << "Rank " << rank << ", my fold: " << myFold << endl;
  	}
  	
  	if (trainWithMultipleFolds) {
  		if (!loadSentences(inputFilesFolds[myFold], inputSentencesFolds[myFold])) {
  			cerr << "Error: Failed to load input sentences from " << inputFilesFolds[myFold] << endl;
  			exit(1);
  		}
  		VERBOSE(1, "Rank " << rank << " reading inputs from " << inputFilesFolds[myFold] << endl);
  		
  		if (!loadSentences(referenceFilesFolds[myFold], referenceSentences[myFold])) {
  			cerr << "Error: Failed to load reference sentences from " << referenceFilesFolds[myFold] << endl;
  			exit(1);
  		}
  		if (referenceSentences[myFold].size() != inputSentencesFolds[myFold].size()) {
  			cerr << "Error: Input file length (" << inputSentencesFolds[myFold].size() << ") != ("
  				<< referenceSentences[myFold].size() << ") reference file length (rank " << rank << ")" << endl;
  			exit(1);
  		}
  		VERBOSE(1, "Rank " << rank << " reading references from " << referenceFilesFolds[myFold] << endl);  		
  	}
  	else {
  		if (!loadSentences(inputFile, inputSentences)) {
  			cerr << "Error: Failed to load input sentences from " << inputFile << endl;
  			return 1;
  		}
  		
  		for (size_t i = 0; i < referenceFiles.size(); ++i) {
  			if (!loadSentences(referenceFiles[i], referenceSentences[i])) {
  				cerr << "Error: Failed to load reference sentences from "
  						<< referenceFiles[i] << endl;
  				return 1;
  			}
  			if (referenceSentences[i].size() != inputSentences.size()) {
  				cerr << "Error: Input file length (" << inputSentences.size() << ") != ("
  						<< referenceSentences[i].size() << ") length of reference file " << i
  						<< endl;
  				return 1;
  			}
  		}
  	}

	if (scaleByAvgInputLength ||  scaleByInverseLength || scaleByAvgInverseLength)
		scaleByInputLength = false;

	if (historyOf1best || historyOfOracles)
		sentenceLevelBleu = false;
	if (!sentenceLevelBleu) {
		if (!historyOf1best && !historyOfOracles) {
			historyOf1best = true;
		}
	}

	// initialise Moses
	// add initial Bleu weight and references to initialize Bleu feature
	decoder_settings += " -weight-bl 1 -references";
	if (trainWithMultipleFolds) {
		decoder_settings += " ";
		decoder_settings += referenceFilesFolds[myFold];
	}
	else {
		for (size_t i=0; i < referenceFiles.size(); ++i) {
			decoder_settings += " ";
			decoder_settings += referenceFiles[i];
		}
	}
	
	vector<string> decoder_params;
	boost::split(decoder_params, decoder_settings, boost::is_any_of("\t "));
	
	string configFile = trainWithMultipleFolds? mosesConfigFilesFolds[myFold] : mosesConfigFile;
	VERBOSE(1, "Rank " << rank << " reading config file from " << configFile << endl);
	MosesDecoder* decoder = new MosesDecoder(configFile, verbosity, decoder_params.size(), decoder_params);
	decoder->setBleuParameters(sentenceLevelBleu, scaleByInputLength, scaleByAvgInputLength,
			scaleByInverseLength, scaleByAvgInverseLength,
			scaleByX, historySmoothing, bleu_smoothing_scheme, relax_BP, useSourceLengthHistory);
	if (normaliseWeights) {
		ScoreComponentCollection startWeights = decoder->getWeights();
		startWeights.L1Normalise();
		decoder->setWeights(startWeights);
	}

	if (decode_hope || decode_fear || decode_model) {
		size_t decode = 1;
		if (decode_fear) decode = 2;
		if (decode_model) decode = 3;
		decodeHopeOrFear(rank, size, decode, decode_filename, inputSentences, decoder, n);
	}

	// Optionally shuffle the sentences
	vector<size_t> order;
	if (trainWithMultipleFolds) {  	
		for (size_t i = 0; i < inputSentencesFolds[myFold].size(); ++i) {
			order.push_back(i);
		}
		
		if (shuffle) {
			cerr << "Shuffling input sentences.." << endl;
			RandomIndex rindex;
			random_shuffle(order.begin(), order.end(), rindex);
		}
	}
	else {
		if (rank == 0) {
			for (size_t i = 0; i < inputSentences.size(); ++i) {
				order.push_back(i);
			}
			
			if (shuffle) {
				cerr << "Shuffling input sentences.." << endl;
				RandomIndex rindex;
				random_shuffle(order.begin(), order.end(), rindex);
			}
		}
	}

	// initialise optimizer
	Optimiser* optimiser = NULL;
	if (learner == "mira") {
		if (rank == 0) {
			cerr << "Optimising using Mira" << endl;
			cerr << "slack: " << slack << ", learning rate: " << mira_learning_rate << endl;
		}
		optimiser = new MiraOptimiser(onlyViolatedConstraints, slack, scale_margin, scale_margin_precision,
				scale_update, scale_update_precision, margin_slack, boost, update_scheme, normaliseMargin);
		learning_rate = mira_learning_rate;
		perceptron_update = false;
	} else if (learner == "perceptron") {
		if (rank == 0) {
			cerr << "Optimising using Perceptron" << endl;
		}
		optimiser = new Perceptron();
		learning_rate = perceptron_learning_rate;
		perceptron_update = true;
		model_hope_fear = false; // mira only
		rank_only = false; // mira only
		hope_fear = false; // mira only
		hope_fear_rank = false; // mira only
		hope_model = false; // mira only
		n = 1;
		hope_n = 1;
		fear_n = 1;
	} else {
		cerr << "Error: Unknown optimiser: " << learner << endl;
		return 1;
	}

	// resolve parameter dependencies
	if (batchSize > 1 && perceptron_update) {
		batchSize = 1;
		cerr << "Info: Setting batch size to 1 for perceptron update" << endl;
	}

	if (hope_n == -1)
		hope_n = n;
	if (fear_n == -1)
		fear_n = n;
	if (rank_n == -1)
		rank_n = n;

	if (model_hope_fear && hope_fear)
		hope_fear = false; // is true by default
	if (rank_only && hope_fear)
		hope_fear = false; // is true by default
	if (hope_fear_rank && hope_fear)
		hope_fear = false; // is true by default
	if (hope_model && hope_fear)
		hope_fear = false; // is true by default
	if (learner == "mira" && !(hope_fear || hope_model || model_hope_fear || rank_only || hope_fear_rank)) {
		cerr << "Error: Need to select an one of parameters --hope-fear/--model-hope-fear for mira update." << endl;
		return 1;
	}

	if (bleuScoreWeight_hope == -1) {
		bleuScoreWeight_hope = bleuScoreWeight;
	}
	if (bleuScoreWeight_fear == -1) {
		bleuScoreWeight_fear = bleuScoreWeight;
	}

	if (max_length_dev_all != -1) {
		max_length_dev_hypos = max_length_dev_all;
		max_length_dev_hope_ref = max_length_dev_all;
		max_length_dev_fear_ref = max_length_dev_all;
	}

#ifdef MPI_ENABLE
	mpi::broadcast(world, order, 0);
#endif

	// Create shards according to the number of processes used
	vector<size_t> shard;
	if (trainWithMultipleFolds) {			
		float shardSize = (float) (order.size())/coresPerFold;
		size_t shardStart = (size_t) (shardSize * (rank % coresPerFold));
		size_t shardEnd = shardStart + shardSize;
		if (rank % coresPerFold == coresPerFold - 1) { // last rank of each fold 
			shardEnd = order.size();
			shardSize = shardEnd - shardStart;
		}		
		VERBOSE(1, "Rank: " << rank << ", shard size: " << shardSize << endl);
		VERBOSE(1, "Rank: " << rank << ", shard start: " << shardStart << " shard end: " << shardEnd << endl);
		shard.resize(shardSize);
		copy(order.begin() + shardStart, order.begin() + shardEnd, shard.begin());
		batchSize = 1;
	}
	else {
		float shardSize = (float) (order.size()) / size;
		size_t shardStart = (size_t) (shardSize * rank);
		size_t shardEnd = (size_t) (shardSize * (rank + 1));
		if (rank == size - 1) {
			shardEnd = order.size();
			shardSize = shardEnd - shardStart;
		}
		VERBOSE(1, "Shard size: " << shardSize << endl);
		VERBOSE(1, "Rank: " << rank << " Shard start: " << shardStart << " Shard end: " << shardEnd << endl);
{'fraction_non_alphanumeric': 0.14581858233026512, 'fraction_numerical': 0.00660844937455747, 'mean_word_length': 5.216530138158699, 'pattern_counts': {'":': 0, '<': 711, '<?xml version=': 0, '>': 395, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7505763', 'n_tokens_mistral': 17806, 'n_tokens_neox': 16417, 'n_words': 4294}