<commit_before>package chart_parser;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;

import lexicon.Category;
import model.Lexicon;
import statistics.HasUpdateStatistics;
import statistics.HypothesisSizeStatistics;
import training.Feature;
import cat_combination.FilledDependency;
import cat_combination.RuleInstancesParams;
import cat_combination.SuperCategory;

public class ChartTrainParserBeam extends ChartParserBeam {
	public HasUpdateStatistics hasUpdateStatistics = new HasUpdateStatistics();
	public HypothesisSizeStatistics hypothesisSizeStatistics = new HypothesisSizeStatistics();

	protected boolean parallelUpdate;
	protected boolean updateLogP;
	protected double maxViolation;

	protected CellCoords maxViolationCell;
	protected LinkedList<CellCoords> violationCells = new LinkedList<CellCoords>();
	protected ArrayList<CellCoords> maxViolationCells = new ArrayList<CellCoords>();

	private Feature[] trainingFeatures;
	private BufferedReader goldDepsPerCell;
	private ArrayList<Category> oracleSupertags;
	private OracleDecoder oracleDecoder;

	public ChartTrainParserBeam(
			String grammarDir,
			boolean altMarkedup,
			boolean eisnerNormalForm,
			int MAX_WORDS,
			int MAX_SUPERCATS,
			boolean output,
			RuleInstancesParams ruleInstancesParams,
			Lexicon lexicon,
			String featuresFile,
			String weightsFile,
			boolean newFeatures,
			boolean cubePruning,
			int beamSize,
			double beta,
			boolean parallelUpdate,
			boolean updateLogP) throws IOException {
		super(grammarDir, altMarkedup, eisnerNormalForm, MAX_WORDS,
				MAX_SUPERCATS, output, ruleInstancesParams,
				lexicon, featuresFile, weightsFile, newFeatures, false, cubePruning,
				beamSize, beta);

		this.chart = new Chart(MAX_WORDS, output, categories.dependencyRelations, false, true);
		this.chart.setWeights(this.weights);

		int numFeatures = features.numFeatures;
		this.trainingFeatures = new Feature[numFeatures];

		for (int i = 0; i < numFeatures; i++) {
			trainingFeatures[i] = new Feature(i, weights.getWeight(i));
		}

		this.parallelUpdate = parallelUpdate;
		this.updateLogP = updateLogP;
		this.oracleSupertags = new ArrayList<Category>();
	}

	/**
	 * Parses one supertagged sentence using chart parser with beam search,
	 * while training using gold dependencies.
	 * 
	 * @param in file containing supertagged sentences to parse
	 * @param goldDepsPerCell file containing gold dependencies per cell
	 * @param stagsIn file containing additional supertags
	 * @param log log file
	 * @param betas array of values of beta
	 * @param oracleDecoder oracle decoder
	 * @return true if sentence is parsed or skipped, false if there are no
	 * sentences left
	 */
	public boolean parseSentence(BufferedReader in, BufferedReader goldDepsPerCell, BufferedReader stagsIn, PrintWriter log, double[] betas, OracleDecoder oracleDecoder) {
		this.goldDepsPerCell = goldDepsPerCell;
		this.oracleDecoder = oracleDecoder;
		return parseSentence(in, stagsIn, log, betas);
	}

	/**
	 * Clears max violation fields and reads in gold dependencies per cell.
	 */
	@Override
	protected boolean preParse() {
		maxViolation = 0;
		maxViolationCell = null;
		maxViolationCells.clear();
		violationCells.clear();

		if (!readDepsPerCell(goldDepsPerCell)) {
			return false;
		}

		System.out.println("Gold dependencies per cell merged.");

		return true;
	}

	/**
	 * Calculates violation of current cell. 
	 * 
	 * Single node max violation update: searches for max violation,
	 * updating it if a larger violation is found.
	 * 
	 * Parallel max violation update: nothing.
	 */
	@Override
	protected void postParse(int pos, int span, int numWords) {
		CellTrainBeam cell = (CellTrainBeam) (chart.cell(pos, span));
		double violation = cell.calcViolation(sentence, oracleDecoder, oracleSupertags, pos);

		if ( !parallelUpdate ) {
			if (violation > maxViolation) {
				maxViolation = violation;
				maxViolationCell = new CellCoords(pos, span);
				System.out.println("New maxViolation found at (" + pos + "," + span + "); maxViolation: " + violation);
			}
		} else {
			if ( violation > 0 ) {
				violationCells.add(new CellCoords(pos, span, violation));
				System.out.println("Adding (" + pos + "," + span + "); to violationCells; violation: " + violation);
			}
		}

		System.out.println("cell (" + pos + "," + span + "); violation: " + violation + "; current maxViolation: " + maxViolation);
	}

	/**
	 * Computes sets of frontiers for parallel max violation update for a
	 * subsentence of a sentence and adds it to maxViolationCells.
	 */
	private void findLocalMaxViolations() {
		Collections.sort(violationCells, new Comparator<CellCoords>(){
			@Override
			public int compare(CellCoords p1, CellCoords p2){
				return -Double.compare(p1.violation, p2.violation);
			}});
<commit_msg>Replace negation of result of compare() with switching order of arguments.<commit_after>package chart_parser;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;

import lexicon.Category;
import model.Lexicon;
import statistics.HasUpdateStatistics;
import statistics.HypothesisSizeStatistics;
import training.Feature;
import cat_combination.FilledDependency;
import cat_combination.RuleInstancesParams;
import cat_combination.SuperCategory;

public class ChartTrainParserBeam extends ChartParserBeam {
	public HasUpdateStatistics hasUpdateStatistics = new HasUpdateStatistics();
	public HypothesisSizeStatistics hypothesisSizeStatistics = new HypothesisSizeStatistics();

	protected boolean parallelUpdate;
	protected boolean updateLogP;
	protected double maxViolation;

	protected CellCoords maxViolationCell;
	protected LinkedList<CellCoords> violationCells = new LinkedList<CellCoords>();
	protected ArrayList<CellCoords> maxViolationCells = new ArrayList<CellCoords>();

	private Feature[] trainingFeatures;
	private BufferedReader goldDepsPerCell;
	private ArrayList<Category> oracleSupertags;
	private OracleDecoder oracleDecoder;

	public ChartTrainParserBeam(
			String grammarDir,
			boolean altMarkedup,
			boolean eisnerNormalForm,
			int MAX_WORDS,
			int MAX_SUPERCATS,
			boolean output,
			RuleInstancesParams ruleInstancesParams,
			Lexicon lexicon,
			String featuresFile,
			String weightsFile,
			boolean newFeatures,
			boolean cubePruning,
			int beamSize,
			double beta,
			boolean parallelUpdate,
			boolean updateLogP) throws IOException {
		super(grammarDir, altMarkedup, eisnerNormalForm, MAX_WORDS,
				MAX_SUPERCATS, output, ruleInstancesParams,
				lexicon, featuresFile, weightsFile, newFeatures, false, cubePruning,
				beamSize, beta);

		this.chart = new Chart(MAX_WORDS, output, categories.dependencyRelations, false, true);
		this.chart.setWeights(this.weights);

		int numFeatures = features.numFeatures;
		this.trainingFeatures = new Feature[numFeatures];

		for (int i = 0; i < numFeatures; i++) {
			trainingFeatures[i] = new Feature(i, weights.getWeight(i));
		}

		this.parallelUpdate = parallelUpdate;
		this.updateLogP = updateLogP;
		this.oracleSupertags = new ArrayList<Category>();
	}

	/**
	 * Parses one supertagged sentence using chart parser with beam search,
	 * while training using gold dependencies.
	 * 
	 * @param in file containing supertagged sentences to parse
	 * @param goldDepsPerCell file containing gold dependencies per cell
	 * @param stagsIn file containing additional supertags
	 * @param log log file
	 * @param betas array of values of beta
	 * @param oracleDecoder oracle decoder
	 * @return true if sentence is parsed or skipped, false if there are no
	 * sentences left
	 */
	public boolean parseSentence(BufferedReader in, BufferedReader goldDepsPerCell, BufferedReader stagsIn, PrintWriter log, double[] betas, OracleDecoder oracleDecoder) {
		this.goldDepsPerCell = goldDepsPerCell;
		this.oracleDecoder = oracleDecoder;
		return parseSentence(in, stagsIn, log, betas);
	}

	/**
	 * Clears max violation fields and reads in gold dependencies per cell.
	 */
	@Override
	protected boolean preParse() {
		maxViolation = 0;
		maxViolationCell = null;
		maxViolationCells.clear();
		violationCells.clear();

		if (!readDepsPerCell(goldDepsPerCell)) {
			return false;
		}

		System.out.println("Gold dependencies per cell merged.");

		return true;
	}

	/**
	 * Calculates violation of current cell. 
	 * 
	 * Single node max violation update: searches for max violation,
	 * updating it if a larger violation is found.
	 * 
	 * Parallel max violation update: nothing.
	 */
	@Override
	protected void postParse(int pos, int span, int numWords) {
		CellTrainBeam cell = (CellTrainBeam) (chart.cell(pos, span));
		double violation = cell.calcViolation(sentence, oracleDecoder, oracleSupertags, pos);

		if ( !parallelUpdate ) {
			if (violation > maxViolation) {
				maxViolation = violation;
				maxViolationCell = new CellCoords(pos, span);
				System.out.println("New maxViolation found at (" + pos + "," + span + "); maxViolation: " + violation);
			}
		} else {
			if ( violation > 0 ) {
				violationCells.add(new CellCoords(pos, span, violation));
				System.out.println("Adding (" + pos + "," + span + "); to violationCells; violation: " + violation);
			}
		}

		System.out.println("cell (" + pos + "," + span + "); violation: " + violation + "; current maxViolation: " + maxViolation);
	}

	/**
	 * Computes sets of frontiers for parallel max violation update for a
	 * subsentence of a sentence and adds it to maxViolationCells.
	 */
	private void findLocalMaxViolations() {
		Collections.sort(violationCells, new Comparator<CellCoords>(){
			@Override
			public int compare(CellCoords p1, CellCoords p2){
				return Double.compare(p2.violation, p1.violation);
			}});

{'fraction_non_alphanumeric': 0.0891542288557214, 'fraction_numerical': 0.0013930348258706469, 'mean_word_length': 4.793083573487031, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '516418', 'n_tokens_mistral': 3218, 'n_tokens_neox': 2849, 'n_words': 840}