<commit_before>package edu.jhu.thrax.features;

import edu.jhu.thrax.datatypes.Rule;

import java.util.Map;
import java.util.HashMap;

public class LexicalProbabilityFeature implements Feature {

    public static final String name = "lex";

    public int length()
    {
        return 2;
    }

    public AggregationStyle [] aggregationStyles()
    {
        return aggregationStyles;
    }

    private static final AggregationStyle [] aggregationStyles = new AggregationStyle[] { AggregationStyle.MIN, AggregationStyle.MIN };

    private CollocationTable table;
    private Map<Rule,Double> maxf2e;
    private Map<Rule,Double> maxe2f;

    public LexicalProbabilityFeature()
    {
        table = new CollocationTable();
        maxf2e = new HashMap<Rule,Double>();
        maxe2f = new HashMap<Rule,Double>();
    }

    public void noteExtraction(Rule r)
    {
        table.add(r.source, r.target, r.alignment);
        return;
    }

    public double [] score(Rule r)
    {
        double [] ret = new double[2];
        ret[0] = -Math.log(targetGivenSource(r));
        ret[1] = -Math.log(sourceGivenTarget(r));
        if (maxf2e.containsKey(r) && ret[0] > maxf2e.get(r))
            ret[0] = maxf2e.get(r);
        else
            maxf2e.put(r, ret[0]);
        if (maxe2f.containsKey(r) && ret[1] > maxe2f.get(r))
            ret[1] = maxe2f.get(r);
        else
            maxe2f.put(r, ret[1]);
        return ret;
    }



    private double targetGivenSource(Rule r)
    {
        double ret = 1.0;
        for (int i = 0; i < r.target.length; i++) {
            if (r.targetLex[i] != 0)
                continue;
            if (i >= r.alignment.e2f.length || r.alignment.e2f[i].length == 0)
                ret *= table.targetGivenSource(CollocationTable.ID_UNALIGNED,
                        r.target[i]);
            else {
                ret /= r.alignment.e2f[i].length;
                double sum = 0;
                for (int x : r.alignment.e2f[i])
                    sum += table.targetGivenSource(r.source[x], r.target[i]);
                ret *= sum;
            }<commit_msg>removed score tracking from lexprob feature<commit_after>package edu.jhu.thrax.features;

import edu.jhu.thrax.datatypes.Rule;

import java.util.Map;
import java.util.HashMap;

public class LexicalProbabilityFeature implements Feature {

    public static final String name = "lex";

    public int length()
    {
        return 2;
    }

    public AggregationStyle [] aggregationStyles()
    {
        return aggregationStyles;
    }

    private static final AggregationStyle [] aggregationStyles = new AggregationStyle[] { AggregationStyle.MIN, AggregationStyle.MIN };

    private CollocationTable table;

    public LexicalProbabilityFeature()
    {
        table = new CollocationTable();
    }

    public void noteExtraction(Rule r)
    {
        table.add(r.source, r.target, r.alignment);
        return;
    }

    public double [] score(Rule r)
    {
        double [] ret = new double[2];
        ret[0] = -Math.log(targetGivenSource(r));
        ret[1] = -Math.log(sourceGivenTarget(r));
        return ret;
    }



    private double targetGivenSource(Rule r)
    {
        double ret = 1.0;
        for (int i = 0; i < r.target.length; i++) {
            if (r.targetLex[i] != 0)
                continue;
            if (i >= r.alignment.e2f.length || r.alignment.e2f[i].length == 0)
                ret *= table.targetGivenSource(CollocationTable.ID_UNALIGNED,
                        r.target[i]);
            else {
                ret /= r.alignment.e2f[i].length;
                double sum = 0;
                for (int x : r.alignment.e2f[i])
                    sum += table.targetGivenSource(r.source[x], r.target[i]);
                ret *= sum;
            }
{'fraction_non_alphanumeric': 0.11947368421052632, 'fraction_numerical': 0.012105263157894737, 'mean_word_length': 1.9741784037558685, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1690205', 'n_tokens_mistral': 1250, 'n_tokens_neox': 1160, 'n_words': 253}