<commit_before>package edu.jhu.featurize;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.TreeSet;

import org.apache.log4j.Logger;

import edu.berkeley.nlp.PCFGLA.smoothing.SrlBerkeleySignatureBuilder;
import edu.jhu.data.DepTree.Dir;
import edu.jhu.data.simple.SimpleAnnoSentence;
import edu.jhu.featurize.TemplateLanguage.BigramTemplate;
import edu.jhu.featurize.TemplateLanguage.FeatTemplate;
import edu.jhu.featurize.TemplateLanguage.FeatTemplate1;
import edu.jhu.featurize.TemplateLanguage.FeatTemplate2;
import edu.jhu.featurize.TemplateLanguage.FeatTemplate3;
import edu.jhu.featurize.TemplateLanguage.FeatTemplate4;
import edu.jhu.featurize.TemplateLanguage.ListModifier;
import edu.jhu.featurize.TemplateLanguage.OtherFeat;
import edu.jhu.featurize.TemplateLanguage.Position;
import edu.jhu.featurize.TemplateLanguage.PositionList;
import edu.jhu.featurize.TemplateLanguage.PositionModifier;
import edu.jhu.featurize.TemplateLanguage.TokPropList;
import edu.jhu.featurize.TemplateLanguage.TokProperty;
import edu.jhu.prim.tuple.Pair;
import edu.jhu.srl.CorpusStatistics;

/**
 * Defines a feature template extractor for templates based on a 'little
 * language'.
 * 
 * @author mgormley
 * @author mmitchell
 */
public class TemplateLanguageExtractor {

    private static final Logger log = Logger.getLogger(TemplateLanguageExtractor.class);

    //private final SimpleAnnoSentence sent;
    //private final CorpusStatistics cs;
    private final SrlBerkeleySignatureBuilder sig;
    private final FeaturizedSentence fSent; 
    
    public TemplateLanguageExtractor(SimpleAnnoSentence sent, CorpusStatistics cs) {
        //this.sent = sent;
        //this.cs = cs;
        this.sig = cs.sig;
        this.fSent = new FeaturizedSentence(sent, cs);
    }
    
    public void addFeatures(List<FeatTemplate> tpls, int pidx, int cidx, List<Object> feats) {
        for (FeatTemplate tpl : tpls) {
            addFeatures(tpl, pidx, cidx, feats);
        }
    }
    
    public void addFeatures(FeatTemplate tpl, int pidx, int cidx, List<Object> feats) {
        int idx = pidx;
        if (tpl instanceof FeatTemplate1) {
            addTokenFeature((FeatTemplate1) tpl, idx, feats);
        } else if (tpl instanceof FeatTemplate2) {
            addTokenFeatures((FeatTemplate2) tpl, idx, feats);            
        } else if (tpl instanceof FeatTemplate3) {
            addListFeature((FeatTemplate3) tpl, pidx, cidx, feats);
        } else if (tpl instanceof FeatTemplate4) {
            addOtherFeature((FeatTemplate4) tpl, pidx, cidx, feats);
        } else if (tpl instanceof BigramTemplate) {
            addBigramFeature((BigramTemplate) tpl, pidx, cidx, feats);
        } else {
            throw new IllegalStateException("Feature not supported: " + tpl);
        }
    }
        
    /**
     * For bigram feature templates of the form:
     *     p.w+c_{-1}.bc0
     *     p.t+c.t
     * @param tpl Structured feature template.
     * @param pidx Token to which the parent position refers.
     * @param cidx Token to which the child position refers.
     * @param feats The feature list to which this will be added.
     */
    public void addBigramFeature(BigramTemplate tpl, int pidx, int cidx, List<Object> feats) {
        ArrayList<Object> feats1 = new ArrayList<Object>();
        ArrayList<Object> feats2 = new ArrayList<Object>();
        addFeatures(tpl.tpl1, pidx, cidx, feats1);
        addFeatures(tpl.tpl2, pidx, cidx, feats2);
        for (Object f1 : feats1) {
            for (Object f2 : feats2) {
                feats.add(toFeat(f1, f2));
            }
        }
    }

    /**
     * Adds features of the form: 
     *     p.bc1
     *     c_{head}.dr
     *     first(t, NOUN, path(p, root)).bc0
     * @param tpl Structured feature template.
     * @param idx Token to which the original position refers.
     * @param feats The feature list to which this will be added.
     */
    public void addTokenFeature(FeatTemplate1 tpl, int idx, List<Object> feats) {
        Position pos = tpl.pos; PositionModifier mod = tpl.mod; TokProperty prop = tpl.prop;
        idx = getModifiedPosition(mod, idx);
        String val = getTokProp(prop, idx);
        if (val != null) {
            Object feat = toFeat(tpl.getName(), val);
            feats.add(feat);
        }
    }
    
    /** Same as above except that it permits properties of the token which expand to multiple strings. */
    public void addTokenFeatures(FeatTemplate2 tpl, int idx, List<Object> feats) {
        Position pos = tpl.pos; PositionModifier mod = tpl.mod; TokPropList prop = tpl.prop;
        idx = getModifiedPosition(mod, idx);
        List<String> vals = getTokPropList(prop, idx);
        for (String val : vals) {
            feats.add(toFeat(tpl.getName(), val));
        }
    }

    /**<commit_msg>Bug fix: was always using Parent position and never Child idx.<commit_after>package edu.jhu.featurize;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.TreeSet;

import org.apache.log4j.Logger;

import edu.berkeley.nlp.PCFGLA.smoothing.SrlBerkeleySignatureBuilder;
import edu.jhu.data.DepTree.Dir;
import edu.jhu.data.simple.SimpleAnnoSentence;
import edu.jhu.featurize.TemplateLanguage.BigramTemplate;
import edu.jhu.featurize.TemplateLanguage.FeatTemplate;
import edu.jhu.featurize.TemplateLanguage.FeatTemplate1;
import edu.jhu.featurize.TemplateLanguage.FeatTemplate2;
import edu.jhu.featurize.TemplateLanguage.FeatTemplate3;
import edu.jhu.featurize.TemplateLanguage.FeatTemplate4;
import edu.jhu.featurize.TemplateLanguage.ListModifier;
import edu.jhu.featurize.TemplateLanguage.OtherFeat;
import edu.jhu.featurize.TemplateLanguage.Position;
import edu.jhu.featurize.TemplateLanguage.PositionList;
import edu.jhu.featurize.TemplateLanguage.PositionModifier;
import edu.jhu.featurize.TemplateLanguage.TokPropList;
import edu.jhu.featurize.TemplateLanguage.TokProperty;
import edu.jhu.prim.tuple.Pair;
import edu.jhu.srl.CorpusStatistics;

/**
 * Defines a feature template extractor for templates based on a 'little
 * language'.
 * 
 * @author mgormley
 * @author mmitchell
 */
public class TemplateLanguageExtractor {

    private static final Logger log = Logger.getLogger(TemplateLanguageExtractor.class);

    //private final SimpleAnnoSentence sent;
    //private final CorpusStatistics cs;
    private final SrlBerkeleySignatureBuilder sig;
    private final FeaturizedSentence fSent; 
    
    public TemplateLanguageExtractor(SimpleAnnoSentence sent, CorpusStatistics cs) {
        //this.sent = sent;
        //this.cs = cs;
        this.sig = cs.sig;
        this.fSent = new FeaturizedSentence(sent, cs);
    }
    
    public void addFeatures(List<FeatTemplate> tpls, int pidx, int cidx, List<Object> feats) {
        for (FeatTemplate tpl : tpls) {
            addFeatures(tpl, pidx, cidx, feats);
        }
    }
    
    public void addFeatures(FeatTemplate tpl, int pidx, int cidx, List<Object> feats) {
        int idx = pidx;
        if (tpl instanceof FeatTemplate1) {
            addTokenFeature((FeatTemplate1) tpl, pidx, cidx, feats);
        } else if (tpl instanceof FeatTemplate2) {
            addTokenFeatures((FeatTemplate2) tpl, pidx, cidx, feats);            
        } else if (tpl instanceof FeatTemplate3) {
            addListFeature((FeatTemplate3) tpl, pidx, cidx, feats);
        } else if (tpl instanceof FeatTemplate4) {
            addOtherFeature((FeatTemplate4) tpl, pidx, cidx, feats);
        } else if (tpl instanceof BigramTemplate) {
            addBigramFeature((BigramTemplate) tpl, pidx, cidx, feats);
        } else {
            throw new IllegalStateException("Feature not supported: " + tpl);
        }
    }
        
    /**
     * For bigram feature templates of the form:
     *     p.w+c_{-1}.bc0
     *     p.t+c.t
     * @param tpl Structured feature template.
     * @param pidx Token to which the parent position refers.
     * @param cidx Token to which the child position refers.
     * @param feats The feature list to which this will be added.
     */
    public void addBigramFeature(BigramTemplate tpl, int pidx, int cidx, List<Object> feats) {
        ArrayList<Object> feats1 = new ArrayList<Object>();
        ArrayList<Object> feats2 = new ArrayList<Object>();
        addFeatures(tpl.tpl1, pidx, cidx, feats1);
        addFeatures(tpl.tpl2, pidx, cidx, feats2);
        for (Object f1 : feats1) {
            for (Object f2 : feats2) {
                feats.add(toFeat(f1, f2));
            }
        }
    }

    /**
     * Adds features of the form: 
     *     p.bc1
     *     c_{head}.dr
     *     first(t, NOUN, path(p, root)).bc0
     * @param tpl Structured feature template.
     * @param pidx Token to which the parent position refers.
     * @param cidx Token to which the child position refers.
     * @param feats The feature list to which this will be added.
     */
    public void addTokenFeature(FeatTemplate1 tpl, int pidx, int cidx, List<Object> feats) {
        Position pos = tpl.pos; PositionModifier mod = tpl.mod; TokProperty prop = tpl.prop;
        int idx = getIndexOfPosition(pidx, cidx, pos);
        idx = getModifiedPosition(mod, idx);
        String val = getTokProp(prop, idx);
        if (val != null) {
            Object feat = toFeat(tpl.getName(), val);
            feats.add(feat);
        }
    }

    private int getIndexOfPosition(int pidx, int cidx, Position pos) {
        int idx;
        switch (pos) {
        case PARENT: idx = pidx; break;
        case CHILD: idx = cidx; break;
        default: throw new IllegalStateException();
        }
        return idx;
    }
    
    /** Same as above except that it permits properties of the token which expand to multiple strings. */
    public void addTokenFeatures(FeatTemplate2 tpl, int pidx, int cidx, List<Object> feats) {
        Position pos = tpl.pos; PositionModifier mod = tpl.mod; TokPropList prop = tpl.prop;
        int idx = getIndexOfPosition(pidx, cidx, pos);
        idx = getModifiedPosition(mod, idx);
        List<String> vals = getTokPropList(prop, idx);
        for (String val : vals) {
            feats.add(toFeat(tpl.getName(), val));
        }
    }

    /**
{'fraction_non_alphanumeric': 0.09815175097276264, 'fraction_numerical': 0.006031128404669261, 'mean_word_length': 2.860683439729628, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3632181', 'n_tokens_mistral': 3219, 'n_tokens_neox': 3223, 'n_words': 827}