<commit_before>package edu.jhu.pacaya.gm.feat;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Serializable;
import java.io.Writer;
import java.util.Iterator;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.Iterables;

import edu.jhu.pacaya.gm.data.FgExampleList;
import edu.jhu.pacaya.gm.data.LFgExample;
import edu.jhu.pacaya.gm.inf.FgInferencer;
import edu.jhu.pacaya.gm.model.Factor;
import edu.jhu.pacaya.gm.model.FactorGraph;
import edu.jhu.pacaya.gm.model.IFgModel;
import edu.jhu.pacaya.gm.model.IndexForVc;
import edu.jhu.pacaya.gm.model.ObsFeatureCarrier;
import edu.jhu.pacaya.gm.model.TemplateFactor;
import edu.jhu.pacaya.gm.model.Var;
import edu.jhu.pacaya.gm.model.VarConfig;
import edu.jhu.pacaya.gm.model.VarTensor;
import edu.jhu.pacaya.gm.model.globalfac.GlobalFactor;
import edu.jhu.pacaya.gm.train.MarginalLogLikelihood;
import edu.jhu.pacaya.gm.util.ArrayIter3D;
import edu.jhu.pacaya.util.FeatureNames;
import edu.jhu.pacaya.util.Prm;
import edu.jhu.pacaya.util.semiring.RealAlgebra;
import edu.jhu.prim.arrays.BoolArrays;
import edu.jhu.prim.iter.IntIter;
import edu.jhu.prim.map.IntDoubleEntry;
import edu.jhu.prim.vector.IntIntDenseVector;

/**
 * Wrapper of ObsFeatureExtractor which returns feature vectors which conjoin
 * the observation function with an indicator of the predicted variables.
 * 
 * @author mgormley
 */
public class ObsFeatureConjoiner implements Serializable {

    public static class ObsFeatureConjoinerPrm extends Prm {
        private static final long serialVersionUID = 1L;
        /** Whether to include unsupported features. */
        public boolean includeUnsupportedFeatures = false;
        /**
         * Minimum number of times (inclusive) a feature must occur in training
         * to be included in the model. Ignored if non-positive. (Using this
         * cutoff implies that unsupported features will not be included.)
         */
        public int featCountCutoff = -1;
    }
    
    private static final Logger log = LoggerFactory.getLogger(ObsFeatureConjoiner.class);
    private static final long serialVersionUID = 1L;
    
    /**
     * The model parameters indices. Indexed by feature template index, variable
     * assignment config index, and observation function feature index.
     */
    private int[][][] indices;
    /**
     * Whether or not the correspondingly indexed model parameter is included in
     * this model.
     */
    private boolean[][][] included;
    /** The number of feature templates. */
    private int numTemplates;
    /** The number of parameters in the model. */
    private int numParams;
    /** The feature templates. */
    private FactorTemplateList templates;
    /** Whether this object is initialized. */
    private boolean initialized;
    /** Alphabet for standard (non-observation function) features. */
    private FeatureNames feAlphabet;
    /** Maximum feature-hashing trick modulo value in use. */
    private int featureHashModMax = 0;
    /** Maximum reserved parameters requested. */
    private int reservedMax = 0;
    /** Offset of the reserved parameters section. */
    private int reservedOffset = -1;
    /** FCM alphabet. Does not directly impact the parameter number. */
    public FeatureNames fcmAlphabet = new FeatureNames();
    
    private ObsFeatureConjoinerPrm prm;
    
    public Object embeddings; // TODO: Remove this hack.
    
    public ObsFeatureConjoiner(ObsFeatureConjoinerPrm prm, FactorTemplateList fts) {
        this.prm = prm;
        initialized = false;
        this.templates = fts;
        this.feAlphabet = new FeatureNames();
    }
    
    /** The case of data == null should only be allowed in testing. */
    public void init(FgExampleList data) {
        if (data == null && !prm.includeUnsupportedFeatures) {
            throw new IllegalArgumentException("Data can only be null if unsupported features are not included.");
        }
        if (data == null && prm.featCountCutoff > 0) {
            throw new IllegalArgumentException("Data can only be null if there is no feature count cutoff.");
        }
        
        // Ensure the FactorTemplateList is initialized, and maybe count features along the way.
        if (templates.isGrowing() && data != null) {
            log.info("Growing feature template list by iterating over examples");
            extractAllFeats(data, templates);
            templates.stopGrowth();
            feAlphabet.stopGrowth();
        }
        numTemplates = templates.size();
        
        // Apply a feature count cutoff.
        this.included = new boolean[numTemplates][][];<commit_msg>Adding WordFeatures interface for use in FCM module.<commit_after>package edu.jhu.pacaya.gm.feat;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Serializable;
import java.io.Writer;
import java.util.Iterator;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.Iterables;

import edu.jhu.pacaya.gm.data.FgExampleList;
import edu.jhu.pacaya.gm.data.LFgExample;
import edu.jhu.pacaya.gm.inf.FgInferencer;
import edu.jhu.pacaya.gm.model.Factor;
import edu.jhu.pacaya.gm.model.FactorGraph;
import edu.jhu.pacaya.gm.model.IFgModel;
import edu.jhu.pacaya.gm.model.IndexForVc;
import edu.jhu.pacaya.gm.model.ObsFeatureCarrier;
import edu.jhu.pacaya.gm.model.TemplateFactor;
import edu.jhu.pacaya.gm.model.Var;
import edu.jhu.pacaya.gm.model.VarConfig;
import edu.jhu.pacaya.gm.model.VarTensor;
import edu.jhu.pacaya.gm.model.globalfac.GlobalFactor;
import edu.jhu.pacaya.gm.train.MarginalLogLikelihood;
import edu.jhu.pacaya.gm.util.ArrayIter3D;
import edu.jhu.pacaya.util.FeatureNames;
import edu.jhu.pacaya.util.Prm;
import edu.jhu.pacaya.util.semiring.RealAlgebra;
import edu.jhu.prim.arrays.BoolArrays;
import edu.jhu.prim.iter.IntIter;
import edu.jhu.prim.map.IntDoubleEntry;
import edu.jhu.prim.vector.IntIntDenseVector;

/**
 * Wrapper of ObsFeatureExtractor which returns feature vectors which conjoin
 * the observation function with an indicator of the predicted variables.
 * 
 * @author mgormley
 */
public class ObsFeatureConjoiner implements Serializable {

    public static class ObsFeatureConjoinerPrm extends Prm {
        private static final long serialVersionUID = 1L;
        /** Whether to include unsupported features. */
        public boolean includeUnsupportedFeatures = false;
        /**
         * Minimum number of times (inclusive) a feature must occur in training
         * to be included in the model. Ignored if non-positive. (Using this
         * cutoff implies that unsupported features will not be included.)
         */
        public int featCountCutoff = -1;
    }
    
    private static final Logger log = LoggerFactory.getLogger(ObsFeatureConjoiner.class);
    private static final long serialVersionUID = 1L;
    
    /**
     * The model parameters indices. Indexed by feature template index, variable
     * assignment config index, and observation function feature index.
     */
    private int[][][] indices;
    /**
     * Whether or not the correspondingly indexed model parameter is included in
     * this model.
     */
    private boolean[][][] included;
    /** The number of feature templates. */
    private int numTemplates;
    /** The number of parameters in the model. */
    private int numParams;
    /** The feature templates. */
    private FactorTemplateList templates;
    /** Whether this object is initialized. */
    private boolean initialized;
    /** Alphabet for standard (non-observation function) features. */
    private FeatureNames feAlphabet;
    /** Maximum feature-hashing trick modulo value in use. */
    private int featureHashModMax = 0;
    /** Maximum reserved parameters requested. */
    private int reservedMax = 0;
    /** Offset of the reserved parameters section. */
    private int reservedOffset = -1;
    /** FCM alphabet. Does not directly impact the parameter number. */
    public FeatureNames fcmAlphabet = new FeatureNames();
    
    private ObsFeatureConjoinerPrm prm;
    
    // HACK: This treats the OFC as a submodel for NLP factors needing embeddings.
    public Object embeddings; // TODO: Remove this hack.
    
    public ObsFeatureConjoiner(ObsFeatureConjoinerPrm prm, FactorTemplateList fts) {
        this.prm = prm;
        initialized = false;
        this.templates = fts;
        this.feAlphabet = new FeatureNames();
    }
    
    /** The case of data == null should only be allowed in testing. */
    public void init(FgExampleList data) {
        if (data == null && !prm.includeUnsupportedFeatures) {
            throw new IllegalArgumentException("Data can only be null if unsupported features are not included.");
        }
        if (data == null && prm.featCountCutoff > 0) {
            throw new IllegalArgumentException("Data can only be null if there is no feature count cutoff.");
        }
        
        // Ensure the FactorTemplateList is initialized, and maybe count features along the way.
        if (templates.isGrowing() && data != null) {
            log.info("Growing feature template list by iterating over examples");
            extractAllFeats(data, templates);
            templates.stopGrowth();
            feAlphabet.stopGrowth();
        }
        numTemplates = templates.size();
        
        // Apply a feature count cutoff.
        this.included = new boolean[numTemplates][][];
{'fraction_non_alphanumeric': 0.09245716673631425, 'fraction_numerical': 0.002089427496865859, 'mean_word_length': 3.091025641025641, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6078044', 'n_tokens_mistral': 2860, 'n_tokens_neox': 2717, 'n_words': 803}