<commit_before>package edu.jhu.gm;

import static org.junit.Assert.*;

import java.util.ArrayList;
import java.util.List;

import org.junit.Test;

import edu.jhu.gm.BeliefPropagation.BeliefPropagationPrm;
import edu.jhu.gm.BeliefPropagation.BpScheduleType;
import edu.jhu.gm.BeliefPropagation.BpUpdateOrder;
import edu.jhu.gm.BeliefPropagation.FgInferencerFactory;
import edu.jhu.gm.Var.VarType;
import edu.jhu.util.Alphabet;
import edu.jhu.util.JUnitUtils;
import edu.jhu.util.vector.IntDoubleEntry;

public class CrfObjectiveTest {
    
    /**
     * A description of a collection of identical log-linear model examples.
     * 
     * @author mgormley
     */
    public static class LogLinearExDesc {
        private int count;
        private FeatureVector features;
        public LogLinearExDesc(int count, FeatureVector features) {
            this.count = count;
            this.features = features;
        }
        public int getCount() {
            return count;
        }
        public FeatureVector getFeatures() {
            return features;
        }         
    }
    
    /**
     * A factor for FgExamples constructed from LogLinearExDesc objects.
     * @author mgormley
     */
    public static class LogLinearEDs {

        private Alphabet<Feature> alphabet = new Alphabet<Feature>();
        private ArrayList<LogLinearExDesc> descList = new ArrayList<LogLinearExDesc>();        

        public void addEx(int count, String... featNames) {
            FeatureVector features = new FeatureVector();
            for (String featName : featNames) {
                features.put(alphabet.lookupIndex(new Feature(featName)), 1.0);
            }
            LogLinearExDesc ex = new LogLinearExDesc(count, features);
            descList.add(ex);
        }
        
        public List<String> getStateNames() {
            List<String> names = new ArrayList<String>();
            for (LogLinearExDesc desc : descList) {
                StringBuilder sb = new StringBuilder();
                for (IntDoubleEntry entry : desc.getFeatures()) {
                    sb.append(entry.index());
                    sb.append(":");
                }
                names.add(sb.toString());
            }
            return names;
        }
        
        public FgExamples getData() {
            FgExamples data = new FgExamples();
            int state=0;
            for (LogLinearExDesc desc : descList) {
                for (int i=0; i<desc.getCount(); i++) {
                    FactorGraph fg = new FactorGraph();
                    final Var v0 = new Var(VarType.PREDICTED, descList.size(), "v0", getStateNames());
                    Factor f0 = new Factor(new VarSet(v0));
                    fg.addFactor(f0);
                    FeatureExtractor featExtractor = new FeatureExtractor() {
                        @Override
                        public FeatureVector calcFeatureVector(int factorId,
                                VarConfig varConfig) {
                            return descList.get(varConfig.getState(v0)).getFeatures();
                        }<commit_msg>Adding unit test of CRF objective.<commit_after>package edu.jhu.gm;

import static org.junit.Assert.*;

import java.util.ArrayList;
import java.util.List;

import org.junit.Test;

import edu.jhu.gm.BeliefPropagation.BeliefPropagationPrm;
import edu.jhu.gm.BeliefPropagation.BpScheduleType;
import edu.jhu.gm.BeliefPropagation.BpUpdateOrder;
import edu.jhu.gm.BeliefPropagation.FgInferencerFactory;
import edu.jhu.gm.Var.VarType;
import edu.jhu.util.Alphabet;
import edu.jhu.util.JUnitUtils;
import edu.jhu.util.vector.IntDoubleEntry;

public class CrfObjectiveTest {
    
    /**
     * A description of a collection of identical log-linear model examples.
     * 
     * @author mgormley
     */
    public static class LogLinearExDesc {
        private int count;
        private FeatureVector features;
        public LogLinearExDesc(int count, FeatureVector features) {
            this.count = count;
            this.features = features;
        }
        public int getCount() {
            return count;
        }
        public FeatureVector getFeatures() {
            return features;
        }         
    }
    
    /**
     * A factor for FgExamples constructed from LogLinearExDesc objects.
     * @author mgormley
     */
    public static class LogLinearEDs {

        private Alphabet<Feature> alphabet = new Alphabet<Feature>();
        private ArrayList<LogLinearExDesc> descList = new ArrayList<LogLinearExDesc>();        

        public void addEx(int count, String... featNames) {
            FeatureVector features = new FeatureVector();
            for (String featName : featNames) {
                features.put(alphabet.lookupIndex(new Feature(featName)), 1.0);
            }
            LogLinearExDesc ex = new LogLinearExDesc(count, features);
            descList.add(ex);
        }
        
        public List<String> getStateNames() {
            List<String> names = new ArrayList<String>();
            for (LogLinearExDesc desc : descList) {
                StringBuilder sb = new StringBuilder();
                for (IntDoubleEntry entry : desc.getFeatures()) {
                    sb.append(entry.index());
                    sb.append(":");
                }
                names.add(sb.toString());
            }
            return names;
        }
        
        public FgExamples getData() {
            FgExamples data = new FgExamples(alphabet);
            int state=0;
            for (LogLinearExDesc desc : descList) {
                for (int i=0; i<desc.getCount(); i++) {
                    FactorGraph fg = new FactorGraph();
                    final Var v0 = new Var(VarType.PREDICTED, descList.size(), "v0", getStateNames());
                    Factor f0 = new Factor(new VarSet(v0));
                    fg.addFactor(f0);
                    FeatureExtractor featExtractor = new FeatureExtractor() {
                        @Override
                        public FeatureVector calcFeatureVector(int factorId,
                                VarConfig varConfig) {
                            return descList.get(varConfig.getState(v0)).getFeatures();
                        }
{'fraction_non_alphanumeric': 0.09119948889953682, 'fraction_numerical': 0.0031943778949049674, 'mean_word_length': 1.837335749886724, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2359370', 'n_tokens_mistral': 1711, 'n_tokens_neox': 1646, 'n_words': 387}