<commit_before>package pl.edu.icm.coansys.metaextr.bibref;

import pl.edu.icm.coansys.metaextr.bibref.BibReferenceParser;
import edu.umass.cs.mallet.base.pipe.Pipe;
import edu.umass.cs.mallet.base.pipe.iterator.LineGroupIterator;
import edu.umass.cs.mallet.base.types.InstanceList;
import edu.umass.cs.mallet.base.types.LabelsSequence;
import edu.umass.cs.mallet.grmm.learning.ACRF;
import java.io.*;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import org.apache.commons.lang.StringUtils;
import pl.edu.icm.coansys.metaextr.exception.AnalysisException;
import pl.edu.icm.coansys.metaextr.bibref.model.BibEntry;
import pl.edu.icm.coansys.metaextr.bibref.parsing.model.Citation;
import pl.edu.icm.coansys.metaextr.bibref.parsing.model.CitationTokenLabel;
import pl.edu.icm.coansys.metaextr.bibref.parsing.tools.CitationUtils;

/**
 *
 * @author <NAME>
 */
public class CRFBibReferenceParser implements BibReferenceParser<BibEntry> {
    
    private String modelFile = "/tmp/acrf.ser.gz";

    @Override
	public BibEntry parseBibReference(String text) throws AnalysisException {
        try {
            Citation citation = CitationUtils.stringToCitation(text);
            String data = StringUtils.join(CitationUtils.citationToMalletInputFormat(citation), "\n");

            ACRF model = readModel();
            Pipe pipe = model.getInputPipe();
            InstanceList instanceList = new InstanceList(pipe);
            instanceList.add(new LineGroupIterator(new StringReader(data), Pattern.compile ("\\s*"), true)); 
            LabelsSequence labelSequence = (LabelsSequence)model.getBestLabels(instanceList).get(0);
            
            for (int i = 0; i < labelSequence.size(); i++) {
                citation.getTokens().get(i).setLabel(CitationTokenLabel.valueOf(labelSequence.get(i).toString()));
            }
            
            return CitationUtils.citationToBibref(citation);
		
        } catch (ClassNotFoundException ex) {
            throw new AnalysisException("Cannot parse reference", ex);
        } catch (IOException ex) {
            throw new AnalysisException("Cannot parse reference", ex);
        }
	}
    
    private ACRF readModel() throws IOException, ClassNotFoundException {
        InputStream is = new FileInputStream(new File(modelFile));
        ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(new GZIPInputStream(is)));
        return (ACRF)(ois.readObject());
    }

    public void setModelFile(String modelFile) {
        this.modelFile = modelFile;
    }
    
}<commit_msg>different way of reading model<commit_after>package pl.edu.icm.coansys.metaextr.bibref;

import edu.umass.cs.mallet.base.pipe.Pipe;
import edu.umass.cs.mallet.base.pipe.iterator.LineGroupIterator;
import edu.umass.cs.mallet.base.types.InstanceList;
import edu.umass.cs.mallet.base.types.LabelsSequence;
import edu.umass.cs.mallet.grmm.learning.ACRF;
import java.io.*;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import org.apache.commons.lang.StringUtils;
import pl.edu.icm.coansys.metaextr.bibref.model.BibEntry;
import pl.edu.icm.coansys.metaextr.bibref.parsing.model.Citation;
import pl.edu.icm.coansys.metaextr.bibref.parsing.model.CitationTokenLabel;
import pl.edu.icm.coansys.metaextr.bibref.parsing.tools.CitationUtils;
import pl.edu.icm.coansys.metaextr.exception.AnalysisException;

/**
 *
 * @author <NAME>
 */
public class CRFBibReferenceParser implements BibReferenceParser<BibEntry> {
    
    private ACRF model;

    @Override
	public BibEntry parseBibReference(String text) throws AnalysisException {
        if (model == null) {
            throw new AnalysisException("Model object is not set!");
        }
        
        Citation citation = CitationUtils.stringToCitation(text);
        String data = StringUtils.join(CitationUtils.citationToMalletInputFormat(citation), "\n");

        Pipe pipe = model.getInputPipe();
        InstanceList instanceList = new InstanceList(pipe);
        instanceList.add(new LineGroupIterator(new StringReader(data), Pattern.compile ("\\s*"), true)); 
        LabelsSequence labelSequence = (LabelsSequence)model.getBestLabels(instanceList).get(0);
           
        for (int i = 0; i < labelSequence.size(); i++) {
            citation.getTokens().get(i).setLabel(CitationTokenLabel.valueOf(labelSequence.get(i).toString()));
        }
            
        return CitationUtils.citationToBibref(citation);
    }
    
    public void setModel(String modelFile) throws AnalysisException {
        InputStream is;
        try {
            is = new FileInputStream(new File(modelFile));
        } catch (FileNotFoundException ex) {
            throw new AnalysisException("Cannot set model!", ex);
        }
        setModel(is);
    }
    
    public void setModel(InputStream modelInputStream) throws AnalysisException {
        try {
            ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(new GZIPInputStream(modelInputStream)));
            model = (ACRF)(ois.readObject());
        } catch (IOException ex) {
            throw new AnalysisException("Cannot set model!", ex);
        } catch (ClassNotFoundException ex) {
            throw new AnalysisException("Cannot set model!", ex);
        }
    }
    
}
{'fraction_non_alphanumeric': 0.11062865773079102, 'fraction_numerical': 0.0007551444213705871, 'mean_word_length': 3.331970564186427, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1896534', 'n_tokens_mistral': 1592, 'n_tokens_neox': 1544, 'n_words': 280}