<commit_before>package pl.edu.icm.cermine;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import pl.edu.icm.cermine.exception.AnalysisException;
import pl.edu.icm.cermine.structure.*;
import pl.edu.icm.cermine.structure.model.BxDocument;
import pl.edu.icm.cermine.structure.tools.BxModelUtils;


/**
 *
 * @author <NAME>
 */
public class PdfGeometricStructureExtractor implements DocumentGeometricStructureExtractor {
    
    CharacterExtractor glyphExtractor;
    
    PageSegmenter pageSegmenter;
    
    ReadingOrderResolver roResolver;
    
    ZoneClassifier initialClassifier;


    public PdfGeometricStructureExtractor() {
        glyphExtractor = new ITextCharacterExtractor();
        pageSegmenter = new DocstrumPageSegmenter();
        roResolver = new HierarchicalReadingOrderResolver();
        
        InputStreamReader modelISRI = new InputStreamReader(PdfGeometricStructureExtractor.class
				.getResourceAsStream("/pl/edu/icm/cermine/structure/svm_initial_classifier"));
        BufferedReader modelFileI = new BufferedReader(modelISRI);
        InputStreamReader rangeISRI = new InputStreamReader(PdfGeometricStructureExtractor.class
		        .getResourceAsStream("/pl/edu/icm/cermine/structure/svm_initial_classifier.range"));
        BufferedReader rangeFileI = new BufferedReader(rangeISRI);
        initialClassifier = new SVMInitialZoneClassifier(modelFileI, rangeFileI);
    }
        
    public BxDocument extractStructure(InputStream stream) throws AnalysisException {
        BxDocument doc = glyphExtractor.extractCharacters(stream);
        doc = pageSegmenter.segmentPages(doc);
        BxModelUtils.setParents(doc);<commit_msg>Override annotations added<commit_after>package pl.edu.icm.cermine;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import pl.edu.icm.cermine.exception.AnalysisException;
import pl.edu.icm.cermine.structure.*;
import pl.edu.icm.cermine.structure.model.BxDocument;
import pl.edu.icm.cermine.structure.tools.BxModelUtils;


/**
 *
 * @author <NAME>
 */
public class PdfGeometricStructureExtractor implements DocumentGeometricStructureExtractor {
    
    CharacterExtractor glyphExtractor;
    
    PageSegmenter pageSegmenter;
    
    ReadingOrderResolver roResolver;
    
    ZoneClassifier initialClassifier;


    public PdfGeometricStructureExtractor() {
        glyphExtractor = new ITextCharacterExtractor();
        pageSegmenter = new DocstrumPageSegmenter();
        roResolver = new HierarchicalReadingOrderResolver();
        
        InputStreamReader modelISRI = new InputStreamReader(PdfGeometricStructureExtractor.class
				.getResourceAsStream("/pl/edu/icm/cermine/structure/svm_initial_classifier"));
        BufferedReader modelFileI = new BufferedReader(modelISRI);
        InputStreamReader rangeISRI = new InputStreamReader(PdfGeometricStructureExtractor.class
		        .getResourceAsStream("/pl/edu/icm/cermine/structure/svm_initial_classifier.range"));
        BufferedReader rangeFileI = new BufferedReader(rangeISRI);
        initialClassifier = new SVMInitialZoneClassifier(modelFileI, rangeFileI);
    }
        
    @Override
    public BxDocument extractStructure(InputStream stream) throws AnalysisException {
        BxDocument doc = glyphExtractor.extractCharacters(stream);
        doc = pageSegmenter.segmentPages(doc);
        BxModelUtils.setParents(doc);
{'fraction_non_alphanumeric': 0.07973325601623658, 'fraction_numerical': 0.0, 'mean_word_length': 4.291411042944786, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4630621', 'n_tokens_mistral': 980, 'n_tokens_neox': 948, 'n_words': 154}