<commit_before>package pl.edu.icm.cermine.pubmed;

import java.io.*;
import java.util.*;
import java.util.Map.Entry;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.hadoop.io.BytesWritable;
import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.xpath.NodeSet;
import org.jruby.compiler.ir.operands.Hash;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import edu.umass.cs.mallet.base.extract.Extraction;
import pl.edu.icm.cermine.PdfBxStructureExtractor;
import pl.edu.icm.cermine.evaluation.tools.CosineDistance;
import pl.edu.icm.cermine.evaluation.tools.SmithWatermanDistance;
import pl.edu.icm.cermine.evaluation.tools.StringTools;
import pl.edu.icm.cermine.evaluation.tools.XMLTools;
import pl.edu.icm.cermine.exception.AnalysisException;
import pl.edu.icm.cermine.exception.TransformationException;
import pl.edu.icm.cermine.metadata.zoneclassification.tools.ZoneLocaliser;
import pl.edu.icm.cermine.structure.model.BxDocument;
import pl.edu.icm.cermine.structure.model.BxPage;
import pl.edu.icm.cermine.structure.model.BxZone;
import pl.edu.icm.cermine.structure.model.BxZoneLabel;
import pl.edu.icm.cermine.structure.transformers.BxDocumentToTrueVizWriter;

public class PubmedXMLGenerator extends EvalFunc<Tuple> {

    private static class LabelTrio {

        private BxZoneLabel label;
        private Double alignment;
        private List<String> entryTokens;
        
        @Override
        public int hashCode() {
            final int prime = 31;
            int result = 1;
            result = prime * result + ((label == null) ? 0 : label.hashCode());
            result = prime * result
                    + ((alignment == null) ? 0 : alignment.hashCode());
            return result;
        }

        @Override
        public boolean equals(Object obj) {
            if (this == obj) {
                return true;
            }
            if (obj == null) {
                return false;
            }
            if (getClass() != obj.getClass()) {
                return false;
            }
            LabelTrio other = (LabelTrio) obj;
            if (label != other.label) {
                return false;
            }
            if (alignment == null) {
                if (other.alignment != null) {
                    return false;
                }
            } else if (!alignment.equals(other.alignment)) {
                return false;
            }
            return true;
        }

        public LabelTrio(BxZoneLabel label, List<String> tokens, Double similarity) {
            this.alignment = similarity;
            this.label = label;
            this.entryTokens = tokens;
        }
    };

    private Tuple output = TupleFactory.getInstance().newTuple(4);
	private boolean verbose = false;

    
    @Override
    public Schema outputSchema(Schema p_input) {
    	try{
    		return Schema.generateNestedSchema(DataType.TUPLE, DataType.CHARARRAY,
    			DataType.BYTEARRAY, DataType.BYTEARRAY, DataType.BYTEARRAY);
    	} catch(FrontendException e) {
    		throw new IllegalStateException(e);
    	}
    }
    
    @Override
    public Tuple exec(Tuple input) throws IOException {
    	System.out.println("doc start");
    	if(input == null || input.size() == 0) {
    		throw new IllegalStateException("Input tuple can't be empty!");
    	}
    	String keyString = (String) input.get(0);
    	DataByteArray nlmByteArray = (DataByteArray) input.get(1);
    	DataByteArray pdfByteArray = (DataByteArray) input.get(2);
    	BxDocument bxDoc = null;
    	try {
    		ByteArrayInputStream nlmIS = new ByteArrayInputStream(nlmByteArray.get());
    		ByteArrayInputStream pdfIS = new ByteArrayInputStream(pdfByteArray.get());
			bxDoc = generateTrueViz(pdfIS, nlmIS);
		} catch (XPathExpressionException e) {
			e.printStackTrace();
			System.exit(1);
		} catch (AnalysisException e) {
			e.printStackTrace();
			System.exit(2);
		} catch (ParserConfigurationException e) {
			e.printStackTrace();
			System.exit(3);
		} catch (SAXException e) {
			e.printStackTrace();
			System.exit(4);
		} catch (TransformationException e) {
			e.printStackTrace();
			System.exit(5);
		}
    	BxDocumentToTrueVizWriter trueVizWriter = new BxDocumentToTrueVizWriter();
    	DataByteArray returnDoc = new DataByteArray(new String()); 
    	//try {
    	//	returnDoc = null; //returnDoc = trueVizWriter.write(bxDoc.getPages());
	//	} catch (TransformationException e) {
	//		e.printStackTrace();
	//		System.exit(6);
	//	}
    	
    	output.set(0, keyString);
    	output.set(1, nlmByteArray);
    	output.set(2, pdfByteArray);
    	output.set(3, returnDoc);
    	System.out.println("doc end");
    	return output;
    }

	private void setVerbose(boolean verbose) {
		this.verbose = verbose;
	}

	private void printlnVerbose(String string) {
		if(verbose) {
			System.out.println(string);
		}
	}
	
	private void printVerbose(String string) {
		if(verbose) {
			printVerbose(string);
		}
	}
	
    private Map<BxZoneLabel, Integer> summarizeDocument(BxDocument doc) {
    	Map<BxZoneLabel, Integer> ret = new HashMap<BxZoneLabel, Integer>();
    	for(BxZone zone: doc.asZones()) {
    		if(ret.containsKey(zone.getLabel())) {
    			ret.put(zone.getLabel(), ret.get(zone.getLabel())+1);
    		} else {
    			ret.put(zone.getLabel(), 1);
    		}
    	}
    	return ret;
    }
    
    public BxDocument generateTrueViz(InputStream pdfStream, InputStream nlmStream) 
    		throws AnalysisException, ParserConfigurationException, SAXException, IOException, XPathExpressionException, TransformationException {
        XPath xpath = XPathFactory.newInstance().newXPath();
        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
        dbf.setValidating(false);
        dbf.setFeature("http://xml.org/sax/features/namespaces", false);
        dbf.setFeature("http://xml.org/sax/features/validation", false);
        dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-dtd-grammar", false);
        dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);

        DocumentBuilder builder = dbf.newDocumentBuilder();
        Document domDoc = builder.parse(nlmStream);

        PdfBxStructureExtractor structureExtractor = new PdfBxStructureExtractor();
        BxDocument bxDoc = structureExtractor.extractStructure(pdfStream);
        Integer bxDocLen = bxDoc.asZones().size();<commit_msg>add debugging info to pubmed UDf<commit_after>package pl.edu.icm.cermine.pubmed;

import java.io.*;
import java.util.*;
import java.util.Map.Entry;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.hadoop.io.BytesWritable;
import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.tools.pigstats.PigStatusReporter;
import org.apache.xpath.NodeSet;
import org.jruby.compiler.ir.operands.Hash;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import edu.umass.cs.mallet.base.extract.Extraction;
import pl.edu.icm.cermine.PdfBxStructureExtractor;
import pl.edu.icm.cermine.evaluation.tools.CosineDistance;
import pl.edu.icm.cermine.evaluation.tools.SmithWatermanDistance;
import pl.edu.icm.cermine.evaluation.tools.StringTools;
import pl.edu.icm.cermine.evaluation.tools.XMLTools;
import pl.edu.icm.cermine.exception.AnalysisException;
import pl.edu.icm.cermine.exception.TransformationException;
import pl.edu.icm.cermine.metadata.zoneclassification.tools.ZoneLocaliser;
import pl.edu.icm.cermine.structure.model.BxDocument;
import pl.edu.icm.cermine.structure.model.BxPage;
import pl.edu.icm.cermine.structure.model.BxZone;
import pl.edu.icm.cermine.structure.model.BxZoneLabel;
import pl.edu.icm.cermine.structure.transformers.BxDocumentToTrueVizWriter;

public class PubmedXMLGenerator extends EvalFunc<Tuple> {

    private static class LabelTrio {

        private BxZoneLabel label;
        private Double alignment;
        private List<String> entryTokens;
        
        @Override
        public int hashCode() {
            final int prime = 31;
            int result = 1;
            result = prime * result + ((label == null) ? 0 : label.hashCode());
            result = prime * result
                    + ((alignment == null) ? 0 : alignment.hashCode());
            return result;
        }

        @Override
        public boolean equals(Object obj) {
            if (this == obj) {
                return true;
            }
            if (obj == null) {
                return false;
            }
            if (getClass() != obj.getClass()) {
                return false;
            }
            LabelTrio other = (LabelTrio) obj;
            if (label != other.label) {
                return false;
            }
            if (alignment == null) {
                if (other.alignment != null) {
                    return false;
                }
            } else if (!alignment.equals(other.alignment)) {
                return false;
            }
            return true;
        }

        public LabelTrio(BxZoneLabel label, List<String> tokens, Double similarity) {
            this.alignment = similarity;
            this.label = label;
            this.entryTokens = tokens;
        }
    };

    private Tuple output = TupleFactory.getInstance().newTuple(4);
	private boolean verbose = false;

    
    @Override
    public Schema outputSchema(Schema p_input) {
    	try{
    		return Schema.generateNestedSchema(DataType.TUPLE, DataType.CHARARRAY,
    			DataType.BYTEARRAY, DataType.BYTEARRAY, DataType.BYTEARRAY);
    	} catch(FrontendException e) {
    		throw new IllegalStateException(e);
    	}
    }
    
    @Override
    public Tuple exec(Tuple input) throws IOException {
    	System.out.println("doc start");
    	if(input == null || input.size() == 0) {
    		throw new IllegalStateException("Input tuple can't be empty!");
    	}
    	String keyString = (String) input.get(0);
    	System.out.println(keyString);
    	DataByteArray nlmByteArray = (DataByteArray) input.get(1);
    	DataByteArray pdfByteArray = (DataByteArray) input.get(2);
    	BxDocument bxDoc = null;
    	try {
    		ByteArrayInputStream nlmIS = new ByteArrayInputStream(nlmByteArray.get());
    		ByteArrayInputStream pdfIS = new ByteArrayInputStream(pdfByteArray.get());
			bxDoc = generateTrueViz(pdfIS, nlmIS);
			System.out.println("generated trueviz");
		} catch (Throwable e) {
			System.err.println(keyString);
			e.printStackTrace();
		}
    	BxDocumentToTrueVizWriter trueVizWriter = new BxDocumentToTrueVizWriter();
    	DataByteArray returnDoc = null;
    	if (bxDoc != null) {
    		try {
    			returnDoc = new DataByteArray(trueVizWriter.write(bxDoc.getPages()));
    			returnDoc = null;
    			System.out.println("wrote down trueviz");
    			PigStatusReporter reporter = PigStatusReporter.getInstance();
    			if (reporter != null) {
    				reporter.getCounter("CERMINE_COUNTERS", "DOC_COUNTER").increment(1);
    			}
    		} catch (TransformationException e) {
    			System.err.println(keyString);
    			e.printStackTrace();
    		}
    	}
    	
    	output.set(0, keyString);
    	output.set(1, nlmByteArray);
    	output.set(2, pdfByteArray);
    	output.set(3, returnDoc);
    	System.out.println("doc end");
    	return output;
    }

	private void setVerbose(boolean verbose) {
		this.verbose = verbose;
	}

	private void printlnVerbose(String string) {
		if(verbose) {
			System.out.println(string);
		}
	}
	
	private void printVerbose(String string) {
		if(verbose) {
			printVerbose(string);
		}
	}
	
    private Map<BxZoneLabel, Integer> summarizeDocument(BxDocument doc) {
    	Map<BxZoneLabel, Integer> ret = new HashMap<BxZoneLabel, Integer>();
    	for(BxZone zone: doc.asZones()) {
    		if(ret.containsKey(zone.getLabel())) {
    			ret.put(zone.getLabel(), ret.get(zone.getLabel())+1);
    		} else {
    			ret.put(zone.getLabel(), 1);
    		}
    	}
    	return ret;
    }
    
    public BxDocument generateTrueViz(InputStream pdfStream, InputStream nlmStream) 
    		throws AnalysisException, ParserConfigurationException, SAXException, IOException, XPathExpressionException, TransformationException {
        XPath xpath = XPathFactory.newInstance().newXPath();
        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
        dbf.setValidating(false);
        dbf.setFeature("http://xml.org/sax/features/namespaces", false);
        dbf.setFeature("http://xml.org/sax/features/validation", false);
        dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-dtd-grammar", false);
        dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);

        DocumentBuilder builder = dbf.newDocumentBuilder();
        Document domDoc = builder.parse(nlmStream);

        PdfBxStructureExtractor structureExtractor = new PdfBxStructureExtractor();
        BxDocument bxDoc = structureExtractor.extractStructure(pdfStream);
        System.out.println("structure extracted");
        Integer bxDocLen = bxDoc.asZones().size();
{'fraction_non_alphanumeric': 0.11341954632181471, 'fraction_numerical': 0.0031799872800508796, 'mean_word_length': 2.9719337636822902, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1921377', 'n_tokens_mistral': 4540, 'n_tokens_neox': 4365, 'n_words': 788}