<commit_before>package parser;

import org.w3c.dom.*;
import org.xml.sax.SAXException;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
import java.io.IOException;

/**
 * Created by Niket on 22/02/15.
 */
public class TEIParser {

    public TEIDocument parseTEIXMLFile(String filePath) throws ParserConfigurationException, IOException, SAXException {
        DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
        DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
        Document document = documentBuilder.parse(new File(filePath));
        document.getDocumentElement().normalize();
        Node bodyNode = document.getElementsByTagName("body").item(0);
        return parseBodyNode(bodyNode);
    }

    private TEIDocument parseBodyNode(Node bodyNode) {
        TEIDocument teiDocument = new TEIDocument();
        NodeList nodeList = bodyNode.getChildNodes();
        for (int i = 0; i < nodeList.getLength(); i++) {
            Node node = nodeList.item(i);
            if (node.getNodeType() == Node.ELEMENT_NODE) {
                String nodeName = node.getNodeName();
                TEIElement teiElement = null;
                if (nodeName.equals("head")) {
                    teiElement = new HeaderTEIElement(node.getTextContent());
                } else if (nodeName.equals("p")) {
                    teiElement = new ParagraphTEIElement(node.getTextContent());
                }
                if (teiElement != null) {
                    teiDocument.addTEIElement(teiElement);
                }
            }
        }
        return teiDocument;
    }

}<commit_msg>initial try at implementing TEI parser hierarchically<commit_after>package parser;

import org.apache.commons.lang3.StringUtils;
import org.w3c.dom.*;
import org.xml.sax.SAXException;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.HashMap;
import java.util.Map;

/**
 * Created by Niket on 22/02/15.
 */
public class TEIParser {

    public TEIDocument parseTEIXMLFile(String filePath) throws ParserConfigurationException, IOException, SAXException {
        DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
        DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
        Document document = documentBuilder.parse(new File(filePath));
        document.getDocumentElement().normalize();
        Node bodyNode = document.getElementsByTagName("body").item(0);
        return parseBodyNode(bodyNode);
    }

    private TEIDocument parseBodyNode(Node bodyNode) {
        TEIDocument teiDocument = new TEIDocument();
        NodeList nodeList = bodyNode.getChildNodes();
        TEIElement currentHeaderElement = null;
        for (int i = 0; i < nodeList.getLength(); i++) {
            Node node = nodeList.item(i);
            int currentLevel = 0;
            Map<Integer, TEIElement> stack = new HashMap<Integer, TEIElement>();
            if (node.getNodeType() == Node.ELEMENT_NODE) {
                String nodeName = node.getNodeName();

                TEIElement teiElement = null;
                if (nodeName.equals("head")) {
                    int headerLevel = getHeaderLevel(node);
                    if (headerLevel > currentLevel) {
                        if (currentHeaderElement != null) {
                            teiElement = new HeaderTEIElement(node.getTextContent(), currentHeaderElement, headerLevel);
                            stack.put(currentLevel, currentHeaderElement);
                            currentHeaderElement = teiElement;
                        }
                        currentLevel++;
                    }
                    if (headerLevel < currentLevel && teiElement.getParentElement() != null) {
                        teiElement = stack.remove(currentLevel);
                        currentLevel--;
                    }
                    if (headerLevel == currentLevel) {
                        if (currentLevel > 0) {
                            currentHeaderElement = new HeaderTEIElement(node.getTextContent(), stack.get(currentLevel -1), currentLevel - 1);
                        } else {
                            currentHeaderElement = new HeaderTEIElement(node.getTextContent());
                            teiDocument.addTEIElement(currentHeaderElement);
                        }
                        stack.put(currentLevel, currentHeaderElement);
                    }
                } else if (nodeName.equals("p")) {
                    teiElement = new ParagraphTEIElement(node.getTextContent(), currentHeaderElement);
                }
            }
        }
        return teiDocument;
    }


    private int getHeaderLevel(Node node) {
        assert node.getNodeName().equals("head");
        NamedNodeMap namedNodeMap = node.getAttributes();
        String levelString = namedNodeMap.getNamedItem("n").getTextContent();
        return StringUtils.countMatches(levelString, ".");
    }

}
{'fraction_non_alphanumeric': 0.09343861628125588, 'fraction_numerical': 0.004324121075390111, 'mean_word_length': 2.2301153612629023, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7302412', 'n_tokens_mistral': 1350, 'n_tokens_neox': 1297, 'n_words': 271}