<commit_before>package ld.ldhomework.crawler;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.semanticweb.yars.nx.Node;
import org.semanticweb.yars.nx.parser.NxParser;
import org.semanticweb.yars.nx.parser.ParseException;
import org.semanticweb.yars2.rdfxml.RDFXMLParser;

public class FileParser {

    private static final Logger LOG = java.util.logging.Logger
	    .getLogger(TripleEntry.class.getName());

    private List<Triple> triples;

    private boolean parsed = false;
    private InputStream is;
    private String baseUrl;

    public FileParser(InputStream is, String baseUrl) {
	this(is, baseUrl, null);
    }

    public FileParser(InputStream is, String baseUrl, String contentType) {
	triples = new ArrayList<Triple>();
	this.baseUrl = baseUrl;
	this.is = is;
    }

    public void parse() throws IOException {
	if (true) {
	    parseXml();
	} else if (false) {
	    parseNx();
	}
    }

    private void parseNx() {
	NxParser nxp = new NxParser(is);
	nxp = new NxParser(is);

	Node[] nxx;
	while (nxp.hasNext()) {
	    nxx = nxp.next();
	    triples.add(new Triple(nxx[0], nxx[1], nxx[2]));
	}
	parsed = true;

    }

    private void parseXml() {
	RDFXMLParser nxp;
	try {

	    nxp = new RDFXMLParser(is, baseUrl);

	    Node[] nxx;
	    while (nxp.hasNext()) {
		nxx = nxp.next();
		triples.add(new Triple(nxx[0], nxx[1], nxx[2]));
	    }

	    parsed = true;
	} catch (ParseException pe) {
	    LOG.log(Level.WARNING, "Error at parsing file", pe);
	} catch (IOException e) {
	    LOG.log(Level.WARNING, "Error at parsing file", e);
	}
    }

    public List<Triple> getTriples() {
	if (parsed) {
	    return triples;
	}
	throw new IllegalStateException("file not yet parsed");
    }

}<commit_msg>undo changes ... <commit_after>package ld.ldhomework.crawler;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.semanticweb.yars.nx.Node;
import org.semanticweb.yars.nx.parser.ParseException;
import org.semanticweb.yars2.rdfxml.RDFXMLParser;

public class FileParser {

    private static final Logger LOG = java.util.logging.Logger
	    .getLogger(TripleEntry.class.getName());

    private RDFXMLParser nxp;
    private List<Triple> triples;

    private boolean parsed = false;
    private InputStream is;
    private String baseUrl;

    public FileParser(InputStream is, String baseUrl) {
	triples = new ArrayList<Triple>();
	this.baseUrl = baseUrl;
	this.is = is;
    }

    public boolean parse() throws IOException {
	try {

	    nxp = new RDFXMLParser(is, baseUrl);

	    Node[] nxx;
	    while (nxp.hasNext()) {

		nxx = nxp.next();

		triples.add(new Triple(nxx[0], nxx[1], nxx[2]));
	    }

	    parsed = true;
	} catch (ParseException pe) {
	    LOG.log(Level.WARNING, "Error at parsing file", pe);
	}

	return parsed;
    }

    public List<Triple> getTriples() {
	if (parsed) {
	    return triples;
	}
	throw new IllegalStateException("file not yet parsed");
    }

}
{'fraction_non_alphanumeric': 0.12562972292191435, 'fraction_numerical': 0.0034634760705289673, 'mean_word_length': 3.430962343096234, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4938856', 'n_tokens_mistral': 1136, 'n_tokens_neox': 1080, 'n_words': 260}