<commit_before>/**
 * A simple API to handle Wikipedia XML dumps
 * @author delip
 *
 */

import org.apache.xerces.parsers.DOMParser;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
import java.io.IOException;




public class WikiXMLParser {
	
	String wikiXMLFile = null;
	boolean useGZip = false;
	
	public WikiXMLParser(String fileName) {
		wikiXMLFile = fileName;
		if(wikiXMLFile.endsWith(".gz")) useGZip = true;
	}
	
	void process() throws Exception {
	}

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		
		if(args.length != 1) {
			System.err.println("Usage: Parser <XML-FILE>");
			System.exit(-1);
		}
		
		WikiXMLParser wxp = new WikiXMLParser(args[0]);
		
		try {
			wxp.process();
		}catch(Exception e) {
			e.printStackTrace();
		}
	}
}<commit_msg>Added code to lazy DOM the XML file.<commit_after>/**
 * A simple API to handle Wikipedia XML dumps
 * @author delip
 *
 */

import org.apache.xerces.parsers.DOMParser;
import org.w3c.dom.Document;
import org.xml.sax.InputSource;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.zip.GZIPInputStream;


public class WikiXMLParser {
	
	String wikiXMLFile = null;
	boolean useGZip = false;
	private DOMParser domParser = new DOMParser();
	private static String FEATURE_URI = 
		"http://apache.org/xml/features/dom/defer-node-expansion";
	
	public WikiXMLParser(String fileName) throws Exception {
		wikiXMLFile = fileName;
		if(wikiXMLFile.endsWith(".gz")) useGZip = true;
		domParser.setFeature(FEATURE_URI, true);
		BufferedReader br = null;
		if(useGZip) {
			br = new BufferedReader(new InputStreamReader(
					new GZIPInputStream(new FileInputStream(wikiXMLFile))));
		} else {
			br = new BufferedReader(new InputStreamReader( 
					new FileInputStream(wikiXMLFile)));
		}
		domParser.parse(new InputSource(br));
	}
	
	void process() throws Exception {
		Document doc = domParser.getDocument();
	}

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		
		if(args.length != 1) {
			System.err.println("Usage: Parser <XML-FILE>");
			System.exit(-1);
		}
		
		try {
			WikiXMLParser wxp = new WikiXMLParser(args[0]);
			wxp.process();
		}catch(Exception e) {
			e.printStackTrace();
		}
	}
}
{'fraction_non_alphanumeric': 0.12808098591549297, 'fraction_numerical': 0.0035211267605633804, 'mean_word_length': 3.815677966101695, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4083551', 'n_tokens_mistral': 865, 'n_tokens_neox': 754, 'n_words': 174}