<commit_before>package jules;

import java.io.File;
import java.io.IOException;

import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.*;
import org.apache.lucene.index.*;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;

import edu.jhu.nlp.wikipedia.*;


public class Indexer {
	public static void Main(String[] args){
		Indexer indexer = new Indexer();
		indexer.index();
	}
	private String wikiFile;
	private String indexDir;
	
	
	public Indexer(){
		wikiFile = "./svwiki-20141031-pages-meta-current.xml";
		indexDir = "./indexDir/";
	}
	
	public void index(){
		//WikiXMLParser wxp = WikiXMLParserFactory.getDOMParser(wikiFile);
		Analyzer analyzer = new StandardAnalyzer();
		IndexWriterConfig iwc = new IndexWriterConfig(Version.LATEST, analyzer);
		try{
			final IndexWriter writer = new IndexWriter(FSDirectory.open(new File(indexDir)), iwc);
			WikiXMLParser wxsp = WikiXMLParserFactory.getSAXParser(wikiFile);
			wxsp.setPageCallback(new PageCallbackHandler() {
				@Override
				public void process(WikiPage page) {
					Document doc = new Document();
					System.out.println(page.getID() + ", title: " + page.getTitle());
					doc.add(new IntField("id", Integer.parseInt(page.getID()), Field.Store.YES));
					doc.add(new TextField("title", page.getTitle(), Field.Store.YES));
					doc.add(new TextField("text", page.getText(), Field.Store.YES));
					try {
						writer.addDocument(doc);
					} catch (IOException e) {
						e.printStackTrace();
					}
					
				}
			});
			wxsp.parse();
			writer.commit();
			writer.close();
		}catch(Exception e){ e.printStackTrace(); }
	}
}<commit_msg>Don't add stub, redirect, special or "disambigious" pages. Also, PLEASE USE 4 (or 2, even 8) SPACES FOR INDENTATION!<commit_after>package jules;

import java.io.File;
import java.io.IOException;

import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.*;
import org.apache.lucene.index.*;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;

import edu.jhu.nlp.wikipedia.*;


public class Indexer {
    public static void Main(String[] args){
        Indexer indexer = new Indexer();
        indexer.index();
    }
    private String wikiFile;
    private String indexDir;


    public Indexer(){
        wikiFile = "./svwiki-20141031-pages-meta-current.xml";
        indexDir = "./indexDir/";
    }

    public void index(){
        //WikiXMLParser wxp = WikiXMLParserFactory.getDOMParser(wikiFile);
        Analyzer analyzer = new StandardAnalyzer();
        IndexWriterConfig iwc = new IndexWriterConfig(Version.LATEST, analyzer);
        try{
            final IndexWriter writer = new IndexWriter(FSDirectory.open(new File(indexDir)), iwc);
            WikiXMLParser wxsp = WikiXMLParserFactory.getSAXParser(wikiFile);
            wxsp.setPageCallback(new PageCallbackHandler() {
                @Override
                public void process(WikiPage page) {
                    // We don't want to store unnecessary pages.
                    if (page.isDisambiguationPage() || page.isRedirect() || page.isSpecialPage() || page.isStub())
                        return;
                    Document doc = new Document();
                    System.out.println(page.getID() + ", title: " + page.getTitle());
                    doc.add(new IntField("id", Integer.parseInt(page.getID()), Field.Store.YES));
                    doc.add(new TextField("title", page.getTitle(), Field.Store.YES));
                    doc.add(new TextField("text", page.getText(), Field.Store.YES));
                    try {
                        writer.addDocument(doc);
                    } catch (IOException e) {
                        e.printStackTrace();
                    }

                }
            });
            wxsp.parse();
            writer.commit();
            writer.close();
        }catch(Exception e){ e.printStackTrace(); }
    }
}
{'fraction_non_alphanumeric': 0.12874692874692875, 'fraction_numerical': 0.004668304668304668, 'mean_word_length': 2.6775067750677506, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4418744', 'n_tokens_mistral': 1318, 'n_tokens_neox': 1158, 'n_words': 249}