<commit_before>package meetlucene;

import java.io.File;
import java.io.IOException;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;

/**
 * chapter 1.4
 * @author <NAME>
 * @date 2014年10月23日
 */
public class Indexer {

	public static void main(String[] args) throws Exception{
		if (args.length != 2){
			throw new IllegalArgumentException("Usage: java "
					+ Indexer.class.getName() + " <index dir> <data dir>");
		}
		String indexDir = args[0];
		String dataDir = args[1];
		long start = System.currentTimeMillis();
		Indexer indexer = new Indexer(indexDir);
		
	}
	
	private IndexWriter writer;
	
	public Indexer(String indexDir) throws IOException{
		Directory dir = FSDirectory.open(new File(indexDir));
		writer = new IndexWriter(dir, new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.UNLIMITED);
		
	}

}
<commit_msg>修改Indexer.java<commit_after>package meetlucene;

import java.io.File;
import java.io.IOException;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;

/**
 * chapter 1.4
 * @author <NAME>
 * @date 2014年10月23日
 */
public class Indexer {

	public static void main(String[] args) throws Exception{
		if (args.length != 2){
			throw new IllegalArgumentException("Usage: java " + Indexer.class.getName() 
					+ " <index dir> <data dir>");
		}
		String indexDir = args[0];
		String dataDir = args[1];
		long start = System.currentTimeMillis();
		Indexer indexer = new Indexer(indexDir);
	}
	
	private IndexWriter writer;
	
	public Indexer(String indexDir) throws IOException{
		Directory dir = FSDirectory.open(new File(indexDir));
		writer = new IndexWriter(dir, new StandardAnalyzer(Version.LUCENE_30), true, 
				IndexWriter.MaxFieldLength.UNLIMITED);
	}
	
	public void close() throws IOException{
		writer.close();
	}

}

{'fraction_non_alphanumeric': 0.11954992967651196, 'fraction_numerical': 0.014064697609001406, 'mean_word_length': 4.645502645502646, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7573424', 'n_tokens_mistral': 762, 'n_tokens_neox': 680, 'n_words': 158}