<commit_before>package model;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
import java.util.Set;

import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.stream.JsonReader;
/**
 * Class for processing documents to build tokens
 * 
 * @author crystalchun
 *
 */
public class DocProcessor {
	String filepath;
	
	private Document getDocument(String filepath) {
		Gson gson = new Gson();
		JsonObject json = null;
		
		try {
			json = parseToJsonObject(new FileInputStream(filepath));
		} catch (IOException e) {
			e.printStackTrace();
			return null;
		}
		
		return gson.fromJson(json, Document.class);
	}
	
	private JsonObject parseToJsonObject(InputStream in) throws IOException {
		JsonParser parser = new JsonParser();
		return parser.parse(new JsonReader(new InputStreamReader(in, "UTF-8"))).getAsJsonObject();
	}
	
	private String normalizeToken(String token) {
		token = Normalizer.trimNonAlphanumeric(token);
		token = Normalizer.removeApostrophe(token);
		token = Normalizer.stemToken(token);
		return token.toLowerCase();
	}
	
	private void populateTerms(String token, List<String> terms) {
		if (token.contains("-")) {
			Set<String> tokens = Normalizer.splitHypenWords(token);
			for (String each : tokens) {
				terms.add(each);
			}
		} else {
			terms.add(token);
		}
	}
	
	private void normalizeTerms(List<String> terms) {
		String normalizedToken;
		for (int i = 0; i < terms.size(); i++) {
			normalizedToken = normalizeToken(terms.get(i));
			terms.set(i, normalizedToken);
			if (normalizedToken.length() == 0) {
				terms.remove(i--);
			}
		}
	}
	
	public DocProcessor(String filepath) {
		this.filepath = filepath;
	}
	
	public List<String> process(String fileName) { // Replace with Document object
		Document doc = getDocument(this.filepath + "/" + fileName);
		Scanner sc = new Scanner(doc.getBody());
		List<String> terms = new ArrayList<String>();
		while (sc.hasNext()) {
			populateTerms(sc.next(), terms);
		}
		normalizeTerms(terms);
		return terms;
	}
	
	public static void main(String[] args) throws IOException {
		DocProcessor dp = new DocProcessor("external/articles/");
		dp.process("article1.json");
		String filepath = "external/articles/article1.json\nHello how are you.";
//		System.out.println(doc);
	}
}<commit_msg>Using File instead of Stirng filepath<commit_after>package model;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
import java.util.Set;

import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.stream.JsonReader;
/**
 * Class for processing documents to build tokens
 * 
 * @author crystalchun
 *
 */
public class DocProcessor {
	File file;
	
	private Document getDocument(File file) {
		Gson gson = new Gson();
		JsonObject json = null;
		
		try {
			json = parseToJsonObject(new FileInputStream(file));
		} catch (IOException e) {
			e.printStackTrace();
			return null;
		}
		
		return gson.fromJson(json, Document.class);
	}
	
	private JsonObject parseToJsonObject(InputStream in) throws IOException {
		JsonParser parser = new JsonParser();
		return parser.parse(new JsonReader(new InputStreamReader(in, "UTF-8"))).getAsJsonObject();
	}
	
	private String normalizeToken(String token) {
		token = Normalizer.trimNonAlphanumeric(token);
		token = Normalizer.removeApostrophe(token);
		token = Normalizer.stemToken(token);
		return token.toLowerCase();
	}
	
	private void populateTerms(String token, List<String> terms) {
		if (token.contains("-")) {
			Set<String> tokens = Normalizer.splitHypenWords(token);
			for (String each : tokens) {
				terms.add(each);
			}
		} else {
			terms.add(token);
		}
	}
	
	private void normalizeTerms(List<String> terms) {
		String normalizedToken;
		for (int i = 0; i < terms.size(); i++) {
			normalizedToken = normalizeToken(terms.get(i));
			terms.set(i, normalizedToken);
			if (normalizedToken.length() == 0) {
				terms.remove(i--);
			}
		}
	}
	
	public DocProcessor(File file) {
		
		this.file = file;
	}
	
	public List<String> process() { // Replace with Document object
		Document doc = getDocument(this.file);
		Scanner sc = new Scanner(doc.getBody());
		List<String> terms = new ArrayList<String>();
		while (sc.hasNext()) {
			populateTerms(sc.next(), terms);
		}
		normalizeTerms(terms);
		return terms;
	}
}
{'fraction_non_alphanumeric': 0.1275913656764266, 'fraction_numerical': 0.0017097670442402222, 'mean_word_length': 4.229050279329609, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4871976', 'n_tokens_mistral': 1665, 'n_tokens_neox': 1474, 'n_words': 348}