<commit_before>import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.Scanner;

/**
 * Takes in a file and removes: HTML, Punctuation and special characters
 * 
 */

/**
 * @author <NAME>
 * 
 *
 */
public class P1 {
	
	private boolean startFlag = false;	
	//private ArrayList<String> fileText = new ArrayList<String>();
	private ArrayList<Integer> occurList = new ArrayList<Integer>();	
	private ArrayList<Term> terms = new ArrayList<Term>();
	private ArrayList<Term> termIndex = new ArrayList<Term>();
	
	
	
	public ArrayList<Term>getTermList() {
		
		return terms;
	}
	
	public ArrayList<String> readFile(String fileName,String docName) {
		String word = null;
		String wordPuncRemoved = null;
		String htmlRemoved = null;
		BinarySearch search = new BinarySearch();		
		
		
				
		try {
			Scanner read = new Scanner(new File(fileName));
			
			
			while(read.hasNext() ) {
				String temp = null;
				String tempTwo = null;
				word = read.next();
				boolean flag = false;
				
				//System.out.println(word);
				
				//System.out.println(word+" =");
				
				if(word.isEmpty() == false)
				{
					htmlRemoved = removeHTML(word).replaceAll("\\s+","");
					wordPuncRemoved = removePunctuation(htmlRemoved).toLowerCase();
					
					
						for(int i=0;i<wordPuncRemoved.length();i++) { 
							<commit_msg>Arraylist<Term><commit_after>import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.Scanner;

/**
 * Takes in a file and removes: HTML, Punctuation and special characters
 * 
 */

/**
 * @author <NAME>
 * 
 *
 */
public class P1 {
	
	private boolean startFlag = false;	
	//private ArrayList<String> fileText = new ArrayList<String>();
	private ArrayList<Integer> occurList = new ArrayList<Integer>();	
	private ArrayList<Term> terms = new ArrayList<Term>();
	private ArrayList<Term> termIndex = new ArrayList<Term>();
	
	
	
	public ArrayList<Term>getTermList() {
		
		return terms;
	}
	
	public ArrayList<Term> readFile(String fileName,String docName) {
		String word = null;
		String wordPuncRemoved = null;
		String htmlRemoved = null;
		BinarySearch search = new BinarySearch();		
		
		
				
		try {
			Scanner read = new Scanner(new File(fileName));
			
			
			while(read.hasNext() ) {
				String temp = null;
				String tempTwo = null;
				word = read.next();
				boolean flag = false;
				
				//System.out.println(word);
				
				//System.out.println(word+" =");
				
				if(word.isEmpty() == false)
				{
					htmlRemoved = removeHTML(word).replaceAll("\\s+","");
					wordPuncRemoved = removePunctuation(htmlRemoved).toLowerCase();
					
					
						for(int i=0;i<wordPuncRemoved.length();i++) { 
							
{'fraction_non_alphanumeric': 0.13215859030837004, 'fraction_numerical': 0.0014684287812041115, 'mean_word_length': 3.2511700468018723, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7494492', 'n_tokens_mistral': 1032, 'n_tokens_neox': 828, 'n_words': 196}