<commit_before>package com.grayben.riskExtractor.htmlScorer;

import java.io.File;
import java.io.IOException;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.jsoup.select.NodeTraversor;
import org.jsoup.select.NodeVisitor;

public class HtmlScorer {
	
	NodeTraversor nt;
	NodeVisitor nv;

	public static void main(String[] args) throws IOException {
		System.out.println("Starting parse");
		File input = null;
		Document doc = null;
		int numDocuments = 100;
		
		// 
					doc = Jsoup.connect("http://google.com/").get();
					Elements elements = doc.select("*");
					String toHtml = elements.text();
					System.out.print(toHtml);
					
					NodeVisitor flattener = new ScoringAndFlatteningNodeVisitor();
					NodeTraversor nt = new NodeTraversor(flattener);
					nt.traverse(doc);
		
		final long startTime = System.currentTimeMillis();
		for(int i = 1; i <= numDocuments; i++){
			input = new File("/Volumes/MBS Data/EDGAR-Form10K/filings/" + i + ".txt");
			doc = Jsoup.parse(input, "ASCII");
			elements = doc.select("*");
			String html = elements.text();
			System.out.print(html);
			
			
		}
		final long endTime = System.currentTimeMillis();
		final long dT = endTime - startTime;
		final long secondsElapsed = dT/1000;
		System.out.println("Finished parsing: " 
				+ numDocuments + " in " + secondsElapsed + " seconds.");
		
		
	}

}<commit_msg>Remove crap from htmlScorer<commit_after>package com.grayben.riskExtractor.htmlScorer;

import java.io.File;
import java.io.IOException;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.jsoup.select.NodeTraversor;
import org.jsoup.select.NodeVisitor;

public class HtmlScorer {
	
	NodeTraversor nt;
	NodeVisitor nv;

	public static void main(String[] args) throws IOException {
		Document doc = Jsoup.connect("http://google.com/").get();
	}
	
	
}
{'fraction_non_alphanumeric': 0.12845953002610966, 'fraction_numerical': 0.005221932114882507, 'mean_word_length': 3.8753180661577606, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 1, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '253235', 'n_tokens_mistral': 726, 'n_tokens_neox': 624, 'n_words': 139}