<commit_before>package com.grayben.riskExtractor.htmlScorer;

import java.io.File;
import java.io.IOException;
import java.util.ListIterator;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.safety.Whitelist;
import org.jsoup.select.Elements;
import org.jsoup.select.NodeTraversor;

public class TreeHtmlScorer implements HtmlScorer {
	
	NodeTraversor nt;
	ScoringAndFlatteningNodeVisitor nv;
	

	/**
	 * @param parser
	 * @param documentScorer
	 */
	public TreeHtmlScorer() {
		super();
		this.nv = new ScoringAndFlatteningNodeVisitor();
		this.nt = new NodeTraversor(this.nv);
	}
	
	private ScoredText traverse(Node node)
			throws NullPointerException {
		if(node != null){
			nt.traverse(node);
		} else {
			throw new NullPointerException();
		}
		return nv.flatText;
	}


	@Override
	public ScoredText scoreHtml(File htmlFile, String charsetName) {
		Document doc = parseHtmlFile(htmlFile, charsetName);
		Element body = getHtmlBody(doc);
		return traverse(body);
	}
	
	private Element getHtmlBody(Document doc) {
		Elements elements = doc.select("html > body");
		ListIterator<Element> iterator = elements.listIterator();
		int i = 1;
		while (iterator.hasNext()){
			System.out.println(i++);
			System.out.println(iterator.next().html());	
		}

		//TODO: implement
		
		return null;
	}

	@Override
	public ScoredText scoreHtml(String url) {
		Document doc = parseHtmlUrl(url);
		return traverse(doc);
	}
	
	private Document parseHtmlUrl(String url) {
		Document doc = null;
		try {
			doc = Jsoup.connect(url).get();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return doc;<commit_msg>Add TODO<commit_after>package com.grayben.riskExtractor.htmlScorer;

import java.io.File;
import java.io.IOException;
import java.util.ListIterator;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.safety.Whitelist;
import org.jsoup.select.Elements;
import org.jsoup.select.NodeTraversor;

public class TreeHtmlScorer implements HtmlScorer {
	
	NodeTraversor nt;
	ScoringAndFlatteningNodeVisitor nv;
	

	/**
	 * @param parser
	 * @param documentScorer
	 */
	public TreeHtmlScorer() {
		super();
		this.nv = new ScoringAndFlatteningNodeVisitor();
		this.nt = new NodeTraversor(this.nv);
	}
	
	private ScoredText traverse(Node node)
			throws NullPointerException {
		if(node != null){
			nt.traverse(node);
		} else {
			throw new NullPointerException();
		}
		return nv.flatText;
	}


	@Override
	public ScoredText scoreHtml(File htmlFile, String charsetName) {
		Document doc = parseHtmlFile(htmlFile, charsetName);
		Element body = getHtmlBody(doc);
		return traverse(body);
	}
	
	private Element getHtmlBody(Document doc) {
		Elements elements = doc.select("html > body");
		ListIterator<Element> iterator = elements.listIterator();
		int i = 1;
		while (iterator.hasNext()){
			System.out.println(i++);
			System.out.println(iterator.next().html());	
		}

		//TODO: implement
		
		return null;
	}

	@Override
	public ScoredText scoreHtml(String url) {
		//TODO: use JSoup to get a Connection.Response so can remove SEC head
		Document doc = parseHtmlUrl(url);
		return traverse(doc);
	}
	
	private Document parseHtmlUrl(String url) {
		Document doc = null;
		try {
			doc = Jsoup.connect(url).get();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return doc;
{'fraction_non_alphanumeric': 0.11584129742253113, 'fraction_numerical': 0.0005792064871126556, 'mean_word_length': 4.225416036308624, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1374361', 'n_tokens_mistral': 1255, 'n_tokens_neox': 1116, 'n_words': 258}