<commit_before>/**
 * 
 */
package de.unima.peoplesearch.extraction;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;

import org.jsoup.Jsoup;

import de.unima.peoplesearch.database.PersonDAO;

/**
 * @author Michi
 * 
 */
public class ExtractorTest {

	/**
	 * @param args
	 * @throws IOException
	 */
	public static void main(String[] args) throws IOException {
		// String url =
		// "http://dws.informatik.uni-mannheim.de/en/people/professors/prof-dr-simone-paolo-ponzetto/";
		// Document doc = Jsoup.parse(new URL(url).openStream(), "ISO-8859-1",
		// url);
		// Document doc = Jsoup.connect(url).get();
		// doc.outputSettings().charset("UTF-8");

		ArrayList<Person> persons = new ArrayList<Person>();
		/*
		 * List<File> files = Arrays.asList(new File("data/pages").listFiles());
		 * 
		 * for(File file: files) { System.out.println(file.getAbsolutePath());
		 * String input = Resources.toString(file.toURL(), Charsets.UTF_8);
		 * input = input.replaceAll("(?i)<br[^>]*>", "br2n"); //
		 * System.out.println(input); Person newPerson = new Person();
		 * newPerson.tryExtract(input, "http://example.com");
		 * persons.add(newPerson); }
		 */
		HtmlExtractor ex = new HtmlExtractor();
		ArrayList<String> list = ex.readLinks();
		final int MAX = 1000;
		int counter = 1;
		int timeoutCounter = 0;
		for (String s : list) {
			try {
				if (counter >= MAX) break;
				System.out.println("GET " + counter++ + " " + s);
				String input = Jsoup.connect(s).ignoreContentType(true).ignoreHttpErrors(true).timeout(1000).get().toString();
				input = input.replaceAll("(?i)<br[^>]*>", " br2n ");
				Person newPerson = new Person();
				String baseUrl = s.replaceAll("/[a-zA-Z0-9]+\\.htm[l]?", "/");
				
				newPerson.tryExtract(input, baseUrl);
				
				if (newPerson.isPerson()) {
					if (newPerson.hasDuplicate(persons) == false) {
						newPerson.setUrl(s);
						persons.add(newPerson);
//						PersonDAO.savePerson(newPerson);
					} else {
						Person duplicate = newPerson.getDuplicate(persons);
						if (newPerson.getFieldsNotNull() > duplicate.getFieldsNotNull()){
							newPerson.setUrl(s);
							persons.add(newPerson);
							persons.remove(duplicate);
//							PersonDAO.savePerson(newPerson);
//							PersonDAO.deletePerson(duplicate);
						} else continue;
					}
				}
				
//				System.err.println(newPerson.getImageUrl());
			} catch (Exception e) {
				timeoutCounter++;
				e.printStackTrace();
			}
		}

		int cPerson = 0;
		int cFN = 0;
		int cLN = 0;
		int cTitle = 0;
		int cEmail = 0;
		int cZip = 0;
		int cStreet = 0;
		int cRoom = 0;
		int cPhone = 0;
		int cImage = 0;

		// Output
		System.out.println();
		System.out.println("Extracted people information: ");
		for (Person p : persons) {<commit_msg>commented in sql commands again<commit_after>/**
 * 
 */
package de.unima.peoplesearch.extraction;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;

import org.jsoup.Jsoup;

import de.unima.peoplesearch.database.PersonDAO;

/**
 * @author Michi
 * 
 */
public class ExtractorTest {

	/**
	 * @param args
	 * @throws IOException
	 */
	public static void main(String[] args) throws IOException {
		// String url =
		// "http://dws.informatik.uni-mannheim.de/en/people/professors/prof-dr-simone-paolo-ponzetto/";
		// Document doc = Jsoup.parse(new URL(url).openStream(), "ISO-8859-1",
		// url);
		// Document doc = Jsoup.connect(url).get();
		// doc.outputSettings().charset("UTF-8");

		ArrayList<Person> persons = new ArrayList<Person>();
		/*
		 * List<File> files = Arrays.asList(new File("data/pages").listFiles());
		 * 
		 * for(File file: files) { System.out.println(file.getAbsolutePath());
		 * String input = Resources.toString(file.toURL(), Charsets.UTF_8);
		 * input = input.replaceAll("(?i)<br[^>]*>", "br2n"); //
		 * System.out.println(input); Person newPerson = new Person();
		 * newPerson.tryExtract(input, "http://example.com");
		 * persons.add(newPerson); }
		 */
		HtmlExtractor ex = new HtmlExtractor();
		ArrayList<String> list = ex.readLinks();
		final int MAX = 1000;
		int counter = 1;
		int timeoutCounter = 0;
		for (String s : list) {
			try {
				if (counter >= MAX) break;
				System.out.println("GET " + counter++ + " " + s);
				String input = Jsoup.connect(s).ignoreContentType(true).ignoreHttpErrors(true).timeout(1000).get().toString();
				input = input.replaceAll("(?i)<br[^>]*>", " br2n ");
				Person newPerson = new Person();
				String baseUrl = s.replaceAll("/[a-zA-Z0-9]+\\.htm[l]?", "/");
				
				newPerson.tryExtract(input, baseUrl);
				
				if (newPerson.isPerson()) {
					if (newPerson.hasDuplicate(persons) == false) {
						newPerson.setUrl(s);
						persons.add(newPerson);
						PersonDAO.savePerson(newPerson);
					} else {
						Person duplicate = newPerson.getDuplicate(persons);
						if (newPerson.getFieldsNotNull() > duplicate.getFieldsNotNull()){
							newPerson.setUrl(s);
							persons.add(newPerson);
							persons.remove(duplicate);
							PersonDAO.savePerson(newPerson);
							PersonDAO.deletePerson(duplicate);
						} else continue;
					}
				}
				
//				System.err.println(newPerson.getImageUrl());
			} catch (Exception e) {
				timeoutCounter++;
				e.printStackTrace();
			}
		}

		int cPerson = 0;
		int cFN = 0;
		int cLN = 0;
		int cTitle = 0;
		int cEmail = 0;
		int cZip = 0;
		int cStreet = 0;
		int cRoom = 0;
		int cPhone = 0;
		int cImage = 0;

		// Output
		System.out.println();
		System.out.println("Extracted people information: ");
		for (Person p : persons) {
{'fraction_non_alphanumeric': 0.16864437252786768, 'fraction_numerical': 0.011147069399496584, 'mean_word_length': 3.1921627731725697, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6989492', 'n_tokens_mistral': 2291, 'n_tokens_neox': 1863, 'n_words': 364}