<commit_before>/**
 * 
 */
package ch.ice.controller;

import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;

import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.poi.EncryptedDocumentException;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.json.JSONArray;
import org.json.JSONObject;

import ch.ice.controller.file.ExcelParser;
import ch.ice.controller.file.ExcelWriter;
import ch.ice.controller.web.BingSearchEngine;
import ch.ice.controller.web.WebCrawler;
import ch.ice.exceptions.IllegalFileExtensionException;
import ch.ice.model.Customer;
import ch.ice.utils.JSONUtil;

/**
 * @author Oliver
 *
 */
public class MainController {
	private static final Logger logger = LogManager
			.getLogger(MainController.class.getName());
	ExcelParser excelParserInstance;

	public static File file;
	public static LinkedList<Customer> customerList;
	public static int i;
	public static String progressText;

	private Integer limitSearchResults = 4;

	public void startMainController() {

		PropertiesConfiguration config;
		List<String> metaTagElements = new ArrayList<String>();

		// For testing if used without GUI
		if (file == null) {
			customerList = retrieveCustomerFromFile(new File("posTest.xlsx"));
		} else {
			customerList = retrieveCustomerFromFile(file);

			// retrieve all customers from file
			logger.info("Retrieve Customers from File "
					+ file.getAbsolutePath());
		}

		// Core settings
		boolean isSearchAvail = false;
		URL defaultUrl = null;

		/*
		 * Load Configuration File
		 */
		try {
			config = new PropertiesConfiguration("conf/app.properties");

			isSearchAvail = config.getBoolean("core.search.isEnabled");
			defaultUrl = new URL(config.getString("core.search.defaultUrl"));
			// this.limitSearchResults =
			// config.getInteger("searchEngine.bing.limitSearchResults", 15);

			metaTagElements = Arrays.asList(config
					.getStringArray("crawler.searchForMetaTags"));
		} catch (ConfigurationException | MalformedURLException e) {
			logger.error("Faild to load config file");
			System.out.println(e.getLocalizedMessage());
			e.printStackTrace();
		}

		WebCrawler wc = new WebCrawler();

		for (Customer customer : customerList) {
			i++;

			// only search via SearchEngine if search is enabled. Disable search
			// for testing purpose
			if (isSearchAvail) {
				// Add url for customer
				URL retrivedUrl = searchForUrl(customer);
				customer.getWebsite().setUrl(retrivedUrl);
				progressText = "Gathering data at: " + retrivedUrl.toString();

			} else {
				customer.getWebsite().setUrl(defaultUrl);
			}

			// add metadata
			try {
				wc.connnect(customer.getWebsite().getUrl().toString());
				customer.getWebsite().setMetaTags(
						wc.getMetaTags(metaTagElements));
			} catch (IOException e) {
				e.printStackTrace();
			}

			logger.info(customer.getWebsite().toString());
		}

		/*
		 * Write every enhanced customer object into a new file
		 */
		this.startWriter(customerList);

		logger.info("end");
	}

	public URL searchForUrl(Customer c) {

		ArrayList<String> params = new ArrayList<String>();
		params.add(c.getFullName().toLowerCase());
		params.add(c.getCountryName().toLowerCase());
		// params.add("loc:"+c.getCountryCode().toLowerCase()); -> delivers 0
		// results sometimes. we have to TEST this!!!!

		String query = BingSearchEngine.buildQuery(params);
		progressText = "Loockup on: " + query;

		logger.info("start searchEngine for URL with query: " + query);

		try {

			// Start Search
			JSONArray results = BingSearchEngine.Search(query,
					this.limitSearchResults);<commit_msg>comment on country code resolves bing search error<commit_after>/**
 * 
 */
package ch.ice.controller;

import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;

import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.poi.EncryptedDocumentException;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.json.JSONArray;
import org.json.JSONObject;

import ch.ice.controller.file.ExcelParser;
import ch.ice.controller.file.ExcelWriter;
import ch.ice.controller.web.BingSearchEngine;
import ch.ice.controller.web.WebCrawler;
import ch.ice.exceptions.IllegalFileExtensionException;
import ch.ice.model.Customer;
import ch.ice.utils.JSONUtil;

/**
 * @author Oliver
 *
 */
public class MainController {
	private static final Logger logger = LogManager
			.getLogger(MainController.class.getName());
	ExcelParser excelParserInstance;

	public static File file;
	public static LinkedList<Customer> customerList;
	public static int i;
	public static String progressText;

	private Integer limitSearchResults = 4;

	public void startMainController() {

		PropertiesConfiguration config;
		List<String> metaTagElements = new ArrayList<String>();

		// For testing if used without GUI
		if (file == null) {
			customerList = retrieveCustomerFromFile(new File("posTest.xlsx"));
		} else {
			customerList = retrieveCustomerFromFile(file);

			// retrieve all customers from file
			logger.info("Retrieve Customers from File "
					+ file.getAbsolutePath());
		}

		// Core settings
		boolean isSearchAvail = false;
		URL defaultUrl = null;

		/*
		 * Load Configuration File
		 */
		try {
			config = new PropertiesConfiguration("conf/app.properties");

			isSearchAvail = config.getBoolean("core.search.isEnabled");
			defaultUrl = new URL(config.getString("core.search.defaultUrl"));
			// this.limitSearchResults =
			// config.getInteger("searchEngine.bing.limitSearchResults", 15);

			metaTagElements = Arrays.asList(config
					.getStringArray("crawler.searchForMetaTags"));
		} catch (ConfigurationException | MalformedURLException e) {
			logger.error("Faild to load config file");
			System.out.println(e.getLocalizedMessage());
			e.printStackTrace();
		}

		WebCrawler wc = new WebCrawler();

		for (Customer customer : customerList) {
			i++;

			// only search via SearchEngine if search is enabled. Disable search
			// for testing purpose
			if (isSearchAvail) {
				// Add url for customer
				URL retrivedUrl = searchForUrl(customer);
				customer.getWebsite().setUrl(retrivedUrl);
				progressText = "Gathering data at: " + retrivedUrl.toString();

			} else {
				customer.getWebsite().setUrl(defaultUrl);
			}

			// add metadata
			try {
				wc.connnect(customer.getWebsite().getUrl().toString());
				customer.getWebsite().setMetaTags(
						wc.getMetaTags(metaTagElements));
			} catch (IOException e) {
				e.printStackTrace();
			}

			logger.info(customer.getWebsite().toString());
		}

		/*
		 * Write every enhanced customer object into a new file
		 */
		this.startWriter(customerList);

		logger.info("end");
	}

	public URL searchForUrl(Customer c) {

		ArrayList<String> params = new ArrayList<String>();
		params.add(c.getFullName().toLowerCase());
		// params.add(c.getCountryName().toLowerCase());
		// params.add("loc:"+c.getCountryCode().toLowerCase()); -> delivers 0
		// results sometimes. we have to TEST this!!!!

		String query = BingSearchEngine.buildQuery(params);
		progressText = "Loockup on: " + query;

		logger.info("start searchEngine for URL with query: " + query);

		try {

			// Start Search
			JSONArray results = BingSearchEngine.Search(query,
					this.limitSearchResults);
{'fraction_non_alphanumeric': 0.11804853387259859, 'fraction_numerical': 0.001769464105156724, 'mean_word_length': 4.292976588628763, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1557722', 'n_tokens_mistral': 2601, 'n_tokens_neox': 2264, 'n_words': 518}