<commit_before>package at.fhj.itm.pswe.pagecrawler.linkcrawler;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.util.Calendar;
import java.util.Date;
import java.util.concurrent.TimeUnit;

import org.json.JSONObject;

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;

public class Init_LinkCrawler {

	private String filename, crawlStorageFolder, path_to_file;
	private Calendar cal;

	public Init_LinkCrawler(String url, int depth) throws Exception {

		// Safe starting time of Crawler
		cal = Calendar.getInstance();
		Date start_date = cal.getTime();

		// Add to Project Root folders result/crawl where our result.txt is
		// saved
		crawlStorageFolder = "./result/crawl/";
		int numberOfCrawlers = 10;

		CrawlConfig config = new CrawlConfig();
		config.setCrawlStorageFolder(crawlStorageFolder);
		config.setPolitenessDelay(25);
		config.setMaxDepthOfCrawling(depth);

		/*
		 * Instantiate the controller for this crawl.
		 */
		PageFetcher pageFetcher = new PageFetcher(config);
		RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
		RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
		CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);

		/*
		 * For each crawl, you need to add some seed urls. These are the first
		 * URLs that are fetched and then the crawler starts following links
		 * which are found in these pages
		 */
		controller.addSeed(url);

		cal = Calendar.getInstance();

		filename = url.replace(".", "_").replace("http://", "") + "_" + (cal.get(Calendar.MONTH) + 1) + "_"
				+ cal.get(Calendar.DAY_OF_MONTH) + "_" + cal.get(Calendar.YEAR) + "-" + cal.get(Calendar.HOUR) + "_"
				+ cal.get(Calendar.MINUTE) + ".txt";
		path_to_file = crawlStorageFolder + filename;

		JSONObject obj = new JSONObject();<commit_msg>Fixed Crawler Error with "Subfolder-URLs"<commit_after>package at.fhj.itm.pswe.pagecrawler.linkcrawler;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.util.Calendar;
import java.util.Date;
import java.util.concurrent.TimeUnit;

import org.json.JSONObject;

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;

public class Init_LinkCrawler {

	private String filename, crawlStorageFolder, path_to_file;
	private Calendar cal;

	public Init_LinkCrawler(String url, int depth) throws Exception {

		// Safe starting time of Crawler
		cal = Calendar.getInstance();
		Date start_date = cal.getTime();

		// Add to Project Root folders result/crawl where our result.txt is
		// saved
		crawlStorageFolder = "./result/crawl/";
		int numberOfCrawlers = 10;

		CrawlConfig config = new CrawlConfig();
		config.setCrawlStorageFolder(crawlStorageFolder);
		config.setPolitenessDelay(25);
		config.setMaxDepthOfCrawling(depth);

		/*
		 * Instantiate the controller for this crawl.
		 */
		PageFetcher pageFetcher = new PageFetcher(config);
		RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
		RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
		CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);

		/*
		 * For each crawl, you need to add some seed urls. These are the first
		 * URLs that are fetched and then the crawler starts following links
		 * which are found in these pages
		 */
		controller.addSeed(url);

		cal = Calendar.getInstance();

		filename = url.replace(".", "_").replace("http://", "").replace("/",  "_") + "_" + (cal.get(Calendar.MONTH) + 1) + "_"
				+ cal.get(Calendar.DAY_OF_MONTH) + "_" + cal.get(Calendar.YEAR) + "-" + cal.get(Calendar.HOUR) + "_"
				+ cal.get(Calendar.MINUTE) + ".txt";
		path_to_file = crawlStorageFolder + filename;

		JSONObject obj = new JSONObject();
{'fraction_non_alphanumeric': 0.11384399156711174, 'fraction_numerical': 0.004684937924572499, 'mean_word_length': 4.618421052631579, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '532527', 'n_tokens_mistral': 1567, 'n_tokens_neox': 1428, 'n_words': 314}