<commit_before>package de.jeha.kame.crawler.core;

import de.jeha.kame.crawler.core.httpclient.HttpClientFactory;
import de.jeha.kame.crawler.core.types.CrawlResult;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

/**
 * @author <EMAIL>
 */
public class Crawler {

    private static final Logger LOG = LoggerFactory.getLogger(Crawler.class);

    private static final String DEFAULT_USER_AGENT = "Mozilla/5.0 (compatible; Kamebot/0.1)";

    private final String url;
    private final String userAgent;

    public Crawler(String url) {
        this(url, DEFAULT_USER_AGENT);
    }

    public Crawler(String url, String userAgent) {
        this.url = url;
        this.userAgent = userAgent;
    }

    public CrawlResult execute() throws IOException {
        LOG.info("Crawling: '{}'", url);

        HttpGet httpGet = new HttpGet(url);
        httpGet.setHeader("User-Agent", userAgent);

        StopWatch stopWatch = new StopWatch();
        stopWatch.start();<commit_msg>javadoc<commit_after>package de.jeha.kame.crawler.core;

import de.jeha.kame.crawler.core.httpclient.HttpClientFactory;
import de.jeha.kame.crawler.core.types.CrawlResult;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

/**
 * @author <EMAIL>
 */
public class Crawler {

    private static final Logger LOG = LoggerFactory.getLogger(Crawler.class);

    private static final String DEFAULT_USER_AGENT = "Mozilla/5.0 (compatible; Kamebot/0.1)";

    private final String url;
    private final String userAgent;

    /**
     * A crawler with default user agent '{@value de.jeha.kame.crawler.core.Crawler#DEFAULT_USER_AGENT}'.
     *
     * @param url url to crawl
     */
    public Crawler(String url) {
        this(url, DEFAULT_USER_AGENT);
    }

    /**
     * @param url       url to crawl
     * @param userAgent custom user agent string
     */
    public Crawler(String url, String userAgent) {
        this.url = url;
        this.userAgent = userAgent;
    }

    /**
     * Crawl the url.
     *
     * @return crawl result
     * @throws IOException
     */
    public CrawlResult execute() throws IOException {
        LOG.info("Crawling: '{}'", url);

        HttpGet httpGet = new HttpGet(url);
        httpGet.setHeader("User-Agent", userAgent);

        StopWatch stopWatch = new StopWatch();
        stopWatch.start();
{'fraction_non_alphanumeric': 0.11489088575096278, 'fraction_numerical': 0.004492939666238768, 'mean_word_length': 3.3472803347280333, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1567231', 'n_tokens_mistral': 1012, 'n_tokens_neox': 997, 'n_words': 213}