<commit_before>package crawling;

import java.util.Set;
import java.util.regex.Pattern;

import org.apache.http.Header;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;

/**
 * Created by Thagus on 14/11/16.
 */
public class Crawler extends WebCrawler{
    private static final Pattern IMAGE_EXTENSIONS = Pattern.compile(".*\\.(bmp|gif|jpg|png)$");
    private static final Pattern SOUND_EXTENSIONS = Pattern.compile(".*\\.(mp3|ogg)$");
    private static final Pattern COMPRESSED_EXTENSIONS = Pattern.compile(".*\\.(zip|rar|gz|tar)$");
    private static final Pattern OTHER_EXTENSIONS = Pattern.compile(".*\\.(css|js)$");

    /**
     * You should implement this function to specify whether the given url
     * should be crawled or not (based on your crawling logic).
     */
    @Override
    public boolean shouldVisit(Page referringPage, WebURL url) {
        String href = url.getURL().toLowerCase();
        //Ignore the url if has the defined extensions
        return !(IMAGE_EXTENSIONS.matcher(href).matches() ||
                SOUND_EXTENSIONS.matcher(href).matches() ||
                COMPRESSED_EXTENSIONS.matcher(href).matches() ||
                OTHER_EXTENSIONS.matcher(href).matches());
    }

    /**
     * This function is called when a page is fetched and ready to be processed
     * by your program.
     */
    @Override
    public void visit(Page page) {
        String url = page.getWebURL().getURL();

        logger.info("URL: {}", url);

        if (page.getParseData() instanceof HtmlParseData) {
            HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
            String text = htmlParseData.getText();
            String html = htmlParseData.getHtml();

            logger.info("Title: {}", htmlParseData.getTitle());

            logger.info("Text length: {}", text.length());
            logger.info("Html length: {}", html.length());
        }


        logger.info("=============");
    }
}<commit_msg>Crawler feeds database<commit_after>package crawling;

import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
import dataObjects.Document;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;
import model.ModelDatabase;

/**
 * Created by Thagus on 14/11/16.
 */
public class Crawler extends WebCrawler{
    private static final Pattern IMAGE_EXTENSIONS = Pattern.compile(".*\\.(bmp|gif|jpg|png)$");
    private static final Pattern SOUND_EXTENSIONS = Pattern.compile(".*\\.(mp3|ogg)$");
    private static final Pattern COMPRESSED_EXTENSIONS = Pattern.compile(".*\\.(zip|rar|gz|tar)$");
    private static final Pattern OTHER_EXTENSIONS = Pattern.compile(".*\\.(css|js)$");
    private ModelDatabase db = ModelDatabase.instance();

    /**
     * You should implement this function to specify whether the given url
     * should be crawled or not (based on your crawling logic).
     */
    @Override
    public boolean shouldVisit(Page referringPage, WebURL url) {
        String href = url.getURL().toLowerCase();
        //Ignore the url if has the defined extensions
        return !(IMAGE_EXTENSIONS.matcher(href).matches() ||
                SOUND_EXTENSIONS.matcher(href).matches() ||
                COMPRESSED_EXTENSIONS.matcher(href).matches() ||
                OTHER_EXTENSIONS.matcher(href).matches());
    }

    /**
     * This function is called when a page is fetched and ready to be processed
     * by your program.
     */
    @Override
    public void visit(Page page) {
        String url = page.getWebURL().getURL();

        logger.info("URL: {}", url);

        if (page.getParseData() instanceof HtmlParseData) {
            HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
            String text = htmlParseData.getText();
            String title = htmlParseData.getTitle();

            String html = htmlParseData.getHtml();

            Document document = new Document(url, title, text);
            feedDatabase(document);

            logger.info("Title: {}", title);
            logger.info("Text length: {}", text.length());
            logger.info("Html length: {}", html.length());
        }

        logger.info("=============");
    }

    /**
     * Counts the words on the document in order to calculate the TF of each term in the document
     * Adds the document and its terms to the database
     *
     * @param doc The document to be added
     */
    private void feedDatabase(Document doc){
        boolean insertCheck = db.opDocuments.addDocument(doc.getUrl(), doc.getTitle(), doc.getText());

        //The document was correctly added if there is no duplicate key
        if(insertCheck) {
            HashMap<String, Integer> wordCountLocal = doc.countWords();    //Request the count of words for the inserted document
            for(Map.Entry<String, Integer> termEntry : wordCountLocal.entrySet()){      //For each term obtained from the document (Key String is the term, Value Integer is the TF)
                //Write the term, with the document id and the TF
                db.opInvertedIndex.addTerm(doc.getUrl(), termEntry.getKey(), termEntry.getValue());
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.1260132645541636, 'fraction_numerical': 0.0040530582166543845, 'mean_word_length': 2.809824561403509, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5813485', 'n_tokens_mistral': 1580, 'n_tokens_neox': 1529, 'n_words': 410}