<commit_before>package org.jbake.app;

import com.orientechnologies.orient.core.record.impl.ODocument;
import org.apache.commons.configuration.CompositeConfiguration;
import org.apache.commons.io.FilenameUtils;
import org.jbake.app.configuration.JBakeConfiguration;
import org.jbake.app.configuration.JBakeConfigurationFactory;
import org.jbake.model.DocumentModel;
import org.jbake.model.DocumentStatus;
import org.jbake.model.DocumentTypes;
import org.jbake.model.ModelAttributes;
import org.jbake.util.HtmlUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Date;

/**
 * Crawls a file system looking for content.
 *
 * @author <NAME> <a href="mailto:<EMAIL>"><EMAIL></a>
 */
public class Crawler {

    private static final Logger logger = LoggerFactory.getLogger(Crawler.class);
    private final ContentStore db;
    private JBakeConfiguration config;
    private Parser parser;

    /**
     * @param db     Database instance for content
     * @param source Base directory where content directory is located
     * @param config Project configuration
     * @deprecated Use {@link #Crawler(ContentStore, JBakeConfiguration)} instead.
     * <p>
     * Creates new instance of Crawler.
     */
    @Deprecated
    public Crawler(ContentStore db, File source, CompositeConfiguration config) {
        this.db = db;
        this.config = new JBakeConfigurationFactory().createDefaultJbakeConfiguration(source, config);
        this.parser = new Parser(this.config);
    }

    /**
     * Creates new instance of Crawler.
     *
     * @param db     Database instance for content
     * @param config Project configuration
     */
    public Crawler(ContentStore db, JBakeConfiguration config) {
        this.db = db;
        this.config = config;
        this.parser = new Parser(config);
    }

    public void crawl() {
        crawl(config.getContentFolder());

        logger.info("Content detected:");
        for (String docType : DocumentTypes.getDocumentTypes()) {
            long count = db.getDocumentCount(docType);
            if (count > 0) {
                logger.info("Parsed {} files of type: {}", count, docType);
            }
        }

    }

    /**
     * Crawl all files and folders looking for content.
     *
     * @param path Folder to start from
     */
    private void crawl(File path) {
        File[] contents = path.listFiles(FileUtil.getFileFilter());
        if (contents != null) {
            Arrays.sort(contents);
            for (File sourceFile : contents) {
                if (sourceFile.isFile()) {
                    crawlFile(sourceFile);
                } else if (sourceFile.isDirectory()) {
                    crawl(sourceFile);
                }
            }
        }
    }

    private void crawlFile(File sourceFile) {
        StringBuilder sb = new StringBuilder();
        sb.append("Processing [").append(sourceFile.getPath()).append("]... ");
        String sha1 = buildHash(sourceFile);
        String uri = buildURI(sourceFile);
        boolean process = true;
        for (String docType : DocumentTypes.getDocumentTypes()) {
            DocumentStatus status = findDocumentStatus(docType, uri, sha1);
            if (status == DocumentStatus.UPDATED) {
                sb.append(" : modified ");
                db.deleteContent(docType, uri);
            } else if (status == DocumentStatus.IDENTICAL) {
                sb.append(" : same ");
                process = false;
            } else if (DocumentStatus.NEW == status) {
                sb.append(" : new ");
            }
            if (!process || status != DocumentStatus.NEW) {
                break;
            }
        }
        if (process) { // new or updated
            processSourceFile(sourceFile, sha1, uri);
        }
        logger.info("{}", sb);
    }

    private String buildHash(final File sourceFile) {
        String sha1;
        try {
            sha1 = FileUtil.sha1(sourceFile);
        } catch (Exception e) {
            logger.error("unable to build sha1 hash for source file '{}'", sourceFile);
            sha1 = "";
        }
        return sha1;
    }

    private String buildURI(final File sourceFile) {<commit_msg>fix broken status concatenation<commit_after>package org.jbake.app;

import com.orientechnologies.orient.core.record.impl.ODocument;
import org.apache.commons.configuration.CompositeConfiguration;
import org.apache.commons.io.FilenameUtils;
import org.jbake.app.configuration.JBakeConfiguration;
import org.jbake.app.configuration.JBakeConfigurationFactory;
import org.jbake.model.DocumentModel;
import org.jbake.model.DocumentStatus;
import org.jbake.model.DocumentTypes;
import org.jbake.model.ModelAttributes;
import org.jbake.util.HtmlUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Date;

/**
 * Crawls a file system looking for content.
 *
 * @author <NAME> <a href="mailto:<EMAIL>"><EMAIL></a>
 */
public class Crawler {

    private static final Logger logger = LoggerFactory.getLogger(Crawler.class);
    private final ContentStore db;
    private JBakeConfiguration config;
    private Parser parser;

    /**
     * @param db     Database instance for content
     * @param source Base directory where content directory is located
     * @param config Project configuration
     * @deprecated Use {@link #Crawler(ContentStore, JBakeConfiguration)} instead.
     * <p>
     * Creates new instance of Crawler.
     */
    @Deprecated
    public Crawler(ContentStore db, File source, CompositeConfiguration config) {
        this.db = db;
        this.config = new JBakeConfigurationFactory().createDefaultJbakeConfiguration(source, config);
        this.parser = new Parser(this.config);
    }

    /**
     * Creates new instance of Crawler.
     *
     * @param db     Database instance for content
     * @param config Project configuration
     */
    public Crawler(ContentStore db, JBakeConfiguration config) {
        this.db = db;
        this.config = config;
        this.parser = new Parser(config);
    }

    public void crawl() {
        crawl(config.getContentFolder());

        logger.info("Content detected:");
        for (String docType : DocumentTypes.getDocumentTypes()) {
            long count = db.getDocumentCount(docType);
            if (count > 0) {
                logger.info("Parsed {} files of type: {}", count, docType);
            }
        }

    }

    /**
     * Crawl all files and folders looking for content.
     *
     * @param path Folder to start from
     */
    private void crawl(File path) {
        File[] contents = path.listFiles(FileUtil.getFileFilter());
        if (contents != null) {
            Arrays.sort(contents);
            for (File sourceFile : contents) {
                if (sourceFile.isFile()) {
                    crawlFile(sourceFile);
                } else if (sourceFile.isDirectory()) {
                    crawl(sourceFile);
                }
            }
        }
    }

    private void crawlFile(File sourceFile) {
        StringBuilder sb = new StringBuilder();
        sb.append("Processing [").append(sourceFile.getPath()).append("]... ");
        String sha1 = buildHash(sourceFile);
        String uri = buildURI(sourceFile);
        boolean process = true;
        DocumentStatus status = DocumentStatus.NEW;

        for (String docType : DocumentTypes.getDocumentTypes()) {
            status = findDocumentStatus(docType, uri, sha1);
            if ( status == null ) continue;
            if (status == DocumentStatus.UPDATED) {
                sb.append(" : modified ");
                db.deleteContent(docType, uri);
                break;
            } else if (status == DocumentStatus.IDENTICAL) {
                sb.append(" : same ");
                process = false;
            }
            if (!process || status != DocumentStatus.NEW) {
                break;
            }
        }
        if (process) { // new or updated
            if (status == DocumentStatus.NEW) {
                sb.append(" : new ");
            }
            processSourceFile(sourceFile, sha1, uri);
        }
        logger.info("{}", sb);
    }

    private String buildHash(final File sourceFile) {
        String sha1;
        try {
            sha1 = FileUtil.sha1(sourceFile);
        } catch (Exception e) {
            logger.error("unable to build sha1 hash for source file '{}'", sourceFile);
            sha1 = "";
        }
        return sha1;
    }

    private String buildURI(final File sourceFile) {
{'fraction_non_alphanumeric': 0.10641891891891891, 'fraction_numerical': 0.002702702702702703, 'mean_word_length': 2.26747608535688, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3429268', 'n_tokens_mistral': 2436, 'n_tokens_neox': 2366, 'n_words': 599}