<commit_before>package org.jbake.app;

import java.io.File;
import java.util.Arrays;
import java.util.Date;
import java.util.Map;

import org.apache.commons.configuration.CompositeConfiguration;
import org.apache.commons.io.FilenameUtils;
import org.jbake.app.ConfigUtil.Keys;
import org.jbake.app.Crawler.Attributes.Status;
import org.jbake.model.DocumentAttributes;
import org.jbake.model.DocumentStatus;
import org.jbake.model.DocumentTypes;
import org.jbake.util.HtmlUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.orientechnologies.orient.core.record.impl.ODocument;

/**
 * Crawls a file system looking for content.
 *
 * @author <NAME> <a href="mailto:<EMAIL>"><EMAIL></a>
 */
public class Crawler {
    public interface Attributes {
        /**
         * Possible values of the {@link Attributes#STATUS} property
         *
         * @author ndx
         */
        interface Status {
            String PUBLISHED_DATE = "published-date";
            String PUBLISHED = "published";
            String DRAFT = "draft";
        }

        String DATE = "date";
        String STATUS = "status";
        String TYPE = "type";
        String TITLE = "title";
        String URI = "uri";
        String FILE = "file";
        String TAGS = "tags";
        String TAG = "tag";
        String ROOTPATH = "rootpath";
        String ID = "id";
        String NO_EXTENSION_URI = "noExtensionUri";
        String ALLTAGS = "alltags";
        String PUBLISHED_DATE = "published_date";
        String BODY = "body";
    }

    private static final Logger LOGGER = LoggerFactory.getLogger(Crawler.class);

    private CompositeConfiguration config;
    private Parser parser;
    private final ContentStore db;
    private String contentPath;

    /**
     * Creates new instance of Crawler.
     *
     * @param db     Database instance for content
     * @param source Base directory where content directory is located
     * @param config Project configuration
     */
    public Crawler(ContentStore db, File source, CompositeConfiguration config) {
        this.db = db;
        this.config = config;
        this.contentPath = FilenameUtils.concat(source.getAbsolutePath(), config.getString(ConfigUtil.Keys.CONTENT_FOLDER));
        this.parser = new Parser(config, contentPath);
    }
<commit_msg>Re-added missing DB attribute - lost during merge.<commit_after>package org.jbake.app;

import java.io.File;
import java.util.Arrays;
import java.util.Date;
import java.util.Map;

import org.apache.commons.configuration.CompositeConfiguration;
import org.apache.commons.io.FilenameUtils;
import org.jbake.app.ConfigUtil.Keys;
import org.jbake.app.Crawler.Attributes.Status;
import org.jbake.model.DocumentAttributes;
import org.jbake.model.DocumentStatus;
import org.jbake.model.DocumentTypes;
import org.jbake.util.HtmlUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.orientechnologies.orient.core.record.impl.ODocument;

/**
 * Crawls a file system looking for content.
 *
 * @author <NAME> <a href="mailto:<EMAIL>"><EMAIL></a>
 */
public class Crawler {
    public interface Attributes {
        /**
         * Possible values of the {@link Attributes#STATUS} property
         *
         * @author ndx
         */
        interface Status {
            String PUBLISHED_DATE = "published-date";
            String PUBLISHED = "published";
            String DRAFT = "draft";
        }

        String DATE = "date";
        String STATUS = "status";
        String TYPE = "type";
        String TITLE = "title";
        String URI = "uri";
        String FILE = "file";
        String TAGS = "tags";
        String TAG = "tag";
        String ROOTPATH = "rootpath";
        String ID = "id";
        String NO_EXTENSION_URI = "noExtensionUri";
        String ALLTAGS = "alltags";
        String PUBLISHED_DATE = "published_date";
        String BODY = "body";
        String DB = "db";
    }

    private static final Logger LOGGER = LoggerFactory.getLogger(Crawler.class);

    private CompositeConfiguration config;
    private Parser parser;
    private final ContentStore db;
    private String contentPath;

    /**
     * Creates new instance of Crawler.
     *
     * @param db     Database instance for content
     * @param source Base directory where content directory is located
     * @param config Project configuration
     */
    public Crawler(ContentStore db, File source, CompositeConfiguration config) {
        this.db = db;
        this.config = config;
        this.contentPath = FilenameUtils.concat(source.getAbsolutePath(), config.getString(ConfigUtil.Keys.CONTENT_FOLDER));
        this.parser = new Parser(config, contentPath);
    }

{'fraction_non_alphanumeric': 0.10192837465564739, 'fraction_numerical': 0.000847637211273575, 'mean_word_length': 2.7729816147082333, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '513910', 'n_tokens_mistral': 1380, 'n_tokens_neox': 1331, 'n_words': 353}