<commit_before>import sqlite3
import ctypes
import ConfigParser
import os, sys, subprocess
import jinja2
import string
from datetime import datetime

import dxr


# Please keep these config objects as simple as possible and in sync with
# docs/configuration.mkd. I'm well aware that this is not the most compact way
# of writing things, but it sure is doomed to fail when user forgets an important
# key. It's also fairly easy to extract default values, and config keys from
# this code, so enjoy.

class Config(object):
    """ Configuration for DXR """
    def __init__(self, configfile, **override):
        # Create parser with sane defaults
        parser = ConfigParser.ConfigParser({
            'dxrroot':          os.path.dirname(dxr.__file__),
            'plugin_folder':    "%(dxrroot)s/plugins",
            'nb_jobs':          "1",
            'temp_folder':      "/tmp/dxr-temp",
            'log_folder':       "%(temp_folder)s/logs",
            'template':         "%(dxrroot)s/templates",
            'wwwroot':          "/",
            'enabled_plugins':  "*",
            'disabled_plugins': " ",
            'directory_index':  ".dxr-directory-index.html",
            'generated_date':   datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
        })
        parser.read(configfile)

        # Set config values
        self.dxrroot          = parser.get('DXR', 'dxrroot',          False, override)
        self.plugin_folder    = parser.get('DXR', 'plugin_folder',    False, override)
        self.nb_jobs          = parser.get('DXR', 'nb_jobs',          False, override)
        self.temp_folder      = parser.get('DXR', 'temp_folder',      False, override)
        self.target_folder    = parser.get('DXR', 'target_folder',    False, override)
        self.log_folder       = parser.get('DXR', 'log_folder',       False, override)
        self.template_folder  = parser.get('DXR', 'template',         False, override)
        self.wwwroot          = parser.get('DXR', 'wwwroot',          False, override)
        self.enabled_plugins  = parser.get('DXR', 'enabled_plugins',  False, override)
        self.disabled_plugins = parser.get('DXR', 'disabled_plugins', False, override)
        self.directory_index  = parser.get('DXR', 'directory_index',  False, override)
        self.generated_date   = parser.get('DXR', 'generated_date',   False, override)
        # Set configfile
        self.configfile       = configfile
        self.trees            = []
        # Set template parameters (using new parser to avoid defaults)
        tmp_cfg = ConfigParser.ConfigParser()
        tmp_cfg.read(configfile)
        self.template_parameters = dict(tmp_cfg.items('Template'))

        # Read all plugin_ keys
        for key, value in tmp_cfg.items('DXR'):
            if key.startswith('plugin_'):
                setattr(self, key, value)

        # Render all paths absolute
        self.dxrroot          = os.path.abspath(self.dxrroot)
        self.plugin_folder    = os.path.abspath(self.plugin_folder)
        self.temp_folder      = os.path.abspath(self.temp_folder)
        self.log_folder       = os.path.abspath(self.log_folder)
        self.target_folder    = os.path.abspath(self.target_folder)
        self.template_folder  = os.path.abspath(self.template_folder)

        # Make sure wwwroot doesn't end in /
        if self.wwwroot[-1] == '/':
            self.wwwroot = self.wwwroot[:-1]

        # Convert disabled plugins to a list
        if self.disabled_plugins == "*":
            self.disabled_plugins = os.listdir(self.plugin_folder)
        else:
            self.disabled_plugins = self.disabled_plugins.split()

        # Convert enabled plugins to a list
        if self.enabled_plugins == "*":
            self.enabled_plugins = [p for p in os.listdir(self.plugin_folder)
                                                                if  p not in self.disabled_plugins]
        else:
            self.enabled_plugins = self.enabled_plugins.split()

        # Test for conflicting plugins settings
        if any((p in self.disabled_plugins for p in self.enabled_plugins)):
            msg = "Plugin: '%s' is both enabled and disabled in '%s'"
            print >> sys.stderr, msg % (p, name)
            sys.exit(1)
<commit_msg>Ignore non-directories when scanning for plugins.<commit_after>import sqlite3
import ctypes
import ConfigParser
import os
from os.path import isdir
import jinja2
import string
import subprocess
import sys
from datetime import datetime

import dxr


# Please keep these config objects as simple as possible and in sync with
# docs/configuration.mkd. I'm well aware that this is not the most compact way
# of writing things, but it sure is doomed to fail when user forgets an important
# key. It's also fairly easy to extract default values, and config keys from
# this code, so enjoy.

class Config(object):
    """ Configuration for DXR """
    def __init__(self, configfile, **override):
        # Create parser with sane defaults
        parser = ConfigParser.ConfigParser({
            'dxrroot':          os.path.dirname(dxr.__file__),
            'plugin_folder':    "%(dxrroot)s/plugins",
            'nb_jobs':          "1",
            'temp_folder':      "/tmp/dxr-temp",
            'log_folder':       "%(temp_folder)s/logs",
            'template':         "%(dxrroot)s/templates",
            'wwwroot':          "/",
            'enabled_plugins':  "*",
            'disabled_plugins': " ",
            'directory_index':  ".dxr-directory-index.html",
            'generated_date':   datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
        })
        parser.read(configfile)

        # Set config values
        self.dxrroot          = parser.get('DXR', 'dxrroot',          False, override)
        self.plugin_folder    = parser.get('DXR', 'plugin_folder',    False, override)
        self.nb_jobs          = parser.get('DXR', 'nb_jobs',          False, override)
        self.temp_folder      = parser.get('DXR', 'temp_folder',      False, override)
        self.target_folder    = parser.get('DXR', 'target_folder',    False, override)
        self.log_folder       = parser.get('DXR', 'log_folder',       False, override)
        self.template_folder  = parser.get('DXR', 'template',         False, override)
        self.wwwroot          = parser.get('DXR', 'wwwroot',          False, override)
        self.enabled_plugins  = parser.get('DXR', 'enabled_plugins',  False, override)
        self.disabled_plugins = parser.get('DXR', 'disabled_plugins', False, override)
        self.directory_index  = parser.get('DXR', 'directory_index',  False, override)
        self.generated_date   = parser.get('DXR', 'generated_date',   False, override)
        # Set configfile
        self.configfile       = configfile
        self.trees            = []
        # Set template parameters (using new parser to avoid defaults)
        tmp_cfg = ConfigParser.ConfigParser()
        tmp_cfg.read(configfile)
        self.template_parameters = dict(tmp_cfg.items('Template'))

        # Read all plugin_ keys
        for key, value in tmp_cfg.items('DXR'):
            if key.startswith('plugin_'):
                setattr(self, key, value)

        # Render all paths absolute
        self.dxrroot          = os.path.abspath(self.dxrroot)
        self.plugin_folder    = os.path.abspath(self.plugin_folder)
        self.temp_folder      = os.path.abspath(self.temp_folder)
        self.log_folder       = os.path.abspath(self.log_folder)
        self.target_folder    = os.path.abspath(self.target_folder)
        self.template_folder  = os.path.abspath(self.template_folder)

        # Make sure wwwroot doesn't end in /
        if self.wwwroot[-1] == '/':
            self.wwwroot = self.wwwroot[:-1]

        # Convert disabled plugins to a list
        if self.disabled_plugins == "*":
            self.disabled_plugins = os.listdir(self.plugin_folder)
        else:
            self.disabled_plugins = self.disabled_plugins.split()

        # Convert enabled plugins to a list
        if self.enabled_plugins == "*":
            self.enabled_plugins = [p for p in os.listdir(self.plugin_folder)
                                    if isdir(p) and
                                    p not in self.disabled_plugins]
        else:
            self.enabled_plugins = self.enabled_plugins.split()

        # Test for conflicting plugins settings
        if any((p in self.disabled_plugins for p in self.enabled_plugins)):
            msg = "Plugin: '%s' is both enabled and disabled in '%s'"
            print >> sys.stderr, msg % (p, name)
            sys.exit(1)

{'fraction_non_alphanumeric': 0.11058579335793357, 'fraction_numerical': 0.0023062730627306273, 'mean_word_length': 2.169956140350877, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 14, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6832836', 'n_tokens_mistral': 2474, 'n_tokens_neox': 2426, 'n_words': 621}