<commit_before>from importlib import import_module
import os
import sys

from .applications import *
from .config import *
from .main import *
from .logging import *
from .auth import *
from .api import *
from .tasks import *

# If PAM_AUTH in plugins enbled, add django_pam
if "pam_auth" in PLUGINS_ENABLED:
    INSTALLED_APPS += ['django_pam']

# If google_build in use, we are required to include GitHub
if "google_build" in PLUGINS_ENABLED:

    # For task discovery by celery
    INSTALLED_APPS += ['shub.plugins.google_build']
    SOCIAL_AUTH_GITHUB_SCOPE = ["admin:repo_hook",
                                "repo:status",
                                "user:email",
                                "read:org",
                                "admin:org_hook",
                                "deployment_status"]
    ENABLE_GITHUB_AUTH=True

# Apply any plugin settings
for plugin in PLUGINS_ENABLED:

    plugin_module = 'shub.plugins.' + plugin
    plugin = import_module(plugin_module)

    # Add the plugin to INSTALLED APPS
    INSTALLED_APPS.append(plugin_module)

    # Add AUTHENTICATION_BACKENDS if defined, for authentication plugins
    if hasattr(plugin, 'AUTHENTICATION_BACKENDS'):
        AUTHENTICATION_BACKENDS = AUTHENTICATION_BACKENDS + plugin.AUTHENTICATION_BACKENDS

    # Add custom context processors, if defines for plugin
    if hasattr(plugin, 'CONTEXT_PROCESSORS'):
        for context_processor in plugin.CONTEXT_PROCESSORS:
            TEMPLATES[0]['OPTIONS']['context_processors'].append(context_processor)<commit_msg>adding anyjson<commit_after>from importlib import import_module
import os
import sys

from .applications import *
from .config import *
from .main import *
from .logging import *
from .auth import *
from .api import *
from .tasks import *

# If PAM_AUTH in plugins enbled, add django_pam
if "pam_auth" in PLUGINS_ENABLED:
    INSTALLED_APPS += ['django_pam']

# If google_build in use, we are required to include GitHub
if "google_build" in PLUGINS_ENABLED:

    # For task discovery by celery
    SOCIAL_AUTH_GITHUB_SCOPE = ["admin:repo_hook",
                                "repo:status",
                                "user:email",
                                "read:org",
                                "admin:org_hook",
                                "deployment_status"]
    ENABLE_GITHUB_AUTH=True

# Apply any plugin settings
for plugin in PLUGINS_ENABLED:

    plugin_module = 'shub.plugins.' + plugin
    plugin = import_module(plugin_module)

    # Add the plugin to INSTALLED APPS
    INSTALLED_APPS.append(plugin_module)

    # Add AUTHENTICATION_BACKENDS if defined, for authentication plugins
    if hasattr(plugin, 'AUTHENTICATION_BACKENDS'):
        AUTHENTICATION_BACKENDS = AUTHENTICATION_BACKENDS + plugin.AUTHENTICATION_BACKENDS

    # Add custom context processors, if defines for plugin
    if hasattr(plugin, 'CONTEXT_PROCESSORS'):
        for context_processor in plugin.CONTEXT_PROCESSORS:
            TEMPLATES[0]['OPTIONS']['context_processors'].append(context_processor)
{'fraction_non_alphanumeric': 0.07373572593800978, 'fraction_numerical': 0.0006525285481239804, 'mean_word_length': 2.8181818181818183, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5799596', 'n_tokens_mistral': 905, 'n_tokens_neox': 875, 'n_words': 238}