<commit_before>import re
from datetime import timedelta
from os.path import (abspath,
                     dirname,
                     join)

import environ
from django.core.exceptions import ImproperlyConfigured
from furl import furl
from kombu import (Exchange,
                   Queue)

from treeherder.config.utils import (connection_should_use_tls,
                                     get_tls_redis_url)

# TODO: Switch to pathlib once using Python 3.
SRC_DIR = dirname(dirname(dirname(abspath(__file__))))

env = environ.Env()

# Top Level configuration
DEBUG = env.bool("TREEHERDER_DEBUG", default=False)

GRAPHQL = env.bool("GRAPHQL", default=True)

# Make this unique, and don't share it with anybody.
SECRET_KEY = env("TREEHERDER_DJANGO_SECRET_KEY")

# Hosts
try:
    SITE_URL = env("SITE_URL")
except ImproperlyConfigured:
    # This is to support Heroku Review apps which host is different for each PR
    SITE_URL = "https://{}.herokuapp.com".format(env("HEROKU_APP_NAME"))

SITE_HOSTNAME = furl(SITE_URL).host
# Including localhost allows using the backend locally
ALLOWED_HOSTS = [SITE_HOSTNAME, 'localhost']

# URL handling
APPEND_SLASH = False
ROOT_URLCONF = "treeherder.config.urls"
WSGI_APPLICATION = 'treeherder.config.wsgi.application'

# Application definition
INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    # Disable Django's own staticfiles handling in favour of WhiteNoise, for
    # greater consistency between gunicorn and `./manage.py runserver`.
    'whitenoise.runserver_nostatic',
    'django.contrib.staticfiles',

    # 3rd party apps
    'rest_framework',
    'corsheaders',
    'django_filters',
    'graphene_django',

    # treeherder apps
    'treeherder.model',
    'treeherder.webapp',
    'treeherder.log_parser',
    'treeherder.etl',
    'treeherder.perf',
    'treeherder.autoclassify',
    'treeherder.seta',
    'treeherder.intermittents_commenter',
]
if DEBUG:
    INSTALLED_APPS.append('django_extensions')

# Middleware
MIDDLEWARE = [middleware for middleware in [
    # Adds custom New Relic annotations. Must be first so all transactions are annotated.
    'treeherder.middleware.NewRelicMiddleware',
    # Redirect to HTTPS/set HSTS and other security headers.
    'django.middleware.security.SecurityMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'corsheaders.middleware.CorsMiddleware',
    # Allows both Django static files and those specified via `WHITENOISE_ROOT`
    # to be served by WhiteNoise, avoiding the need for Apache/nginx on Heroku.
    'treeherder.middleware.CustomWhiteNoise',
    'django.middleware.gzip.GZipMiddleware',
    'debug_toolbar.middleware.DebugToolbarMiddleware' if DEBUG else False,
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
] if middleware]

# Templating
TEMPLATES = [{
    'BACKEND': 'django.template.backends.django.DjangoTemplates',
    'APP_DIRS': True,
}]

# Database
# The database config is defined using environment variables of form:
#
#   'mysql://username:password@host:optional_port/database_name'
#
# which django-environ converts into the Django DB settings dict format.
DATABASES = {
    'default': env.db_url('DATABASE_URL'),
}

# Only used when syncing local database with production replicas
UPSTREAM_DATABASE_URL = env('UPSTREAM_DATABASE_URL', default=None)
if UPSTREAM_DATABASE_URL:
    DATABASES['upstream'] = env.db_url_config(UPSTREAM_DATABASE_URL)

# We're intentionally not using django-environ's query string options feature,
# since it hides configuration outside of the repository, plus could lead to
# drift between environments.
for alias in DATABASES:
    # Persist database connections for 5 minutes, to avoid expensive reconnects.
    DATABASES[alias]['CONN_MAX_AGE'] = 300
    DATABASES[alias]['OPTIONS'] = {
        # Override Django's default connection charset of 'utf8', otherwise it's
        # still not possible to insert non-BMP unicode into utf8mb4 tables.
        'charset': 'utf8mb4',
        # From MySQL 5.7 onwards and on fresh installs of MySQL 5.6, the default value of the sql_mode
        # option contains STRICT_TRANS_TABLES. That option escalates warnings into errors when data are
        # truncated upon insertion, so Django highly recommends activating a strict mode for MySQL to
        # prevent data loss (either STRICT_TRANS_TABLES or STRICT_ALL_TABLES).
        'init_command': "SET sql_mode='STRICT_TRANS_TABLES'"
    }
    if connection_should_use_tls(DATABASES[alias]['HOST']):
        # Use TLS when connecting to RDS.
        # https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_MySQL.html#MySQL.Concepts.SSLSupport
        # https://mysqlclient.readthedocs.io/user_guide.html#functions-and-attributes
        DATABASES[alias]['OPTIONS']['ssl'] = {
            'ca': 'deployment/aws/rds-combined-ca-bundle.pem',
        }

# Caches
REDIS_URL = env('REDIS_URL')
if connection_should_use_tls(REDIS_URL):
    # Connect using TLS on Heroku.
    REDIS_URL = get_tls_redis_url(REDIS_URL)

CACHES = {
    'default': {
        'BACKEND': 'django_redis.cache.RedisCache',
        'LOCATION': REDIS_URL,
        'OPTIONS': {
            # Override the default of no timeout, to avoid connection hangs.
            'SOCKET_CONNECT_TIMEOUT': 5,
        },
    },
}

# Internationalization
TIME_ZONE = "UTC"
USE_I18N = False
USE_L10N = True

# Static files (CSS, JavaScript, Images)
STATIC_ROOT = join(SRC_DIR, ".django-static")
STATIC_URL = "/static/"

# Create hashed+gzipped versions of assets during collectstatic,
# which will then be served by WhiteNoise with a suitable max-age.
# http://whitenoise.evans.io/en/stable/django.html#add-compression-and-caching-support
STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'

# Authentication
AUTHENTICATION_BACKENDS = [
    'django.contrib.auth.backends.ModelBackend',
    'treeherder.auth.backends.AuthBackend',
]

# Use the cache-based backend rather than the default of database.
SESSION_ENGINE = 'django.contrib.sessions.backends.cache'

# Path to redirect to on successful login.
LOGIN_REDIRECT_URL = '/'

# Path to redirect to on unsuccessful login attempt.
LOGIN_REDIRECT_URL_FAILURE = '/'

# Path to redirect to on logout.
LOGOUT_REDIRECT_URL = '/'

# Logging
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_true': {
            '()': 'django.utils.log.RequireDebugTrue',
        },
    },
    'formatters': {
        'standard': {
            'format': "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s",
        },
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
            'formatter': 'standard'
        },
    },
    'loggers': {
        'django': {
            'filters': ['require_debug_true'],
            'handlers': ['console'],
            'level': 'INFO',
            'propagate': True,
        },
        'django.request': {
            'handlers': ['console'],
            'level': 'WARNING',
            'propagate': True,
        },
        'treeherder': {
            'handlers': ['console'],
            'level': 'DEBUG' if DEBUG else 'WARNING',
            'propagate': not DEBUG,
        },
        'kombu': {
            'handlers': ['console'],
            'level': 'WARNING',
        },
    }
}

# SECURITY
USE_X_FORWARDED_HOST = True
USE_X_FORWARDED_PORT = True
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')

if SITE_URL.startswith('https://'):
    SECURE_SSL_REDIRECT = True
    SECURE_HSTS_INCLUDE_SUBDOMAINS = True
    SECURE_HSTS_PRELOAD = True
    SECURE_HSTS_SECONDS = int(timedelta(days=365).total_seconds())
    # Mark session and CSRF cookies as being HTTPS-only.<commit_msg>Support changing logging level with environment variable (#5803)<commit_after>import re
from datetime import timedelta
from os.path import (abspath,
                     dirname,
                     join)

import environ
from django.core.exceptions import ImproperlyConfigured
from furl import furl
from kombu import (Exchange,
                   Queue)

from treeherder.config.utils import (connection_should_use_tls,
                                     get_tls_redis_url)

# TODO: Switch to pathlib once using Python 3.
SRC_DIR = dirname(dirname(dirname(abspath(__file__))))

env = environ.Env()

# Top Level configuration
DEBUG = env.bool("TREEHERDER_DEBUG", default=False)

# Papertrail logs WARNING messages. This env variable allows modifying the behaviour
LOGGING_LEVEL = 'DEBUG' if DEBUG else env.bool("LOGGING_LEVEL", default='WARNING')

GRAPHQL = env.bool("GRAPHQL", default=True)

# Make this unique, and don't share it with anybody.
SECRET_KEY = env("TREEHERDER_DJANGO_SECRET_KEY")

# Hosts
try:
    SITE_URL = env("SITE_URL")
except ImproperlyConfigured:
    # This is to support Heroku Review apps which host is different for each PR
    SITE_URL = "https://{}.herokuapp.com".format(env("HEROKU_APP_NAME"))

SITE_HOSTNAME = furl(SITE_URL).host
# Including localhost allows using the backend locally
ALLOWED_HOSTS = [SITE_HOSTNAME, 'localhost']

# URL handling
APPEND_SLASH = False
ROOT_URLCONF = "treeherder.config.urls"
WSGI_APPLICATION = 'treeherder.config.wsgi.application'

# Application definition
INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    # Disable Django's own staticfiles handling in favour of WhiteNoise, for
    # greater consistency between gunicorn and `./manage.py runserver`.
    'whitenoise.runserver_nostatic',
    'django.contrib.staticfiles',

    # 3rd party apps
    'rest_framework',
    'corsheaders',
    'django_filters',
    'graphene_django',

    # treeherder apps
    'treeherder.model',
    'treeherder.webapp',
    'treeherder.log_parser',
    'treeherder.etl',
    'treeherder.perf',
    'treeherder.autoclassify',
    'treeherder.seta',
    'treeherder.intermittents_commenter',
]
if DEBUG:
    INSTALLED_APPS.append('django_extensions')

# Middleware
MIDDLEWARE = [middleware for middleware in [
    # Adds custom New Relic annotations. Must be first so all transactions are annotated.
    'treeherder.middleware.NewRelicMiddleware',
    # Redirect to HTTPS/set HSTS and other security headers.
    'django.middleware.security.SecurityMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'corsheaders.middleware.CorsMiddleware',
    # Allows both Django static files and those specified via `WHITENOISE_ROOT`
    # to be served by WhiteNoise, avoiding the need for Apache/nginx on Heroku.
    'treeherder.middleware.CustomWhiteNoise',
    'django.middleware.gzip.GZipMiddleware',
    'debug_toolbar.middleware.DebugToolbarMiddleware' if DEBUG else False,
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
] if middleware]

# Templating
TEMPLATES = [{
    'BACKEND': 'django.template.backends.django.DjangoTemplates',
    'APP_DIRS': True,
}]

# Database
# The database config is defined using environment variables of form:
#
#   'mysql://username:password@host:optional_port/database_name'
#
# which django-environ converts into the Django DB settings dict format.
DATABASES = {
    'default': env.db_url('DATABASE_URL'),
}

# Only used when syncing local database with production replicas
UPSTREAM_DATABASE_URL = env('UPSTREAM_DATABASE_URL', default=None)
if UPSTREAM_DATABASE_URL:
    DATABASES['upstream'] = env.db_url_config(UPSTREAM_DATABASE_URL)

# We're intentionally not using django-environ's query string options feature,
# since it hides configuration outside of the repository, plus could lead to
# drift between environments.
for alias in DATABASES:
    # Persist database connections for 5 minutes, to avoid expensive reconnects.
    DATABASES[alias]['CONN_MAX_AGE'] = 300
    DATABASES[alias]['OPTIONS'] = {
        # Override Django's default connection charset of 'utf8', otherwise it's
        # still not possible to insert non-BMP unicode into utf8mb4 tables.
        'charset': 'utf8mb4',
        # From MySQL 5.7 onwards and on fresh installs of MySQL 5.6, the default value of the sql_mode
        # option contains STRICT_TRANS_TABLES. That option escalates warnings into errors when data are
        # truncated upon insertion, so Django highly recommends activating a strict mode for MySQL to
        # prevent data loss (either STRICT_TRANS_TABLES or STRICT_ALL_TABLES).
        'init_command': "SET sql_mode='STRICT_TRANS_TABLES'"
    }
    if connection_should_use_tls(DATABASES[alias]['HOST']):
        # Use TLS when connecting to RDS.
        # https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_MySQL.html#MySQL.Concepts.SSLSupport
        # https://mysqlclient.readthedocs.io/user_guide.html#functions-and-attributes
        DATABASES[alias]['OPTIONS']['ssl'] = {
            'ca': 'deployment/aws/rds-combined-ca-bundle.pem',
        }

# Caches
REDIS_URL = env('REDIS_URL')
if connection_should_use_tls(REDIS_URL):
    # Connect using TLS on Heroku.
    REDIS_URL = get_tls_redis_url(REDIS_URL)

CACHES = {
    'default': {
        'BACKEND': 'django_redis.cache.RedisCache',
        'LOCATION': REDIS_URL,
        'OPTIONS': {
            # Override the default of no timeout, to avoid connection hangs.
            'SOCKET_CONNECT_TIMEOUT': 5,
        },
    },
}

# Internationalization
TIME_ZONE = "UTC"
USE_I18N = False
USE_L10N = True

# Static files (CSS, JavaScript, Images)
STATIC_ROOT = join(SRC_DIR, ".django-static")
STATIC_URL = "/static/"

# Create hashed+gzipped versions of assets during collectstatic,
# which will then be served by WhiteNoise with a suitable max-age.
# http://whitenoise.evans.io/en/stable/django.html#add-compression-and-caching-support
STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'

# Authentication
AUTHENTICATION_BACKENDS = [
    'django.contrib.auth.backends.ModelBackend',
    'treeherder.auth.backends.AuthBackend',
]

# Use the cache-based backend rather than the default of database.
SESSION_ENGINE = 'django.contrib.sessions.backends.cache'

# Path to redirect to on successful login.
LOGIN_REDIRECT_URL = '/'

# Path to redirect to on unsuccessful login attempt.
LOGIN_REDIRECT_URL_FAILURE = '/'

# Path to redirect to on logout.
LOGOUT_REDIRECT_URL = '/'

# Logging
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_true': {
            '()': 'django.utils.log.RequireDebugTrue',
        },
    },
    'formatters': {
        'standard': {
            'format': "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s",
        },
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
            'formatter': 'standard'
        },
    },
    'loggers': {
        'django': {
            'filters': ['require_debug_true'],
            'handlers': ['console'],
            'level': 'INFO',
            'propagate': True,
        },
        'django.request': {
            'handlers': ['console'],
            'level': 'WARNING',
            'propagate': True,
        },
        'treeherder': {
            'handlers': ['console'],
            'level': LOGGING_LEVEL,
            'propagate': LOGGING_LEVEL != 'WARNING',
        },
        'kombu': {
            'handlers': ['console'],
            'level': 'WARNING',
        },
    }
}

# SECURITY
USE_X_FORWARDED_HOST = True
USE_X_FORWARDED_PORT = True
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')

if SITE_URL.startswith('https://'):
    SECURE_SSL_REDIRECT = True
    SECURE_HSTS_INCLUDE_SUBDOMAINS = True
    SECURE_HSTS_PRELOAD = True
    SECURE_HSTS_SECONDS = int(timedelta(days=365).total_seconds())
    # Mark session and CSRF cookies as being HTTPS-only.
{'fraction_non_alphanumeric': 0.10724277276966117, 'fraction_numerical': 0.003232825613926018, 'mean_word_length': 3.4633740288568258, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2464182', 'n_tokens_mistral': 5170, 'n_tokens_neox': 5038, 'n_words': 1184}