<commit_before>import os
import sys
import time
from copy import deepcopy
from typing import Any, Dict, List, Tuple, Union
from urllib.parse import urljoin

from django.template.loaders import app_directories

import zerver.lib.logging_util
from scripts.lib.zulip_tools import get_tornado_ports
from zerver.lib.db import TimeTrackingConnection, TimeTrackingCursor

from .config import (
    DEPLOY_ROOT,
    DEVELOPMENT,
    PRODUCTION,
    config_file,
    get_config,
    get_from_file_if_exists,
    get_secret,
)
from .configured_settings import (
    ADMINS,
    ALLOWED_HOSTS,
    AUTH_LDAP_BIND_DN,
    AUTH_LDAP_CONNECTION_OPTIONS,
    AUTH_LDAP_SERVER_URI,
    AUTHENTICATION_BACKENDS,
    CAMO_URI,
    CUSTOM_HOME_NOT_LOGGED_IN,
    DEBUG,
    DEBUG_ERROR_REPORTING,
    EMAIL_BACKEND,
    EMAIL_HOST,
    ERROR_REPORTING,
    EXTERNAL_HOST,
    EXTERNAL_HOST_WITHOUT_PORT,
    EXTERNAL_URI_SCHEME,
    EXTRA_INSTALLED_APPS,
    GOOGLE_OAUTH2_CLIENT_ID,
    IS_DEV_DROPLET,
    LOCAL_UPLOADS_DIR,
    MEMCACHED_LOCATION,
    MEMCACHED_USERNAME,
    REALM_HOSTS,
    REGISTER_LINK_DISABLED,
    REMOTE_POSTGRES_HOST,
    REMOTE_POSTGRES_PORT,
    REMOTE_POSTGRES_SSLMODE,
    ROOT_SUBDOMAIN_ALIASES,
    SENDFILE_BACKEND,
    SENTRY_DSN,
    SOCIAL_AUTH_APPLE_APP_ID,
    SOCIAL_AUTH_APPLE_SERVICES_ID,
    SOCIAL_AUTH_GITHUB_KEY,
    SOCIAL_AUTH_GITHUB_ORG_NAME,
    SOCIAL_AUTH_GITHUB_TEAM_ID,
    SOCIAL_AUTH_GOOGLE_KEY,
    SOCIAL_AUTH_SAML_ENABLED_IDPS,
    SOCIAL_AUTH_SAML_SECURITY_CONFIG,
    SOCIAL_AUTH_SUBDOMAIN,
    STATSD_HOST,
    TORNADO_PORTS,
    USING_PGROONGA,
    ZULIP_ADMINISTRATOR,
)

########################################################################
# INITIAL SETTINGS
########################################################################

# Make this unique, and don't share it with anybody.
SECRET_KEY = get_secret("secret_key")

# A shared secret, used to authenticate different parts of the app to each other.
SHARED_SECRET = get_secret("shared_secret")

# We use this salt to hash a user's email into a filename for their user-uploaded
# avatar.  If this salt is discovered, attackers will only be able to determine
# that the owner of an email account has uploaded an avatar to Zulip, which isn't
# the end of the world.  Don't use the salt where there is more security exposure.
AVATAR_SALT = get_secret("avatar_salt")

# SERVER_GENERATION is used to track whether the server has been
# restarted for triggering browser clients to reload.
SERVER_GENERATION = int(time.time())

# Key to authenticate this server to zulip.org for push notifications, etc.
ZULIP_ORG_KEY = get_secret("zulip_org_key")
ZULIP_ORG_ID = get_secret("zulip_org_id")

if DEBUG:
    INTERNAL_IPS = ("127.0.0.1",)

# Detect whether we're running as a queue worker; this impacts the logging configuration.
if len(sys.argv) > 2 and sys.argv[0].endswith("manage.py") and sys.argv[1] == "process_queue":
    IS_WORKER = True
else:
    IS_WORKER = False


# This is overridden in test_settings.py for the test suites
TEST_SUITE = False
# This is overridden in test_settings.py for the test suites
PUPPETEER_TESTS = False
# This is overridden in test_settings.py for the test suites
RUNNING_OPENAPI_CURL_TEST = False
# This is overridden in test_settings.py for the test suites
GENERATE_STRIPE_FIXTURES = False
# This is overridden in test_settings.py for the test suites
BAN_CONSOLE_OUTPUT = False

# These are the settings that we will check that the user has filled in for
# production deployments before starting the app.  It consists of a series
# of pairs of (setting name, default value that it must be changed from)
REQUIRED_SETTINGS = [
    ("EXTERNAL_HOST", "zulip.example.com"),
    ("ZULIP_ADMINISTRATOR", "<EMAIL>"),
    # SECRET_KEY doesn't really need to be here, in
    # that we set it automatically, but just in
    # case, it seems worth having in this list
    ("SECRET_KEY", ""),
    ("AUTHENTICATION_BACKENDS", ()),
]

MANAGERS = ADMINS

########################################################################
# STANDARD DJANGO SETTINGS
########################################################################

# Local time zone for this installation. Choices can be found here:
# https://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = "UTC"

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = "en-us"

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not use time-zone-aware datetimes.
USE_TZ = True

# this directory will be used to store logs for development environment
DEVELOPMENT_LOG_DIRECTORY = os.path.join(DEPLOY_ROOT, "var", "log")

# Extend ALLOWED_HOSTS with localhost (needed to RPC to Tornado),
ALLOWED_HOSTS += ["127.0.0.1", "localhost"]
# ... with hosts corresponding to EXTERNAL_HOST,
ALLOWED_HOSTS += [EXTERNAL_HOST_WITHOUT_PORT, "." + EXTERNAL_HOST_WITHOUT_PORT]
# ... and with the hosts in REALM_HOSTS.
ALLOWED_HOSTS += REALM_HOSTS.values()


class TwoFactorLoader(app_directories.Loader):
    def get_dirs(self) -> List[Union[bytes, str]]:
        dirs = super().get_dirs()
        return [d for d in dirs if d.match("two_factor/*")]


MIDDLEWARE = (
    "zerver.middleware.TagRequests",
    "zerver.middleware.SetRemoteAddrFromRealIpHeader",
    "zerver.middleware.RequestContext",
    "django.contrib.sessions.middleware.SessionMiddleware",
    "django.contrib.auth.middleware.AuthenticationMiddleware",
    # Important: All middleware before LogRequests should be
    # inexpensive, because any time spent in that middleware will not
    # be counted in the LogRequests instrumentation of how time was
    # spent while processing a request.
    "zerver.middleware.LogRequests",
    "zerver.middleware.JsonErrorHandler",
    "zerver.middleware.RateLimitMiddleware",
    "zerver.middleware.FlushDisplayRecipientCache",
    "zerver.middleware.ZulipCommonMiddleware",<commit_msg>settings: Add isinstance check before filtering.<commit_after>import os
import sys
import time
from copy import deepcopy
from pathlib import Path
from typing import Any, Dict, List, Tuple, Union
from urllib.parse import urljoin

from django.template.loaders import app_directories

import zerver.lib.logging_util
from scripts.lib.zulip_tools import get_tornado_ports
from zerver.lib.db import TimeTrackingConnection, TimeTrackingCursor

from .config import (
    DEPLOY_ROOT,
    DEVELOPMENT,
    PRODUCTION,
    config_file,
    get_config,
    get_from_file_if_exists,
    get_secret,
)
from .configured_settings import (
    ADMINS,
    ALLOWED_HOSTS,
    AUTH_LDAP_BIND_DN,
    AUTH_LDAP_CONNECTION_OPTIONS,
    AUTH_LDAP_SERVER_URI,
    AUTHENTICATION_BACKENDS,
    CAMO_URI,
    CUSTOM_HOME_NOT_LOGGED_IN,
    DEBUG,
    DEBUG_ERROR_REPORTING,
    EMAIL_BACKEND,
    EMAIL_HOST,
    ERROR_REPORTING,
    EXTERNAL_HOST,
    EXTERNAL_HOST_WITHOUT_PORT,
    EXTERNAL_URI_SCHEME,
    EXTRA_INSTALLED_APPS,
    GOOGLE_OAUTH2_CLIENT_ID,
    IS_DEV_DROPLET,
    LOCAL_UPLOADS_DIR,
    MEMCACHED_LOCATION,
    MEMCACHED_USERNAME,
    REALM_HOSTS,
    REGISTER_LINK_DISABLED,
    REMOTE_POSTGRES_HOST,
    REMOTE_POSTGRES_PORT,
    REMOTE_POSTGRES_SSLMODE,
    ROOT_SUBDOMAIN_ALIASES,
    SENDFILE_BACKEND,
    SENTRY_DSN,
    SOCIAL_AUTH_APPLE_APP_ID,
    SOCIAL_AUTH_APPLE_SERVICES_ID,
    SOCIAL_AUTH_GITHUB_KEY,
    SOCIAL_AUTH_GITHUB_ORG_NAME,
    SOCIAL_AUTH_GITHUB_TEAM_ID,
    SOCIAL_AUTH_GOOGLE_KEY,
    SOCIAL_AUTH_SAML_ENABLED_IDPS,
    SOCIAL_AUTH_SAML_SECURITY_CONFIG,
    SOCIAL_AUTH_SUBDOMAIN,
    STATSD_HOST,
    TORNADO_PORTS,
    USING_PGROONGA,
    ZULIP_ADMINISTRATOR,
)

########################################################################
# INITIAL SETTINGS
########################################################################

# Make this unique, and don't share it with anybody.
SECRET_KEY = get_secret("secret_key")

# A shared secret, used to authenticate different parts of the app to each other.
SHARED_SECRET = get_secret("shared_secret")

# We use this salt to hash a user's email into a filename for their user-uploaded
# avatar.  If this salt is discovered, attackers will only be able to determine
# that the owner of an email account has uploaded an avatar to Zulip, which isn't
# the end of the world.  Don't use the salt where there is more security exposure.
AVATAR_SALT = get_secret("avatar_salt")

# SERVER_GENERATION is used to track whether the server has been
# restarted for triggering browser clients to reload.
SERVER_GENERATION = int(time.time())

# Key to authenticate this server to zulip.org for push notifications, etc.
ZULIP_ORG_KEY = get_secret("zulip_org_key")
ZULIP_ORG_ID = get_secret("zulip_org_id")

if DEBUG:
    INTERNAL_IPS = ("127.0.0.1",)

# Detect whether we're running as a queue worker; this impacts the logging configuration.
if len(sys.argv) > 2 and sys.argv[0].endswith("manage.py") and sys.argv[1] == "process_queue":
    IS_WORKER = True
else:
    IS_WORKER = False


# This is overridden in test_settings.py for the test suites
TEST_SUITE = False
# This is overridden in test_settings.py for the test suites
PUPPETEER_TESTS = False
# This is overridden in test_settings.py for the test suites
RUNNING_OPENAPI_CURL_TEST = False
# This is overridden in test_settings.py for the test suites
GENERATE_STRIPE_FIXTURES = False
# This is overridden in test_settings.py for the test suites
BAN_CONSOLE_OUTPUT = False

# These are the settings that we will check that the user has filled in for
# production deployments before starting the app.  It consists of a series
# of pairs of (setting name, default value that it must be changed from)
REQUIRED_SETTINGS = [
    ("EXTERNAL_HOST", "zulip.example.com"),
    ("ZULIP_ADMINISTRATOR", "<EMAIL>"),
    # SECRET_KEY doesn't really need to be here, in
    # that we set it automatically, but just in
    # case, it seems worth having in this list
    ("SECRET_KEY", ""),
    ("AUTHENTICATION_BACKENDS", ()),
]

MANAGERS = ADMINS

########################################################################
# STANDARD DJANGO SETTINGS
########################################################################

# Local time zone for this installation. Choices can be found here:
# https://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = "UTC"

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = "en-us"

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not use time-zone-aware datetimes.
USE_TZ = True

# this directory will be used to store logs for development environment
DEVELOPMENT_LOG_DIRECTORY = os.path.join(DEPLOY_ROOT, "var", "log")

# Extend ALLOWED_HOSTS with localhost (needed to RPC to Tornado),
ALLOWED_HOSTS += ["127.0.0.1", "localhost"]
# ... with hosts corresponding to EXTERNAL_HOST,
ALLOWED_HOSTS += [EXTERNAL_HOST_WITHOUT_PORT, "." + EXTERNAL_HOST_WITHOUT_PORT]
# ... and with the hosts in REALM_HOSTS.
ALLOWED_HOSTS += REALM_HOSTS.values()


class TwoFactorLoader(app_directories.Loader):
    def get_dirs(self) -> List[Union[str, Path]]:
        dirs = super().get_dirs()
        # app_directories.Loader returns only a list of
        # Path objects by calling get_app_template_dirs
        two_factor_dirs: List[Union[str, Path]] = []
        for d in dirs:
            assert isinstance(d, Path)
            if d.match("two_factor/*"):
                two_factor_dirs.append(d)
        return two_factor_dirs


MIDDLEWARE = (
    "zerver.middleware.TagRequests",
    "zerver.middleware.SetRemoteAddrFromRealIpHeader",
    "zerver.middleware.RequestContext",
    "django.contrib.sessions.middleware.SessionMiddleware",
    "django.contrib.auth.middleware.AuthenticationMiddleware",
    # Important: All middleware before LogRequests should be
    # inexpensive, because any time spent in that middleware will not
    # be counted in the LogRequests instrumentation of how time was
    # spent while processing a request.
    "zerver.middleware.LogRequests",
    "zerver.middleware.JsonErrorHandler",
    "zerver.middleware.RateLimitMiddleware",
    "zerver.middleware.FlushDisplayRecipientCache",
    "zerver.middleware.ZulipCommonMiddleware",
{'fraction_non_alphanumeric': 0.11407976181148632, 'fraction_numerical': 0.0031340593904254487, 'mean_word_length': 4.53272648461205, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4014523', 'n_tokens_mistral': 4234, 'n_tokens_neox': 4094, 'n_words': 1156}