<commit_before>from celery.schedules import crontab
from tempfile import gettempdir
from os import environ as env, path

DEBUG = True
ASSETS_DEBUG = True
CACHE = True

APP_TITLE = 'Aleph'
APP_NAME = 'aleph'
APP_LOGO = '/static/images/aleph_small.png'
APP_FAVICON = '/static/images/aleph_small.png'
APP_BASEURL = 'http://localhost:5000/'

# Archive type (either S3 or file, i.e. local file system):
ARCHIVE_TYPE = 'file'
ARCHIVE_AWS_KEY_ID = env.get('AWS_ACCESS_KEY_ID')
ARCHIVE_AWS_SECRET = env.get('AWS_SECRET_ACCESS_KEY')
# ARCHIVE_BUCKET = 'aleph2-dev.pudo.org'
ARCHIVE_PATH = env.get('ARCHIVE_PATH', '/srv/data/aleph')

# A temporary folder for uploads:
UPLOAD_FOLDER = path.join(gettempdir(), 'aleph_uploads')

# Maximum upload size:
MAX_CONTENT_LENGTH = 500 * 1024 * 1024

# Set up a custom SCSS file with additional style rules here.
CUSTOM_SCSS_PATH = None
CUSTOM_TEMPLATES_DIR = []

# Shown on the home page as a few sample queries:
SAMPLE_SEARCHES = ['Serbia', 'TeliaSonera', '<NAME>']

# Language configuration
DEFAULT_LANGUAGE = 'en'

# Category schema for collections.
# TODO: add extra weight info.
COLLECTION_CATEGORIES = {
    'news': 'News archives',
    'leak': 'Leaks',
    'gazette': 'Gazettes',
    'court': 'Court archives',
    'company': 'Company registries',
    'watchlist': 'Watchlists',
    'investigation': 'Personal collections',
    'sanctions': 'Sanctions lists',
    'scrape': 'Scrapes',
    'procurement': 'Procurement',
    'grey': 'Grey literature'
}

# Binary paths for programs to which the ingestor shells out:
TESSDATA_PREFIX = env.get('TESSDATA_PREFIX')
PDFTOHTML_BIN = env.get('PDFTOHTML_BIN', 'pdftohtml')
CONVERT_BIN = env.get('CONVERT_BIN', 'convert')
SOFFICE_BIN = env.get('SOFFICE_BIN', 'soffice')
WKHTMLTOPDF_BIN = env.get('WKHTMLTOPDF_BIN', 'wkhtmltopdf')
DDJVU_BIN = env.get('DDJVU_BIN', 'ddjvu')
MDB_TABLES_BIN = env.get('MDB_TABLES_BIN', 'mdb-tables')
MDB_EXPORT_BIN = env.get('MDB_EXPORT_BIN', 'mdb-export')
SEVENZ_BIN = env.get('SEVENZ_BIN', '7z')
# OCR_PDF_PAGES = True
OCR_DEFAULTS = ['en']

# Use Apache Tika for PDF extraction:
TIKA_URI = env.get('TIKA_URI')

# Enable the Aho Corasick based entity string matcher:
REGEX_ENTITIES = True

# Disable all crawlers (temporarily?)
DISABLE_CRAWLERS = False

SECRET_KEY = env.get('SECRET_KEY')

SQLALCHEMY_DATABASE_URI = env.get('DATABASE_URL', 'sqlite:///aleph.sqlite3')
SQLALCHEMY_TRACK_MODIFICATIONS = False

ELASTICSEARCH_URL = env.get('BONSAI_URL', 'http://localhost:9200')

ALEMBIC_DIR = path.join(path.dirname(__file__), 'migrate')
ALEMBIC_DIR = path.abspath(ALEMBIC_DIR)

CELERY_ALWAYS_EAGER = False
CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
CELERYD_MAX_TASKS_PER_CHILD = 200
CELERYD_PREFETCH_MULTIPLIER = 5
CELERY_DISABLE_RATE_LIMITS = True
CELERY_TASK_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TIMEZONE = 'UTC'
CELERY_IGNORE_RESULT = True
CELERY_RESULT_BACKEND = 'amqp'
CELERY_RESULT_PERSISTENT = False
# CELERY_ACKS_LATE = True
# ultra-high time limit to shoot hung tasks:
CELERYD_TASK_TIME_LIMIT = 3600 * 3
CELERY_BROKER_URL = env.get('RABBITMQ_BIGWIG_URL',
                            'amqp://guest:guest@localhost:5672//')<commit_msg>do not enable a result backend, fixes #107<commit_after>from celery.schedules import crontab
from tempfile import gettempdir
from os import environ as env, path

DEBUG = True
ASSETS_DEBUG = True
CACHE = True

APP_TITLE = 'Aleph'
APP_NAME = 'aleph'
APP_LOGO = '/static/images/aleph_small.png'
APP_FAVICON = '/static/images/aleph_small.png'
APP_BASEURL = 'http://localhost:5000/'

# Archive type (either S3 or file, i.e. local file system):
ARCHIVE_TYPE = 'file'
ARCHIVE_AWS_KEY_ID = env.get('AWS_ACCESS_KEY_ID')
ARCHIVE_AWS_SECRET = env.get('AWS_SECRET_ACCESS_KEY')
# ARCHIVE_BUCKET = 'aleph2-dev.pudo.org'
ARCHIVE_PATH = env.get('ARCHIVE_PATH', '/srv/data/aleph')

# A temporary folder for uploads:
UPLOAD_FOLDER = path.join(gettempdir(), 'aleph_uploads')

# Maximum upload size:
MAX_CONTENT_LENGTH = 500 * 1024 * 1024

# Set up a custom SCSS file with additional style rules here.
CUSTOM_SCSS_PATH = None
CUSTOM_TEMPLATES_DIR = []

# Shown on the home page as a few sample queries:
SAMPLE_SEARCHES = ['Serbia', 'TeliaSonera', 'Vladimir Putin']

# Language configuration
DEFAULT_LANGUAGE = 'en'

# Category schema for collections.
# TODO: add extra weight info.
COLLECTION_CATEGORIES = {
    'news': 'News archives',
    'leak': 'Leaks',
    'gazette': 'Gazettes',
    'court': 'Court archives',
    'company': 'Company registries',
    'watchlist': 'Watchlists',
    'investigation': 'Personal collections',
    'sanctions': 'Sanctions lists',
    'scrape': 'Scrapes',
    'procurement': 'Procurement',
    'grey': 'Grey literature'
}

# Binary paths for programs to which the ingestor shells out:
TESSDATA_PREFIX = env.get('TESSDATA_PREFIX')
PDFTOHTML_BIN = env.get('PDFTOHTML_BIN', 'pdftohtml')
CONVERT_BIN = env.get('CONVERT_BIN', 'convert')
SOFFICE_BIN = env.get('SOFFICE_BIN', 'soffice')
WKHTMLTOPDF_BIN = env.get('WKHTMLTOPDF_BIN', 'wkhtmltopdf')
DDJVU_BIN = env.get('DDJVU_BIN', 'ddjvu')
MDB_TABLES_BIN = env.get('MDB_TABLES_BIN', 'mdb-tables')
MDB_EXPORT_BIN = env.get('MDB_EXPORT_BIN', 'mdb-export')
SEVENZ_BIN = env.get('SEVENZ_BIN', '7z')
# OCR_PDF_PAGES = True
OCR_DEFAULTS = ['en']

# Use Apache Tika for PDF extraction:
TIKA_URI = env.get('TIKA_URI')

# Enable the Aho Corasick based entity string matcher:
REGEX_ENTITIES = True

# Disable all crawlers (temporarily?)
DISABLE_CRAWLERS = False

SECRET_KEY = env.get('SECRET_KEY')

SQLALCHEMY_DATABASE_URI = env.get('DATABASE_URL', 'sqlite:///aleph.sqlite3')
SQLALCHEMY_TRACK_MODIFICATIONS = False

ELASTICSEARCH_URL = env.get('BONSAI_URL', 'http://localhost:9200')

ALEMBIC_DIR = path.join(path.dirname(__file__), 'migrate')
ALEMBIC_DIR = path.abspath(ALEMBIC_DIR)

CELERY_ALWAYS_EAGER = False
CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
CELERYD_MAX_TASKS_PER_CHILD = 200
CELERYD_PREFETCH_MULTIPLIER = 5
CELERY_DISABLE_RATE_LIMITS = True
CELERY_TASK_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TIMEZONE = 'UTC'
CELERY_IGNORE_RESULT = True
# CELERY_RESULT_BACKEND = 'amqp'
CELERY_RESULT_PERSISTENT = False
# CELERY_ACKS_LATE = True
# ultra-high time limit to shoot hung tasks:
CELERYD_TASK_TIME_LIMIT = 3600 * 3
CELERY_BROKER_URL = env.get('RABBITMQ_BIGWIG_URL',
                            'amqp://guest:guest@localhost:5672//')
{'fraction_non_alphanumeric': 0.12223442648674093, 'fraction_numerical': 0.011768397928761965, 'mean_word_length': 5.639583333333333, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7428039', 'n_tokens_mistral': 2617, 'n_tokens_neox': 2498, 'n_words': 513}