<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import, print_function, unicode_literals

from fnmatch import fnmatch
import os.path
from string import lower

import paste.deploy
from pylons import config

from ckan.plugins import toolkit
from ckan.config.environment import load_environment


DEFAULTS = {
    'ckanext.extractor.indexed_formats': 'pdf',
    'ckanext.extractor.indexed_fields': 'contents',
}

TRANSFORMATIONS = {
    'ckanext.extractor.indexed_formats': [lower, toolkit.aslist],
    'ckanext.extractor.indexed_fields': [lower, toolkit.aslist],
}


def get(setting):
    """
    Get configuration setting.

    ``setting`` is the setting without the ``ckanext.extractor.``
    prefix.

    Handles defaults and transformations.
    """
    setting = 'ckanext.extractor.' + setting
    value = config.get(setting, DEFAULTS[setting])
    for transformation in TRANSFORMATIONS[setting]:
        value = transformation(value)
    return value


# Adapted from ckanext-archiver
def load_config(ini_path):
    """
    Load CKAN configuration.
    """
    ini_path = os.path.abspath(ini_path)
    conf = paste.deploy.appconfig('config:' + ini_path)
    load_environment(conf.global_conf, conf.local_conf)


def _any_match(s, patterns):
    """
    Check if a string matches at least one pattern.
    """
    return any(fnmatch(s, pattern) for pattern in patterns)


def is_field_indexed(field):
    """
    Check if a metadata field is configured to be indexed.
    """
    return _any_match(field.lower(), get('indexed_fields'))


def is_format_indexed(format):
    """
    Check if a resource format is configured to be indexed.
    """
    return _any_match(format.lower(), get('indexed_formats'))
<commit_msg>Fix: No translator was registered during background tasks.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import, print_function, unicode_literals

from fnmatch import fnmatch
import os.path
from string import lower

import paste.deploy
from paste.registry import Registry
from pylons import config, translator

from ckan.plugins import toolkit
from ckan.config.environment import load_environment
from ckan.lib.cli import MockTranslator


DEFAULTS = {
    'ckanext.extractor.indexed_formats': 'pdf',
    'ckanext.extractor.indexed_fields': 'contents',
}

TRANSFORMATIONS = {
    'ckanext.extractor.indexed_formats': [lower, toolkit.aslist],
    'ckanext.extractor.indexed_fields': [lower, toolkit.aslist],
}


def get(setting):
    """
    Get configuration setting.

    ``setting`` is the setting without the ``ckanext.extractor.``
    prefix.

    Handles defaults and transformations.
    """
    setting = 'ckanext.extractor.' + setting
    value = config.get(setting, DEFAULTS[setting])
    for transformation in TRANSFORMATIONS[setting]:
        value = transformation(value)
    return value


# Adapted from ckanext-archiver
def load_config(ini_path):
    """
    Load CKAN configuration.
    """
    ini_path = os.path.abspath(ini_path)
    conf = paste.deploy.appconfig('config:' + ini_path)
    load_environment(conf.global_conf, conf.local_conf)
    _register_translator()


# Adapted from ckanext-archiver
def _register_translator():
    """
    Register a translator in this thread.
    """
    global registry
    try:
        registry
    except NameError:
        registry = Registry()
    registry.prepare()
    global translator_obj
    try:
        translator_obj
    except NameError:
        translator_obj = MockTranslator()
    registry.register(translator, translator_obj)


def _any_match(s, patterns):
    """
    Check if a string matches at least one pattern.
    """
    return any(fnmatch(s, pattern) for pattern in patterns)


def is_field_indexed(field):
    """
    Check if a metadata field is configured to be indexed.
    """
    return _any_match(field.lower(), get('indexed_fields'))


def is_format_indexed(format):
    """
    Check if a resource format is configured to be indexed.
    """
    return _any_match(format.lower(), get('indexed_formats'))

{'fraction_non_alphanumeric': 0.10343149184716476, 'fraction_numerical': 0.0004867364322219518, 'mean_word_length': 3.686431014823261, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4410705', 'n_tokens_mistral': 1285, 'n_tokens_neox': 1238, 'n_words': 329}