<commit_before>from ckan.common import _

import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit


def eurovoc_subjects():
    '''
    Return a list of (id, label) tuples representing toplevel Eurovoc
    subjects.

    http://eurovoc.europa.eu/drupal/?q=download/subject_oriented&cl=en
    '''
    eurovoc_subjects = (
        (None, ''),
        ('politics', _('Politics')),
        ('international-relations', _('International Relations')),
        ('european-union', _('European Union')),
        ('law', _('Law')),
        ('economics', _('Economics')),
        ('trade', _('Trade')),
        ('finance', _('Finance')),
        ('social-questions', _('Social Questions')),
        ('education-and-communications', _('Education and Communications')),
        ('science', _('Science')),
        ('business-and-competition', _('Business and Competition')),
        ('employment-and-working-conditions', _('Employment and Working Conditions')),
        ('transport', _('Transport')),
        ('environment', _('Environment')),
        ('agriculture-forestry-and-fisheries', _('Agriculture, Forestry and Fisheries')),
        ('agri-foodstuffs', _('Agri-foodstuffs')),
        ('production-technology-and-research', _('Production, Technology and Research')),
        ('energy', _('Energy')),
        ('industry', _('Industry')),
        ('geography', _('Geography')),
        ('international-organisations', _('International Organisations')),
    )
    return eurovoc_subjects


def _eurovoc_text_output(id):
    '''Return the label value for a given subject id.'''
    subject_dict = dict(eurovoc_subjects())
    return subject_dict.get(id, None)


class EurovocPlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm):
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.IDatasetForm)
    plugins.implements(plugins.ITemplateHelpers, inherit=True)
    plugins.implements(plugins.IValidators)

    # IConfigurer
    def update_config(self, config):
        toolkit.add_template_directory(config, 'templates')

    # IDatasetForm

    def _modify_package_schema(self, schema):
        schema.update({
            'eurovoc_subject': [
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('convert_to_extras')
            ]
        })
        return schema

    def create_package_schema(self):
        schema = super(EurovocPlugin, self).create_package_schema()
        schema = self._modify_package_schema(schema)
        return schema

    def update_package_schema(self):
        schema = super(EurovocPlugin, self).update_package_schema()
        schema = self._modify_package_schema(schema)
        return schema

    def show_package_schema(self):
        schema = super(EurovocPlugin, self).show_package_schema()
        schema.update({
            'eurovoc_subject': [
                toolkit.get_converter('convert_from_extras'),
                toolkit.get_validator('eurovoc_text_output'),
                toolkit.get_validator('ignore_missing')
            ]
        })
        return schema

    def is_fallback(self):
        return True

    def package_types(self):
        return []

    # ITemplateHelpers

    def get_helpers(self):
        return {'eurovoc_subjects': eurovoc_subjects}

    # IValidators

    def get_validators(self):
        return {
            'eurovoc_text_output': _eurovoc_text_output,
        }<commit_msg>Search and filter by eurovoc category<commit_after>import json

from ckan.common import _

import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit


def eurovoc_subjects():
    '''
    Return a list of (id, label) tuples representing toplevel Eurovoc
    subjects.

    http://eurovoc.europa.eu/drupal/?q=download/subject_oriented&cl=en
    '''
    eurovoc_subjects = (
        (None, ''),
        ('politics', _('Politics')),
        ('international-relations', _('International Relations')),
        ('european-union', _('European Union')),
        ('law', _('Law')),
        ('economics', _('Economics')),
        ('trade', _('Trade')),
        ('finance', _('Finance')),
        ('social-questions', _('Social Questions')),
        ('education-and-communications', _('Education and Communications')),
        ('science', _('Science')),
        ('business-and-competition', _('Business and Competition')),
        ('employment-and-working-conditions', _('Employment and Working Conditions')),
        ('transport', _('Transport')),
        ('environment', _('Environment')),
        ('agriculture-forestry-and-fisheries', _('Agriculture, Forestry and Fisheries')),
        ('agri-foodstuffs', _('Agri-foodstuffs')),
        ('production-technology-and-research', _('Production, Technology and Research')),
        ('energy', _('Energy')),
        ('industry', _('Industry')),
        ('geography', _('Geography')),
        ('international-organisations', _('International Organisations')),
    )
    return eurovoc_subjects


def _eurovoc_text_output(id):
    '''Return the label value for a given subject id.'''
    subject_dict = dict(eurovoc_subjects())
    return subject_dict.get(id, None)


class EurovocPlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm):
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.IDatasetForm)
    plugins.implements(plugins.ITemplateHelpers, inherit=True)
    plugins.implements(plugins.IValidators)
    plugins.implements(plugins.IPackageController, inherit=True)
    plugins.implements(plugins.IFacets)

    # IConfigurer
    def update_config(self, config):
        toolkit.add_template_directory(config, 'templates')

    # IDatasetForm

    def _modify_package_schema(self, schema):
        schema.update({
            'eurovoc_subject': [
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('convert_to_extras')
            ]
        })
        return schema

    def create_package_schema(self):
        schema = super(EurovocPlugin, self).create_package_schema()
        schema = self._modify_package_schema(schema)
        return schema

    def update_package_schema(self):
        schema = super(EurovocPlugin, self).update_package_schema()
        schema = self._modify_package_schema(schema)
        return schema

    def show_package_schema(self):
        schema = super(EurovocPlugin, self).show_package_schema()
        schema.update({
            'eurovoc_subject': [
                toolkit.get_converter('convert_from_extras'),
                toolkit.get_validator('eurovoc_text_output'),
                toolkit.get_validator('ignore_missing')
            ]
        })
        return schema

    def is_fallback(self):
        return True

    def package_types(self):
        return []

    # ITemplateHelpers

    def get_helpers(self):
        return {'eurovoc_subjects': eurovoc_subjects}

    # IValidators

    def get_validators(self):
        return {
            'eurovoc_text_output': _eurovoc_text_output,
        }

    # IFacets

    def dataset_facets(self, facets_dict, package_type):
        self._update_facets(facets_dict)
        return facets_dict

    def group_facets(self, facets_dict, group_type, package_type):
        self._update_facets(facets_dict)
        return facets_dict

    def organization_facets(self, facets_dict, organization_type, package_type):
        self._update_facets(facets_dict)
        return facets_dict

    def _update_facets(self, facets_dict):
        facets_dict.update({
            'eurovoc_subject_label': plugins.toolkit._('Eurovoc Categories')
        })

    # IPackageController

    def before_index(self, dataset_dict):
        '''
        Insert `eurovoc_subject_label` into solr index derived from the
        dataset_dict's `eurovoc_subject` field.
        '''
        if dataset_dict.get('eurovoc_subject'):
            label = _eurovoc_text_output(dataset_dict.get('eurovoc_subject'))
            if label is not None:
                dataset_dict['eurovoc_subject_label'] = label

        return dataset_dict
{'fraction_non_alphanumeric': 0.12158808933002481, 'fraction_numerical': 0.0, 'mean_word_length': 3.000496277915633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7626760', 'n_tokens_mistral': 2478, 'n_tokens_neox': 2356, 'n_words': 423}