<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
Created by 'bens3' on 2013-06-21.
Copyright (c) 2013 'bens3'. All rights reserved.
"""

import json
import ckan.logic as logic
import ckan.plugins.toolkit as toolkit
from ckan.common import request, response

NotFound = logic.NotFound

# Filter used for filter groups
FIELD_DISPLAY_FILTER = '_f'

# Name of the field display cookie for sotring hidden fields
HIDDEN_FIELDS_COOKIE_NAME = 'hidden_fields'


def resource_get_ordered_fields(resource_id):
    """
    This is a replacement for resource_view_get_fields, but this function
    handles errors internally, and return the fields in their original order
    @param resource_id:
    @return:
    """
    data = {'resource_id': resource_id, 'limit': 0}
    try:
        result = toolkit.get_action('datastore_search')({}, data)
    except NotFound:
        return []

    return [field['id'] for field in result.get('fields', [])]


def resource_filter_options(resource):
    """
    Return the list of filter options for the given resource.

    Note that this is the master source for the list of available options.

    We may want to have this dependent on available fields (rather than
    resource format), so it's useful to keep this as a function (rather
    than a static dict).

    @type resource: dict
    @param resource: Dictionary representing a resource
    @rtype: dict
    @return: A dictionary associating each option's name to a dict
            defining:
                - label: The label to display to users;
                - hide: Optional. If present and True, the option will not
                        be presented to users (but the filter will work and
                        the pill displayed if the filter is applied)
                - sql: Optional. The SQL WHERE statement to use when this
                       option is checked (as a tuple containing statement
                       and value replacements);
                - sql_false: Optional. The SQL WHERE statement to use when
                             this option is unchecked (as a tuple containing
                             statement and value replacements).
    """
    table = resource['id']
    # Get the resource-dependent filter option list
    if resource['format'].lower() == 'dwc':
        return {
            '_has_type': {
                'label': 'Has type',
                'sql': ('"{}"."typeStatus" IS NOT NULL'.format(table),),
                'solr': "typeStatus:[* TO *]"
            },
            '_has_image': {
                'label': 'Has image',
                'sql': ('"{}"."associatedMedia" IS NOT NULL'.format(table),),
                'solr': "associatedMedia:[* TO *]"
            },
            '_has_lat_long': {
                'label': 'Has lat/long',
                 # BS: Changed to look for latitude field,as _geom is only available after a map has been added
                 # As this works for all DwC, we might get datasets without a map
                'sql': ('"{}"."decimalLatitude" IS NOT NULL'.format(table),),
                'solr': ['decimalLatitude:[* TO *]']
            },
            '_exclude_centroid': {
                'label': 'Exclude centroids',
                'sql': ('NOT (LOWER("{}"."centroid"::text) = ANY(\'{{true,yes,1}}\'))'.format(table),),
                'solr': '(centroid:true OR centroid:yes OR centroid:1)'
            },
            '_exclude_mineralogy': {
                'label': 'Exclude Mineralogy',
                'hide': True,
                'sql': ('"{}"."collectionCode" <> \'MIN\''.format(table),),
                'solr': '-collectionCode:MIN'
            }
        }
    else:
        return {}<commit_msg>Fix solr expressions for filter searches<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
Created by 'bens3' on 2013-06-21.
Copyright (c) 2013 'bens3'. All rights reserved.
"""

import json
import ckan.logic as logic
import ckan.plugins.toolkit as toolkit
from ckan.common import request, response

NotFound = logic.NotFound

# Filter used for filter groups
FIELD_DISPLAY_FILTER = '_f'

# Name of the field display cookie for sotring hidden fields
HIDDEN_FIELDS_COOKIE_NAME = 'hidden_fields'


def resource_get_ordered_fields(resource_id):
    """
    This is a replacement for resource_view_get_fields, but this function
    handles errors internally, and return the fields in their original order
    @param resource_id:
    @return:
    """
    data = {'resource_id': resource_id, 'limit': 0}
    try:
        result = toolkit.get_action('datastore_search')({}, data)
    except NotFound:
        return []

    return [field['id'] for field in result.get('fields', [])]


def resource_filter_options(resource):
    """
    Return the list of filter options for the given resource.

    Note that this is the master source for the list of available options.

    We may want to have this dependent on available fields (rather than
    resource format), so it's useful to keep this as a function (rather
    than a static dict).

    @type resource: dict
    @param resource: Dictionary representing a resource
    @rtype: dict
    @return: A dictionary associating each option's name to a dict
            defining:
                - label: The label to display to users;
                - hide: Optional. If present and True, the option will not
                        be presented to users (but the filter will work and
                        the pill displayed if the filter is applied)
                - sql: Optional. The SQL WHERE statement to use when this
                       option is checked (as a tuple containing statement
                       and value replacements);
                - sql_false: Optional. The SQL WHERE statement to use when
                             this option is unchecked (as a tuple containing
                             statement and value replacements).
    """
    table = resource['id']
    # Get the resource-dependent filter option list
    if resource['format'].lower() == 'dwc':
        return {
            '_has_type': {
                'label': 'Has type',
                'sql': ('"{}"."typeStatus" IS NOT NULL'.format(table),),
                'solr': "typeStatus:[* TO *]"
            },
            '_has_image': {
                'label': 'Has image',
                'sql': ('"{}"."associatedMedia" IS NOT NULL'.format(table),),
                'solr': "associatedMedia:[* TO *]"
            },
            '_has_lat_long': {
                'label': 'Has lat/long',
                 # BS: Changed to look for latitude field,as _geom is only available after a map has been added
                 # As this works for all DwC, we might get datasets without a map
                'sql': ('"{}"."decimalLatitude" IS NOT NULL'.format(table),),
                'solr': 'decimalLatitude:[* TO *]'
            },
            '_exclude_centroid': {
                'label': 'Exclude centroids',
                'sql': ('NOT (LOWER("{}"."centroid"::text) = ANY(\'{{true,yes,1}}\'))'.format(table),),
                'solr': 'centroid:true'
            },
            '_exclude_mineralogy': {
                'label': 'Exclude Mineralogy',
                'hide': True,
                'sql': ('"{}"."collectionCode" <> \'MIN\''.format(table),),
                'solr': '-collectionCode:MIN'
            }
        }
    else:
        return {}
{'fraction_non_alphanumeric': 0.10912030176478513, 'fraction_numerical': 0.004715074767614172, 'mean_word_length': 2.017886178861789, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4981749', 'n_tokens_mistral': 2051, 'n_tokens_neox': 1922, 'n_words': 705}