<commit_before>from collections import OrderedDict

import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit
import ckan.lib.helpers as h

from webhelpers.html import literal

from ckan.common import request

import os
import json

qchoices = {}

with open(os.path.dirname(os.path.realpath(__file__)) + '/schema.json') as jsonfile:    
    schema = json.load(jsonfile)
    for field in schema['dataset_fields']:
        if field['field_name'] == 'question':
            for item in field['choices']:
                qchoices[item['value']] = item['label']
            break

def get_from_flat_dict(list_of_dicts, key, default=None):
    '''Extract data from a list of dicts with keys 'key' and 'value'
    e.g. pkg_dict['extras'] = [{'key': 'language', 'value': '"french"'}, ... ]
    '''
    for dict_ in list_of_dicts:
        if dict_.get('key', '') == key:
            return (dict_.get('value', default) or '').strip('"')
    return default


def extended_build_nav(*args):
    # we go through the args, add links for raw links,
    # and then pass the rest to core build_nav_main
    output = ''
    for item in args:
        menu_item, title = item[:2]

        if len(item) == 3 and not toolkit.check_access(item[2]):
            continue

        if menu_item.startswith('http') or menu_item.startswith('/'):
            # it's a link
            output += literal(
                '<li><a href="%s">%s</a></li>' % (menu_item, title)
            )
        else:
            # give it to the core helper for this
            output += h._make_menu_item(menu_item, title)
    return output

def dataset_count():
    try:
        q = toolkit.get_action('package_search')({}, {'q': 'type:dataset'})
        return q.get('count')
    except Exception:
        return ''

def document_count():
    try:
        q = toolkit.get_action('package_search')({}, {'q': 'type:document'})
        return q.get('count')
    except Exception:
        return ''

def country_count():
    try:
        q = toolkit.get_action('package_search')({}, {'facet.field': ['country'], 'facet.limit': -1})
        return len(q.get('facets').get('country').keys())
    except Exception:
        return ''

def get_facet_items_dict_questions(facet, limit=None, exclude_active=False):
    if facet != 'question':
        return []
    
    facets = h.get_facet_items_dict(facet, limit=None, exclude_active=False)

    newfacets = []
    facet_counts = {}

    for facet in facets:
        if not facet or len(facet) == 0:
            continue
        newfacet = facet
        #Clean up
        num = newfacet['display_name'].replace('[', '').replace(']', '').replace('"', '').replace(' ', '')
        if num == '':
            continue
        #Handle multiple questions; should actually be doable in SOLR
        if ',' in num:
            parts = num.split(',')
            #For counts of items having both, add the total to both counts
            for part in parts:
                if part in facet_counts:
                    facet_counts[part] = facet_counts[part] + newfacet['count']
                else:
                    facet_counts[part] = newfacet['count']
            #And now don't show it
            continue
        if num in facet_counts:
            facet_counts[num] = facet_counts[num] + newfacet['count']
        else:
            facet_counts[num] = newfacet['count']

        newfacet['display_name'] = qchoices[num] 
        newfacet['num'] = num
        newfacet['name'] = '[\\"' + num  + '\\"]'
        newfacets.append(newfacet)

    for facet in newfacets:
       facet['count'] = facet_counts[facet['num']]
       del facet_counts[facet['num']]

    #The remaining ones don't have a facet to update, create new ones
    for count in facet_counts:
       name = '[\\"' + count  + '\\"]'
       cfacet = {'count': facet_counts[count], 'display_name': qchoices[count], 'name': name}
       newfacets.append(cfacet)

    #Now we have to resort
    sortedfacets = sorted(newfacets, key=lambda k: k['count'], reverse=True)

    #Mark selected
    selectedfacet = None
    for item in request.params.items():<commit_msg>Workaround missing question(s) in schema<commit_after>from collections import OrderedDict

import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit
import ckan.lib.helpers as h

from webhelpers.html import literal

from ckan.common import request

import os
import json

qchoices = {}

with open(os.path.dirname(os.path.realpath(__file__)) + '/schema.json') as jsonfile:    
    schema = json.load(jsonfile)
    for field in schema['dataset_fields']:
        if field['field_name'] == 'question':
            for item in field['choices']:
                qchoices[item['value']] = item['label']
            break

def get_from_flat_dict(list_of_dicts, key, default=None):
    '''Extract data from a list of dicts with keys 'key' and 'value'
    e.g. pkg_dict['extras'] = [{'key': 'language', 'value': '"french"'}, ... ]
    '''
    for dict_ in list_of_dicts:
        if dict_.get('key', '') == key:
            return (dict_.get('value', default) or '').strip('"')
    return default


def extended_build_nav(*args):
    # we go through the args, add links for raw links,
    # and then pass the rest to core build_nav_main
    output = ''
    for item in args:
        menu_item, title = item[:2]

        if len(item) == 3 and not toolkit.check_access(item[2]):
            continue

        if menu_item.startswith('http') or menu_item.startswith('/'):
            # it's a link
            output += literal(
                '<li><a href="%s">%s</a></li>' % (menu_item, title)
            )
        else:
            # give it to the core helper for this
            output += h._make_menu_item(menu_item, title)
    return output

def dataset_count():
    try:
        q = toolkit.get_action('package_search')({}, {'q': 'type:dataset'})
        return q.get('count')
    except Exception:
        return ''

def document_count():
    try:
        q = toolkit.get_action('package_search')({}, {'q': 'type:document'})
        return q.get('count')
    except Exception:
        return ''

def country_count():
    try:
        q = toolkit.get_action('package_search')({}, {'facet.field': ['country'], 'facet.limit': -1})
        return len(q.get('facets').get('country').keys())
    except Exception:
        return ''

def get_facet_items_dict_questions(facet, limit=None, exclude_active=False):
    if facet != 'question':
        return []
    
    facets = h.get_facet_items_dict(facet, limit=None, exclude_active=False)

    newfacets = []
    facet_counts = {}

    for facet in facets:
        if not facet or len(facet) == 0:
            continue
        newfacet = facet
        #Clean up
        num = newfacet['display_name'].replace('[', '').replace(']', '').replace('"', '').replace(' ', '')
        if num == '':
            continue
        #Handle multiple questions; should actually be doable in SOLR
        if ',' in num:
            parts = num.split(',')
            #For counts of items having both, add the total to both counts
            for part in parts:
                if part in facet_counts:
                    facet_counts[part] = facet_counts[part] + newfacet['count']
                else:
                    facet_counts[part] = newfacet['count']
            #And now don't show it
            continue
        if num in facet_counts:
            facet_counts[num] = facet_counts[num] + newfacet['count']
        else:
            facet_counts[num] = newfacet['count']
        
        #Questions can get deleted from the schema, workaround
        if num in qchoices:
            newfacet['display_name'] = qchoices[num] 
            newfacet['num'] = num
            newfacet['name'] = '[\\"' + num  + '\\"]'
            newfacets.append(newfacet)

    for facet in newfacets:
       facet['count'] = facet_counts[facet['num']]
       del facet_counts[facet['num']]

    #The remaining ones don't have a facet to update, create new ones
    for count in facet_counts:
       name = '[\\"' + count  + '\\"]'
       cfacet = {'count': facet_counts[count], 'display_name': qchoices[count], 'name': name}
       newfacets.append(cfacet)

    #Now we have to resort
    sortedfacets = sorted(newfacets, key=lambda k: k['count'], reverse=True)

    #Mark selected
    selectedfacet = None
    for item in request.params.items():
{'fraction_non_alphanumeric': 0.1358450202044212, 'fraction_numerical': 0.001188495364868077, 'mean_word_length': 2.4, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2188156', 'n_tokens_mistral': 2651, 'n_tokens_neox': 2569, 'n_words': 693}