<commit_before>#!/usr/bin/env python

import os
import ckanapi
from ckanapi.errors import CKANAPIError
from urlparse import urlparse


host = os.environ['CKAN_HOST']
token = os.environ['CKAN_API_TOKEN']


def update_dataset(remote, dataset, attributes):
    try:
        # Remove duplicate metadata fields
        # NOTE: This happened in a few experiments in the extra fields
        unique_extras = set(str(e) for e in dataset['extras'])
        dataset['extras'] = [eval(e) for e in unique_extras]

        # Merge new attributes and update package
        dataset = dict(dataset.items() + attributes.items())
        remote.call_action('package_update',
                data_dict=dataset,
                apikey=token)
        return True
    except CKANAPIError, e:
        print "CKANAPIError for dataset", "'%s'" % dataset['title']
        print e
    return False


def update_resource(remote, resource, attributes):
    try:
        # Merge new attributes and update package
        resource = dict(resource.items() + attributes.items())
        remote.call_action('resource_update',
                data_dict=resource,
                apikey=token)
        return True
    except CKANAPIError, e:
        print "CKANAPIError for resource", "'%s'" % resource['url']
        print e
    return False


# Extract DCAT publisher name from metadata
def get_dcat_publisher(dataset):
    extra_metadata = dataset.get('extras', [])
    for e in extra_metadata:
        if e['key'] == 'dcat_publisher_name':
            return e['value']
    print "Couldn't find dcat fields metadata for", "'%s'" % dataset['title']
    return ''


# Update the owner package to be the same as DCAT publisher
# NOTE: publisher name comes in uppercase, org name in lowercase
def update_dataset_owner_as_dcat_publisher(remote, dataset):
    organizations = remote.action.organization_list()
    publisher = get_dcat_publisher(dataset).lower()
    # TODO: owner_org is a hash id so it will never match publisher name
    if (publisher in organizations and dataset['owner_org'] != publisher):
        success = update_dataset(remote,
                dataset,
                { 'owner_org': publisher })
        if success:
            print publisher.upper(), "now owns", "'%s'" % dataset['title']


def clear_dataset_license(remote, dataset):
    attributes = { 'license_id': 'notspecified' }
    if dataset['license_id'] == 'notspecified':
        return True
    if update_dataset(remote, dataset, attributes):
        title = "'%s'" % dataset['title']
        print "Updated", dataset['license_id'], "to notspecified", "for", title


def update_resources_format_based_on_extension(remote, dataset):
    formats = {
        'csv':      'CSV',
        'geojson':  'GeoJSON',
        'gif':      'GIF',
        'json':     'JSON',
        'kml+xml':  'KML',
        'kmz':      'KMZ',
        'pdf':      'PDF',
        'png':      'PNG',
        'xls':      'XLS',
        'xlsx':     'XLSX',
        'xml':      'XML',
        'zip':      'ZIP',
    }
    for r in dataset['resources']:
        if len(r['format']) > 0:
            continue
        extension = get_extension_from_url(r['url'])
        try:
            f = formats[extension]
            update_resource(remote,
                    r,
                    { 'format': f })
        except KeyError, e:
            print extension, "is not a known file extension"


def get_extension_from_url(u):
    # SEE: https://docs.python.org/2/library/urlparse.html
    path = urlparse(u)[2]
    return path.split('.')[-1].strip().lower()


def update_bbox(remote, dataset, region):
    regions = {
        'Mexico': [
            {'key':'spatial-text', 'value':'Mexico'},
            {'key':'spatial', 'value':'{"type":"Polygon","coordinates":[[[-118.30078125,13.667338259654947],[-118.30078125,33.35806161277885],[-85.95703125,33.35806161277885],[-85.95703125,13.667338259654947],[-118.30078125,13.667338259654947]]]}'},
            {'key':'spatial-uri', 'value':'http://www.geonames.org/3996063'}
        ]
    }

    print "Updating", "'%s'" % dataset['title']
    extras = dataset['extras']
    for h in regions[region]:
        extras.append(h)
    update_dataset(remote, dataset, {'extras': extras})


def main():
    remote = ckanapi.RemoteCKAN(host, user_agent='ckanops/1.0', apikey=token)
    datasets = remote.action.package_list()
    # datasets = [d['name'] for d in remote.action.package_search(q='nacional')['results']]

    print "Will update", len(datasets), "datasets"
    for d in datasets:<commit_msg>Get all tags by organization<commit_after>#!/usr/bin/env python

import os
import operator
import ckanapi
from ckanapi.errors import CKANAPIError
from urlparse import urlparse


host = os.environ['CKAN_HOST']
token = os.environ['CKAN_API_TOKEN']


def update_dataset(remote, dataset, attributes):
    try:
        # Remove duplicate metadata fields
        # NOTE: This happened in a few experiments in the extra fields
        unique_extras = set(str(e) for e in dataset['extras'])
        dataset['extras'] = [eval(e) for e in unique_extras]

        # Merge new attributes and update package
        dataset = dict(dataset.items() + attributes.items())
        remote.call_action('package_update',
                data_dict=dataset,
                apikey=token)
        return True
    except CKANAPIError, e:
        print "CKANAPIError for dataset", "'%s'" % dataset['title']
        print e
    return False


def update_resource(remote, resource, attributes):
    try:
        # Merge new attributes and update package
        resource = dict(resource.items() + attributes.items())
        remote.call_action('resource_update',
                data_dict=resource,
                apikey=token)
        return True
    except CKANAPIError, e:
        print "CKANAPIError for resource", "'%s'" % resource['url']
        print e
    return False


# Extract DCAT publisher name from metadata
def get_dcat_publisher(dataset):
    extra_metadata = dataset.get('extras', [])
    for e in extra_metadata:
        if e['key'] == 'dcat_publisher_name':
            return e['value']
    print "Couldn't find dcat fields metadata for", "'%s'" % dataset['title']
    return ''


# Update the owner package to be the same as DCAT publisher
# NOTE: publisher name comes in uppercase, org name in lowercase
def update_dataset_owner_as_dcat_publisher(remote, dataset):
    organizations = remote.action.organization_list()
    publisher = get_dcat_publisher(dataset).lower()
    # TODO: owner_org is a hash id so it will never match publisher name
    if (publisher in organizations and dataset['owner_org'] != publisher):
        success = update_dataset(remote,
                dataset,
                { 'owner_org': publisher })
        if success:
            print publisher.upper(), "now owns", "'%s'" % dataset['title']


def clear_dataset_license(remote, dataset):
    attributes = { 'license_id': 'notspecified' }
    if dataset['license_id'] == 'notspecified':
        return True
    if update_dataset(remote, dataset, attributes):
        title = "'%s'" % dataset['title']
        print "Updated", dataset['license_id'], "to notspecified", "for", title


def update_resources_format_based_on_extension(remote, dataset):
    formats = {
        'csv':      'CSV',
        'geojson':  'GeoJSON',
        'gif':      'GIF',
        'json':     'JSON',
        'kml+xml':  'KML',
        'kmz':      'KMZ',
        'pdf':      'PDF',
        'png':      'PNG',
        'xls':      'XLS',
        'xlsx':     'XLSX',
        'xml':      'XML',
        'zip':      'ZIP',
    }
    for r in dataset['resources']:
        if len(r['format']) > 0:
            continue
        extension = get_extension_from_url(r['url'])
        try:
            f = formats[extension]
            update_resource(remote,
                    r,
                    { 'format': f })
        except KeyError, e:
            print extension, "is not a known file extension"


def get_extension_from_url(u):
    # SEE: https://docs.python.org/2/library/urlparse.html
    path = urlparse(u)[2]
    return path.split('.')[-1].strip().lower()


def update_bbox(remote, dataset, region):
    regions = {
        'Mexico': [
            {'key':'spatial-text', 'value':'Mexico'},
            {'key':'spatial', 'value':'{"type":"Polygon","coordinates":[[[-118.30078125,13.667338259654947],[-118.30078125,33.35806161277885],[-85.95703125,33.35806161277885],[-85.95703125,13.667338259654947],[-118.30078125,13.667338259654947]]]}'},
            {'key':'spatial-uri', 'value':'http://www.geonames.org/3996063'}
        ]
    }

    print "Updating", "'%s'" % dataset['title']
    extras = dataset['extras']
    for h in regions[region]:
        extras.append(h)
    update_dataset(remote, dataset, {'extras': extras})


def tags_covered_by_an_organization(o):
    tags = []
    for p in o['packages']:
        tag_names = [t['name'] for t in p['tags']]
        tags.append(tag_names)
    if len(tags) == 0:
        return []
    else:
        return reduce(operator.add, tags)


def all_tags_by_organization(remote):
    tags_by_organization = {}

    organizations_names = remote.action.organization_list()
    for name in organizations_names:
        organization = remote.action.organization_show(id=name)
        tags = tags_covered_by_an_organization(organization)
        tags_by_organization[name] = tags

    return tags_by_organization


def main():
    remote = ckanapi.RemoteCKAN(host, user_agent='ckanops/1.0', apikey=token)

    datasets = remote.action.package_list()
    # datasets = [d['name'] for d in remote.action.package_search(q='nacional')['results']]

    print "Will update", len(datasets), "datasets"
    for d in datasets:
{'fraction_non_alphanumeric': 0.13312186636652001, 'fraction_numerical': 0.030594495037347794, 'mean_word_length': 2.7163498098859318, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5948072', 'n_tokens_mistral': 3186, 'n_tokens_neox': 2923, 'n_words': 720}