<commit_before>#!/usr/bin/env python

"""

Download data from statweb.provincia.tn.it into a folder containing
json files, for use by the import script.

Will also convert original datasets into datasets suitable for
insertion in Ckan.

"""

from __future__ import print_function

import json
import logging
import sys
import os

from client_statistica import StatisticaClient, CATEGORIES


## todo: we need to create organization / group json files, reading
##       from some configuration that will also contain information
##       on how to map categories source -> ckan.

_logger = logging.getLogger()  # root logger
_logger.addHandler(logging.StreamHandler(sys.stderr))
_logger.setLevel(logging.DEBUG)

logger = logging.getLogger(__name__)


destination = sys.argv[1]
for name in ('dataset', 'group', 'organization'):
    dirname = os.path.join(destination, name)
    if not os.path.exists(dirname):
        os.makedirs(dirname)
del dirname


def download_datasets(destination):
    ## Download datasets
    client = StatisticaClient()
    for dataset in client.iter_datasets():
        logger.info("Dataset: {0}".format(dataset['id']))
        destfile = os.path.join(destination, 'dataset',
                                '{0}.json'.format(dataset['id']))
        with open(destfile, 'wb') as f:
            json.dump(dataset, f)


def write_organizations(destination):
    ## Write organization to file
    destfile = os.path.join(destination, 'organization',
                            'pat-s-statistica.json')
    with open(destfile, 'wb') as f:
        json.dump({
            "name": "pat-s-statistica",
            "description":
            "Censimenti, analisi, indagine statistiche, indicatori, ...",
            "display_name": "<NAME>",
            "image_url": "http://dati.trentino.it/images/logo.png",
            "is_organization": True,
            "state": "active",
            "tags": [],
            "title": "PAT S. Statistica",
            "type": "organization"
        }, f)


def write_groups(destination):
    ## Write groups to file
    for key, val in CATEGORIES.iteritems():
        destfile = os.path.join(destination, 'group', '{0}.json'.format(key))
        with open(destfile, 'wb') as f:
            json.dump(val, f)


write_organizations(destination)
write_groups(destination)
# download_datasets(destination)
<commit_msg>Uncommented line<commit_after>#!/usr/bin/env python

"""

Download data from statweb.provincia.tn.it into a folder containing
json files, for use by the import script.

Will also convert original datasets into datasets suitable for
insertion in Ckan.

"""

from __future__ import print_function

import json
import logging
import sys
import os

from client_statistica import StatisticaClient, CATEGORIES


## todo: we need to create organization / group json files, reading
##       from some configuration that will also contain information
##       on how to map categories source -> ckan.

_logger = logging.getLogger()  # root logger
_logger.addHandler(logging.StreamHandler(sys.stderr))
_logger.setLevel(logging.DEBUG)

logger = logging.getLogger(__name__)


destination = sys.argv[1]
for name in ('dataset', 'group', 'organization'):
    dirname = os.path.join(destination, name)
    if not os.path.exists(dirname):
        os.makedirs(dirname)
del dirname


def download_datasets(destination):
    ## Download datasets
    client = StatisticaClient()
    for dataset in client.iter_datasets():
        logger.info("Dataset: {0}".format(dataset['id']))
        destfile = os.path.join(destination, 'dataset',
                                '{0}.json'.format(dataset['id']))
        with open(destfile, 'wb') as f:
            json.dump(dataset, f)


def write_organizations(destination):
    ## Write organization to file
    destfile = os.path.join(destination, 'organization',
                            'pat-s-statistica.json')
    with open(destfile, 'wb') as f:
        json.dump({
            "name": "pat-s-statistica",
            "description":
            "Censimenti, analisi, indagine statistiche, indicatori, ...",
            "display_name": "<NAME>",
            "image_url": "http://dati.trentino.it/images/logo.png",
            "is_organization": True,
            "state": "active",
            "tags": [],
            "title": "<NAME>",
            "type": "organization"
        }, f)


def write_groups(destination):
    ## Write groups to file
    for key, val in CATEGORIES.iteritems():
        destfile = os.path.join(destination, 'group', '{0}.json'.format(key))
        with open(destfile, 'wb') as f:
            json.dump(val, f)


write_organizations(destination)
write_groups(destination)
download_datasets(destination)

{'fraction_non_alphanumeric': 0.12098138747884941, 'fraction_numerical': 0.001692047377326565, 'mean_word_length': 2.9018151815181517, 'pattern_counts': {'":': 18, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4027113', 'n_tokens_mistral': 1373, 'n_tokens_neox': 1347, 'n_words': 369}