<commit_before>import os
import uuid
import json
import shutil
import ckanapi
import zipfile
import logging
import requests
import shapefile

from subprocess import call

TEMPDIR = os.path.join(os.path.dirname(__file__), '..', 'tmp')
OUTDIR = os.path.join(TEMPDIR, 'out')

class BadResourceFileException(Exception):
    def __init__(self, extra_msg=None):
        self.extra_msg = extra_msg

    def __str__(self):
        return self.extra_msg

class FileTooLargeError(Exception):

    def __init__(self, extra_msg=None):
        self.extra_msg = extra_msg

    def __str__(self):
        return self.extra_msg

def convert_and_import(ckan, datasets, file_format):
    shutil.rmtree(TEMPDIR)

    for d in datasets:
        dataset = ckan.action.package_show(id=d)
        resources = dataset['resources']
        for resource in resources:
            res_name = resource['name'].encode('ascii', 'ignore')
            if resource['format'] == file_format:
                try:
                    print "processing {0}:{1}".format(
                                            d, res_name.encode('utf-8'))
                    process(ckan, resource, file_format)
                except FileTooLargeError:
                    print "skipping {0}:{1} - too large".format(
                                            d, res_name.encode('utf-8'))

def process(ckan, resource, file_format):
    if not os.path.isdir(OUTDIR):<commit_msg>regenerate temp dir<commit_after>import os
import uuid
import json
import shutil
import ckanapi
import zipfile
import logging
import requests
import shapefile

from subprocess import call

TEMPDIR = os.path.join(os.path.dirname(__file__), '..', 'tmp')
OUTDIR = os.path.join(TEMPDIR, 'out')

class BadResourceFileException(Exception):
    def __init__(self, extra_msg=None):
        self.extra_msg = extra_msg

    def __str__(self):
        return self.extra_msg

class FileTooLargeError(Exception):

    def __init__(self, extra_msg=None):
        self.extra_msg = extra_msg

    def __str__(self):
        return self.extra_msg

def convert_and_import(ckan, datasets, file_format):
    shutil.rmtree(TEMPDIR)
    os.makedirs(TEMPDIR)

    for d in datasets:
        dataset = ckan.action.package_show(id=d)
        resources = dataset['resources']
        for resource in resources:
            res_name = resource['name'].encode('ascii', 'ignore')
            if resource['format'] == file_format:
                try:
                    print "processing {0}:{1}".format(
                                            d, res_name.encode('utf-8'))
                    process(ckan, resource, file_format)
                except FileTooLargeError:
                    print "skipping {0}:{1} - too large".format(
                                            d, res_name.encode('utf-8'))

def process(ckan, resource, file_format):
    if not os.path.isdir(OUTDIR):
{'fraction_non_alphanumeric': 0.10328151986183075, 'fraction_numerical': 0.004145077720207254, 'mean_word_length': 2.332566168009206, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6564309', 'n_tokens_mistral': 836, 'n_tokens_neox': 819, 'n_words': 198}