<commit_before>'''
Test the whole process. This requires a running CKAN instance
with the datastore activated.
'''

import os
import json
import uuid

from nose import SkipTest
from nose.tools import assert_equal
import requests

import httpretty

import ckanserviceprovider.web as web
import datapusher.main as main
import datapusher.jobs as jobs

import settings_test as config

os.environ['JOB_CONFIG'] = os.path.join(os.path.dirname(__file__),
                                        'settings_test.py')

web.configure()
app = main.serve_test()


def join_static_path(filename):
    return os.path.join(os.path.dirname(__file__), 'static', filename)


def get_static_file(filename):
    return open(join_static_path(filename)).read()


class TestImport():
    resource_ids = []

    @classmethod
    def setup_class(cls):
        try:
            r = requests.get('http://{0}/api/3/action/datastore_search?resource_id=_table_metadata'.format(config.CKAN_HOST))
            if r.status_code not in [200, 201]:
                raise SkipTest("Need a CKAN with the datastore enabled")
        except requests.ConnectionError:
            raise SkipTest("No CKAN running")
        cls.host = config.CKAN_HOST
        cls.api_key = config.USER_API_KEY

    def teardown(self):
        self.clean_up()

    def clean_up(self):
        ''' Go through all created resources and delete them
        '''
        while self.resource_ids:
            res_id = self.resource_ids.pop()
            request = {'resource_id': res_id}
            r = requests.post('http://%s/api/3/action/datastore_delete' % self.host,
                              data=json.dumps(request),
                              headers={'Content-Type': 'application/json',
                                       'Authorization': self.api_key}
                              )
            if r.status_code not in [200, 404]:
                raise Exception('Error deleting datastore for resource %s' % res_id)

    def make_resource_id(self, url):
        ''' Create a resource in ckan and get back the id
        '''
        r = requests.post(
            'http://%s/api/3/action/package_create' % self.host,
            data=json.dumps(
                {'name': str(uuid.uuid4()),
                 'resources': [{'url': url, 'format': 'csv'}]}
            ),
            headers={'Authorization': self.api_key, 'content-type': 'application/json'}
        )
        if r.status_code != 200:
                raise Exception('Error creating datastore for resource. Check the CKAN output.')

        res_id = json.loads(r.content)['result']['resources'][0]['id']
        self.resource_ids.append(res_id)
        return res_id

    @httpretty.activate
    def test_simple_csv_directly(self):
        url = 'http://www.ckan.org/static/simple.csv'
        httpretty.register_uri(httpretty.GET, url,
                               body=get_static_file('simple.csv'),
                               content_type="application/csv")
        resource_id = self.make_resource_id(url)

        data = {
            'api_key': self.api_key,
            'job_type': 'push_to_datastore',
            'metadata': {
                'ckan_url': 'http://%s/' % self.host,
                'resource_id': resource_id
            }
        }

        jobs.push_to_datastore(None, data, web.queue)

        response = requests.get(
            'http://%s/api/action/datastore_search?resource_id=%s' % (self.host, resource_id),
            headers={"content-type": "application/json"})

        result = json.loads(response.content)

        assert not 'error' in result, result['error']

        value = result['result']['records'][0][u'temperature']
        assert_equal(int(value), 1)
        assert_equal(result['result']['total'], 6)
        assert_equal(result['result']['fields'],
                     [{u'type': u'int4', u'id': u'_id'},
                      {u'type': u'timestamp', u'id': u'date'},
                      {u'type': u'numeric', u'id': u'temperature'},
                      {u'type': u'text', u'id': u'place'}])<commit_msg>Test long and messy files in integration testing<commit_after>'''
Test the whole process. This requires a running CKAN instance
with the datastore activated.
'''

import os
import json
import uuid

from nose import SkipTest
from nose.tools import assert_equal
import requests

import httpretty

import ckanserviceprovider.web as web
import datapusher.main as main
import datapusher.jobs as jobs

import settings_test as config

os.environ['JOB_CONFIG'] = os.path.join(os.path.dirname(__file__),
                                        'settings_test.py')

web.configure()
app = main.serve_test()

headers = {"content-type": "application/json"}


def join_static_path(filename):
    return os.path.join(os.path.dirname(__file__), 'static', filename)


def get_static_file(filename):
    return open(join_static_path(filename)).read()


class TestImport():
    resource_ids = []

    @classmethod
    def setup_class(cls):
        try:
            r = requests.get('http://{0}/api/3/action/datastore_search?resource_id=_table_metadata'.format(config.CKAN_HOST))
            if r.status_code not in [200, 201]:
                raise SkipTest("Need a CKAN with the datastore enabled")
        except requests.ConnectionError:
            raise SkipTest("No CKAN running")
        cls.host = config.CKAN_HOST
        cls.api_key = config.USER_API_KEY

    def teardown(self):
        self.clean_up()

    def clean_up(self):
        ''' Go through all created resources and delete them
        '''
        while self.resource_ids:
            res_id = self.resource_ids.pop()
            request = {'resource_id': res_id}
            r = requests.post('http://%s/api/3/action/datastore_delete' % self.host,
                              data=json.dumps(request),
                              headers={'Content-Type': 'application/json',
                                       'Authorization': self.api_key}
                              )
            if r.status_code not in [200, 404]:
                raise Exception('Error deleting datastore for resource %s' % res_id)

    def make_resource_id(self, url):
        ''' Create a resource in ckan and get back the id
        '''
        r = requests.post(
            'http://%s/api/3/action/package_create' % self.host,
            data=json.dumps(
                {'name': str(uuid.uuid4()),
                 'resources': [{'url': url, 'format': 'csv'}]}
            ),
            headers={'Authorization': self.api_key, 'content-type': 'application/json'}
        )
        if r.status_code != 200:
                raise Exception('Error creating datastore for resource. Check the CKAN output.')

        res_id = json.loads(r.text)['result']['resources'][0]['id']
        self.resource_ids.append(res_id)
        return res_id

    @httpretty.activate
    def test_simple_csv(self):
        url = 'http://www.ckan.org/static/simple.csv'
        httpretty.register_uri(httpretty.GET, url,
                               body=get_static_file('simple.csv'),
                               content_type="application/csv")
        resource_id = self.make_resource_id(url)

        data = {
            'api_key': self.api_key,
            'job_type': 'push_to_datastore',
            'metadata': {
                'ckan_url': 'http://%s/' % self.host,
                'resource_id': resource_id
            }
        }

        jobs.push_to_datastore(None, data, web.queue)

        response = requests.get(
            'http://%s/api/action/datastore_search?resource_id=%s' % (self.host, resource_id),
            headers=headers)

        assert_equal(response.status_code, 200)
        try:
            result = response.json()
        except:
            print "Could not parse", response.text
            raise

        assert not 'error' in result, result['error']

        value = result['result']['records'][0][u'temperature']
        assert_equal(int(value), 1)
        assert_equal(result['result']['total'], 6)
        assert_equal(result['result']['fields'],
                     [{u'type': u'int4', u'id': u'_id'},
                      {u'type': u'timestamp', u'id': u'date'},
                      {u'type': u'numeric', u'id': u'temperature'},
                      {u'type': u'text', u'id': u'place'}])

    @httpretty.activate
    def test_long_csv(self):
        url = 'http://www.ckan.org/static/long.csv'
        httpretty.register_uri(httpretty.GET, url,
                               body=get_static_file('long.csv'),
                               content_type="application/csv")
        resource_id = self.make_resource_id(url)

        data = {
            'api_key': self.api_key,
            'job_type': 'push_to_datastore',
            'metadata': {
                'ckan_url': 'http://%s/' % self.host,
                'resource_id': resource_id
            }
        }

        jobs.push_to_datastore(None, data, web.queue)

        response = requests.get(
            'http://%s/api/action/datastore_search?resource_id=%s' % (self.host, resource_id),
            headers=headers)

        assert_equal(response.status_code, 200)
        try:
            result = response.json()
        except:
            print response.text
            raise

        assert not 'error' in result, result['error']
        assert result['result']['total'] == 4000, (result['result']['total'], resource_id)

    @httpretty.activate
    def test_messier_file(self):
        url = 'http://www.ckan.org/static/messy.csv'
        httpretty.register_uri(httpretty.GET, url,
                               body=get_static_file('3ffdcd42-5c63-4089-84dd-c23876259973.csv'),
                               content_type="application/csv")
        resource_id = self.make_resource_id(url)

        data = {
            'api_key': self.api_key,
            'job_type': 'push_to_datastore',
            'metadata': {
                'ckan_url': 'http://%s/' % self.host,
                'resource_id': resource_id
            }
        }

        jobs.push_to_datastore(None, data, web.queue)

        response = requests.get(
            'http://%s/api/action/datastore_search?resource_id=%s' % (self.host, resource_id),
            headers=headers)

        assert_equal(response.status_code, 200)
        try:
            result = response.json()
        except:
            print response.text
            raise

        assert not 'error' in result, result['error']
        value = result['result']['records'][0][u'Transaction Number']
        assert_equal(int(value), 136980)
        assert_equal(result['result']['total'], 564)

        assert_equal(result['result']['fields'],
                     [{u'type': u'int4', u'id': u'_id'},
                      {u'type': u'text', u'id': u'Body Name'},
                      {u'type': u'timestamp', u'id': u'Date'},
                      {u'type': u'numeric', u'id': u'Transaction Number'},
                      {u'type': u'numeric', u'id': u'Amount'},
                      {u'type': u'text', u'id': u'Supplier'},
                      {u'type': u'text', u'id': u'Expense Area'}])
{'fraction_non_alphanumeric': 0.13060603347954525, 'fraction_numerical': 0.008683197565123982, 'mean_word_length': 1.9879646964428992, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3680153', 'n_tokens_mistral': 3355, 'n_tokens_neox': 3237, 'n_words': 654}