<commit_before>import json
import os

import cherrypy

from controllers.datasets import Datasets
from controllers.calculations import Calculations
from lib.constants import CREATED_AT, ERROR, ID, MODE_INFO,\
    MODE_SUMMARY, MONGO_RESERVED_KEYS, MONGO_RESERVED_KEY_PREFIX,\
    SCHEMA, SUCCESS, SUMMARY, UPDATED_AT
from lib.decorators import requires_internet
from lib.io import create_dataset_from_url
from lib.utils import build_labels_to_slugs
from models.dataset import Dataset
from models.calculation import Calculation
from tests.test_base import TestBase
from tests.mock import MockUploadedFile


class TestDatasets(TestBase):

    # NOTE: NUM_COLS should be 15 (we are not currently returning _id column)
    NUM_COLS = 14
    NUM_ROWS = 19

    def setUp(self):
        TestBase.setUp(self)
        self._file_name = 'good_eats.csv'
        self._file_path = 'tests/fixtures/%s' % self._file_name
        self._file_uri = 'file://%s' % self._file_path
        self.url = 'http://formhub.org/mberg/forms/good_eats/data.csv'
        self._update_file_name = 'good_eats_update.json'
        self._update_file_path = 'tests/fixtures/%s' % self._update_file_name
        self.controller = Datasets()

    def _post_file(self):
        self.dataset_id = create_dataset_from_url(self._file_uri,
                                                  allow_local_file=True)[ID]

    def _post_calculations(self):
        # must call after _post_file
        controller = Calculations()
        formulae = [
            'amount + 1',
            'amount - 5',
        ]
        # NOTE: this may not work due to lack of synchronization
        for formula in formulae:
            controller.POST(self.dataset_id, formula, formula)

    def _test_summary_results(self, results):
        results = json.loads(results)
        self.assertTrue(isinstance(results, dict))
        return results

    def _test_summary_no_group(self, results):
        result_keys = results.keys()
        self.assertEqual(len(result_keys), self.NUM_COLS)
        columns = [col for col in
                   self.test_data[self._file_name].columns.tolist()
                   if not col in MONGO_RESERVED_KEYS]
        dataset = Dataset.find_one(self.dataset_id)
        labels_to_slugs = build_labels_to_slugs(dataset)
        for col in columns:
            slug = labels_to_slugs[col]
            self.assertTrue(slug in result_keys,
                            'col (slug): %s in: %s' % (slug, result_keys))
            self.assertTrue(SUMMARY in results[slug].keys())

    def _test_get_with_query_or_select(self, query='{}', select=None):
        self._post_file()
        results = json.loads(self.controller.GET(self.dataset_id, query=query,
                             select=select))
        self.assertTrue(isinstance(results, list))
        self.assertTrue(isinstance(results[3], dict))
        if select:
            self.assertEqual(sorted(results[0].keys()), ['rating'])
        if query != '{}':
            self.assertEqual(len(results), 11)

    def test_POST_dataset_id_update(self):
        self._post_file()
        self._post_calculations()
        num_rows = len(json.loads(self.controller.GET(self.dataset_id)))
        # mock the cherrypy server by setting the POST request body
        cherrypy.request.body = open(self._update_file_path, 'r')
        result = json.loads(self.controller.POST(dataset_id=self.dataset_id))
        num_rows_after_update = len(json.loads(
            self.controller.GET(self.dataset_id)))
        self.assertTrue(isinstance(result, dict))
        self.assertTrue(ID in result)
        self.assertEqual(num_rows_after_update, num_rows + 1)
        results = json.loads(self.controller.GET(self.dataset_id))
        schema = json.loads(self.controller.GET(self.dataset_id,<commit_msg>test update post with bad id<commit_after>import json
import os

import cherrypy

from controllers.datasets import Datasets
from controllers.calculations import Calculations
from lib.constants import CREATED_AT, ERROR, ID, MODE_INFO,\
    MODE_SUMMARY, MONGO_RESERVED_KEYS, MONGO_RESERVED_KEY_PREFIX,\
    SCHEMA, SUCCESS, SUMMARY, UPDATED_AT
from lib.decorators import requires_internet
from lib.io import create_dataset_from_url
from lib.utils import build_labels_to_slugs
from models.dataset import Dataset
from models.calculation import Calculation
from tests.test_base import TestBase
from tests.mock import MockUploadedFile


class TestDatasets(TestBase):

    # NOTE: NUM_COLS should be 15 (we are not currently returning _id column)
    NUM_COLS = 14
    NUM_ROWS = 19

    def setUp(self):
        TestBase.setUp(self)
        self._file_name = 'good_eats.csv'
        self._file_path = 'tests/fixtures/%s' % self._file_name
        self._file_uri = 'file://%s' % self._file_path
        self.url = 'http://formhub.org/mberg/forms/good_eats/data.csv'
        self._update_file_name = 'good_eats_update.json'
        self._update_file_path = 'tests/fixtures/%s' % self._update_file_name
        self.controller = Datasets()

    def _post_file(self):
        self.dataset_id = create_dataset_from_url(self._file_uri,
                                                  allow_local_file=True)[ID]

    def _post_calculations(self):
        # must call after _post_file
        controller = Calculations()
        formulae = [
            'amount + 1',
            'amount - 5',
        ]
        # NOTE: this may not work due to lack of synchronization
        for formula in formulae:
            controller.POST(self.dataset_id, formula, formula)

    def _test_summary_results(self, results):
        results = json.loads(results)
        self.assertTrue(isinstance(results, dict))
        return results

    def _test_summary_no_group(self, results):
        result_keys = results.keys()
        self.assertEqual(len(result_keys), self.NUM_COLS)
        columns = [col for col in
                   self.test_data[self._file_name].columns.tolist()
                   if not col in MONGO_RESERVED_KEYS]
        dataset = Dataset.find_one(self.dataset_id)
        labels_to_slugs = build_labels_to_slugs(dataset)
        for col in columns:
            slug = labels_to_slugs[col]
            self.assertTrue(slug in result_keys,
                            'col (slug): %s in: %s' % (slug, result_keys))
            self.assertTrue(SUMMARY in results[slug].keys())

    def _test_get_with_query_or_select(self, query='{}', select=None):
        self._post_file()
        results = json.loads(self.controller.GET(self.dataset_id, query=query,
                             select=select))
        self.assertTrue(isinstance(results, list))
        self.assertTrue(isinstance(results[3], dict))
        if select:
            self.assertEqual(sorted(results[0].keys()), ['rating'])
        if query != '{}':
            self.assertEqual(len(results), 11)

    def test_POST_dataset_id_update_bad_dataset_id(self):
        result = json.loads(self.controller.POST(dataset_id=111))
        assert(ERROR in result)

    def test_POST_dataset_id_update(self):
        self._post_file()
        self._post_calculations()
        num_rows = len(json.loads(self.controller.GET(self.dataset_id)))
        # mock the cherrypy server by setting the POST request body
        cherrypy.request.body = open(self._update_file_path, 'r')
        result = json.loads(self.controller.POST(dataset_id=self.dataset_id))
        num_rows_after_update = len(json.loads(
            self.controller.GET(self.dataset_id)))
        self.assertTrue(isinstance(result, dict))
        self.assertTrue(ID in result)
        self.assertEqual(num_rows_after_update, num_rows + 1)
        results = json.loads(self.controller.GET(self.dataset_id))
        schema = json.loads(self.controller.GET(self.dataset_id,
{'fraction_non_alphanumeric': 0.09459113047936427, 'fraction_numerical': 0.0037169956421430404, 'mean_word_length': 2.755052935514918, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3933625', 'n_tokens_mistral': 2396, 'n_tokens_neox': 2347, 'n_words': 483}