<commit_before>#coding=UTF8
"""
Unit tests for the hxl.io module
<NAME>
October 2014

License: Public Domain
"""

import unittest
import os
import sys
import io
import json
from urllib.error import HTTPError
from io import StringIO

import hxl
from hxl.io import make_input, HXLParseException, HXLReader, CSVInput


def _resolve_file(filename):
    return os.path.join(os.path.dirname(__file__), filename)

DATA = [
   ['Sector', 'Organisation', 'Province name'],
   ['#sector', '#org', '#adm1'],
   ['WASH', 'Org A', 'Coast'],
   ['Health', 'Org B', 'Plains']
]

FILE_CSV = _resolve_file('./files/test_io/input-valid.csv')
FILE_CSV_OUT = _resolve_file('./files/test_io/output-valid.csv')
FILE_EXCEL = _resolve_file('./files/test_io/input-valid.xlsx')
FILE_JSON = _resolve_file('./files/test_io/input-valid.json')
FILE_JSON_TXT = _resolve_file('./files/test_io/input-valid-json.txt')
FILE_JSON_UNTAGGED = _resolve_file('./files/test_io/input-untagged.json')
FILE_JSON_OUT = _resolve_file('./files/test_io/output-valid.json')
FILE_JSON_OBJECTS = _resolve_file('./files/test_io/input-valid-objects.json')
FILE_JSON_OBJECTS_UNTAGGED = _resolve_file('./files/test_io/input-untagged-objects.json')
FILE_JSON_OBJECTS_OUT = _resolve_file('./files/test_io/output-valid-objects.json')
FILE_JSON_NESTED = _resolve_file('./files/test_io/input-valid-nested.json')
FILE_JSON_SELECTOR = _resolve_file('./files/test_io/input-valid-json-selector.json')
FILE_MULTILINE = _resolve_file('./files/test_io/input-multiline.csv')
FILE_FUZZY = _resolve_file('./files/test_io/input-fuzzy.csv')
FILE_INVALID = _resolve_file('./files/test_io/input-invalid.csv')
URL_CSV = 'https://raw.githubusercontent.com/HXLStandard/libhxl-python/master/tests/files/test_io/input-valid.csv'
URL_EXCEL = 'https://raw.githubusercontent.com/HXLStandard/libhxl-python/master/tests/files/test_io/input-valid.xlsx'
URL_JSON = 'https://raw.githubusercontent.com/HXLStandard/libhxl-python/master/tests/files/test_io/input-valid.json'
URL_GOOGLE_NOHASH = 'https://docs.google.com/spreadsheets/d/1VTswL-w9EI0IdGIBFZoZ-2RmIiebXKsrhv03yd7LlIg/edit'
URL_GOOGLE_HASH = 'https://docs.google.com/spreadsheets/d/1VTswL-w9EI0IdGIBFZoZ-2RmIiebXKsrhv03yd7LlIg/edit#gid=299366282'


class TestInput(unittest.TestCase):

    def test_array(self):
        self.assertTrue(make_input(DATA).is_repeatable)
        self.assertTrue('#sector' in hxl.data(DATA).tags)

    def test_csv(self):
        input = make_input(FILE_CSV, True)
        self.assertFalse(input.is_repeatable)
        self.assertTrue('#sector' in hxl.data(input).tags)

    def test_json_lists(self):
        input = make_input(FILE_JSON, True)
        self.assertFalse(input.is_repeatable)
        self.assertTrue('#sector' in hxl.data(input).tags)

    def test_json_objects(self):
        input = make_input(FILE_JSON_OBJECTS, True)
        self.assertFalse(input.is_repeatable)
        self.assertTrue('#sector' in hxl.data(input).tags)

    def test_json_selector(self):
        input = make_input(FILE_JSON_SELECTOR, True, selector="sel1")
        pass

    def test_excel(self):
        input = make_input(FILE_EXCEL, True)
        self.assertTrue(input.is_repeatable)

    def test_ckan_resource(self):
        source = hxl.data('https://data.humdata.org/dataset/hxl-master-vocabulary-list/resource/d22dd1b6-2ff0-47ab-85c6-08aeb911a832')
        self.assertTrue('#vocab' in source.tags)

    def test_ckan_dataset(self):
        source = hxl.data('https://data.humdata.org/dataset/hxl-master-vocabulary-list')
        self.assertTrue('#vocab' in source.tags)

class TestUntaggedInput(unittest.TestCase):<commit_msg>Add test for string (actually byte) input.<commit_after>#coding=UTF8
"""
Unit tests for the hxl.io module
<NAME>
October 2014

License: Public Domain
"""

import unittest
import os
import sys
import io
import json
from urllib.error import HTTPError
from io import StringIO

import hxl
from hxl.io import make_input, HXLParseException, HXLReader, CSVInput


def _resolve_file(filename):
    return os.path.join(os.path.dirname(__file__), filename)

DATA = [
   ['Sector', 'Organisation', 'Province name'],
   ['#sector', '#org', '#adm1'],
   ['WASH', 'Org A', 'Coast'],
   ['Health', 'Org B', 'Plains']
]

FILE_CSV = _resolve_file('./files/test_io/input-valid.csv')
FILE_CSV_OUT = _resolve_file('./files/test_io/output-valid.csv')
FILE_EXCEL = _resolve_file('./files/test_io/input-valid.xlsx')
FILE_JSON = _resolve_file('./files/test_io/input-valid.json')
FILE_JSON_TXT = _resolve_file('./files/test_io/input-valid-json.txt')
FILE_JSON_UNTAGGED = _resolve_file('./files/test_io/input-untagged.json')
FILE_JSON_OUT = _resolve_file('./files/test_io/output-valid.json')
FILE_JSON_OBJECTS = _resolve_file('./files/test_io/input-valid-objects.json')
FILE_JSON_OBJECTS_UNTAGGED = _resolve_file('./files/test_io/input-untagged-objects.json')
FILE_JSON_OBJECTS_OUT = _resolve_file('./files/test_io/output-valid-objects.json')
FILE_JSON_NESTED = _resolve_file('./files/test_io/input-valid-nested.json')
FILE_JSON_SELECTOR = _resolve_file('./files/test_io/input-valid-json-selector.json')
FILE_MULTILINE = _resolve_file('./files/test_io/input-multiline.csv')
FILE_FUZZY = _resolve_file('./files/test_io/input-fuzzy.csv')
FILE_INVALID = _resolve_file('./files/test_io/input-invalid.csv')
URL_CSV = 'https://raw.githubusercontent.com/HXLStandard/libhxl-python/master/tests/files/test_io/input-valid.csv'
URL_EXCEL = 'https://raw.githubusercontent.com/HXLStandard/libhxl-python/master/tests/files/test_io/input-valid.xlsx'
URL_JSON = 'https://raw.githubusercontent.com/HXLStandard/libhxl-python/master/tests/files/test_io/input-valid.json'
URL_GOOGLE_NOHASH = 'https://docs.google.com/spreadsheets/d/1VTswL-w9EI0IdGIBFZoZ-2RmIiebXKsrhv03yd7LlIg/edit'
URL_GOOGLE_HASH = 'https://docs.google.com/spreadsheets/d/1VTswL-w9EI0IdGIBFZoZ-2RmIiebXKsrhv03yd7LlIg/edit#gid=299366282'


class TestInput(unittest.TestCase):

    def test_array(self):
        self.assertTrue(make_input(DATA).is_repeatable)
        self.assertTrue('#sector' in hxl.data(DATA).tags)

    def test_csv(self):
        input = make_input(FILE_CSV, True)
        self.assertFalse(input.is_repeatable)
        self.assertTrue('#sector' in hxl.data(input).tags)

    def test_json_lists(self):
        input = make_input(FILE_JSON, True)
        self.assertFalse(input.is_repeatable)
        self.assertTrue('#sector' in hxl.data(input).tags)

    def test_json_objects(self):
        input = make_input(FILE_JSON_OBJECTS, True)
        self.assertFalse(input.is_repeatable)
        self.assertTrue('#sector' in hxl.data(input).tags)

    def test_json_selector(self):
        input = make_input(FILE_JSON_SELECTOR, True, selector="sel1")
        pass

    def test_excel(self):
        input = make_input(FILE_EXCEL, True)
        self.assertTrue(input.is_repeatable)

    def test_ckan_resource(self):
        source = hxl.data('https://data.humdata.org/dataset/hxl-master-vocabulary-list/resource/d22dd1b6-2ff0-47ab-85c6-08aeb911a832')
        self.assertTrue('#vocab' in source.tags)

    def test_ckan_dataset(self):
        source = hxl.data('https://data.humdata.org/dataset/hxl-master-vocabulary-list')
        self.assertTrue('#vocab' in source.tags)

    def test_bytes_buffer(self):
        """Test reading from a string via BytesIO"""
        source = hxl.data(io.BytesIO("#org\nOrg A".encode('utf-8')))
        self.assertTrue('#org' in source.tags)

class TestUntaggedInput(unittest.TestCase):
{'fraction_non_alphanumeric': 0.14033680834001605, 'fraction_numerical': 0.013231756214915798, 'mean_word_length': 4.634789156626506, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 14, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2047868', 'n_tokens_mistral': 2810, 'n_tokens_neox': 2699, 'n_words': 350}