<commit_before>"""Tests for the EcoData Retriever"""
from future import standard_library
standard_library.install_aliases()
from imp import reload

import os

import sys
reload(sys)
if hasattr(sys, 'setdefaultencoding'):
    sys.setdefaultencoding('utf-8')
from io import StringIO
from retriever.lib.engine import Engine
from retriever.lib.table import Table
from retriever.lib.templates import BasicTextTemplate
from retriever.lib.tools import getmd5
from retriever.lib.cleanup import correct_invalid_value
from retriever.lib.tools import xml2csv
from retriever.lib.tools import json2csv
from retriever.lib.tools import sort_file
from retriever.lib.tools import sort_csv
from retriever.lib.tools import create_file
from retriever.lib.tools import file_2string


# Create simple engine fixture
test_engine = Engine()
test_engine.table = Table("test")
test_engine.script = BasicTextTemplate(tables={'test': test_engine.table},
                                       shortname='test')
test_engine.opts = {'database_name': '{db}_abc'}
HOMEDIR = os.path.expanduser('~')


def setup_module():
    """"change directory to test directory"""
    os.chdir(os.path.dirname(os.path.abspath(__file__)))


def teardown_method():
    """Cleanup temporary output files after testing"""
    os.chdir("..")

def test_auto_get_columns():
    """Basic test of getting column labels from header"""
    test_engine.table.delimiter = ","
    columns, column_values = test_engine.table.auto_get_columns("a,b,c,d")
    assert columns == [['a', None], ['b', None], ['c', None], ['d', None]]


def test_auto_get_columns_cleanup():
    """Test of automatically cleaning up column labels from header"""
    test_engine.table.delimiter = ","
    columns, column_values = test_engine.table.auto_get_columns("a),b.b,c/c,d___d,group")
    assert columns == [['a', None], ['b_b', None], ['c_c', None], ['d_d', None],
                       ['grp', None]]


def test_auto_get_delimiter_comma():
    """Test if commas are properly detected as delimiter"""
    test_engine.auto_get_delimiter("a,b,c;,d")
    assert test_engine.table.delimiter == ","


def test_auto_get_delimiter_tab():
    """Test if commas are properly detected as delimiter"""
    test_engine.auto_get_delimiter("a\tb\tc\td,")<commit_msg>Add test for properly removing whitespace from column headers<commit_after>"""Tests for the EcoData Retriever"""
from future import standard_library
standard_library.install_aliases()
from imp import reload

import os

import sys
reload(sys)
if hasattr(sys, 'setdefaultencoding'):
    sys.setdefaultencoding('utf-8')
from io import StringIO
from retriever.lib.engine import Engine
from retriever.lib.table import Table
from retriever.lib.templates import BasicTextTemplate
from retriever.lib.tools import getmd5
from retriever.lib.cleanup import correct_invalid_value
from retriever.lib.tools import xml2csv
from retriever.lib.tools import json2csv
from retriever.lib.tools import sort_file
from retriever.lib.tools import sort_csv
from retriever.lib.tools import create_file
from retriever.lib.tools import file_2string


# Create simple engine fixture
test_engine = Engine()
test_engine.table = Table("test")
test_engine.script = BasicTextTemplate(tables={'test': test_engine.table},
                                       shortname='test')
test_engine.opts = {'database_name': '{db}_abc'}
HOMEDIR = os.path.expanduser('~')


def setup_module():
    """"change directory to test directory"""
    os.chdir(os.path.dirname(os.path.abspath(__file__)))


def teardown_method():
    """Cleanup temporary output files after testing"""
    os.chdir("..")


def test_auto_get_columns():
    """Basic test of getting column labels from header"""
    test_engine.table.delimiter = ","
    columns, column_values = test_engine.table.auto_get_columns("a,b,c,d")
    assert columns == [['a', None], ['b', None], ['c', None], ['d', None]]


def test_auto_get_columns_extra_whitespace():
    """Test getting column labels from header with extra whitespace"""
    test_engine.table.delimiter = ","
    columns, column_values = test_engine.table.auto_get_columns(" a ,b, c,d  ")
    assert columns == [['a', None], ['b', None], ['c', None], ['d', None]]


def test_auto_get_columns_cleanup():
    """Test of automatically cleaning up column labels from header"""
    test_engine.table.delimiter = ","
    columns, column_values = test_engine.table.auto_get_columns("a),b.b,c/c,d___d,group")
    assert columns == [['a', None], ['b_b', None], ['c_c', None], ['d_d', None],
                       ['grp', None]]


def test_auto_get_delimiter_comma():
    """Test if commas are properly detected as delimiter"""
    test_engine.auto_get_delimiter("a,b,c;,d")
    assert test_engine.table.delimiter == ","


def test_auto_get_delimiter_tab():
    """Test if commas are properly detected as delimiter"""
    test_engine.auto_get_delimiter("a\tb\tc\td,")
{'fraction_non_alphanumeric': 0.12823649337410806, 'fraction_numerical': 0.0020387359836901123, 'mean_word_length': 4.66512702078522, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4516992', 'n_tokens_mistral': 1520, 'n_tokens_neox': 1500, 'n_words': 395}