<commit_before>import pytest
import pandas as pd

import urbanaccess.gtfs.load as gtfs_load
from urbanaccess.gtfs.gtfsfeeds_dataframe import urbanaccess_gtfs_df


@pytest.fixture
def expected_urbanaccess_gtfs_df_keys():
    expected_keys = ['stops', 'routes', 'trips', 'stop_times',
                     'calendar', 'calendar_dates', 'stop_times_int',
                     'headways']
    return expected_keys.sort()


def test_loadgtfsfeed_to_df_wo_calendar(<commit_msg>added unit tests for functions in _standardize_txt()<commit_after># coding=utf-8
import pytest
import pandas as pd
import os
import six
import codecs
import sys

import urbanaccess.gtfs.load as gtfs_load
from urbanaccess.gtfs.gtfsfeeds_dataframe import urbanaccess_gtfs_df


@pytest.fixture
def expected_urbanaccess_gtfs_df_keys():
    expected_keys = ['stops', 'routes', 'trips', 'stop_times',
                     'calendar', 'calendar_dates', 'stop_times_int',
                     'headways']
    return expected_keys.sort()


@pytest.fixture
def test_txt_files(tmpdir):
    # test file that does not need to be fixed
    do_not_fix_txt = os.path.join(tmpdir.strpath, 'agency.txt')
    data = ['name,text\n', '  Circulação  , áéíóúüñ¿¡ \n']
    if six.PY2:
        with open(do_not_fix_txt, 'w') as f:
            f.writelines(data)
    else:
        with open(do_not_fix_txt, 'w', encoding='utf-8') as f:
            f.writelines(data)

    # test file that does need to be fixed
    fix_txt = os.path.join(tmpdir.strpath, 'calendar.txt')
    data = ['  name  , text \n', '  Circulação  , áéíóúüñ¿¡ \n']
    if six.PY2:
        with open(fix_txt, 'w') as f:
            f.writelines(data)
    else:
        with open(fix_txt, 'w', encoding='utf-8') as f:
            f.writelines(data)

    fix_txt_wBOM = os.path.join(tmpdir.strpath, 'calendar_dates.txt')
    if six.PY2:
        data = [codecs.BOM_UTF8,
                '  name  , text \n',
                '  Circulação  , áéíóúüñ¿¡ \n']
        with open(fix_txt_wBOM, 'w') as f:
            f.writelines(data)
    else:
        data = [str(codecs.BOM_UTF8),
                '  name  , text \n',
                '  Circulação  , áéíóúüñ¿¡ \n']
        with open(fix_txt_wBOM, 'w', encoding='utf-8') as f:
            f.writelines(data)

    return tmpdir.strpath, do_not_fix_txt, fix_txt, fix_txt_wBOM


@pytest.fixture
def test_txt_files_to_use():
    gtfsfiles_to_use = ['stops.txt', 'routes.txt', 'trips.txt',
                        'stop_times.txt', 'calendar.txt',
                        'agency.txt', 'calendar_dates.txt']
    return gtfsfiles_to_use


def test_txt_standardization(test_txt_files):
    root_dir, do_not_fix_txt, fix_txt, fix_txt_wBOM = test_txt_files

    gtfs_load._standardize_txt(csv_rootpath=root_dir)

    df = pd.read_csv(fix_txt)
    assert list(df.columns) == list(df.columns.str.strip())

    df = pd.read_csv(fix_txt_wBOM)
    assert list(df.columns) == list(df.columns.str.strip())


def test_txt_header_whitespace_check(test_txt_files, test_txt_files_to_use):
    root_dir, do_not_fix_txt, fix_txt, fix_txt_wBOM = test_txt_files

    gtfs_load._txt_header_whitespace_check(
        gtfsfiles_to_use=test_txt_files_to_use,
        csv_rootpath=root_dir)

    # only check 'fix_txt' as 'fix_txt_wBOM' would need to be
    # fixed by _txt_encoder_check first
    df = pd.read_csv(fix_txt)
    assert list(df.columns) == list(df.columns.str.strip())


@pytest.mark.skipif(
    sys.version_info >= (3, 0), reason="requires python < 3.0")
def test_txt_encoder_check(test_txt_files, test_txt_files_to_use):
    root_dir, do_not_fix_txt, fix_txt, fix_txt_wBOM = test_txt_files

    gtfs_load._txt_encoder_check(
        gtfsfiles_to_use=test_txt_files_to_use,
        csv_rootpath=root_dir)

    with open(fix_txt_wBOM, 'r') as f:
        raw = f.read()
    assert raw.startswith(codecs.BOM_UTF8) is False


def test_loadgtfsfeed_to_df_wo_calendar(
{'fraction_non_alphanumeric': 0.10980291783977476, 'fraction_numerical': 0.0035833120040952137, 'mean_word_length': 3.0708333333333333, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4759446', 'n_tokens_mistral': 1482, 'n_tokens_neox': 1444, 'n_words': 274}