<commit_before>import nycdb
import os


def test_extract_csvs_from_zip():
    test_csv_file = os.path.join(os.path.dirname(__file__), 'cats.zip')
    csv_content = nycdb.transform.extract_csvs_from_zip(test_csv_file)
    result = ["name,superpower\n", "alice,eating\n", "fluffy,purring\n", "meowses,sitting\n", "pickles,looking out the window\n"]
    assert list(csv_content) == result


def test_to_csv(tmpdir):
    f = tmpdir.join('test.csv')
    f.write("name,borough,block,lot\nalice,queens,1,2\nbob,bronx,3,4")

    output_list = list(nycdb.transform.to_csv(f.strpath))
    assert output_list[0] == {'name': 'alice', 'borough': 'queens', 'block': '1', 'lot': '2'}
    assert output_list[1] == {'name': 'bob', 'borough': 'bronx', 'block': '3', 'lot': '4'}


def test_zip_to_csv():
    test_csv_file = os.path.join(os.path.dirname(__file__), 'cats.zip')
    out = list(nycdb.transform.to_csv(nycdb.transform.extract_csvs_from_zip(test_csv_file)))
    assert len(out) == 4
    assert out[0] == { 'name': 'alice', 'superpower': 'eating' }
    assert out[1] == { 'name': 'fluffy', 'superpower': 'purring' }
    assert out[2] == { 'name': 'meowses', 'superpower': 'sitting' }


def test_to_bbl():
    table = [ { 'borough': 'queens', 'block': '1', 'lot': '1' },{ 'borough': 'queens', 'block': '1', 'lot': '2' } ]
    out = list(nycdb.transform.with_bbl(table))
    assert out[0] == { 'borough': 'queens', 'block': '1', 'lot': '1', 'bbl': '4000010001' }
    assert out[1] == { 'borough': 'queens', 'block': '1', 'lot': '2', 'bbl': '4000010002' }


def test_flip_numbers_nothing_to_flip():
    # assert nycdb.transform.flip_numbers(['one1', 'two2']) == ['one1', 'two2']
    assert nycdb.transform.flip_numbers('one1') == 'one1'
    assert nycdb.transform.flip_numbers('one1234') == 'one1234'
    assert nycdb.transform.flip_numbers('one1234two') == 'one1234two'


def test_flip_numbers():
    assert nycdb.transform.flip_numbers('1one') == 'one1'
    assert nycdb.transform.flip_numbers('123one') == 'one123'


def test_skip_fields():
    table = [{'a': 1, 'b': 2, 'c': 3}, {'a': 'x', 'b': 'y', 'c': 'y'}]
    fields_to_skip = frozenset(['c'])
    out = list(nycdb.transform.skip_fields(table, fields_to_skip))
    assert out == [{'a': 1, 'b': 2 }, {'a': 'x', 'b': 'y'}]<commit_msg>add unit tests for updated transform functions<commit_after>import nycdb
import os
from datetime import datetime


def test_extract_csvs_from_zip():
    test_csv_file = os.path.join(os.path.dirname(__file__), 'cats.zip')
    csv_content = nycdb.transform.extract_csvs_from_zip(test_csv_file)
    result = ["name,superpower\n", "alice,eating\n", "fluffy,purring\n",
              "meowses,sitting\n", "pickles,looking out the window\n"]
    assert list(csv_content) == result


def test_to_csv(tmpdir):
    f = tmpdir.join('test.csv')
    f.write("name,borough,block,lot\nalice,queens,1,2\nbob,bronx,3,4")

    output_list = list(nycdb.transform.to_csv(f.strpath))
    assert output_list[0] == {'name': 'alice', 'borough': 'queens', 'block': '1', 'lot': '2'}
    assert output_list[1] == {'name': 'bob', 'borough': 'bronx', 'block': '3', 'lot': '4'}


def test_zip_to_csv():
    test_csv_file = os.path.join(os.path.dirname(__file__), 'cats.zip')
    out = list(nycdb.transform.to_csv(nycdb.transform.extract_csvs_from_zip(test_csv_file)))
    assert len(out) == 4
    assert out[0] == {'name': 'alice', 'superpower': 'eating'}
    assert out[1] == {'name': 'fluffy', 'superpower': 'purring'}
    assert out[2] == {'name': 'meowses', 'superpower': 'sitting'}


def test_to_bbl_with_borough_field():
    table = [{'borough': 'queens', 'block': '1', 'lot': '1'}, {'borough': 'queens', 'block': '1', 'lot': '2'}]
    out = list(nycdb.transform.with_bbl(table))
    assert out[0] == {'borough': 'queens', 'block': '1', 'lot': '1', 'bbl': '4000010001'}
    assert out[1] == {'borough': 'queens', 'block': '1', 'lot': '2', 'bbl': '4000010002'}


def test_to_bbl_with_boro_field():
    table = [{'boro': 'queens', 'block': '1', 'lot': '1'}, {'boro': 'queens', 'block': '1', 'lot': '2'}]
    out = list(nycdb.transform.with_bbl(table))
    assert out[0] == {'boro': 'queens', 'block': '1', 'lot': '1', 'bbl': '4000010001'}
    assert out[1] == {'boro': 'queens', 'block': '1', 'lot': '2', 'bbl': '4000010002'}


def test_text_to_date():
    table = [{'issuedate': '19991231', 'anotherdatefield': '20000101'},
             {'issuedate': 'malformed', 'anotherdatefield': '20000101'}]
    out = list(nycdb.transform.text_to_date(['issuedate', 'anotherdatefield'], table))

    assert out[0] == {'issuedate': datetime(
        1999, 12, 31, 0, 0), 'anotherdatefield': datetime(2000, 1, 1, 0, 0)}
    assert out[1] == {'issuedate': datetime(1, 1, 1, 0, 0), 'anotherdatefield': datetime(2000, 1, 1, 0, 0)}


def test_flip_numbers_nothing_to_flip():
    # assert nycdb.transform.flip_numbers(['one1', 'two2']) == ['one1', 'two2']
    assert nycdb.transform.flip_numbers('one1') == 'one1'
    assert nycdb.transform.flip_numbers('one1234') == 'one1234'
    assert nycdb.transform.flip_numbers('one1234two') == 'one1234two'


def test_flip_numbers():
    assert nycdb.transform.flip_numbers('1one') == 'one1'
    assert nycdb.transform.flip_numbers('123one') == 'one123'


def test_skip_fields():
    table = [{'a': 1, 'b': 2, 'c': 3}, {'a': 'x', 'b': 'y', 'c': 'y'}]
    fields_to_skip = frozenset(['c'])
    out = list(nycdb.transform.skip_fields(table, fields_to_skip))
    assert out == [{'a': 1, 'b': 2}, {'a': 'x', 'b': 'y'}]
{'fraction_non_alphanumeric': 0.2188638018936635, 'fraction_numerical': 0.04461034231609614, 'mean_word_length': 4.856076759061834, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4647561', 'n_tokens_mistral': 2313, 'n_tokens_neox': 2108, 'n_words': 428}