<commit_before>import os
import time

import pandas
import pytest

import ulmo
import test_util


TEST_FILE_PATH = '/tmp/ulmo_test.h5'


@pytest.fixture
def delete_test_file():
    if os.path.exists(TEST_FILE_PATH):
        os.remove(TEST_FILE_PATH)


def test_update_site_list(delete_test_file):
    site_files = ['usgs/nwis/RI_daily.xml', 'usgs/nwis/RI_instantaneous.xml']
    for site_file in site_files:
        test_site_file = test_util.get_test_file_path(site_file)
        ulmo.usgs.nwis.hdf5.update_site_list(path=TEST_FILE_PATH,
                input_file=test_site_file)

    sites = ulmo.usgs.nwis.hdf5.get_sites(TEST_FILE_PATH)
    assert len(sites) == 64

    test_sites = {
        # uses_dst == False
        '01111410': {
            'agency': 'USGS',
            'code': '01111410',
            'county': '44007',
            'huc': '01090003',
            'location': {
                'latitude': '41.9409318',
                'longitude': '-71.6481214',
                'srs': 'EPSG:4326'
            },
            'name': 'CHEPACHET RIVER WEST OF GAZZA RD AT GAZZAVILLE, RI',
            'state_code': '44',
            'network': 'NWIS',
            'site_type': 'ST',
            'timezone_info': {
                'default_tz': {
                    'abbreviation': 'EST',
                    'offset': '-05:00'
                },
                'dst_tz': {
                    'abbreviation': 'EDT',
                    'offset': '-04:00',
                },
                'uses_dst': False,
            }
        },
        # only in RI_daily
        '01116300': {
            'agency': 'USGS',
            'code': '01116300',
            'county': '44007',
            'huc': '01090004',
            'location': {
                'latitude': '41.7564892',
                'longitude': '-71.4972824',
                'srs': 'EPSG:4326'
            },
            'name': '<NAME>, RI',
            'network': 'NWIS',
            'site_type': 'ST',
            'state_code': '44',
            'timezone_info': {
                'default_tz': {'abbreviation': 'EST', 'offset': '-05:00'},
                'dst_tz': {'abbreviation': 'EDT', 'offset': '-04:00'},
                'uses_dst': True
            },
        },
        # only in RI_instantaneous
        '01115170': {
            'agency': 'USGS',
            'code': '01115170',
            'county': '44007',
            'huc': '01090004',
            'location': {
                'latitude': '41.84093269',
                'longitude': '-71.584508',
                'srs': 'EPSG:4326',
            },
            'name': '<NAME> NORTH SCITUATE, RI',
            'network': 'NWIS',
            'site_type': 'ST',
            'state_code': '44',
            'timezone_info': {
                'default_tz': {'abbreviation': 'EST', 'offset': '-05:00'},
                'dst_tz': {'abbreviation': 'EDT', 'offset': '-04:00'},
                'uses_dst': True
            },
        },
    }

    for test_code, test_value in test_sites.iteritems():
        assert sites[test_code] == test_value


def test_sites_table_remains_unique(delete_test_file):
    site_files = ['usgs/nwis/RI_daily.xml', 'usgs/nwis/RI_instantaneous.xml']
    for site_file in site_files:
        test_site_file = test_util.get_test_file_path(site_file)
        ulmo.usgs.nwis.hdf5.update_site_list(path=TEST_FILE_PATH,
            input_file=test_site_file)

    store = pandas.io.pytables.HDFStore(TEST_FILE_PATH)
    sites_df = store.select('sites')
    assert len(sites_df) == len(set(sites_df.index))


def test_get_site(delete_test_file):
    site_code = '08068500'
    site_data_file = 'usgs/nwis/site_%s_daily.xml' % site_code
    input_file = test_util.get_test_file_path(site_data_file)
    ulmo.usgs.nwis.hdf5.update_site_list(path=TEST_FILE_PATH,
            input_file=input_file)

    site = ulmo.usgs.nwis.hdf5.get_site(site_code, path=TEST_FILE_PATH)
    assert site == {
        'agency': 'USGS',
        'code': '08068500',
        'county': '48339',<commit_msg>test that updated site attributes (e.g. site['name']) propagate to stored site list<commit_after>import os
import time

import pandas
import pytest

import ulmo
import test_util


TEST_FILE_PATH = '/tmp/ulmo_test.h5'


@pytest.fixture
def delete_test_file():
    if os.path.exists(TEST_FILE_PATH):
        os.remove(TEST_FILE_PATH)


def test_update_site_list(delete_test_file):
    site_files = ['usgs/nwis/RI_daily.xml', 'usgs/nwis/RI_instantaneous.xml']
    for site_file in site_files:
        test_site_file = test_util.get_test_file_path(site_file)
        ulmo.usgs.nwis.hdf5.update_site_list(path=TEST_FILE_PATH,
                input_file=test_site_file)

    sites = ulmo.usgs.nwis.hdf5.get_sites(TEST_FILE_PATH)
    assert len(sites) == 64

    test_sites = {
        # uses_dst == False
        '01111410': {
            'agency': 'USGS',
            'code': '01111410',
            'county': '44007',
            'huc': '01090003',
            'location': {
                'latitude': '41.9409318',
                'longitude': '-71.6481214',
                'srs': 'EPSG:4326'
            },
            'name': '<NAME> AT GAZZAVILLE, RI',
            'state_code': '44',
            'network': 'NWIS',
            'site_type': 'ST',
            'timezone_info': {
                'default_tz': {
                    'abbreviation': 'EST',
                    'offset': '-05:00'
                },
                'dst_tz': {
                    'abbreviation': 'EDT',
                    'offset': '-04:00',
                },
                'uses_dst': False,
            }
        },
        # only in RI_daily
        '01116300': {
            'agency': 'USGS',
            'code': '01116300',
            'county': '44007',
            'huc': '01090004',
            'location': {
                'latitude': '41.7564892',
                'longitude': '-71.4972824',
                'srs': 'EPSG:4326'
            },
            'name': '<NAME> AT CRANSTON, RI',
            'network': 'NWIS',
            'site_type': 'ST',
            'state_code': '44',
            'timezone_info': {
                'default_tz': {'abbreviation': 'EST', 'offset': '-05:00'},
                'dst_tz': {'abbreviation': 'EDT', 'offset': '-04:00'},
                'uses_dst': True
            },
        },
        # only in RI_instantaneous
        '01115170': {
            'agency': 'USGS',
            'code': '01115170',
            'county': '44007',
            'huc': '01090004',
            'location': {
                'latitude': '41.84093269',
                'longitude': '-71.584508',
                'srs': 'EPSG:4326',
            },
            'name': 'MOSWANSICUT STREAM NR NORTH SCITUATE, RI',
            'network': 'NWIS',
            'site_type': 'ST',
            'state_code': '44',
            'timezone_info': {
                'default_tz': {'abbreviation': 'EST', 'offset': '-05:00'},
                'dst_tz': {'abbreviation': 'EDT', 'offset': '-04:00'},
                'uses_dst': True
            },
        },
    }

    for test_code, test_value in test_sites.iteritems():
        assert sites[test_code] == test_value


def test_update_site_list_with_changes(delete_test_file):
    site_files = [
        ('usgs/nwis/RI_daily.xml', {
            'agency': 'USGS',
            'code': '01106000',
            'county': '44005',
            'huc': '01090002',
            'location': {'latitude': '41.5584366',
                        'longitude': '-71.12921047',
                        'srs': 'EPSG:4326'},
            'name': 'ADAMSVILLE BROOK AT ADAMSVILLE, RI',
            'network': 'NWIS',
            'site_type': 'ST',
            'state_code': '44',
            'timezone_info': {
                'default_tz': {'abbreviation': 'EST', 'offset': '-05:00'},
                'dst_tz': {'abbreviation': 'EDT', 'offset': '-04:00'},
                'uses_dst': True}}),
        ('usgs/nwis/RI_daily_update.xml', {
            'agency': 'USGS',
            'code': '01106000',
            'county': '44005',
            'huc': '01090002',
            'location': {'latitude': '41.5584366',
                        'longitude': '-71.12921047',
                        'srs': 'EPSG:4326'},
            'name': '<NAME>',
            'network': 'NWIS',
            'site_type': 'ST',
            'state_code': '44',
            'timezone_info': {
                'default_tz': {'abbreviation': 'EST', 'offset': '-05:00'},
                'dst_tz': {'abbreviation': 'EDT', 'offset': '-04:00'},
                'uses_dst': True}}),
    ]
    for test_file, test_site in site_files:
        test_site_file = test_util.get_test_file_path(test_file)
        ulmo.usgs.nwis.hdf5.update_site_list(path=TEST_FILE_PATH,
                input_file=test_site_file)
        sites = ulmo.usgs.nwis.hdf5.get_sites(path=TEST_FILE_PATH)
        test_code = test_site['code']
        assert sites[test_code] == test_site


def test_sites_table_remains_unique(delete_test_file):
    site_files = ['usgs/nwis/RI_daily.xml', 'usgs/nwis/RI_instantaneous.xml']
    for site_file in site_files:
        test_site_file = test_util.get_test_file_path(site_file)
        ulmo.usgs.nwis.hdf5.update_site_list(path=TEST_FILE_PATH,
            input_file=test_site_file)

    store = pandas.io.pytables.HDFStore(TEST_FILE_PATH)
    sites_df = store.select('sites')
    assert len(sites_df) == len(set(sites_df.index))


def test_get_site(delete_test_file):
    site_code = '08068500'
    site_data_file = 'usgs/nwis/site_%s_daily.xml' % site_code
    input_file = test_util.get_test_file_path(site_data_file)
    ulmo.usgs.nwis.hdf5.update_site_list(path=TEST_FILE_PATH,
            input_file=input_file)

    site = ulmo.usgs.nwis.hdf5.get_site(site_code, path=TEST_FILE_PATH)
    assert site == {
        'agency': 'USGS',
        'code': '08068500',
        'county': '48339',
{'fraction_non_alphanumeric': 0.14892760044305708, 'fraction_numerical': 0.05377102003826402, 'mean_word_length': 1.8009024252679076, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5420853', 'n_tokens_mistral': 3800, 'n_tokens_neox': 3329, 'n_words': 546}