<commit_before>import numpy as np
import pandas

from ulmo import util
from ulmo.ncdc import ghcn_daily

import test_util


test_stations = [
    {
        'country': 'US',
        'elevation': 286.5,
        'gsn_flag': 'GSN',
        'hcn_flag': 'HCN',
        'id': 'USW00003870',
        'latitude': 34.8831,
        'longitude': -82.2203,
        'name': 'GREER',
        'network': 'W',
        'network_id': '00003870',
        'state': 'SC',
        'wm_oid': '72312'
    }, {
        'country': 'US',
        'elevation': 91.1,
        'gsn_flag': None,
        'hcn_flag': None,
        'id': 'USW00003850',
        'latitude': 31.2667,
        'longitude': -85.7167,
        'name': '<NAME>',
        'network': 'W',
        'network_id': '00003850',
        'state': 'AL',
        'wm_oid': None
    },
]

test_data = {
    'USW00003870': {
        'ACMH': {
            '1965-03-27': (50, np.nan, np.nan, 'X'),
        },
        'PRCP': {
            '1962-10-01': (np.nan, np.nan, np.nan, np.nan),
            '1962-10-14': (np.nan, np.nan, np.nan, np.nan),
            '1962-10-15': (0, np.nan, np.nan, 'X'),
            '1962-10-16': (0, 'T', np.nan, 'X'),
            '1962-10-17': (0, np.nan, np.nan, 'X'),
            '1962-10-21': (102, np.nan, np.nan, 'X'),
            '1962-10-31': (30, np.nan, np.nan, 'X'),
            '1963-02-01': (15, np.nan, np.nan, '0'),
            '1963-02-28': (0, np.nan, np.nan, '0'),
            '1963-03-01': (249, np.nan, np.nan, '0'),
            '1984-02-01': (0, np.nan, np.nan, '0'),
            '1984-02-06': (8, np.nan, np.nan, '0'),
            '1984-02-28': (20, np.nan, np.nan, '0'),
            '1984-02-29': (0, np.nan, np.nan, '0'),
            '1984-03-01': (0, np.nan, np.nan, '0'),
            '1984-03-05': (287, np.nan, np.nan, '0'),
            '2012-03-01': (0, 'T', np.nan, 'X'),
            '2012-03-02': (10, np.nan, np.nan, 'X'),
            '2012-03-03': (422, np.nan, np.nan, 'X'),
        },
    },
    'USC00411885': {
        'PRCP': {
            '1912-09-01': (0, 'P', np.nan, '6'),
            '1912-09-05': (0, 'P', np.nan, '6'),
            '1912-09-06': (0, 'P', np.nan, '6'),
            '1912-09-07': (0, 'P', np.nan, '6'),
            '1912-09-08': (0, 'P', np.nan, '6'),
            '1912-09-30': (0, 'P', np.nan, '6'),
        },
    },
}


def test_get_data_as_dataframes():
    for station_id, sample_data in test_data.iteritems():
        elements = sample_data.keys()
        with test_util.mocked_urls('ncdc/ghcnd/%s.dly' % station_id):
            station_data = ghcn_daily.get_data(
                station_id, elements=elements, as_dataframe=True)

            for element_id, element_test_data in sample_data.iteritems():
                element_df = station_data[element_id]
                for date, test_value in element_test_data.iteritems():
                    value = element_df.xs(date)
                    test_array = np.array(test_value, dtype=value.dtype)

                    nulls = pandas.isnull(value)
<commit_msg>TST: fix ghcn_daily_test so it works with pandas 0.11 by casting explicitly to a daily period instead of implictly with date string<commit_after>import numpy as np
import pandas

from ulmo import util
from ulmo.ncdc import ghcn_daily

import test_util


test_stations = [
    {
        'country': 'US',
        'elevation': 286.5,
        'gsn_flag': 'GSN',
        'hcn_flag': 'HCN',
        'id': 'USW00003870',
        'latitude': 34.8831,
        'longitude': -82.2203,
        'name': 'GREER',
        'network': 'W',
        'network_id': '00003870',
        'state': 'SC',
        'wm_oid': '72312'
    }, {
        'country': 'US',
        'elevation': 91.1,
        'gsn_flag': None,
        'hcn_flag': None,
        'id': 'USW00003850',
        'latitude': 31.2667,
        'longitude': -85.7167,
        'name': '<NAME>',
        'network': 'W',
        'network_id': '00003850',
        'state': 'AL',
        'wm_oid': None
    },
]

test_data = {
    'USW00003870': {
        'ACMH': {
            '1965-03-27': (50, np.nan, np.nan, 'X'),
        },
        'PRCP': {
            '1962-10-01': (np.nan, np.nan, np.nan, np.nan),
            '1962-10-14': (np.nan, np.nan, np.nan, np.nan),
            '1962-10-15': (0, np.nan, np.nan, 'X'),
            '1962-10-16': (0, 'T', np.nan, 'X'),
            '1962-10-17': (0, np.nan, np.nan, 'X'),
            '1962-10-21': (102, np.nan, np.nan, 'X'),
            '1962-10-31': (30, np.nan, np.nan, 'X'),
            '1963-02-01': (15, np.nan, np.nan, '0'),
            '1963-02-28': (0, np.nan, np.nan, '0'),
            '1963-03-01': (249, np.nan, np.nan, '0'),
            '1984-02-01': (0, np.nan, np.nan, '0'),
            '1984-02-06': (8, np.nan, np.nan, '0'),
            '1984-02-28': (20, np.nan, np.nan, '0'),
            '1984-02-29': (0, np.nan, np.nan, '0'),
            '1984-03-01': (0, np.nan, np.nan, '0'),
            '1984-03-05': (287, np.nan, np.nan, '0'),
            '2012-03-01': (0, 'T', np.nan, 'X'),
            '2012-03-02': (10, np.nan, np.nan, 'X'),
            '2012-03-03': (422, np.nan, np.nan, 'X'),
        },
    },
    'USC00411885': {
        'PRCP': {
            '1912-09-01': (0, 'P', np.nan, '6'),
            '1912-09-05': (0, 'P', np.nan, '6'),
            '1912-09-06': (0, 'P', np.nan, '6'),
            '1912-09-07': (0, 'P', np.nan, '6'),
            '1912-09-08': (0, 'P', np.nan, '6'),
            '1912-09-30': (0, 'P', np.nan, '6'),
        },
    },
}


def test_get_data_as_dataframes():
    for station_id, sample_data in test_data.iteritems():
        elements = sample_data.keys()
        with test_util.mocked_urls('ncdc/ghcnd/%s.dly' % station_id):
            station_data = ghcn_daily.get_data(
                station_id, elements=elements, as_dataframe=True)

            for element_id, element_test_data in sample_data.iteritems():
                element_df = station_data[element_id]
                for date, test_value in element_test_data.iteritems():
                    date = pandas.Period(pandas.Timestamp(date), freq='D')
                    value = element_df.xs(date)
                    test_array = np.array(test_value, dtype=value.dtype)

                    nulls = pandas.isnull(value)

{'fraction_non_alphanumeric': 0.20555821753713463, 'fraction_numerical': 0.11036575626896662, 'mean_word_length': 1.7190620929222753, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1516408', 'n_tokens_mistral': 2821, 'n_tokens_neox': 2407, 'n_words': 498}