<commit_before>"""
module that defines pytables cache
"""
from __future__ import absolute_import

import datetime
import isodate
import warnings

import tables
from tables.exceptions import NoSuchNodeError

from ulmo import util
from ulmo.usgs import core

# default hdf5 file path
HDF5_FILE_PATH = util.get_default_h5file_path()


class USGSSite(tables.IsDescription):
    agency = tables.StringCol(20)
    code = tables.StringCol(20)
    county = tables.StringCol(30)
    huc = tables.StringCol(20)
    name = tables.StringCol(250)
    network = tables.StringCol(20)
    site_type = tables.StringCol(20)
    state_code = tables.StringCol(2)

    class location(tables.IsDescription):
        srs = tables.StringCol(20)
        latitude = tables.Float32Col()
        longitude = tables.Float32Col()

    class timezone_info(tables.IsDescription):
        uses_dst = tables.BoolCol()

        class default_tz(tables.IsDescription):
            abbreviation = tables.StringCol(5)
            offset = tables.StringCol(7)

        class dst_tz(tables.IsDescription):
            abbreviation = tables.StringCol(5)
            offset = tables.StringCol(7)


class USGSValue(tables.IsDescription):
    datetime = tables.StringCol(26)
    qualifiers = tables.StringCol(20)
    value = tables.StringCol(20)
    last_checked = tables.StringCol(26)
    last_modified = tables.StringCol(26)


def get_sites(path=None):
    """gets a dict of sites from an hdf5 file"""
    if not path:
        path = HDF5_FILE_PATH
    with util.open_h5file(path, 'r') as h5file:
        sites_table = _get_sites_table(h5file)
        if not sites_table:
            return {}

        return dict([
            (row['code'], _row_to_dict(row, sites_table))
            for row in sites_table.iterrows()])


def get_site(site_code, path=None):
    """gets a site dict for a specific site_code from an hdf5 file"""
    if not path:
        path = HDF5_FILE_PATH
    # XXX: this is really dumb
    sites = get_sites(path)
    if site_code in sites:
        site = sites.get(site_code)
    else:
        # TODO: this might be a bad idea - it requires writing to the file and
        # we might want to guarantee that gets don't require a write lock
        update_site_list(sites=site_code, path=path)
        sites = get_sites(path)
        site = sites.get(site_code)
        if not site:
            raise LookupError("Could not find site %s" % site_code)
    return site


def get_site_data(site_code, agency_code=None, path=None):
    """gets the data for a given site"""
    if not path:
        path = HDF5_FILE_PATH
    # walk the agency groups looking for site code
    with util.open_h5file(path, mode='r') as h5file:
        sites_table = _get_sites_table(h5file)
        if sites_table is None:
            return {}

        if agency_code:
            where_clause = "(code == '%s') & (agency == '%s')" % (site_code, agency_code)
        else:
            where_clause = '(code == "%s")' % site_code

        agency = None
        for count, row in enumerate(sites_table.where(where_clause)):
            agency = row['agency']

        if not agency:
            return {}

        if count >= 1 and not agency_code:
            raise ('more than one site found, limit your query using an agency code')

        site_path = '/usgs/values/%s/%s' % (agency, site_code)
        try:
            site_group = h5file.getNode(site_path)
        except NoSuchNodeError:
            raise Exception("no site found for code: %s" % site_code)

        values_dict = dict([
            _values_table_as_dict(table)
            for table in site_group._f_walkNodes('Table')])
        return values_dict


def update_site_list(sites=None, state_code=None, service=None, path=None):
    """update list of sites for a given state_code"""
    if not path:
        path = HDF5_FILE_PATH
    sites = core.get_sites(sites=sites, state_code=state_code, service=service)
    _update_sites_table(sites.itervalues(), path)


def update_site_data(site_code, date_range=None, path=None):
    """updates data for a given site
    """
    if not path:
        path = HDF5_FILE_PATH
    update_site_list(sites=site_code, path=path)
    site = get_site(site_code, path=path)

    query_isodate = isodate.datetime_isoformat(datetime.datetime.now())

    # XXX: use some sort of mutex or file lock to guard against concurrent
    # processes writing to the file
    with util.open_h5file(path, mode="r+") as h5file:
        if not date_range:
            last_site_refresh = _last_refresh(site, h5file)
            if last_site_refresh:
                date_range = datetime.datetime.now() - isodate.parse_datetime(last_site_refresh)
            else:
                date_range = 'all'

        site_data = core.get_site_data(site_code, date_range=date_range)

        for d in site_data.itervalues():
            variable = d['variable']
            update_values = d['values']
            # add last_modified date to the values we're updating
            for value in update_values:
                value.update({'last_checked': query_isodate})
            value_table = _get_value_table(h5file, site, variable)
            value_table.attrs.variable = variable
            value_table.attrs.last_refresh = query_isodate
            _update_or_append_sortable(value_table, update_values, 'datetime', query_isodate)
            value_table.flush()


def _flatten_nested_dict(d, prepend=''):
    """flattens a nested dict structure into structure suitable for inserting
    into a pytables table; assumes that no keys in the nested dict structure
    contain the character '/'
    """
    return_dict = {}

    for k, v in d.iteritems():
        if isinstance(v, dict):
            flattened = _flatten_nested_dict(v, prepend=prepend + k + '/')
            return_dict.update(flattened)
        else:
            return_dict[prepend + k] = v

    return return_dict


def _get_agency_group(h5file, agency_name):
    values_group = _get_values_group(h5file)
    if values_group is None:
        return None
<commit_msg>move updating of last_refresh until after data has been updated<commit_after>"""
module that defines pytables cache
"""
from __future__ import absolute_import

import datetime
import isodate
import warnings

import tables
from tables.exceptions import NoSuchNodeError

from ulmo import util
from ulmo.usgs import core

# default hdf5 file path
HDF5_FILE_PATH = util.get_default_h5file_path()


class USGSSite(tables.IsDescription):
    agency = tables.StringCol(20)
    code = tables.StringCol(20)
    county = tables.StringCol(30)
    huc = tables.StringCol(20)
    name = tables.StringCol(250)
    network = tables.StringCol(20)
    site_type = tables.StringCol(20)
    state_code = tables.StringCol(2)

    class location(tables.IsDescription):
        srs = tables.StringCol(20)
        latitude = tables.Float32Col()
        longitude = tables.Float32Col()

    class timezone_info(tables.IsDescription):
        uses_dst = tables.BoolCol()

        class default_tz(tables.IsDescription):
            abbreviation = tables.StringCol(5)
            offset = tables.StringCol(7)

        class dst_tz(tables.IsDescription):
            abbreviation = tables.StringCol(5)
            offset = tables.StringCol(7)


class USGSValue(tables.IsDescription):
    datetime = tables.StringCol(26)
    qualifiers = tables.StringCol(20)
    value = tables.StringCol(20)
    last_checked = tables.StringCol(26)
    last_modified = tables.StringCol(26)


def get_sites(path=None):
    """gets a dict of sites from an hdf5 file"""
    if not path:
        path = HDF5_FILE_PATH
    with util.open_h5file(path, 'r') as h5file:
        sites_table = _get_sites_table(h5file)
        if not sites_table:
            return {}

        return dict([
            (row['code'], _row_to_dict(row, sites_table))
            for row in sites_table.iterrows()])


def get_site(site_code, path=None):
    """gets a site dict for a specific site_code from an hdf5 file"""
    if not path:
        path = HDF5_FILE_PATH
    # XXX: this is really dumb
    sites = get_sites(path)
    if site_code in sites:
        site = sites.get(site_code)
    else:
        # TODO: this might be a bad idea - it requires writing to the file and
        # we might want to guarantee that gets don't require a write lock
        update_site_list(sites=site_code, path=path)
        sites = get_sites(path)
        site = sites.get(site_code)
        if not site:
            raise LookupError("Could not find site %s" % site_code)
    return site


def get_site_data(site_code, agency_code=None, path=None):
    """gets the data for a given site"""
    if not path:
        path = HDF5_FILE_PATH
    # walk the agency groups looking for site code
    with util.open_h5file(path, mode='r') as h5file:
        sites_table = _get_sites_table(h5file)
        if sites_table is None:
            return {}

        if agency_code:
            where_clause = "(code == '%s') & (agency == '%s')" % (site_code, agency_code)
        else:
            where_clause = '(code == "%s")' % site_code

        agency = None
        for count, row in enumerate(sites_table.where(where_clause)):
            agency = row['agency']

        if not agency:
            return {}

        if count >= 1 and not agency_code:
            raise ('more than one site found, limit your query using an agency code')

        site_path = '/usgs/values/%s/%s' % (agency, site_code)
        try:
            site_group = h5file.getNode(site_path)
        except NoSuchNodeError:
            raise Exception("no site found for code: %s" % site_code)

        values_dict = dict([
            _values_table_as_dict(table)
            for table in site_group._f_walkNodes('Table')])
        return values_dict


def update_site_list(sites=None, state_code=None, service=None, path=None):
    """update list of sites for a given state_code"""
    if not path:
        path = HDF5_FILE_PATH
    sites = core.get_sites(sites=sites, state_code=state_code, service=service)
    _update_sites_table(sites.itervalues(), path)


def update_site_data(site_code, date_range=None, path=None):
    """updates data for a given site
    """
    if not path:
        path = HDF5_FILE_PATH
    update_site_list(sites=site_code, path=path)
    site = get_site(site_code, path=path)

    query_isodate = isodate.datetime_isoformat(datetime.datetime.now())

    # XXX: use some sort of mutex or file lock to guard against concurrent
    # processes writing to the file
    with util.open_h5file(path, mode="r+") as h5file:
        if not date_range:
            last_site_refresh = _last_refresh(site, h5file)
            if last_site_refresh:
                date_range = datetime.datetime.now() - isodate.parse_datetime(last_site_refresh)
            else:
                date_range = 'all'

        site_data = core.get_site_data(site_code, date_range=date_range)

        for d in site_data.itervalues():
            variable = d['variable']
            update_values = d['values']
            # add last_modified date to the values we're updating
            for value in update_values:
                value.update({'last_checked': query_isodate})
            value_table = _get_value_table(h5file, site, variable)
            value_table.attrs.variable = variable
            _update_or_append_sortable(value_table, update_values, 'datetime', query_isodate)
            value_table.attrs.last_refresh = query_isodate
            value_table.flush()


def _flatten_nested_dict(d, prepend=''):
    """flattens a nested dict structure into structure suitable for inserting
    into a pytables table; assumes that no keys in the nested dict structure
    contain the character '/'
    """
    return_dict = {}

    for k, v in d.iteritems():
        if isinstance(v, dict):
            flattened = _flatten_nested_dict(v, prepend=prepend + k + '/')
            return_dict.update(flattened)
        else:
            return_dict[prepend + k] = v

    return return_dict


def _get_agency_group(h5file, agency_name):
    values_group = _get_values_group(h5file)
    if values_group is None:
        return None

{'fraction_non_alphanumeric': 0.08934426229508197, 'fraction_numerical': 0.009836065573770493, 'mean_word_length': 2.6794330518697227, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6574349', 'n_tokens_mistral': 3818, 'n_tokens_neox': 3681, 'n_words': 995}