<commit_before>"""
module that defines pytables cache
"""
from __future__ import absolute_import

import datetime
import isodate
import os
import tempfile
import warnings

import tables
from tables.exceptions import NoSuchNodeError

from pyhis.usgs import core

# default hdf5 file path
HDF5_FILE_PATH = os.path.join(tempfile.gettempdir(), "pyhis.h5")


class USGSSite(tables.IsDescription):
    agency = tables.StringCol(20)
    code = tables.StringCol(20)
    county = tables.StringCol(30)
    huc = tables.StringCol(20)
    name = tables.StringCol(250)
    network = tables.StringCol(20)
    site_type = tables.StringCol(20)
    state_code = tables.StringCol(2)
    last_refresh = tables.StringCol(26)

    class location(tables.IsDescription):
        srs = tables.StringCol(20)
        latitude = tables.Float32Col()
        longitude = tables.Float32Col()

    class timezone_info(tables.IsDescription):
        uses_dst = tables.BoolCol()

        class default_tz(tables.IsDescription):
            abbreviation = tables.StringCol(5)
            offset = tables.StringCol(7)

        class dst_tz(tables.IsDescription):
            abbreviation = tables.StringCol(5)
            offset = tables.StringCol(7)


class USGSValue(tables.IsDescription):
    datetime = tables.StringCol(26)
    qualifiers = tables.StringCol(20)
    value = tables.StringCol(20)
    last_modified = tables.StringCol(26)


def get_sites(path=HDF5_FILE_PATH):
    """gets a dict of sites from an hdf5 file"""
    h5file = tables.openFile(path, mode='r')
    site_table = h5file.root.usgs.sites
    return_dict = dict([(row['code'], _row_to_dict(row, site_table)) for row in site_table.iterrows()])
    h5file.close()
    return return_dict


def get_site(site_code, path=HDF5_FILE_PATH):
    """gets a site dict for a specific site_code from an hdf5 file"""
    # XXX: this is really dumb
    return get_sites().get(site_code)


def get_site_data(site_code, agency_code=None, path=HDF5_FILE_PATH):
    """gets the data for a given site"""
    # walk the agency groups looking for site code
    h5file = tables.openFile(path, mode='r')
    site_table = h5file.root.usgs.sites
    if agency_code:
        where_clause = "(code == '%s') & (agency == '%s')" % (site_code, agency_code)
    else:
        where_clause = '(code == "%s")' % site_code

    agency = None
    for count, row in enumerate(site_table.where(where_clause)):
        agency = row['agency']

    if not agency:
        h5file.close()
        return {}
    if count >= 1 and not agency_code:
        raise ('more than one site found, limit your query using an agency code')

    site_path = '/usgs/values/%s/%s' % (agency, site_code)
    try:
        site_group = h5file.getNode(site_path)
    except NoSuchNodeError:
        raise "no site found for code: %s" % site_code

    values_dict = dict([
        _values_table_as_dict(table)
        for table in site_group._f_walkNodes('Table')])
    h5file.close()
    return values_dict


def init_h5(path=HDF5_FILE_PATH, mode='w'):
    """creates an hdf5 file an initialized it with relevant tables, etc"""
    h5file = tables.openFile(path, mode=mode, title="pyHIS data")

    usgs = h5file.createGroup('/', 'usgs', 'USGS Data')
    sites = h5file.createTable(usgs, 'sites', USGSSite, "USGS Sites")
    sites.cols.code.createIndex()
    sites.cols.network.createIndex()

    h5file.createGroup(usgs, 'values', "USGS Values")

    h5file.close()


def update_site_list(state_code, service=None, path=HDF5_FILE_PATH):
    """update list of sites for a given state_code"""
    sites = core.get_sites(state_code, service=service)
    _update_site_table(sites, path)


def update_site_data(site_code, date_range=None, path=HDF5_FILE_PATH):
    """updates data for a given site
    """
    site = get_site(site_code)

    if not date_range:
        if site['last_refresh']:
            date_range = datetime.datetime.now() - isodate.parse_datetime(site['last_refresh'])
        else:
            date_range = 'all'

    query_isodate = isodate.datetime_isoformat(datetime.datetime.now())
    site_data = core.get_site_data(site_code, date_range=date_range)

    # XXX: use some sort of mutex or file lock to guard against concurrent
    # processes writing to the file
    h5file = tables.openFile(path, mode="r+")
    sites_table = h5file.root.usgs.sites

    for d in site_data.itervalues():
        variable = d['variable']
        update_values = d['values']
        # add last_modified date to the values we're updating
        for value in update_values:
            value.update({'last_modified': query_isodate})
        value_table = _get_value_table(h5file, site, variable)
        _update_or_append_sortable(value_table, update_values, 'datetime')
        value_table.flush()

    for site_row in sites_table.where('(code == "%s")' % site['code']):
        site_row['last_refresh'] = query_isodate
        site_row.update()
        sites_table.flush()

    h5file.close()


def _flatten_nested_dict(d, prepend=''):
    """flattens a nested dict structure into structure suitable for inserting
    into a pytables table; assumes that no keys in the nested dict structure
    contain the character '/'<commit_msg>make it easy to change default hdf5 file path if we look up the default HDF5 file path inside each function then it doesn't matter when HDF5_FILE_PATH is set<commit_after>"""
module that defines pytables cache
"""
from __future__ import absolute_import

import datetime
import isodate
import os
import tempfile
import warnings

import tables
from tables.exceptions import NoSuchNodeError

from pyhis.usgs import core

# default hdf5 file path
HDF5_FILE_PATH = os.path.join(tempfile.gettempdir(), "pyhis.h5")


class USGSSite(tables.IsDescription):
    agency = tables.StringCol(20)
    code = tables.StringCol(20)
    county = tables.StringCol(30)
    huc = tables.StringCol(20)
    name = tables.StringCol(250)
    network = tables.StringCol(20)
    site_type = tables.StringCol(20)
    state_code = tables.StringCol(2)
    last_refresh = tables.StringCol(26)

    class location(tables.IsDescription):
        srs = tables.StringCol(20)
        latitude = tables.Float32Col()
        longitude = tables.Float32Col()

    class timezone_info(tables.IsDescription):
        uses_dst = tables.BoolCol()

        class default_tz(tables.IsDescription):
            abbreviation = tables.StringCol(5)
            offset = tables.StringCol(7)

        class dst_tz(tables.IsDescription):
            abbreviation = tables.StringCol(5)
            offset = tables.StringCol(7)


class USGSValue(tables.IsDescription):
    datetime = tables.StringCol(26)
    qualifiers = tables.StringCol(20)
    value = tables.StringCol(20)
    last_modified = tables.StringCol(26)


def get_sites(path=None):
    """gets a dict of sites from an hdf5 file"""
    if not path:
        path = HDF5_FILE_PATH
    h5file = tables.openFile(path, mode='r')
    site_table = h5file.root.usgs.sites
    return_dict = dict([(row['code'], _row_to_dict(row, site_table)) for row in site_table.iterrows()])
    h5file.close()
    return return_dict


def get_site(site_code, path=None):
    """gets a site dict for a specific site_code from an hdf5 file"""
    if not path:
        path = HDF5_FILE_PATH
    # XXX: this is really dumb
    return get_sites(path).get(site_code)


def get_site_data(site_code, agency_code=None, path=None):
    """gets the data for a given site"""
    if not path:
        path = HDF5_FILE_PATH
    # walk the agency groups looking for site code
    h5file = tables.openFile(path, mode='r')
    site_table = h5file.root.usgs.sites
    if agency_code:
        where_clause = "(code == '%s') & (agency == '%s')" % (site_code, agency_code)
    else:
        where_clause = '(code == "%s")' % site_code

    agency = None
    for count, row in enumerate(site_table.where(where_clause)):
        agency = row['agency']

    if not agency:
        h5file.close()
        return {}
    if count >= 1 and not agency_code:
        raise ('more than one site found, limit your query using an agency code')

    site_path = '/usgs/values/%s/%s' % (agency, site_code)
    try:
        site_group = h5file.getNode(site_path)
    except NoSuchNodeError:
        raise "no site found for code: %s" % site_code

    values_dict = dict([
        _values_table_as_dict(table)
        for table in site_group._f_walkNodes('Table')])
    h5file.close()
    return values_dict


def init_h5(path=None, mode='w'):
    """creates an hdf5 file an initialized it with relevant tables, etc"""
    if not path:
        path = HDF5_FILE_PATH
    h5file = tables.openFile(path, mode=mode, title="pyHIS data")

    usgs = h5file.createGroup('/', 'usgs', 'USGS Data')
    sites = h5file.createTable(usgs, 'sites', USGSSite, "USGS Sites")
    sites.cols.code.createIndex()
    sites.cols.network.createIndex()

    h5file.createGroup(usgs, 'values', "USGS Values")

    h5file.close()


def update_site_list(state_code, service=None, path=None):
    """update list of sites for a given state_code"""
    if not path:
        path = HDF5_FILE_PATH
    sites = core.get_sites(state_code, service=service)
    _update_site_table(sites, path)


def update_site_data(site_code, date_range=None, path=None):
    """updates data for a given site
    """
    if not path:
        path = HDF5_FILE_PATH
    site = get_site(site_code)

    if not date_range:
        if site['last_refresh']:
            date_range = datetime.datetime.now() - isodate.parse_datetime(site['last_refresh'])
        else:
            date_range = 'all'

    query_isodate = isodate.datetime_isoformat(datetime.datetime.now())
    site_data = core.get_site_data(site_code, date_range=date_range)

    # XXX: use some sort of mutex or file lock to guard against concurrent
    # processes writing to the file
    h5file = tables.openFile(path, mode="r+")
    sites_table = h5file.root.usgs.sites

    for d in site_data.itervalues():
        variable = d['variable']
        update_values = d['values']
        # add last_modified date to the values we're updating
        for value in update_values:
            value.update({'last_modified': query_isodate})
        value_table = _get_value_table(h5file, site, variable)
        value_table.attrs.variable = variable
        _update_or_append_sortable(value_table, update_values, 'datetime')
        value_table.flush()

    for site_row in sites_table.where('(code == "%s")' % site['code']):
        site_row['last_refresh'] = query_isodate
        site_row.update()
        sites_table.flush()

    h5file.close()


def _flatten_nested_dict(d, prepend=''):
    """flattens a nested dict structure into structure suitable for inserting
    into a pytables table; assumes that no keys in the nested dict structure
    contain the character '/'
{'fraction_non_alphanumeric': 0.10606899096107729, 'fraction_numerical': 0.01263604501014573, 'mean_word_length': 3.306195393169182, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2721803', 'n_tokens_mistral': 3560, 'n_tokens_neox': 3440, 'n_words': 886}