<commit_before>"""
    PyHIS.cache
    ~~~~~~~~~~

    SQLAlchemy-based caching support
"""
#----------------------------------------------------------------------------
# cache should work like this:
#  1. check in-memory cache, if found return the obj
#  2. if not check database, if found return the obj and update in-memory cache
#  3. if not in database, make new network request, update database
#     with results and update the in-memory cache
#----------------------------------------------------------------------------
from datetime import datetime, timedelta
import logging
import os
import tempfile
import warnings

import pandas
import sqlalchemy as sa
from sqlalchemy import (Column, Integer, String, Float, DateTime)
from sqlalchemy.ext.declarative import (declarative_base, declared_attr,
                                        DeclarativeMeta)
from sqlalchemy.orm import relationship, sessionmaker
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.schema import ForeignKey, Index, UniqueConstraint
from sqlalchemy.sql.expression import asc, desc
import suds

import pyhis
from pyhis import waterml
from pyhis.exceptions import NoDataError


USE_SPATIAL = False
try:
    from geoalchemy import (GeometryColumn, GeometryDDL, Point,
                            WKTSpatialElement)
except ImportError:
    USE_SPATIAL = False

CACHE_DATABASE_FILE = os.path.join(tempfile.gettempdir(), "pyhis_cache.db")
ECHO_SQLALCHEMY = False

#XXX: this should be programmatically generated in some clever way
#     (e.g. based on some config)
USE_CACHE = True

# If the difference between now and the last time a these parts of the
# cache were last refreshed, then make a new request and update the
# cache
CACHE_EXPIRES = {
    'get_sites': timedelta(days=7),
    'site': timedelta(days=7),
    'timeseries': timedelta(days=7),
    }

# If there are more than these values in a value count, break the
# request into a set of smaller requests - this is important for
# services which can return huge amounts of data which can swamp
# memory
MAX_VALUE_COUNT = 50000
DEFAULT_SMALL_REQUEST_INTERVAL = timedelta(days=180)

# configure logging
LOG_FORMAT = '%(message)s'
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
log = logging.getLogger(__name__)


Session = sessionmaker(autocommit=False, autoflush=False)
Base = declarative_base()

# initialize db_session to None so we can easily test for its
# existence and close it if necessary
db_session = None

_cache = {}


def init_cache(cache_database_uri=CACHE_DATABASE_FILE,
               echo=ECHO_SQLALCHEMY, use_spatial=False, schema=None):
    global engine
    global db_session
    global _cache
    global USE_SPATIAL

    # USE_SPATIAL = use_spatial

    if schema:
        update_models_schema(schema)

    if not '://' in cache_database_uri:
        cache_database_uri = 'sqlite:///' + cache_database_uri

    engine = sa.create_engine(cache_database_uri, convert_unicode=True,
                              echo=echo)

    # if we are reinitializing cache, close the old session
    if db_session:
        db_session.commit()
        db_session.close()

    # bind new engine to the db_session and Base metadata objects
    db_session = Session(bind=engine)
    Base.metadata.bind = engine

    # in-memory cache dict that will keep us from creating multiple cache
    # objects that represent the same DB objects
    _cache = {
        'service': {},       # key: url
        'site': {},         # key: (service_url, network, site_code)
        'timeseries': {},   # key: (service_url, network, site_code, variable_code)
        'variable': {},     # key: (vocabulary, variable_code)
        'units': {},        # key: (name, code)
        }

    log.info('cache initialized with database: %s' % cache_database_uri)

    # there is probably a cleaner way to do this, but the idea is to
    # try to run create_all_tables; this won't work if the DB objects
    # haven't been initialized (i.e. on import), but if init_cache is
    # called post-import (meaning the database file is changed), then
    # the tables need to be created
    try:
        Base.metadata.create_all()
    except NameError:
        pass


init_cache()


def clear_memory_cache():
    """Clean out the in-memory cache dict. This is useful for
    large/long-running programs that might be using up all available
    memory.
    """
    global _cache
    _cache = {
        'service': {},       # key: url
        'site': {},         # key: (service_url, network, site_code)
        'timeseries': {},   # key: (service_url, network, site_code, variable_code)
        'variable': {},     # key: (vocabulary, variable_code)
        'units': {},        # key: (name, code)
        }


#----------------------------------------------------------------------------
# cache SQLAlchemy models
#----------------------------------------------------------------------------
def create_cache_obj(db_model, cache_key, lookup_key_func, db_lookup_func):
    """Create a cache object that searches the in-memory cache dict
    for the key returned by calling lookup_key_func. If no object is
    found matching this key, then call db_lookup_func to search the
    database. If no result is found in the database, then create a new
    database object, save it to the database and return the new<commit_msg>remove implicit init_cache() call; this isn't the right place for this the proper place for this should probably be when the cache is first accessed<commit_after>"""
    PyHIS.cache
    ~~~~~~~~~~

    SQLAlchemy-based caching support
"""
#----------------------------------------------------------------------------
# cache should work like this:
#  1. check in-memory cache, if found return the obj
#  2. if not check database, if found return the obj and update in-memory cache
#  3. if not in database, make new network request, update database
#     with results and update the in-memory cache
#----------------------------------------------------------------------------
from datetime import datetime, timedelta
import logging
import os
import tempfile
import warnings

import pandas
import sqlalchemy as sa
from sqlalchemy import (Column, Integer, String, Float, DateTime)
from sqlalchemy.ext.declarative import (declarative_base, declared_attr,
                                        DeclarativeMeta)
from sqlalchemy.orm import relationship, sessionmaker
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.schema import ForeignKey, Index, UniqueConstraint
from sqlalchemy.sql.expression import asc, desc
import suds

import pyhis
from pyhis import waterml
from pyhis.exceptions import NoDataError


USE_SPATIAL = False
try:
    from geoalchemy import (GeometryColumn, GeometryDDL, Point,
                            WKTSpatialElement)
except ImportError:
    USE_SPATIAL = False

CACHE_DATABASE_FILE = os.path.join(tempfile.gettempdir(), "pyhis_cache.db")
ECHO_SQLALCHEMY = False

#XXX: this should be programmatically generated in some clever way
#     (e.g. based on some config)
USE_CACHE = True

# If the difference between now and the last time a these parts of the
# cache were last refreshed, then make a new request and update the
# cache
CACHE_EXPIRES = {
    'get_sites': timedelta(days=7),
    'site': timedelta(days=7),
    'timeseries': timedelta(days=7),
    }

# If there are more than these values in a value count, break the
# request into a set of smaller requests - this is important for
# services which can return huge amounts of data which can swamp
# memory
MAX_VALUE_COUNT = 50000
DEFAULT_SMALL_REQUEST_INTERVAL = timedelta(days=180)

# configure logging
LOG_FORMAT = '%(message)s'
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
log = logging.getLogger(__name__)


Session = sessionmaker(autocommit=False, autoflush=False)
Base = declarative_base()

# initialize db_session to None so we can easily test for its
# existence and close it if necessary
db_session = None

_cache = {}


def init_cache(cache_database_uri=CACHE_DATABASE_FILE,
               echo=ECHO_SQLALCHEMY, use_spatial=False, schema=None):
    global engine
    global db_session
    global _cache
    global USE_SPATIAL

    # USE_SPATIAL = use_spatial

    if schema:
        update_models_schema(schema)

    if not '://' in cache_database_uri:
        cache_database_uri = 'sqlite:///' + cache_database_uri

    engine = sa.create_engine(cache_database_uri, convert_unicode=True,
                              echo=echo)

    # if we are reinitializing cache, close the old session
    if db_session:
        db_session.commit()
        db_session.close()

    # bind new engine to the db_session and Base metadata objects
    db_session = Session(bind=engine)
    Base.metadata.bind = engine

    # in-memory cache dict that will keep us from creating multiple cache
    # objects that represent the same DB objects
    _cache = {
        'service': {},       # key: url
        'site': {},         # key: (service_url, network, site_code)
        'timeseries': {},   # key: (service_url, network, site_code, variable_code)
        'variable': {},     # key: (vocabulary, variable_code)
        'units': {},        # key: (name, code)
        }

    log.info('cache initialized with database: %s' % cache_database_uri)

    # there is probably a cleaner way to do this, but the idea is to
    # try to run create_all_tables; this won't work if the DB objects
    # haven't been initialized (i.e. on import), but if init_cache is
    # called post-import (meaning the database file is changed), then
    # the tables need to be created
    try:
        Base.metadata.create_all()
    except NameError:
        pass


def clear_memory_cache():
    """Clean out the in-memory cache dict. This is useful for
    large/long-running programs that might be using up all available
    memory.
    """
    global _cache
    _cache = {
        'service': {},       # key: url
        'site': {},         # key: (service_url, network, site_code)
        'timeseries': {},   # key: (service_url, network, site_code, variable_code)
        'variable': {},     # key: (vocabulary, variable_code)
        'units': {},        # key: (name, code)
        }


#----------------------------------------------------------------------------
# cache SQLAlchemy models
#----------------------------------------------------------------------------
def create_cache_obj(db_model, cache_key, lookup_key_func, db_lookup_func):
    """Create a cache object that searches the in-memory cache dict
    for the key returned by calling lookup_key_func. If no object is
    found matching this key, then call db_lookup_func to search the
    database. If no result is found in the database, then create a new
    database object, save it to the database and return the new
{'fraction_non_alphanumeric': 0.1311263177362678, 'fraction_numerical': 0.0025892361753282783, 'mean_word_length': 2.5834990059642147, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6836368', 'n_tokens_mistral': 3006, 'n_tokens_neox': 2885, 'n_words': 1078}