<commit_before>"""Some useful functions that did not fit into the other modules.

Copyright: OGGM developers, 2014-2015

License: GPLv3+
"""
from __future__ import absolute_import, division

import glob
import six.moves.cPickle as pickle

import pandas as pd
from salem import lazy_property
from six.moves.urllib.request import urlretrieve

# Builtins
import os
import gzip
import shutil
import zipfile
import sys
from collections import OrderedDict
from functools import partial, wraps

# External libs
import geopandas as gpd
import numpy as np
import netCDF4
from scipy import stats
from joblib import Memory
from shapely.ops import transform as shp_trafo
from salem import wgs84

# Locals
import oggm.cfg as cfg

GH_ZIP = 'https://github.com/OGGM/oggm-sample-data/archive/master.zip'

# Joblib
MEMORY = Memory(cachedir=cfg.CACHE_DIR, verbose=0)

# Function
tuple2int = partial(np.array, dtype=np.int64)


def empty_cache():  # pragma: no cover
    """Empty oggm's cache directory."""

    if os.path.exists(cfg.CACHE_DIR):
        shutil.rmtree(cfg.CACHE_DIR)
    os.makedirs(cfg.CACHE_DIR)


def _download_demo_files():
    """Checks if the demo data is already on the cache and downloads it.

    TODO: Currently there's no check to see of the server file has changed
    this is bad. In the mean time, with empty_cache() you can ensure that the
    files are up-to-date.
    """

    ofile = os.path.join(cfg.CACHE_DIR, 'oggm-sample-data.zip')
    odir = os.path.join(cfg.CACHE_DIR)
    if not os.path.exists(ofile):  # pragma: no cover
        urlretrieve(GH_ZIP, ofile)
        with zipfile.ZipFile(ofile) as zf:
            zf.extractall(odir)

    out = dict()
    sdir = os.path.join(cfg.CACHE_DIR, 'oggm-sample-data-master')
    for root, directories, filenames in os.walk(sdir):
        for filename in filenames:
            out[filename] = os.path.join(root, filename)
    return out


def get_demo_file(fname):
    """Returns the path to the desired demo file."""

    d = _download_demo_files()
    if fname in d:
        return d[fname]
    else:
        return None


def query_yes_no(question, default="yes"):
    """Ask a yes/no question via raw_input() and return their answer.
<commit_msg>Check oggm-sample-data github commit for updates<commit_after>"""Some useful functions that did not fit into the other modules.

Copyright: OGGM developers, 2014-2015

License: GPLv3+
"""
from __future__ import absolute_import, division

import glob
import six.moves.cPickle as pickle

import pandas as pd
from salem import lazy_property
from six.moves.urllib.request import urlretrieve, urlopen
from six.moves.urllib.error import HTTPError

# Builtins
import os
import gzip
import shutil
import zipfile
import sys
import json
import time
from collections import OrderedDict
from functools import partial, wraps

# External libs
import geopandas as gpd
import numpy as np
import netCDF4
from scipy import stats
from joblib import Memory
from shapely.ops import transform as shp_trafo
from salem import wgs84

# Locals
import oggm.cfg as cfg

SAMPLE_DATA_GH_REPO = 'OGGM/oggm-sample-data'

# Joblib
MEMORY = Memory(cachedir=cfg.CACHE_DIR, verbose=0)

# Function
tuple2int = partial(np.array, dtype=np.int64)


def empty_cache():  # pragma: no cover
    """Empty oggm's cache directory."""

    if os.path.exists(cfg.CACHE_DIR):
        shutil.rmtree(cfg.CACHE_DIR)
    os.makedirs(cfg.CACHE_DIR)


def _download_demo_files():
    """Checks if the demo data is already on the cache and downloads it."""

    master_sha_url = 'https://api.github.com/repos/%s/commits/master' % SAMPLE_DATA_GH_REPO
    master_zip_url = 'https://github.com/%s/archive/master.zip' % SAMPLE_DATA_GH_REPO
    ofile = os.path.join(cfg.CACHE_DIR, 'oggm-sample-data.zip')
    shafile = os.path.join(cfg.CACHE_DIR, 'oggm-sample-data-commit.txt')
    odir = os.path.join(cfg.CACHE_DIR)

    if os.path.exists(shafile):
        with open(shafile, 'r') as sfile:
            local_sha = sfile.read().strip()
        last_mod = os.path.getmtime(shafile)
    else:
        local_sha = '0000'
        last_mod = 0

    if time.time() - last_mod > 1800:
        write_sha = True

        try:
            with urlopen(master_sha_url) as resp:
                json_str = resp.read().decode('utf-8')
                json_obj = json.loads(json_str)
                master_sha = json_obj['sha']

            if local_sha != master_sha:
                empty_cache()
        except HTTPError:
            master_sha = 'error'
    else:
        write_sha = False

    if not os.path.exists(ofile):  # pragma: no cover
        urlretrieve(master_zip_url, ofile)
        with zipfile.ZipFile(ofile) as zf:
            zf.extractall(odir)

    out = dict()
    sdir = os.path.join(cfg.CACHE_DIR, 'oggm-sample-data-master')
    for root, directories, filenames in os.walk(sdir):
        for filename in filenames:
            out[filename] = os.path.join(root, filename)

    if write_sha:
        with open(shafile, 'w') as sfile:
            sfile.write(master_sha)

    return out


def get_demo_file(fname):
    """Returns the path to the desired demo file."""

    d = _download_demo_files()
    if fname in d:
        return d[fname]
    else:
        return None


def query_yes_no(question, default="yes"):
    """Ask a yes/no question via raw_input() and return their answer.

{'fraction_non_alphanumeric': 0.0945440956651719, 'fraction_numerical': 0.007847533632286996, 'mean_word_length': 3.1496124031007753, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6907681', 'n_tokens_mistral': 1836, 'n_tokens_neox': 1764, 'n_words': 515}