<commit_before>"""Some useful functions that did not fit into the other modules.
"""
from __future__ import absolute_import, division

import six.moves.cPickle as pickle
from six import string_types
from six.moves.urllib.request import urlretrieve, urlopen
from six.moves.urllib.error import HTTPError, URLError, ContentTooShortError
from six.moves.urllib.parse import urlparse

# Builtins
import glob
import os
import gzip
import shutil
import zipfile
import sys
import math
import datetime
import logging
from collections import OrderedDict
from functools import partial, wraps
import json
import time
import fnmatch
import subprocess

# External libs
import geopandas as gpd
import pandas as pd
import salem
from salem import lazy_property, read_shapefile
import numpy as np
import netCDF4
from scipy import stats
from joblib import Memory
from shapely.ops import transform as shp_trafo
from salem import wgs84
import xarray as xr
import rasterio
try:
    # rasterio V > 1.0
    from rasterio.merge import merge as merge_tool
except ImportError:
    from rasterio.tools.merge import merge as merge_tool
import multiprocessing as mp
import filelock

# Locals
import oggm.cfg as cfg
from oggm.cfg import CUMSEC_IN_MONTHS, SEC_IN_YEAR, BEGINSEC_IN_MONTHS

SAMPLE_DATA_GH_REPO = 'OGGM/oggm-sample-data'
CRU_SERVER = 'https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_3.24.01/cruts' \
             '.1701201703.v3.24.01/'

DEM3REG = {
    'ISL': [-25., -12., 63., 67.],  # Iceland
    'SVALBARD': [10., 34., 76., 81.],
    'JANMAYEN': [-10., -7., 70., 72.],
    'FJ': [36., 66., 79., 82.],  # <NAME>
    'FAR': [-8., -6., 61., 63.],  # Faroer
    'BEAR': [18., 20., 74., 75.],  # Bear Island
    'SHL': [-3., 0., 60., 61.],  # Shetland
    # Antarctica tiles as UTM zones, large files
    # '01-15': [-180., -91., -90, -60.],
    # '16-30': [-91., -1., -90., -60.],
    # '31-45': [-1., 89., -90., -60.],
    # '46-60': [89., 189., -90., -60.],
    # Greenland tiles
    # 'GL-North': [-78., -11., 75., 84.],
    # 'GL-West': [-68., -42., 64., 76.],
    # 'GL-South': [-52., -40., 59., 64.],
    # 'GL-East': [-42., -17., 64., 76.]
}

# Joblib
MEMORY = Memory(cachedir=cfg.CACHE_DIR, verbose=0)

# Function
tuple2int = partial(np.array, dtype=np.int64)


def _get_download_lock():
    try:
        lock_dir = cfg.PATHS['working_dir']
    except:
        lock_dir = cfg.CACHE_DIR
    mkdir(lock_dir)
    lockfile = os.path.join(lock_dir, 'oggm_data_download.lock')
    try:
        return filelock.FileLock(lockfile).acquire()
    except:
        return filelock.SoftFileLock(lockfile).acquire()


def _urlretrieve(url, ofile, *args, **kwargs):
    try:
        cache_dir = cfg.PATHS['dl_cache_dir']
        if cache_dir and os.path.isdir(cache_dir):
            p = urlparse(url)
            p = os.path.join(cache_dir, p.netloc, p.path[1:])
            if not os.path.exists(os.path.dirname(p)):
                os.makedirs(os.path.dirname(p))
            # TODO: Maybe figure out a way to verify the integrity of the cached file?
            if not os.path.isfile(p):
                urlretrieve(url, p, *args, **kwargs)
            shutil.copyfile(p, ofile)
        else:
            urlretrieve(url, ofile, *args, **kwargs)
        return ofile
    except:
        if os.path.exists(ofile):
            os.remove(ofile)
        raise


def progress_urlretrieve(url, ofile):
    logging.getLogger('download').info("Downloading %s ..." % url)
    try:
        from progressbar import DataTransferBar, UnknownLength<commit_msg>Also remove cache file on download error<commit_after>"""Some useful functions that did not fit into the other modules.
"""
from __future__ import absolute_import, division

import six.moves.cPickle as pickle
from six import string_types
from six.moves.urllib.request import urlretrieve, urlopen
from six.moves.urllib.error import HTTPError, URLError, ContentTooShortError
from six.moves.urllib.parse import urlparse

# Builtins
import glob
import os
import gzip
import shutil
import zipfile
import sys
import math
import datetime
import logging
from collections import OrderedDict
from functools import partial, wraps
import json
import time
import fnmatch
import subprocess

# External libs
import geopandas as gpd
import pandas as pd
import salem
from salem import lazy_property, read_shapefile
import numpy as np
import netCDF4
from scipy import stats
from joblib import Memory
from shapely.ops import transform as shp_trafo
from salem import wgs84
import xarray as xr
import rasterio
try:
    # rasterio V > 1.0
    from rasterio.merge import merge as merge_tool
except ImportError:
    from rasterio.tools.merge import merge as merge_tool
import multiprocessing as mp
import filelock

# Locals
import oggm.cfg as cfg
from oggm.cfg import CUMSEC_IN_MONTHS, SEC_IN_YEAR, BEGINSEC_IN_MONTHS

SAMPLE_DATA_GH_REPO = 'OGGM/oggm-sample-data'
CRU_SERVER = 'https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_3.24.01/cruts' \
             '.1701201703.v3.24.01/'

DEM3REG = {
    'ISL': [-25., -12., 63., 67.],  # Iceland
    'SVALBARD': [10., 34., 76., 81.],
    'JANMAYEN': [-10., -7., 70., 72.],
    'FJ': [36., 66., 79., 82.],  # <NAME>
    'FAR': [-8., -6., 61., 63.],  # Faroer
    'BEAR': [18., 20., 74., 75.],  # Bear Island
    'SHL': [-3., 0., 60., 61.],  # Shetland
    # Antarctica tiles as UTM zones, large files
    # '01-15': [-180., -91., -90, -60.],
    # '16-30': [-91., -1., -90., -60.],
    # '31-45': [-1., 89., -90., -60.],
    # '46-60': [89., 189., -90., -60.],
    # Greenland tiles
    # 'GL-North': [-78., -11., 75., 84.],
    # 'GL-West': [-68., -42., 64., 76.],
    # 'GL-South': [-52., -40., 59., 64.],
    # 'GL-East': [-42., -17., 64., 76.]
}

# Joblib
MEMORY = Memory(cachedir=cfg.CACHE_DIR, verbose=0)

# Function
tuple2int = partial(np.array, dtype=np.int64)


def _get_download_lock():
    try:
        lock_dir = cfg.PATHS['working_dir']
    except:
        lock_dir = cfg.CACHE_DIR
    mkdir(lock_dir)
    lockfile = os.path.join(lock_dir, 'oggm_data_download.lock')
    try:
        return filelock.FileLock(lockfile).acquire()
    except:
        return filelock.SoftFileLock(lockfile).acquire()


def _urlretrieve(url, ofile, *args, **kwargs):
    p = None
    try:
        cache_dir = cfg.PATHS['dl_cache_dir']
        if cache_dir and os.path.isdir(cache_dir):
            p = urlparse(url)
            p = os.path.join(cache_dir, p.netloc, p.path[1:])
            if not os.path.exists(os.path.dirname(p)):
                os.makedirs(os.path.dirname(p))
            # TODO: Maybe figure out a way to verify the integrity of the cached file?
            if not os.path.isfile(p):
                urlretrieve(url, p, *args, **kwargs)
            shutil.copyfile(p, ofile)
        else:
            urlretrieve(url, ofile, *args, **kwargs)
        return ofile
    except:
        if os.path.exists(ofile):
            os.remove(ofile)
        if p and os.path.exists(p):
            os.remove(p)
        raise


def progress_urlretrieve(url, ofile):
    logging.getLogger('download').info("Downloading %s ..." % url)
    try:
        from progressbar import DataTransferBar, UnknownLength
{'fraction_non_alphanumeric': 0.13785864240727783, 'fraction_numerical': 0.04534639608117565, 'mean_word_length': 2.768987341772152, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5471433', 'n_tokens_mistral': 2780, 'n_tokens_neox': 2524, 'n_words': 708}