<commit_before>import logging
log = logging.getLogger(__name__)

import os
import time
import h5py
import datetime
import numpy as np
# used by compare_istrument_settings_timestamp():
from numpy import array  # DO not remove;
from copy import deepcopy
from matplotlib.colors import LogNorm
from matplotlib.colors import LinearSegmentedColormap as lscmap
from sklearn.mixture import GaussianMixture as GM
from pycqed.utilities.get_default_datadir import get_default_datadir
from scipy.interpolate import griddata
from mpl_toolkits.axes_grid1 import make_axes_locatable
from scipy.optimize import Bounds, LinearConstraint, minimize
from .tools.plotting import *
from matplotlib import cm


datadir = get_default_datadir()
print('Data directory set to:', datadir)


######################################################################
#     File-handling tools
######################################################################


def nearest_idx(array, value):
    '''
    find the index of the value closest to the specified value.
    '''
    return np.abs(array-value).argmin()


def nearest_value(array, value):
    '''
    find the value in the array that is closest to the specified value.
    '''
    return array[nearest_idx(array, value)]


def verify_timestamp(timestamp):
    if len(timestamp) == 6:
        daystamp = time.strftime('%Y%m%d')
        tstamp = timestamp
    elif len(timestamp) == 14:
        daystamp = timestamp[:8]
        tstamp = timestamp[8:]
    elif len(timestamp) == 15:  # In case day and timestamp separted by _
        daystamp = timestamp[:8]
        tstamp = timestamp[9:]
    else:
        raise Exception("Cannot interpret timestamp '%s'" % timestamp)

    return daystamp, tstamp


def is_older(ts0, ts1, or_equal=False):
    '''
    returns True if timestamp ts0 is an earlier data than timestamp ts1,
    False otherwise.
    '''
    if ts0 is None or ts1 is None:
        return True
    else:

        dstamp0, tstamp0 = verify_timestamp(ts0)
        dstamp1, tstamp1 = verify_timestamp(ts1)
        if not or_equal:
            # print 'isolder', (dstamp0+tstamp0) < (dstamp1+tstamp1)
            return (dstamp0+tstamp0) < (dstamp1+tstamp1)
        else:
            return ((dstamp0+tstamp0) <= (dstamp1+tstamp1))


def is_equal(ts0, ts1, or_equal=False):
    '''
    returns True if timestamp ts0 is an the same data as timestamp ts1,
    False otherwise.
    '''
    dstamp0, tstamp0 = verify_timestamp(ts0)
    dstamp1, tstamp1 = verify_timestamp(ts1)

    return (dstamp0+tstamp0) == (dstamp1+tstamp1)


def get_last_n_timestamps(n, contains=''):
    timestamps = []
    for i in range(n):
        if i == 0:
            timestamp = latest_data(contains=contains,
                                    return_timestamp=True)[0]
            timestamps.append(timestamp)
        else:
            timestamps.append(latest_data(contains=contains,
                                          older_than=timestamps[-1],
                                          return_timestamp=True)[0])
    return timestamps


def latest_data(contains='', older_than=None, newer_than=None, or_equal=False,
                return_timestamp=False, return_path=True, raise_exc=True,
                folder=None, n_matches=None, return_all=False,
                match_whole_words=False):
    """
        Finds the latest taken data with <contains> in its name.
        Returns the full path of the data directory and/or the timestamp.
        If no fitting data is found, an exception is raised (unless raise_exc=False).

        Args:
            contains: filter for the labels. Any label that does not include
                this string will be ignored. (default: no filter)
            older_than: if this is not None, the latest data that fits and that is
                older than the date given by the timestamp older_than is returned.
            newer_than: if this is not None, the latest data that fits and that is
                newer than the date given by the timestamp newer_than is returned.
            or_equal: switches "older/newer than" comparisons to  "older/newer than
                or equal" comparisons to (default: False)
            return_timestamp: return the timestamp(s) as first return value
                (default: False)
            return_path: return the path(s) as the only or the second return
                value (default: True)
            raise_exc: Return False instead of raising an exception if no data is
                found. (default: True)
            folder: search directory (default: the stored datadir)
            n_matches: If set to an integer n, a list of the n latest matches
                is returned. If None, a single item is returned.
            return_all: returns all the folders satisfying the requirements found
                in the last day folder in which any match is found
            match_whole_words: If True, filtering with the attribute contains
                only matches folders where contains matches a whole word (with
                underscores treated as word separators). (default: False)
        Returns: (list of) path and/or timestamps. Return format depends on the
            choice of return_timestamp, return_path, list_timestamps, return_all.
    """

    assert return_timestamp or return_path, \
        'No return value chosen (return_timestamp=return_path=False).'

    if folder is None:
        search_dir = datadir
    else:
        search_dir = folder

    daydirs = os.listdir(search_dir)
    if len(daydirs) == 0:
        log.warning('No data found in datadir')
        return None

    daydirs.sort()

    measdirs = []
    paths = []
    timestamps = []
    i = len(daydirs)-1
    if n_matches is None or return_all:
        n_matches = 0

    timestamp = None<commit_msg>a_tools.latest_data: make it possible to specify the default behavior on a setup by putting a_tools.latest_data_match_whole_words = True into the init script<commit_after>import logging
log = logging.getLogger(__name__)

import os
import time
import h5py
import datetime
import numpy as np
# used by compare_istrument_settings_timestamp():
from numpy import array  # DO not remove;
from copy import deepcopy
from matplotlib.colors import LogNorm
from matplotlib.colors import LinearSegmentedColormap as lscmap
from sklearn.mixture import GaussianMixture as GM
from pycqed.utilities.get_default_datadir import get_default_datadir
from scipy.interpolate import griddata
from mpl_toolkits.axes_grid1 import make_axes_locatable
from scipy.optimize import Bounds, LinearConstraint, minimize
from .tools.plotting import *
from matplotlib import cm

latest_data_match_whole_words = False
datadir = get_default_datadir()
print('Data directory set to:', datadir)


######################################################################
#     File-handling tools
######################################################################


def nearest_idx(array, value):
    '''
    find the index of the value closest to the specified value.
    '''
    return np.abs(array-value).argmin()


def nearest_value(array, value):
    '''
    find the value in the array that is closest to the specified value.
    '''
    return array[nearest_idx(array, value)]


def verify_timestamp(timestamp):
    if len(timestamp) == 6:
        daystamp = time.strftime('%Y%m%d')
        tstamp = timestamp
    elif len(timestamp) == 14:
        daystamp = timestamp[:8]
        tstamp = timestamp[8:]
    elif len(timestamp) == 15:  # In case day and timestamp separted by _
        daystamp = timestamp[:8]
        tstamp = timestamp[9:]
    else:
        raise Exception("Cannot interpret timestamp '%s'" % timestamp)

    return daystamp, tstamp


def is_older(ts0, ts1, or_equal=False):
    '''
    returns True if timestamp ts0 is an earlier data than timestamp ts1,
    False otherwise.
    '''
    if ts0 is None or ts1 is None:
        return True
    else:

        dstamp0, tstamp0 = verify_timestamp(ts0)
        dstamp1, tstamp1 = verify_timestamp(ts1)
        if not or_equal:
            # print 'isolder', (dstamp0+tstamp0) < (dstamp1+tstamp1)
            return (dstamp0+tstamp0) < (dstamp1+tstamp1)
        else:
            return ((dstamp0+tstamp0) <= (dstamp1+tstamp1))


def is_equal(ts0, ts1, or_equal=False):
    '''
    returns True if timestamp ts0 is an the same data as timestamp ts1,
    False otherwise.
    '''
    dstamp0, tstamp0 = verify_timestamp(ts0)
    dstamp1, tstamp1 = verify_timestamp(ts1)

    return (dstamp0+tstamp0) == (dstamp1+tstamp1)


def get_last_n_timestamps(n, contains=''):
    timestamps = []
    for i in range(n):
        if i == 0:
            timestamp = latest_data(contains=contains,
                                    return_timestamp=True)[0]
            timestamps.append(timestamp)
        else:
            timestamps.append(latest_data(contains=contains,
                                          older_than=timestamps[-1],
                                          return_timestamp=True)[0])
    return timestamps


def latest_data(contains='', older_than=None, newer_than=None, or_equal=False,
                return_timestamp=False, return_path=True, raise_exc=True,
                folder=None, n_matches=None, return_all=False,
                match_whole_words=None):
    """
        Finds the latest taken data with <contains> in its name.
        Returns the full path of the data directory and/or the timestamp.
        If no fitting data is found, an exception is raised (unless raise_exc=False).

        Args:
            contains: filter for the labels. Any label that does not include
                this string will be ignored. (default: no filter)
            older_than: if this is not None, the latest data that fits and that is
                older than the date given by the timestamp older_than is returned.
            newer_than: if this is not None, the latest data that fits and that is
                newer than the date given by the timestamp newer_than is returned.
            or_equal: switches "older/newer than" comparisons to  "older/newer than
                or equal" comparisons to (default: False)
            return_timestamp: return the timestamp(s) as first return value
                (default: False)
            return_path: return the path(s) as the only or the second return
                value (default: True)
            raise_exc: Return False instead of raising an exception if no data is
                found. (default: True)
            folder: search directory (default: the stored datadir)
            n_matches: If set to an integer n, a list of the n latest matches
                is returned. If None, a single item is returned.
            return_all: returns all the folders satisfying the requirements found
                in the last day folder in which any match is found
            match_whole_words: If True, filtering with the attribute contains
                only matches folders where contains matches a whole word (with
                underscores treated as word separators).
                (default: use the value latest_data_match_whole_words
                specified in the module)
        Returns: (list of) path and/or timestamps. Return format depends on the
            choice of return_timestamp, return_path, list_timestamps, return_all.
    """
    if match_whole_words is None:
        match_whole_words = latest_data_match_whole_words

    assert return_timestamp or return_path, \
        'No return value chosen (return_timestamp=return_path=False).'

    if folder is None:
        search_dir = datadir
    else:
        search_dir = folder

    daydirs = os.listdir(search_dir)
    if len(daydirs) == 0:
        log.warning('No data found in datadir')
        return None

    daydirs.sort()

    measdirs = []
    paths = []
    timestamps = []
    i = len(daydirs)-1
    if n_matches is None or return_all:
        n_matches = 0

    timestamp = None
{'fraction_non_alphanumeric': 0.09182484182484182, 'fraction_numerical': 0.009324009324009324, 'mean_word_length': 2.5956300508829693, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '230271', 'n_tokens_mistral': 3398, 'n_tokens_neox': 3266, 'n_words': 1151}