<commit_before># coding: utf-8
"""
Collection of classes and functions to help reading HDF5 file generated at
The European XFEL.

Copyright (c) 2017, European X-Ray Free-Electron Laser Facility GmbH
All rights reserved.

You should have received a copy of the 3-Clause BSD License along with this
program. If not, see <https://opensource.org/licenses/BSD-3-Clause>
"""

from collections import defaultdict
import datetime
from fnmatch import fnmatchcase
from glob import glob
import h5py
import numpy as np
import os.path as osp
import pandas as pd
import re
import xarray as xr


__all__ = ['H5File', 'RunDirectory', 'RunHandler', 'stack_data',
           'stack_detector_data', 'by_id', 'by_index', 'SourceNameError',
           'PropertyNameError',
          ]


RUN_DATA = 'RUN'
INDEX_DATA = 'INDEX'
METADATA = 'METADATA'

DETECTOR_NAMES = {'AGIPD', 'LPD'}


class FilenameInfo:
    is_detector = False
    detector_name = None
    detector_moduleno = -1

    _rawcorr_descr = {'RAW': 'Raw', 'CORR': 'Corrected'}

    def __init__(self, path):
        self.basename = osp.basename(path)
        nameparts = self.basename[:-3].split('-')
        assert len(nameparts) == 4, self.basename
        rawcorr, runno, datasrc, segment = nameparts
        m = re.match(r'([A-Z]+)(\d+)', datasrc)

        if m and m.group(1) == 'DA':
            self.description = "Aggregated data"
        elif m and m.group(1) in DETECTOR_NAMES:
            self.is_detector = True
            name, moduleno = m.groups()
            self.detector_name = name
            self.detector_moduleno = moduleno
            self.description = "{} detector data from {} module {}".format(
                self._rawcorr_descr.get(rawcorr, '?'), name, moduleno
            )
        else:
            self.description = "Unknown data source ({})", datasrc

class _SliceConstructor(type):
    """Allows instantiation like subclass[1:5]
    """
    def __getitem__(self, item):
        return self(item)

class by_id(metaclass=_SliceConstructor):
    def __init__(self, value):
        self.value = value

class by_index(metaclass=_SliceConstructor):
    def __init__(self, value):
        self.value = value

def _tid_to_slice_ix(tid, dataset, stop=False):
    """Convert a train ID to an integer index for slicing the dataset

    Throws ValueError if the slice won't overlap the trains in the data.
    The *stop* parameter tells it which end of the slice it is making.
    """
    if tid is None:
        return None

    try:
        return dataset.train_indices[tid]
    except KeyError:
        if tid < dataset.train_ids[0]:
            if stop:
                raise ValueError("Train ID {} is before this run (starts at {})"
                                 .format(tid, dataset.train_ids[0]))
            else:
                return None
        elif tid > dataset.train_ids[-1]:
            if stop:
                return None
            else:
                raise ValueError("Train ID {} is after this run (ends at {})"
                                 .format(tid, dataset.train_ids[-1]))
        else:
            # This train ID is within the run, but doesn't have an entry
            for ix, tid_cmp in enumerate(dataset.train_ids):
                if tid_cmp > tid:
                    return ix

    raise Exception("Shouldn't reach here. tid={}".format(tid))


def _normalize_data_selection(selection, dataset):
    """Normalize selected data fields

    We offer different ways to select sources and keys from the data.
    This function normalises the different inputs to a set containing
    (source, key) tuples.

    dataset is meant to be a H5File or RunDirectory object
    """
    if isinstance(selection, set):
        return selection

    res = set()
    if isinstance(selection, dict):
        # {source: {key1, key2}}
        # {source: {}} -> all keys for this source
        for source, keys in selection.items():#
            if source not in (dataset.control_sources | dataset.instrument_sources):
                raise ValueError("Source {} not in this run".format(source))

            for k in (keys or dataset._keys_for_source(source)):
                res.add((source, k))

    elif isinstance(selection, list):
        # [('src_glob', 'key_glob'), ...]
        for src_glob, key_glob in selection:
            matched = set()
            for source in (dataset.control_sources | dataset.instrument_sources):
                if not fnmatchcase(source, src_glob):
                    continue

                for key in dataset._keys_for_source(source):
                    if fnmatchcase(key, key_glob):
                        matched.add((source, key))

            if not matched:
                raise ValueError("No matches for pattern {}"
                                 .format((src_glob, key_glob)))

            res.update(matched)
    else:
        TypeError("Unknown selection type: {}".format(type(selection)))

    return res


class SourceNameError(KeyError):
    def __init__(self, source, run=True):
        self.source = source
        self.run = run

    def __str__(self):
        run_file = 'run' if self.run else 'file'
        return "This {0} has no source named {1!r}.\n" \
               "See {0}.all_sources for available sources.".format(
            run_file, self.source
        )

class PropertyNameError(KeyError):
    def __init__(self, prop, source):
        self.prop = prop
        self.source = source

    def __str__(self):<commit_msg>Convert glob patterns to regexes before matching source/key names<commit_after># coding: utf-8
"""
Collection of classes and functions to help reading HDF5 file generated at
The European XFEL.

Copyright (c) 2017, European X-Ray Free-Electron Laser Facility GmbH
All rights reserved.

You should have received a copy of the 3-Clause BSD License along with this
program. If not, see <https://opensource.org/licenses/BSD-3-Clause>
"""

from collections import defaultdict
import datetime
import fnmatch
from fnmatch import fnmatchcase
from glob import glob
import h5py
import numpy as np
import os.path as osp
import pandas as pd
import re
import xarray as xr


__all__ = ['H5File', 'RunDirectory', 'RunHandler', 'stack_data',
           'stack_detector_data', 'by_id', 'by_index', 'SourceNameError',
           'PropertyNameError',
          ]


RUN_DATA = 'RUN'
INDEX_DATA = 'INDEX'
METADATA = 'METADATA'

DETECTOR_NAMES = {'AGIPD', 'LPD'}


class FilenameInfo:
    is_detector = False
    detector_name = None
    detector_moduleno = -1

    _rawcorr_descr = {'RAW': 'Raw', 'CORR': 'Corrected'}

    def __init__(self, path):
        self.basename = osp.basename(path)
        nameparts = self.basename[:-3].split('-')
        assert len(nameparts) == 4, self.basename
        rawcorr, runno, datasrc, segment = nameparts
        m = re.match(r'([A-Z]+)(\d+)', datasrc)

        if m and m.group(1) == 'DA':
            self.description = "Aggregated data"
        elif m and m.group(1) in DETECTOR_NAMES:
            self.is_detector = True
            name, moduleno = m.groups()
            self.detector_name = name
            self.detector_moduleno = moduleno
            self.description = "{} detector data from {} module {}".format(
                self._rawcorr_descr.get(rawcorr, '?'), name, moduleno
            )
        else:
            self.description = "Unknown data source ({})", datasrc

class _SliceConstructor(type):
    """Allows instantiation like subclass[1:5]
    """
    def __getitem__(self, item):
        return self(item)

class by_id(metaclass=_SliceConstructor):
    def __init__(self, value):
        self.value = value

class by_index(metaclass=_SliceConstructor):
    def __init__(self, value):
        self.value = value

def _tid_to_slice_ix(tid, dataset, stop=False):
    """Convert a train ID to an integer index for slicing the dataset

    Throws ValueError if the slice won't overlap the trains in the data.
    The *stop* parameter tells it which end of the slice it is making.
    """
    if tid is None:
        return None

    try:
        return dataset.train_indices[tid]
    except KeyError:
        if tid < dataset.train_ids[0]:
            if stop:
                raise ValueError("Train ID {} is before this run (starts at {})"
                                 .format(tid, dataset.train_ids[0]))
            else:
                return None
        elif tid > dataset.train_ids[-1]:
            if stop:
                return None
            else:
                raise ValueError("Train ID {} is after this run (ends at {})"
                                 .format(tid, dataset.train_ids[-1]))
        else:
            # This train ID is within the run, but doesn't have an entry
            for ix, tid_cmp in enumerate(dataset.train_ids):
                if tid_cmp > tid:
                    return ix

    raise Exception("Shouldn't reach here. tid={}".format(tid))


def _normalize_data_selection(selection, dataset):
    """Normalize selected data fields

    We offer different ways to select sources and keys from the data.
    This function normalises the different inputs to a set containing
    (source, key) tuples.

    dataset is meant to be a H5File or RunDirectory object
    """
    if isinstance(selection, set):
        return selection

    res = set()
    if isinstance(selection, dict):
        # {source: {key1, key2}}
        # {source: {}} -> all keys for this source
        for source, keys in selection.items():#
            if source not in (dataset.control_sources | dataset.instrument_sources):
                raise ValueError("Source {} not in this run".format(source))

            for k in (keys or dataset._keys_for_source(source)):
                res.add((source, k))

    elif isinstance(selection, list):
        # [('src_glob', 'key_glob'), ...]
        for src_glob, key_glob in selection:
            # Convert glob patterns to regexes
            src_re = re.compile(fnmatch.translate(src_glob))
            key_re = re.compile(fnmatch.translate(key_glob))
            if key_glob.endswith('.value'):
                ctrl_key_re = key_re
            else:
                ctrl_key_re = re.compile(key_re.pattern + r'(\.value)?')

            matched = set()
            for source in (dataset.control_sources | dataset.instrument_sources):
                if not src_re.match(source):
                    continue

                use_key_re = ctrl_key_re if (source in dataset.control_sources) else key_re
                for key in dataset._keys_for_source(source):
                    if use_key_re.match(key):
                        matched.add((source, key))

            if not matched:
                raise ValueError("No matches for pattern {}"
                                 .format((src_glob, key_glob)))

            res.update(matched)
    else:
        TypeError("Unknown selection type: {}".format(type(selection)))

    return res


class SourceNameError(KeyError):
    def __init__(self, source, run=True):
        self.source = source
        self.run = run

    def __str__(self):
        run_file = 'run' if self.run else 'file'
        return "This {0} has no source named {1!r}.\n" \
               "See {0}.all_sources for available sources.".format(
            run_file, self.source
        )

class PropertyNameError(KeyError):
    def __init__(self, prop, source):
        self.prop = prop
        self.source = source

    def __str__(self):
{'fraction_non_alphanumeric': 0.09936372352479735, 'fraction_numerical': 0.004706702693279875, 'mean_word_length': 2.1996653653095373, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3060368', 'n_tokens_mistral': 3325, 'n_tokens_neox': 3196, 'n_words': 1022}