<commit_before>import os
import io
import re
import sys
import csv
import datetime
import itertools

import pelops.datasets.chip as chip

# ================================================================================
#  SLiCE Test Dataset (labeled by STR)
# ================================================================================


class SliceDataset(chip.ChipDataset):

    def __init__(self, dataset_path, set_type=None, debug=False):
        super().__init__(dataset_path, set_type)
        self.__obset_txt = '(?P<obSetId>ObSet(?P<obSetIdx>\d+)_(?P<epoch>\d+)_(?P<obSetName>.+?)' \
                           '[/\\\\]images[/\\\\]ObSet\d+-(?P<chipId>\d+)-.*(?:png|jpg))'
        self.__obset_ptn = re.compile(self.__obset_txt, re.S | re.I)
        self.__noise_seq = 0
        self.__debug = debug
        self.__set_chips()

    @staticmethod
    def __decode_truth_file(truth_file):
        """The labels for the STR processed SLiCE chips are in a 'truth.txt' file which this function parses."""

        try:
            with open(truth_file) as truth_hdl:
                truth_text = truth_hdl.read()
                for char in [' ', '%']:
                    truth_text = truth_text.replace(char, '')
                truth_fobj = io.StringIO(truth_text)
                return {(int(dct['obSetIdx']), int(dct['chipIdx'])): int(dct['targetID'])
                        for dct in csv.DictReader(truth_fobj)}

        except IOError as io_err:
            sys.stderr.write("Error occurred when attempting to read slice truth file ({}).".format(io_err))
        except KeyError as key_err:
            sys.stderr.write("Truth file headers may not be set appropriately ({}).".format(key_err))

        return {}<commit_msg>Remove extra `-` in Regexp<commit_after>import os
import io
import re
import sys
import csv
import datetime
import itertools

import pelops.datasets.chip as chip

# ================================================================================
#  SLiCE Test Dataset (labeled by STR)
# ================================================================================


class SliceDataset(chip.ChipDataset):

    def __init__(self, dataset_path, set_type=None, debug=False):
        super().__init__(dataset_path, set_type)
        self.__obset_txt = '(?P<obSetId>ObSet(?P<obSetIdx>\d+)_(?P<epoch>\d+)_(?P<obSetName>.+?)' \
                           '[/\\\\]images[/\\\\]ObSet\d+-(?P<chipId>\d+).*(?:png|jpg))'
        self.__obset_ptn = re.compile(self.__obset_txt, re.S | re.I)
        self.__noise_seq = 0
        self.__debug = debug
        self.__set_chips()

    @staticmethod
    def __decode_truth_file(truth_file):
        """The labels for the STR processed SLiCE chips are in a 'truth.txt' file which this function parses."""

        try:
            with open(truth_file) as truth_hdl:
                truth_text = truth_hdl.read()
                for char in [' ', '%']:
                    truth_text = truth_text.replace(char, '')
                truth_fobj = io.StringIO(truth_text)
                return {(int(dct['obSetIdx']), int(dct['chipIdx'])): int(dct['targetID'])
                        for dct in csv.DictReader(truth_fobj)}

        except IOError as io_err:
            sys.stderr.write("Error occurred when attempting to read slice truth file ({}).".format(io_err))
        except KeyError as key_err:
            sys.stderr.write("Truth file headers may not be set appropriately ({}).".format(key_err))

        return {}
{'fraction_non_alphanumeric': 0.21326472269868496, 'fraction_numerical': 0.0005717552887364208, 'mean_word_length': 2.7827027027027027, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4087057', 'n_tokens_mistral': 1081, 'n_tokens_neox': 1003, 'n_words': 233}