<commit_before>import os.path

from cbagent.collectors import Collector


class SecondaryLatencyStats(Collector):

    COLLECTOR = "secondaryscan_latency"

    def _get_secondaryscan_latency(self):
        stats = {}
        if os.path.isfile(self.secondary_statsfile):
            with open(self.secondary_statsfile, 'rb') as fh:
                try:
                    next(fh).decode()
                    fh.seek(-400, 2)
                    last = fh.readlines()[-1].decode()
                    duration = last.split(',')[-1]
                    stats = {}
                    latency = duration.split(':')[1]
                    latency = latency.rstrip()
                    latency_key = duration.split(':')[0]
                    latency_key = latency_key.strip()
                    stats[latency_key] = int(latency)
                except StopIteration:
                    pass
        return stats

    def sample(self):
        stats = self._get_secondaryscan_latency()
        if stats:
            self.update_metric_metadata(stats.keys())
            self.store.append(stats, cluster=self.cluster, collector=self.COLLECTOR)

    def update_metadata(self):
        self.mc.add_cluster()<commit_msg>CBPS-237: Catching IOError as when the file is created fh.seek(-200) is not possible immediatly and results in invalid argument error<commit_after>import os.path

from cbagent.collectors import Collector


class SecondaryLatencyStats(Collector):

    COLLECTOR = "secondaryscan_latency"

    def _get_secondaryscan_latency(self):
        stats = {}
        if os.path.isfile(self.secondary_statsfile):
            with open(self.secondary_statsfile, 'rb') as fh:
                try:
                    next(fh).decode()
                    fh.seek(-200, 2)
                    last = fh.readlines()[-1].decode()
                    duration = last.split(',')[-1]
                    stats = {}
                    latency = duration.split(':')[1]
                    latency = latency.rstrip()
                    latency_key = duration.split(':')[0]
                    latency_key = latency_key.strip()
                    stats[latency_key] = int(latency)
                except (StopIteration, IOError):
                    pass
        return stats

    def sample(self):
        stats = self._get_secondaryscan_latency()
        if stats:
            self.update_metric_metadata(stats.keys())
            self.store.append(stats, cluster=self.cluster, collector=self.COLLECTOR)

    def update_metadata(self):
        self.mc.add_cluster()
{'fraction_non_alphanumeric': 0.10261412407335155, 'fraction_numerical': 0.008583690987124463, 'mean_word_length': 1.7451820128479658, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4877915', 'n_tokens_mistral': 741, 'n_tokens_neox': 681, 'n_words': 133}