<commit_before>"""The Intel MPI Benchmarks
    https://software.intel.com/en-us/articles/intel-mpi-benchmarks
"""
import os.path as osp
import re

from cached_property import cached_property

from hpcbench.api import (
    Benchmark,
    Metrics,
    MetricsExtractor,
)
from hpcbench.toolbox.process import find_executable


class IMBExtractor(MetricsExtractor):
    """Ignore stdout until this line"""
    STDOUT_IGNORE_PRIOR = (
        "# Benchmarking PingPong"
    )

    latency_bandwidth = re.compile(
        r'^\s*(\d)+\s+\d+\s+([0-9]*\.?[0-9]+)[\s]+([0-9]*\.?[0-9]+)'
    )

    METRICS = dict(
        latency=Metrics.Second,
        bandwidth=Metrics.Flops,
    )

    METRICS_NAMES = set(METRICS)

    @property
    def metrics(self):
        """ The metrics to be extracted.
            This property can not be replaced, but can be mutated as required
        """
        return IMBExtractor.METRICS

    def extract(self, outdir, metas):
        metrics = {}
        s_latency = set()
        s_bandwidth = set()
        # parse stdout and extract desired metrics
        with open(self.stdout(outdir)) as istr:
            for line in istr:
                if line.strip() == IMBExtractor.STDOUT_IGNORE_PRIOR:
                    break
            for line in istr:
                line = line.strip()
                print(repr(line))
                search = IMBExtractor.latency_bandwidth.search(line)
                if search:
                    byte = int(search.group(1))
                    if byte != 0:
                        s_latency.add(float(search.group(2)))
                        s_bandwidth.add(float(search.group(3)))

        metrics["latency"] = min(s_latency)
        metrics["bandwidth"] = max(s_bandwidth)
        # ensure all metrics have been extracted
        unset_attributes = IMBExtractor.METRICS_NAMES - set(metrics)
        if any(unset_attributes):
            error = \
                'Could not extract some metrics: %s\n' \
                'metrics setted are: %s'
            raise Exception(error % (' ,'.join(unset_attributes),
                                     ' ,'.join(set(metrics))))
        return metrics


class IMB(Benchmark):
    """Benchmark wrapper for the IMBbench utility
    """
    DEFAULT_THREADS = [1]
    DEFAULT_DEVICE = 'cpu'
    DEFAULT_EXECUTABLE = 'IMB-MPI1'

    def __init__(self):
        # locate `stream_c` executable
        super(IMB, self).__init__(
            attributes=dict(
                threads=IMB.DEFAULT_THREADS,
                data="",
                device=IMB.DEFAULT_DEVICE,
                executable=IMB.DEFAULT_EXECUTABLE
            )
        )
    name = 'imb'

    description = "Provides latency/bandwidth of the newtork."

    @cached_property
    def executable(self):
        """Get absolute path to executable
        """
        return find_executable(self.attributes['executable'])

    @property
    def execution_matrix(self):
        yield dict(
            category=IMB.DEFAULT_DEVICE,
            command=[
                './' + osp.basename(self.executable),
            ],
        )

    @cached_property
    def metrics_extractors(self):
        return IMBExtractor()
<commit_msg>Fix path to executable / fix typos<commit_after>"""The Intel MPI Benchmarks
    https://software.intel.com/en-us/articles/intel-mpi-benchmarks
"""
import os.path as osp
import re

from cached_property import cached_property

from hpcbench.api import (
    Benchmark,
    Metrics,
    MetricsExtractor,
)
from hpcbench.toolbox.process import find_executable


class IMBExtractor(MetricsExtractor):
    """Ignore stdout until this line"""
    STDOUT_IGNORE_PRIOR = (
        "# Benchmarking PingPong"
    )

    latency_bandwidth = re.compile(
        r'^\s*(\d)+\s+\d+\s+([0-9]*\.?[0-9]+)[\s]+([0-9]*\.?[0-9]+)'
    )

    METRICS = dict(
        latency=Metrics.Second,
        bandwidth=Metrics.Flops,
    )

    METRICS_NAMES = set(METRICS)

    @property
    def metrics(self):
        """ The metrics to be extracted.
            This property can not be replaced, but can be mutated as required
        """
        return IMBExtractor.METRICS

    def extract(self, outdir, metas):
        metrics = {}
        s_latency = set()
        s_bandwidth = set()
        # parse stdout and extract desired metrics
        with open(self.stdout(outdir)) as istr:
            for line in istr:
                if line.strip() == IMBExtractor.STDOUT_IGNORE_PRIOR:
                    break
            for line in istr:
                line = line.strip()
                print(repr(line))
                search = IMBExtractor.latency_bandwidth.search(line)
                if search:
                    byte = int(search.group(1))
                    if byte != 0:
                        s_latency.add(float(search.group(2)))
                        s_bandwidth.add(float(search.group(3)))

        metrics["latency"] = min(s_latency)
        metrics["bandwidth"] = max(s_bandwidth)
        # ensure all metrics have been extracted
        unset_attributes = IMBExtractor.METRICS_NAMES - set(metrics)
        if any(unset_attributes):
            error = \
                'Could not extract some metrics: %s\n' \
                'metrics setted are: %s'
            raise Exception(error % (' ,'.join(unset_attributes),
                                     ' ,'.join(set(metrics))))
        return metrics


class IMB(Benchmark):
    """Benchmark wrapper for the IMBbench utility
    """
    DEFAULT_THREADS = [1]
    DEFAULT_DEVICE = 'cpu'
    DEFAULT_EXECUTABLE = 'IMB-MPI1'

    def __init__(self):
        super(IMB, self).__init__(
            attributes=dict(
                threads=IMB.DEFAULT_THREADS,
                data="",
                device=IMB.DEFAULT_DEVICE,
                executable=IMB.DEFAULT_EXECUTABLE
            )
        )
    name = 'imb'

    description = "Provides latency/bandwidth of the network."

    @cached_property
    def executable(self):
        """Get absolute path to executable
        """
        return find_executable(self.attributes['executable'])

    @property
    def execution_matrix(self):
        yield dict(
            category=IMB.DEFAULT_DEVICE,
            command=[self.executable],
        )

    @cached_property
    def metrics_extractors(self):
        return IMBExtractor()

{'fraction_non_alphanumeric': 0.10588420390182504, 'fraction_numerical': 0.004405286343612335, 'mean_word_length': 1.8908594815825375, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1846642', 'n_tokens_mistral': 1929, 'n_tokens_neox': 1835, 'n_words': 401}