<commit_before>#!/usr/bin/env python3
"""Machine model and helper functions."""
from distutils.spawn import find_executable
from distutils.version import LooseVersion
import re

import ruamel
import cachesim
from sympy.parsing.sympy_parser import parse_expr

from . import prefixedunit
from . import __version__


MIN_SUPPORTED_VERSION = "0.6.5"


def sanitize_symbolname(name):
    """
    Sanitize all characters not matched to a symbol by sympy's parse_expr.

    Based on same rules as used for python variables.
    """
    return re.subn('(^[0-9])|[^0-9a-zA-Z_]', '_', name)[0]


class MachineModel(object):
    """Representation of the hardware and machine architecture."""

    def __init__(self, path_to_yaml=None, machine_yaml=None, args=None):
        """Create machine representation from yaml file."""
        if not path_to_yaml and not machine_yaml:
            raise ValueError('Either path_to_yaml or machine_yaml is required')
        if path_to_yaml and machine_yaml:
            raise ValueError('Only one of path_to_yaml and machine_yaml is allowed')
        self._path = path_to_yaml
        self._data = machine_yaml
        self._args = args
        if path_to_yaml:
            with open(path_to_yaml, 'r') as f:
                # Ignore ruamel unsafe loading warning, by supplying Loader parameter
                self._data = ruamel.yaml.load(f, Loader=ruamel.yaml.RoundTripLoader)

        assert 'kerncraft version' in self._data, \
            "Machine description requires a 'kerncraft version' entry, containg the kerncraft " \
            "version it was written for."
        if not (MIN_SUPPORTED_VERSION <= LooseVersion(self._data['kerncraft version'])
                <= LooseVersion(__version__)):
            raise ValueError("The provided machine description is incompatible with this version. "
                             "Supported versions are from {} to {}. Check change logs and examples "
                             " to update your own machine description file format.".format(
                                MIN_SUPPORTED_VERSION, __version__))

    def __getitem__(self, key):
        """Return configuration entry."""
        return self._data[key]

    def __repr__(self):
        """Return object representation."""
        return '{}({})'.format(
            self.__class__.__name__,
            repr(self._path or self._data['model name']),
        )

    def get_cachesim(self, cores=1):
        """
        Return a cachesim.CacheSimulator object based on the machine description.

        :param cores: core count (default: 1)
        """
        cache_dict = {}<commit_msg>added changelog of machine file format<commit_after>#!/usr/bin/env python3
"""Machine model and helper functions."""
from distutils.spawn import find_executable
from distutils.version import LooseVersion
import re
from collections import OrderedDict

import ruamel
import cachesim
from sympy.parsing.sympy_parser import parse_expr

from . import prefixedunit
from . import __version__


MIN_SUPPORTED_VERSION = "0.6.5"

CHANGES_SINCE = OrderedDict([
    ("0.6.5",
     """
     Removed 'cycles per cache line transfer' and replaced it by 
     'non-overlap upstream throughput' in cache levels. The new parameter
     takes the following arguments and is now associated with the cache level 
     that is read from or written to: 
     [$TP B/cy or 'full socket memory bandwidth', 'half-duplex' or 'full-duplex']
     """),
])


def sanitize_symbolname(name):
    """
    Sanitize all characters not matched to a symbol by sympy's parse_expr.

    Based on same rules as used for python variables.
    """
    return re.subn('(^[0-9])|[^0-9a-zA-Z_]', '_', name)[0]


class MachineModel(object):
    """Representation of the hardware and machine architecture."""

    def __init__(self, path_to_yaml=None, machine_yaml=None, args=None):
        """Create machine representation from yaml file."""
        if not path_to_yaml and not machine_yaml:
            raise ValueError('Either path_to_yaml or machine_yaml is required')
        if path_to_yaml and machine_yaml:
            raise ValueError('Only one of path_to_yaml and machine_yaml is allowed')
        self._path = path_to_yaml
        self._data = machine_yaml
        self._args = args
        if path_to_yaml:
            with open(path_to_yaml, 'r') as f:
                # Ignore ruamel unsafe loading warning, by supplying Loader parameter
                self._data = ruamel.yaml.load(f, Loader=ruamel.yaml.RoundTripLoader)

        assert 'kerncraft version' in self._data, \
            "Machine description requires a 'kerncraft version' entry, containg the kerncraft " \
            "version it was written for."
        file_version = LooseVersion(self._data['kerncraft version'])
        if not (MIN_SUPPORTED_VERSION <= file_version
                <= LooseVersion(__version__)):
            print("Relevant changes to the machine description file format:")
            print('\n'.join(['{}: {}'.format(version, help_text)
                             for version, help_text in CHANGES_SINCE.items()
                             if LooseVersion(version) >= file_version]))
            raise ValueError("Machine description is incompatible with this version. "
                             "Supported versions are from {} to {}. Check change logs and examples "
                             " to update your own machine description file format.".format(
                                MIN_SUPPORTED_VERSION, __version__))

    def __getitem__(self, key):
        """Return configuration entry."""
        return self._data[key]

    def __repr__(self):
        """Return object representation."""
        return '{}({})'.format(
            self.__class__.__name__,
            repr(self._path or self._data['model name']),
        )

    def get_cachesim(self, cores=1):
        """
        Return a cachesim.CacheSimulator object based on the machine description.

        :param cores: core count (default: 1)
        """
        cache_dict = {}
{'fraction_non_alphanumeric': 0.09146442133069176, 'fraction_numerical': 0.0041274558362225525, 'mean_word_length': 2.5509964830011724, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6855031', 'n_tokens_mistral': 1652, 'n_tokens_neox': 1595, 'n_words': 527}