<commit_before>#    Copyright 2015-2017 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


# pylint can't see any of the dynamically allocated classes of FTrace
# pylint: disable=no-member

import itertools
import json
import os
import re
import pandas as pd
import hashlib
import shutil
import warnings

from trappy.bare_trace import BareTrace
from trappy.utils import listify

class FTraceParseError(Exception):
    pass

def _plot_freq_hists(allfreqs, what, axis, title):
    """Helper function for plot_freq_hists

    allfreqs is the output of a Cpu*Power().get_all_freqs() (for
    example, CpuInPower.get_all_freqs()).  what is a string: "in" or
    "out"

    """
    import trappy.plot_utils

    for ax, actor in zip(axis, allfreqs):
        this_title = "freq {} {}".format(what, actor)
        this_title = trappy.plot_utils.normalize_title(this_title, title)
        xlim = (0, allfreqs[actor].max())

        trappy.plot_utils.plot_hist(allfreqs[actor], ax, this_title, "KHz", 20,
                             "Frequency", xlim, "default")

SPECIAL_FIELDS_RE = re.compile(
                        r"^\s*(?P<comm>.*)-(?P<pid>\d+)(?:\s+\(.*\))"\
                        r"?\s+\[(?P<cpu>\d+)\](?:\s+....)?\s+"\
                        r"(?P<timestamp>[0-9]+(?P<us>\.[0-9]+)?): (\w+:\s+)+(?P<data>.+)"
)

class GenericFTrace(BareTrace):
    """Generic class to parse output of FTrace.  This class is meant to be
subclassed by FTrace (for parsing FTrace coming from trace-cmd) and SysTrace."""

    thermal_classes = {}

    sched_classes = {}

    dynamic_classes = {}

    disable_cache = True

    def __init__(self, name="", normalize_time=True, scope="all",
                 events=[], window=(0, None), abs_window=(0, None)):
        super(GenericFTrace, self).__init__(name)

        self.class_definitions.update(self.dynamic_classes.items())
        self.__add_events(listify(events))

        if scope == "thermal":
            self.class_definitions.update(self.thermal_classes.items())
        elif scope == "sched":
            self.class_definitions.update(self.sched_classes.items())
        elif scope != "custom":
            self.class_definitions.update(self.thermal_classes.items() +
                                          self.sched_classes.items())

        for attr, class_def in self.class_definitions.iteritems():
            trace_class = class_def()
            setattr(self, attr, trace_class)
            self.trace_classes.append(trace_class)

        # save parameters to complete init later
        self.normalize_time = normalize_time
        self.window = window
        self.abs_window = abs_window

        self._do_parse()

    @classmethod
    def register_parser(cls, cobject, scope):
        """Register the class as an Event. This function
        can be used to register a class which is associated
        with an FTrace unique word.

        .. seealso::

            :mod:`trappy.dynamic.register_dynamic_ftrace` :mod:`trappy.dynamic.register_ftrace_parser`

        """

        if not hasattr(cobject, "name"):
            cobject.name = cobject.unique_word.split(":")[0]

        # Add the class to the classes dictionary
        if scope == "all":
            cls.dynamic_classes[cobject.name] = cobject
        else:
            getattr(cls, scope + "_classes")[cobject.name] = cobject

    @classmethod
    def unregister_parser(cls, cobject):
        """Unregister a parser

        This is the opposite of FTrace.register_parser(), it removes a class
        from the list of classes that will be parsed on the trace

        """

        # TODO: scopes should not be hardcoded (nor here nor in the FTrace object)
        all_scopes = [cls.thermal_classes, cls.sched_classes,
                      cls.dynamic_classes]
        known_events = ((n, c, sc) for sc in all_scopes for n, c in sc.items())

        for name, obj, scope_classes in known_events:
            if cobject == obj:
                del scope_classes[name]

    def _trace_cache_path(self):
        trace_file = self.trace_path
        cache_dir  = '.' +  os.path.basename(trace_file) + '.cache'
        tracefile_dir = os.path.dirname(os.path.abspath(trace_file))
        cache_path = os.path.join(tracefile_dir, cache_dir)
        return cache_path

    def _get_csv_path(self, trace_class):
        path = self._trace_cache_path()
        return os.path.join(path, trace_class.__class__.__name__ + '.csv')

    def _check_trace_cache(self, params):
        cache_path = self._trace_cache_path()
        trace_metadata_path = os.path.join(cache_path, 'metadata.json')

        if not os.path.exists(trace_metadata_path):
            return False

        with open(trace_metadata_path) as f:
            trace_metadata = json.load(f)

        with open(self.trace_path, 'rb') as f:
            trace_md5sum = hashlib.md5(f.read()).hexdigest()

        self.basetime = float(trace_metadata["basetime"])

        # Check if cache is valid
        if trace_metadata["md5sum"] != trace_md5sum:
            warnstr = "Cached data is from another trace, invalidating cache."
            warnings.warn(warnstr)
            shutil.rmtree(cache_path)
            return False

        # Check if cache can be used with given parameters
        # Convert to a json string for comparison
        if json.dumps(trace_metadata["params"]) != json.dumps(params):
            warnstr = "Cached trace parameters differ from those given, invalidating cache."
            warnings.warn(warnstr)
            shutil.rmtree(cache_path)
            return False
        return True

    def _create_trace_cache(self, params):
        cache_path = self._trace_cache_path()
        trace_metadata_path = os.path.join(cache_path, 'metadata.json')

        if os.path.exists(cache_path):
            shutil.rmtree(cache_path)
        os.mkdir(cache_path)

        trace_metadata = {}

        trace_metadata["md5sum"] = hashlib.md5(open(self.trace_path, 'rb').read()).hexdigest()
        trace_metadata["basetime"] = self.basetime
        trace_metadata["params"] = params

        with open(trace_metadata_path, 'w') as f:
            json.dump(trace_metadata, f)

    def _get_params_to_cache(self):
        return {'window': self.window, 'abs_window': self.abs_window}

    def _update_cache(self):
        params = self._get_params_to_cache()

        try:
            # Recreate basic cache directories only if nothing cached
            if not any([c.cached for c in self.trace_classes]):
                self._create_trace_cache(params)

            # Write out only events that weren't cached before
            for trace_class in self.trace_classes:
                if trace_class.cached:
                    continue
                csv_file = self._get_csv_path(trace_class)
                trace_class.write_csv(csv_file)
        except OSError as err:
            warnings.warn(
                "TRAPpy: Cache not created due to OS error: {0}".format(err))

    def _load_cache(self):
        params = self._get_params_to_cache()

        if self._check_trace_cache(params):
             # Read csv into frames
            for trace_class in self.trace_classes:
                try:
                    csv_file = self._get_csv_path(trace_class)
                    trace_class.read_csv(csv_file)
                    trace_class.cached = True
                except:
                    warnstr = "TRAPpy: Couldn't read {} from cache, reading it from trace".format(trace_class)
                    warnings.warn(warnstr)

    def _do_parse(self):
        if not self.__class__.disable_cache:
            self._load_cache()

            # Check if cache data is enough
            if all([c.cached for c in self.trace_classes]):
                if self.normalize_time:
                    self._normalize_time()
                return

        self._parsing_setup()

        self.__parse_trace_file(self.trace_path)
        self.finalize_objects()

        # Update (or create) cache directory
        if not self.__class__.disable_cache:
            self._update_cache()

        if self.normalize_time:
             self._normalize_time()

        self._parsing_teardown()

    def _parsing_setup(self):
        pass

    def _parsing_teardown(self):
        pass

    def __add_events(self, events):
        """Add events to the class_definitions

        If the events are known to trappy just add that class to the
        class definitions list.  Otherwise, register a class to parse
        that event

        """

        from trappy.dynamic import DynamicTypeFactory, default_init
        from trappy.base import Base

        # TODO: scopes should not be hardcoded (nor here nor in the FTrace object)
        all_scopes = [self.thermal_classes, self.sched_classes,
                      self.dynamic_classes]
        known_events = {k: v for sc in all_scopes for k, v in sc.iteritems()}

        for event_name in events:
            for cls in known_events.itervalues():
                if (event_name == cls.unique_word) or \
                   (event_name + ":" == cls.unique_word):
                    self.class_definitions[event_name] = cls
                    break
            else:
                kwords = {
                    "__init__": default_init,
                    "unique_word": event_name + ":",
                    "name": event_name,
                }
                trace_class = DynamicTypeFactory(event_name, (Base,), kwords)
                self.class_definitions[event_name] = trace_class

    def __get_trace_class(self, line, cls_word):
        trace_class = None
        for unique_word, cls in cls_word.iteritems():
            if unique_word in line:
                trace_class = cls
                if not cls.fallback:
                    return trace_class
        return trace_class

    def __populate_data(self, fin, cls_for_unique_word):
        """Append to trace data from a txt trace"""

        actual_trace = itertools.dropwhile(self.trace_hasnt_started(), fin)
        actual_trace = itertools.takewhile(self.trace_hasnt_finished(),
                                           actual_trace)

        for line in actual_trace:
            trace_class = self.__get_trace_class(line, cls_for_unique_word)
            if not trace_class:
                self.lines += 1
                continue

            line = line[:-1]

            fields_match = SPECIAL_FIELDS_RE.match(line)
            if not fields_match:
                raise FTraceParseError("Couldn't match fields in '{}'".format(line))
            comm = fields_match.group('comm')
            pid = int(fields_match.group('pid'))
            cpu = int(fields_match.group('cpu'))

            # The timestamp, depending on the trace_clock configuration, can be
            # reported either in [s].[us] or [ns] format. Let's ensure that we
            # always generate DF which have the index expressed in:
            #    [s].[decimals]
            timestamp = float(fields_match.group('timestamp'))
            if not fields_match.group('us'):
                timestamp /= 1e9
            data_str = fields_match.group('data')

            if not self.basetime:
                self.basetime = timestamp

            if (timestamp < self.window[0] + self.basetime) or \
               (timestamp < self.abs_window[0]):
                self.lines += 1
                continue

            if (self.window[1] and timestamp > self.window[1] + self.basetime) or \
               (self.abs_window[1] and timestamp > self.abs_window[1]):
                return

            # Remove empty arrays from the trace<commit_msg>ftrace: Always disregard window when parsing<commit_after>#    Copyright 2015-2017 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


# pylint can't see any of the dynamically allocated classes of FTrace
# pylint: disable=no-member

import itertools
import json
import os
import re
import pandas as pd
import hashlib
import shutil
import warnings

from trappy.bare_trace import BareTrace
from trappy.utils import listify

class FTraceParseError(Exception):
    pass

def _plot_freq_hists(allfreqs, what, axis, title):
    """Helper function for plot_freq_hists

    allfreqs is the output of a Cpu*Power().get_all_freqs() (for
    example, CpuInPower.get_all_freqs()).  what is a string: "in" or
    "out"

    """
    import trappy.plot_utils

    for ax, actor in zip(axis, allfreqs):
        this_title = "freq {} {}".format(what, actor)
        this_title = trappy.plot_utils.normalize_title(this_title, title)
        xlim = (0, allfreqs[actor].max())

        trappy.plot_utils.plot_hist(allfreqs[actor], ax, this_title, "KHz", 20,
                             "Frequency", xlim, "default")

SPECIAL_FIELDS_RE = re.compile(
                        r"^\s*(?P<comm>.*)-(?P<pid>\d+)(?:\s+\(.*\))"\
                        r"?\s+\[(?P<cpu>\d+)\](?:\s+....)?\s+"\
                        r"(?P<timestamp>[0-9]+(?P<us>\.[0-9]+)?): (\w+:\s+)+(?P<data>.+)"
)

class GenericFTrace(BareTrace):
    """Generic class to parse output of FTrace.  This class is meant to be
subclassed by FTrace (for parsing FTrace coming from trace-cmd) and SysTrace."""

    thermal_classes = {}

    sched_classes = {}

    dynamic_classes = {}

    disable_cache = True

    def __init__(self, name="", normalize_time=True, scope="all",
                 events=[], window=(0, None), abs_window=(0, None)):
        super(GenericFTrace, self).__init__(name)

        self.class_definitions.update(self.dynamic_classes.items())
        self.__add_events(listify(events))

        if scope == "thermal":
            self.class_definitions.update(self.thermal_classes.items())
        elif scope == "sched":
            self.class_definitions.update(self.sched_classes.items())
        elif scope != "custom":
            self.class_definitions.update(self.thermal_classes.items() +
                                          self.sched_classes.items())

        for attr, class_def in self.class_definitions.iteritems():
            trace_class = class_def()
            setattr(self, attr, trace_class)
            self.trace_classes.append(trace_class)

        # save parameters to complete init later
        self.normalize_time = normalize_time
        self.window = window
        self.abs_window = abs_window
        self.max_window = (0, None)

        self._do_parse()

    @classmethod
    def register_parser(cls, cobject, scope):
        """Register the class as an Event. This function
        can be used to register a class which is associated
        with an FTrace unique word.

        .. seealso::

            :mod:`trappy.dynamic.register_dynamic_ftrace` :mod:`trappy.dynamic.register_ftrace_parser`

        """

        if not hasattr(cobject, "name"):
            cobject.name = cobject.unique_word.split(":")[0]

        # Add the class to the classes dictionary
        if scope == "all":
            cls.dynamic_classes[cobject.name] = cobject
        else:
            getattr(cls, scope + "_classes")[cobject.name] = cobject

    @classmethod
    def unregister_parser(cls, cobject):
        """Unregister a parser

        This is the opposite of FTrace.register_parser(), it removes a class
        from the list of classes that will be parsed on the trace

        """

        # TODO: scopes should not be hardcoded (nor here nor in the FTrace object)
        all_scopes = [cls.thermal_classes, cls.sched_classes,
                      cls.dynamic_classes]
        known_events = ((n, c, sc) for sc in all_scopes for n, c in sc.items())

        for name, obj, scope_classes in known_events:
            if cobject == obj:
                del scope_classes[name]

    def _calc_max_window(self):
        """
        Compute the maximum window of the 'window' & 'abs_window' intersection
        """
        max_window = [0, None]
        max_window[0] = max(self.window[0] + self.basetime, self.abs_window[0])

        if (self.window[1] is not None) and (self.abs_window[1] is not None):
            max_window[1] = max(self.window[1] + self.basetime, self.abs_window[1])
        elif self.window[1] is not None:
            max_window[1] = self.window[1] + self.basetime
        elif self.abs_window[1] is not None:
            max_window[1] = self.abs_window[1]

        return max_window

    def _windowify_class(self, trace_class, window):
        if len(trace_class.data_frame) < 1:
            return

        if window[1]:
            trace_class.data_frame = trace_class.data_frame[
                window[0]:window[1]]
        elif window[0]:
            trace_class.data_frame = trace_class.data_frame[
                window[0]:]

    def _trace_cache_path(self):
        trace_file = self.trace_path
        cache_dir  = '.' +  os.path.basename(trace_file) + '.cache'
        tracefile_dir = os.path.dirname(os.path.abspath(trace_file))
        cache_path = os.path.join(tracefile_dir, cache_dir)
        return cache_path

    def _get_csv_path(self, trace_class):
        path = self._trace_cache_path()
        return os.path.join(path, trace_class.__class__.__name__ + '.csv')

    def _check_trace_cache(self):
        cache_path = self._trace_cache_path()
        trace_metadata_path = os.path.join(cache_path, 'metadata.json')

        if not os.path.exists(trace_metadata_path):
            return False

        with open(trace_metadata_path) as f:
            trace_metadata = json.load(f)

        with open(self.trace_path, 'rb') as f:
            trace_md5sum = hashlib.md5(f.read()).hexdigest()

        self.basetime = float(trace_metadata["basetime"])

        # Check if cache is valid
        if trace_metadata["md5sum"] != trace_md5sum:
            warnstr = "Cached data is from another trace, invalidating cache."
            warnings.warn(warnstr)
            shutil.rmtree(cache_path)
            return False

        return True

    def _create_trace_cache(self):
        cache_path = self._trace_cache_path()
        trace_metadata_path = os.path.join(cache_path, 'metadata.json')

        if os.path.exists(cache_path):
            shutil.rmtree(cache_path)
        os.mkdir(cache_path)

        trace_metadata = {}

        trace_metadata["md5sum"] = hashlib.md5(open(self.trace_path, 'rb').read()).hexdigest()
        trace_metadata["basetime"] = self.basetime

        with open(trace_metadata_path, 'w') as f:
            json.dump(trace_metadata, f)

    def _update_cache(self):
        try:
            # Recreate basic cache directories only if nothing cached
            if not any([c.cached for c in self.trace_classes]):
                self._create_trace_cache()

            # Write out only events that weren't cached before
            for trace_class in self.trace_classes:
                if trace_class.cached:
                    continue
                csv_file = self._get_csv_path(trace_class)
                trace_class.write_csv(csv_file)
        except OSError as err:
            warnings.warn(
                "TRAPpy: Cache not created due to OS error: {0}".format(err))

    def _load_cache(self):
        if self._check_trace_cache():
            self.max_window = self._calc_max_window()
             # Read csv into frames
            for trace_class in self.trace_classes:
                try:
                    csv_file = self._get_csv_path(trace_class)
                    trace_class.read_csv(csv_file)
                    trace_class.cached = True
                except:
                    warnstr = "TRAPpy: Couldn't read {} from cache, reading it from trace".format(trace_class)
                    warnings.warn(warnstr)

    def _apply_user_parameters(self):
        # Traces are read without any window consideration, so we apply
        # the window after reading the cache or parsing the trace
        for trace_class in self.trace_classes:
            self._windowify_class(trace_class, self.max_window)

        if self.normalize_time:
            self._normalize_time()

    def _do_parse(self):
        if not self.__class__.disable_cache:
            self._load_cache()

            # Check if cache data is enough
            if all([c.cached for c in self.trace_classes]):
                self._apply_user_parameters()
                return

        self._parsing_setup()

        self.__parse_trace_file(self.trace_path)
        self.finalize_objects()

        # Update (or create) cache directory
        if not self.__class__.disable_cache:
            self._update_cache()

        self._apply_user_parameters()

        self._parsing_teardown()

    def _parsing_setup(self):
        pass

    def _parsing_teardown(self):
        pass

    def __add_events(self, events):
        """Add events to the class_definitions

        If the events are known to trappy just add that class to the
        class definitions list.  Otherwise, register a class to parse
        that event

        """

        from trappy.dynamic import DynamicTypeFactory, default_init
        from trappy.base import Base

        # TODO: scopes should not be hardcoded (nor here nor in the FTrace object)
        all_scopes = [self.thermal_classes, self.sched_classes,
                      self.dynamic_classes]
        known_events = {k: v for sc in all_scopes for k, v in sc.iteritems()}

        for event_name in events:
            for cls in known_events.itervalues():
                if (event_name == cls.unique_word) or \
                   (event_name + ":" == cls.unique_word):
                    self.class_definitions[event_name] = cls
                    break
            else:
                kwords = {
                    "__init__": default_init,
                    "unique_word": event_name + ":",
                    "name": event_name,
                }
                trace_class = DynamicTypeFactory(event_name, (Base,), kwords)
                self.class_definitions[event_name] = trace_class

    def __get_trace_class(self, line, cls_word):
        trace_class = None
        for unique_word, cls in cls_word.iteritems():
            if unique_word in line:
                trace_class = cls
                if not cls.fallback:
                    return trace_class
        return trace_class

    def __populate_data(self, fin, cls_for_unique_word):
        """Append to trace data from a txt trace"""

        actual_trace = itertools.dropwhile(self.trace_hasnt_started(), fin)
        actual_trace = itertools.takewhile(self.trace_hasnt_finished(),
                                           actual_trace)

        for line in actual_trace:
            trace_class = self.__get_trace_class(line, cls_for_unique_word)
            if not trace_class:
                self.lines += 1
                continue

            line = line[:-1]

            fields_match = SPECIAL_FIELDS_RE.match(line)
            if not fields_match:
                raise FTraceParseError("Couldn't match fields in '{}'".format(line))
            comm = fields_match.group('comm')
            pid = int(fields_match.group('pid'))
            cpu = int(fields_match.group('cpu'))

            # The timestamp, depending on the trace_clock configuration, can be
            # reported either in [s].[us] or [ns] format. Let's ensure that we
            # always generate DF which have the index expressed in:
            #    [s].[decimals]
            timestamp = float(fields_match.group('timestamp'))
            if not fields_match.group('us'):
                timestamp /= 1e9
            data_str = fields_match.group('data')

            if not self.basetime:
                self.basetime = timestamp
                # Now that we know the basetime, we can derive max_window
                self.max_window = self._calc_max_window()

            # Remove empty arrays from the trace
{'fraction_non_alphanumeric': 0.09436313927307816, 'fraction_numerical': 0.0038195561273721454, 'mean_word_length': 2.1239638281838733, 'pattern_counts': {'":': 20, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6296630', 'n_tokens_mistral': 7237, 'n_tokens_neox': 6923, 'n_words': 1857}