<commit_before>#    Copyright 2015-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


# pylint can't see any of the dynamically allocated classes of Run
# pylint: disable=no-member

import os
import re
import pandas as pd

import trappy.plot_utils

def _plot_freq_hists(allfreqs, what, axis, title):
    """Helper function for plot_freq_hists

    allfreqs is the output of a Cpu*Power().get_all_freqs() (for
    example, CpuInPower.get_all_freqs()).  what is a string: "in" or
    "out"

    """
    for ax, actor in zip(axis, allfreqs):
        this_title = "freq {} {}".format(what, actor)
        this_title = trappy.plot_utils.normalize_title(this_title, title)
        xlim = (0, allfreqs[actor].max())

        trappy.plot_utils.plot_hist(allfreqs[actor], ax, this_title, "KHz", 20,
                             "Frequency", xlim, "default")

class Run(object):
    """A wrapper class that initializes all the classes of a given run

The run class can receive the following optional parameters.

path contains the path to the trace file.  If no path is given, it
uses the current directory by default.  If path is a file, and ends in
.dat, it's run through "trace-cmd report".  If it doesn't end in
".dat", then it must be the output of a trace-cmd report run.  If path
is a directory that contains a trace.txt, that is assumed to be the
output of "trace-cmd report".  If path is a directory that doesn't
have a trace.txt but has a trace.dat, it runs trace-cmd report on the
trace.dat, saves it in trace.txt and then uses that.

name is a string describing the trace.

normalize_time is used to make all traces start from time 0 (the
default).  If normalize_time is False, the trace times are the same as
in the trace file.

scope can be used to limit the parsing done on the trace.  The default
scope parses all the traces known to trappy.  If scope is thermal, only
the thermal classes are parsed.  If scope is sched, only the sched
classes are parsed.

    """

    thermal_classes = {}

    sched_classes = {}

    dynamic_classes = {}

    def __init__(self, path=".", name="", normalize_time=True, scope="all"):
        self.name = name
        self.trace_path, self.trace_path_raw = self.__process_path(path)
        self.class_definitions = self.dynamic_classes.copy()

        if scope == "thermal":
            self.class_definitions.update(self.thermal_classes.items())
        elif scope == "sched":
            self.class_definitions.update(self.sched_classes.items())
        else:
            self.class_definitions.update(self.thermal_classes.items() +
                                          self.sched_classes.items())

        self.trace_classes = []
        for attr, class_name in self.class_definitions.iteritems():
            trace_class = globals()[class_name]()
            setattr(self, attr, trace_class)
            self.trace_classes.append(trace_class)

        self.__parse_trace_file()
        self.__parse_trace_file(raw=True)
        self.__finalize_objects()

        if normalize_time:
            basetime = self.get_basetime()
            self.normalize_time(basetime)

    def __process_path(self, basepath):
        """Process the path and return the path to the trace text file"""

        if os.path.isfile(basepath):
            trace_txt = os.path.splitext(basepath)[0]
            raw_trace_path = trace_txt + ".raw.txt"
            trace_path = trace_txt + ".txt"

            if basepath.endswith(".dat"):
                self.__run_trace_cmd_report(basepath)

            elif basepath.endswith(".txt"):
                trace_path = basepath
                if not os.path.isfile(raw_trace_path):
                    raw_trace_path = None
        else:
            trace_path = os.path.join(basepath, "trace.txt")
            raw_trace_path = os.path.join(basepath, "trace.raw.txt")
            dat_file = os.path.join(basepath, "trace.dat")

            if not os.path.isfile(trace_path):
                self.__run_trace_cmd_report(dat_file)

            # The condition below handles the the following cases
            # trace.dat and trace.txt are both present
            # We can still generate the trace.raw.txt
            if not os.path.isfile(raw_trace_path):
                if os.path.isfile(dat_file):
                    self.__run_trace_cmd_report(dat_file)
                else:
                    raw_trace_path = None

        return trace_path, raw_trace_path

    def __run_trace_cmd_report(self, fname):
        """Run "trace-cmd report fname > fname.txt"
           and "trace-cmd report -R fname > fname.raw.txt"

        The resulting traces are stored in files with extension ".txt"
        and ".raw.txt" respectively.  If fname is "my_trace.dat", the
        trace is stored in "my_trace.txt" and "my_trace.raw.txt".  The
        contents of the destination files are overwritten if they
        exist.

        """
        from subprocess import check_output

        cmd = ["trace-cmd", "report"]

        if not os.path.isfile(fname):
            raise IOError("No such file or directory: {}".format(fname))

        raw_trace_output = os.path.splitext(fname)[0] + ".raw.txt"
        trace_output = os.path.splitext(fname)[0] + ".txt"
        cmd.append(fname)

        with open(os.devnull) as devnull:
            out = check_output(cmd, stderr=devnull)<commit_msg>run: update trace naming to improve consistency<commit_after>#    Copyright 2015-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


# pylint can't see any of the dynamically allocated classes of Run
# pylint: disable=no-member

import os
import re
import pandas as pd

import trappy.plot_utils

def _plot_freq_hists(allfreqs, what, axis, title):
    """Helper function for plot_freq_hists

    allfreqs is the output of a Cpu*Power().get_all_freqs() (for
    example, CpuInPower.get_all_freqs()).  what is a string: "in" or
    "out"

    """
    for ax, actor in zip(axis, allfreqs):
        this_title = "freq {} {}".format(what, actor)
        this_title = trappy.plot_utils.normalize_title(this_title, title)
        xlim = (0, allfreqs[actor].max())

        trappy.plot_utils.plot_hist(allfreqs[actor], ax, this_title, "KHz", 20,
                             "Frequency", xlim, "default")

class Run(object):
    """A wrapper class that initializes all the classes of a given run

The run class can receive the following optional parameters.

path contains the path to the trace file.  If no path is given, it
uses the current directory by default.  If path is a file, and ends in
.dat, it's run through "trace-cmd report".  If it doesn't end in
".dat", then it must be the output of a trace-cmd report run.  If path
is a directory that contains a trace.txt, that is assumed to be the
output of "trace-cmd report".  If path is a directory that doesn't
have a trace.txt but has a trace.dat, it runs trace-cmd report on the
trace.dat, saves it in trace.txt and then uses that.

name is a string describing the trace.

normalize_time is used to make all traces start from time 0 (the
default).  If normalize_time is False, the trace times are the same as
in the trace file.

scope can be used to limit the parsing done on the trace.  The default
scope parses all the traces known to trappy.  If scope is thermal, only
the thermal classes are parsed.  If scope is sched, only the sched
classes are parsed.

    """

    thermal_classes = {}

    sched_classes = {}

    dynamic_classes = {}

    def __init__(self, path=".", name="", normalize_time=True, scope="all"):
        self.name = name
        self.trace_path, self.trace_path_raw = self.__process_path(path)
        self.class_definitions = self.dynamic_classes.copy()

        if scope == "thermal":
            self.class_definitions.update(self.thermal_classes.items())
        elif scope == "sched":
            self.class_definitions.update(self.sched_classes.items())
        else:
            self.class_definitions.update(self.thermal_classes.items() +
                                          self.sched_classes.items())

        self.trace_classes = []
        for attr, class_name in self.class_definitions.iteritems():
            trace_class = globals()[class_name]()
            setattr(self, attr, trace_class)
            self.trace_classes.append(trace_class)

        self.__parse_trace_file()
        self.__parse_trace_file(raw=True)
        self.__finalize_objects()

        if normalize_time:
            basetime = self.get_basetime()
            self.normalize_time(basetime)

    def __process_path(self, basepath):
        """Process the path and return the path to the trace text file"""

        if os.path.isfile(basepath):
            trace_name = os.path.splitext(basepath)[0]
            trace_raw = trace_name + ".raw.txt"
            trace_txt = trace_name + ".txt"

            if basepath.endswith(".dat"):
                self.__run_trace_cmd_report(basepath)

            elif basepath.endswith(".txt"):
                trace_txt = basepath
                if not os.path.isfile(trace_raw):
                    trace_raw = None
        else:
            trace_txt = os.path.join(basepath, "trace.txt")
            trace_raw = os.path.join(basepath, "trace.raw.txt")
            trace_dat = os.path.join(basepath, "trace.dat")

            if not os.path.isfile(trace_txt):
                self.__run_trace_cmd_report(trace_dat)

            # The condition below handles the the following cases
            # trace.dat and trace.txt are both present
            # We can still generate the trace.raw.txt
            if not os.path.isfile(trace_raw):
                if os.path.isfile(trace_dat):
                    self.__run_trace_cmd_report(trace_dat)
                else:
                    trace_raw = None

        return trace_txt, trace_raw

    def __run_trace_cmd_report(self, fname):
        """Run "trace-cmd report fname > fname.txt"
           and "trace-cmd report -R fname > fname.raw.txt"

        The resulting traces are stored in files with extension ".txt"
        and ".raw.txt" respectively.  If fname is "my_trace.dat", the
        trace is stored in "my_trace.txt" and "my_trace.raw.txt".  The
        contents of the destination files are overwritten if they
        exist.

        """
        from subprocess import check_output

        cmd = ["trace-cmd", "report"]

        if not os.path.isfile(fname):
            raise IOError("No such file or directory: {}".format(fname))

        raw_trace_output = os.path.splitext(fname)[0] + ".raw.txt"
        trace_output = os.path.splitext(fname)[0] + ".txt"
        cmd.append(fname)

        with open(os.devnull) as devnull:
            out = check_output(cmd, stderr=devnull)
{'fraction_non_alphanumeric': 0.08998042053290202, 'fraction_numerical': 0.0032348684770579722, 'mean_word_length': 2.5121076233183857, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1945243', 'n_tokens_mistral': 3449, 'n_tokens_neox': 3322, 'n_words': 1137}