<commit_before># SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2015, ARM Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import glob
import matplotlib.gridspec as gridspec
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import pylab as pl
import re
import sys
import trappy

# Configure logging
import logging

class Trace(object):

    def __init__(self, platform, datadir, events, tasks=None, window=(0,None)):

        # The platform used to run the experiments
        self.platform = platform

        # Folder containing all perf data
        self.datadir = None

        # TRAPpy FTrace object
        self.ftrace = None

        # The time window used to limit trace parsing to
        self.window = window

        # Dynamically registered TRAPpy events
        self.trappy_cls = {}

        # Maximum timespan for all collected events
        self.time_range = 0

        # Time the system was overutilzied
        self.overutilized_time = 0
        self.overutilized_prc = 0

        # The dictionary of tasks descriptors available in the dataset
        self.tasks = {}

        # List of events required by user
        self.events = []

        # List of events available in the parsed trace
        self.available_events = []

        # Folder containing trace
        if not os.path.isdir(datadir):
             self.datadir = os.path.dirname(datadir)
        else:
            self.datadir = datadir

        self.__registerTraceEvents(events)
        self.__parseTrace(datadir, tasks, window)
        self.__computeTimeSpan()

    def __registerTraceEvents(self, events):

        if isinstance(events, basestring):
            self.events = events.split(' ')
        elif isinstance(events, list):
            self.events = events
        else:
            raise ValueError('Events must be a string or a list of strings')


    def __parseTrace(self, path, tasks, window):
        logging.debug('Loading [sched] events from trace in [%s]...', path)
        logging.debug("Parsing events: %s", self.events)
        self.ftrace = trappy.FTrace(path, scope="custom", events=self.events, window=window)

        # Check for events available on the parsed trace
        self.__checkAvailableEvents()
        if len(self.available_events) == 0:
            raise ValueError('The trace does not contain useful events')

        # Setup internal data reference to interesting events/dataframes

        self._sanitize_SchedLoadAvgCpu()
        self._sanitize_SchedLoadAvgTask()
        self._sanitize_SchedCpuCapacity()
        self._sanitize_SchedBoostCpu()
        self._sanitize_SchedBoostTask()
        self._sanitize_SchedEnergyDiff()
        self._sanitize_SchedOverutilized()

        self.__loadTasksNames(tasks)


    def __checkAvailableEvents(self):
        for val in trappy.FTrace.get_filters(self.ftrace):
            obj = getattr(self.ftrace, val)
            if len(obj.data_frame):
                self.available_events.append(val)
        logging.debug('Events found on trace:')
        for evt in self.available_events:
            logging.debug(' - %s', evt)


    def __loadTasksNames(self, tasks):<commit_msg>libs/utils/trace: add support for systrace trace format<commit_after># SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2015, ARM Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import glob
import matplotlib.gridspec as gridspec
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import pylab as pl
import re
import sys
import trappy

# Configure logging
import logging

class Trace(object):

    def __init__(self, platform, datadir, events,
                 tasks=None, window=(0,None),
                 trace_format='FTrace'):

        # The platform used to run the experiments
        self.platform = platform

        # Folder containing all perf data
        self.datadir = None

        # TRAPpy Trace object
        self.ftrace = None

        # Trace format
        self.trace_format = trace_format

        # The time window used to limit trace parsing to
        self.window = window

        # Dynamically registered TRAPpy events
        self.trappy_cls = {}

        # Maximum timespan for all collected events
        self.time_range = 0

        # Time the system was overutilzied
        self.overutilized_time = 0
        self.overutilized_prc = 0

        # The dictionary of tasks descriptors available in the dataset
        self.tasks = {}

        # List of events required by user
        self.events = []

        # List of events available in the parsed trace
        self.available_events = []

        # Folder containing trace
        if not os.path.isdir(datadir):
             self.datadir = os.path.dirname(datadir)
        else:
            self.datadir = datadir

        self.__registerTraceEvents(events)
        self.__parseTrace(datadir, tasks, window, trace_format)
        self.__computeTimeSpan()

    def __registerTraceEvents(self, events):

        if isinstance(events, basestring):
            self.events = events.split(' ')
        elif isinstance(events, list):
            self.events = events
        else:
            raise ValueError('Events must be a string or a list of strings')


    def __parseTrace(self, path, tasks, window, trace_format):
        logging.debug('Loading [sched] events from trace in [%s]...', path)
        logging.debug("Parsing events: %s", self.events)
        if trace_format.upper() == 'SYSTRACE' or path.endswith('html'):
            logging.info('Parsing SysTrace format...')
            self.ftrace = trappy.SysTrace(path, scope="custom",
                                          events=self.events,
                                          window=window)
            self.trace_format = 'SysTrace'
        elif trace_format.upper() == 'FTRACE':
            logging.info('Parsing FTrace format...')
            self.ftrace = trappy.FTrace(path, scope="custom",
                                        events=self.events,
                                        window=window)
            self.trace_format = 'FTrace'

        # Check for events available on the parsed trace
        self.__checkAvailableEvents()
        if len(self.available_events) == 0:
            raise ValueError('The trace does not contain useful events')

        # Setup internal data reference to interesting events/dataframes

        self._sanitize_SchedLoadAvgCpu()
        self._sanitize_SchedLoadAvgTask()
        self._sanitize_SchedCpuCapacity()
        self._sanitize_SchedBoostCpu()
        self._sanitize_SchedBoostTask()
        self._sanitize_SchedEnergyDiff()
        self._sanitize_SchedOverutilized()

        self.__loadTasksNames(tasks)


    def __checkAvailableEvents(self, key=""):
        for val in self.ftrace.get_filters(key):
            obj = getattr(self.ftrace, val)
            if len(obj.data_frame):
                self.available_events.append(val)
        logging.debug('Events found on trace:')
        for evt in self.available_events:
            logging.debug(' - %s', evt)


    def __loadTasksNames(self, tasks):
{'fraction_non_alphanumeric': 0.07945238968341278, 'fraction_numerical': 0.003667033370003667, 'mean_word_length': 2.42056856187291, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5282554', 'n_tokens_mistral': 2221, 'n_tokens_neox': 2140, 'n_words': 706}