<commit_before># SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2018, Arm Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

""" Scheduler load tracking analysis module """

import operator
import itertools
import pandas as pd

from lisa.analysis.base import TraceAnalysisBase
from lisa.analysis.status import StatusAnalysis
from lisa.trace import requires_one_event_of, may_use_events, TaskID, CPU, MissingTraceEventError, OrTraceEventChecker
from lisa.utils import deprecate
from lisa.datautils import df_refit_index, series_refit_index, df_filter_task_ids, df_split_signals
from lisa.generic import TypedList


class LoadTrackingAnalysis(TraceAnalysisBase):
    """
    Support for scheduler load tracking analysis

    :param trace: input Trace object
    :type trace: lisa.trace.Trace
    """

    name = 'load_tracking'

    _SCHED_PELT_SE_NAMES = [
        'sched_pelt_se',
        'sched_load_se',
        'sched_load_avg_task'
    ]
    """
    All the names that the per-task load tracking event ever had in various
    kernel versions (Android, mainline etc)
    """

    _SCHED_PELT_CFS_NAMES = [
        'sched_pelt_cfs',
        'sched_load_cfs_rq',
        'sched_load_avg_cpu',
    ]
    """
    All the names that the per-CPU load tracking event ever had in various
    kernel versions (Android, mainline etc)
    """

    @classmethod
    def _columns_renaming(cls, event):
        """
        Columns to rename to unify dataframes between trace event versions
        """
        if event in ['sched_load_avg_cpu', 'sched_load_avg_task']:
            return {
                'load_avg': 'load',
                'util_avg': 'util',
                "utilization": 'util',
                'avg_period': 'period_contrib',
                'runnable_avg_sum': 'load_sum',
                'running_avg_sum': 'util_sum',
            }
        elif event == 'cpu_capacity':
            return {
                'cpu_id': 'cpu',
                'state': 'capacity',
            }
        else:
            return {}

    @classmethod
    def _columns_to_drop(cls, event):
        """
        The extra columns not shared between trace event versions
        """
        if event in [*cls._SCHED_PELT_CFS_NAMES, 'sched_load_se', 'sched_pelt_se']:
            return ['path', 'rbl_load', 'runnable']

        if event in ['sched_load_avg_task']:
            return ['load_sum', 'period_contrib', 'util_sum']

        return []

    def _df_uniformized_signal(self, event):
        df = self.trace.df_event(event)

        df = df.rename(columns=self._columns_renaming(event), copy=True)

        # Legacy sched_load_avg_* events don't have a `path` field.
        if not event.startswith('sched_load_avg_'):
            if event in self._SCHED_PELT_SE_NAMES:
                df = df[df.path == "(null)"]

            if event in self._SCHED_PELT_CFS_NAMES:
                df = df[df.path == "/"]

        to_drop = self._columns_to_drop(event)
        df.drop(columns=to_drop, inplace=True, errors='ignore')

        return df

    def _df_either_event(self, events):
        missing = []
        for event in events:
            try:
                return self._df_uniformized_signal(event)
            except MissingTraceEventError as e:
                missing.append(e.missing_events)

        missing = OrTraceEventChecker.from_events(missing)
        raise MissingTraceEventError(missing, self.trace.available_events)

    @may_use_events(
        requires_one_event_of(*_SCHED_PELT_CFS_NAMES),
        'sched_util_est_cfs',
        'sched_cpu_capacity',
    )
    def df_cpus_signal(self, signal, cpus: TypedList[CPU]=None):
        """
        Get the load-tracking signals for the CPUs

        :returns: a :class:`pandas.DataFrame` with a column of the same name as
            the specified ``signal``, and additional context columns such as
            ``cpu``.

        :param signal: Signal name to get. Can be any of:

            * ``util``
            * ``load``
            * ``enqueued`` (util est enqueued)
            * ``capacity``

        :type signal: str

        :param cpus: If specified, list of CPUs to select.
        :type cpus: list(lisa.trace.CPU) or None
        """

        if signal in ('util', 'load'):
            df = self._df_either_event(self._SCHED_PELT_CFS_NAMES)
        elif signal == 'enqueued':
            df = self._df_uniformized_signal('sched_util_est_cfs')
        elif signal == 'capacity':
            df = self._df_uniformized_signal('sched_cpu_capacity')
        else:
            raise ValueError(f'Signal "{signal}" not supported')

        df = df[['cpu', signal]]
        if cpus is not None:
            df = df[df['cpu'].isin(cpus)]
        return df

    @deprecate(replaced_by=df_cpus_signal, deprecated_in='2.0', removed_in='3.0')
    @requires_one_event_of(*_SCHED_PELT_CFS_NAMES)
    def df_cpus_signals(self):
        """
        Get the load-tracking signals for the CPUs
<commit_msg>lisa.analysis.load_tracking: Add `update_time` column for signal `util` & `load` in df_cpus_signal()<commit_after># SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2018, Arm Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

""" Scheduler load tracking analysis module """

import operator
import itertools
import pandas as pd

from lisa.analysis.base import TraceAnalysisBase
from lisa.analysis.status import StatusAnalysis
from lisa.trace import requires_one_event_of, may_use_events, TaskID, CPU, MissingTraceEventError, OrTraceEventChecker
from lisa.utils import deprecate
from lisa.datautils import df_refit_index, series_refit_index, df_filter_task_ids, df_split_signals
from lisa.generic import TypedList


class LoadTrackingAnalysis(TraceAnalysisBase):
    """
    Support for scheduler load tracking analysis

    :param trace: input Trace object
    :type trace: lisa.trace.Trace
    """

    name = 'load_tracking'

    _SCHED_PELT_SE_NAMES = [
        'sched_pelt_se',
        'sched_load_se',
        'sched_load_avg_task'
    ]
    """
    All the names that the per-task load tracking event ever had in various
    kernel versions (Android, mainline etc)
    """

    _SCHED_PELT_CFS_NAMES = [
        'sched_pelt_cfs',
        'sched_load_cfs_rq',
        'sched_load_avg_cpu',
    ]
    """
    All the names that the per-CPU load tracking event ever had in various
    kernel versions (Android, mainline etc)
    """

    @classmethod
    def _columns_renaming(cls, event):
        """
        Columns to rename to unify dataframes between trace event versions
        """
        if event in ['sched_load_avg_cpu', 'sched_load_avg_task']:
            return {
                'load_avg': 'load',
                'util_avg': 'util',
                "utilization": 'util',
                'avg_period': 'period_contrib',
                'runnable_avg_sum': 'load_sum',
                'running_avg_sum': 'util_sum',
            }
        elif event == 'cpu_capacity':
            return {
                'cpu_id': 'cpu',
                'state': 'capacity',
            }
        else:
            return {}

    @classmethod
    def _columns_to_drop(cls, event):
        """
        The extra columns not shared between trace event versions
        """
        if event in [*cls._SCHED_PELT_CFS_NAMES, 'sched_load_se', 'sched_pelt_se']:
            return ['path', 'rbl_load', 'runnable']

        if event in ['sched_load_avg_task']:
            return ['load_sum', 'period_contrib', 'util_sum']

        return []

    def _df_uniformized_signal(self, event):
        df = self.trace.df_event(event)

        df = df.rename(columns=self._columns_renaming(event), copy=True)

        # Legacy sched_load_avg_* events don't have a `path` field.
        if not event.startswith('sched_load_avg_'):
            if event in self._SCHED_PELT_SE_NAMES:
                df = df[df.path == "(null)"]

            if event in self._SCHED_PELT_CFS_NAMES:
                df = df[df.path == "/"]

        to_drop = self._columns_to_drop(event)
        df.drop(columns=to_drop, inplace=True, errors='ignore')

        return df

    def _df_either_event(self, events):
        missing = []
        for event in events:
            try:
                return self._df_uniformized_signal(event)
            except MissingTraceEventError as e:
                missing.append(e.missing_events)

        missing = OrTraceEventChecker.from_events(missing)
        raise MissingTraceEventError(missing, self.trace.available_events)

    @may_use_events(
        requires_one_event_of(*_SCHED_PELT_CFS_NAMES),
        'sched_util_est_cfs',
        'sched_cpu_capacity',
    )
    def df_cpus_signal(self, signal, cpus: TypedList[CPU]=None):
        """
        Get the load-tracking signals for the CPUs

        :returns: a :class:`pandas.DataFrame` with a column of the same name as
            the specified ``signal``, and additional context columns such as
            ``cpu``.

        :param signal: Signal name to get. Can be any of:

            * ``util``
            * ``load``
            * ``enqueued`` (util est enqueued)
            * ``capacity``

        :type signal: str

        :param cpus: If specified, list of CPUs to select.
        :type cpus: list(lisa.trace.CPU) or None
        """

        if signal in ('util', 'load'):
            df = self._df_either_event(self._SCHED_PELT_CFS_NAMES)
        elif signal == 'enqueued':
            df = self._df_uniformized_signal('sched_util_est_cfs')
        elif signal == 'capacity':
            df = self._df_uniformized_signal('sched_cpu_capacity')
        else:
            raise ValueError(f'Signal "{signal}" not supported')

        if signal in ('util', 'load'):
            columns = {'cpu', 'update_time', signal}
        else:
            columns = {'cpu', signal}

        # Select the available columns among
        columns = sorted(set(df.columns) & columns)
        df = df[columns]

        if cpus is not None:
            df = df[df['cpu'].isin(cpus)]
        return df

    @deprecate(replaced_by=df_cpus_signal, deprecated_in='2.0', removed_in='3.0')
    @requires_one_event_of(*_SCHED_PELT_CFS_NAMES)
    def df_cpus_signals(self):
        """
        Get the load-tracking signals for the CPUs

{'fraction_non_alphanumeric': 0.09702812665841146, 'fraction_numerical': 0.0024765611179904477, 'mean_word_length': 2.4758684291423303, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4867564', 'n_tokens_mistral': 3434, 'n_tokens_neox': 3312, 'n_words': 925}