<commit_before># SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2020, Arm Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from collections.abc import Mapping
from collections import defaultdict
import inspect
import os
import abc
import contextlib
import sqlite3
import pathlib

import pandas as pd

from wa import discover_wa_outputs, Status

from lisa.version import VERSION_TOKEN
from lisa.stats import Stats
from lisa.utils import Loggable, memoized, get_subclasses
from lisa.git import find_shortest_symref, get_commit_message
from lisa.trace import Trace

def _df_concat(dfs):
    return pd.concat(dfs, ignore_index=True, copy=False, sort=False)


class WAOutputNotFoundError(Exception):
    def __init__(self, collectors):
        # pylint: disable=super-init-not-called
        self.collectors = collectors

    def __str__(self):
        sep = '\n    '
        return 'Could not find output for collectors{}{}'.format(
            ':' + sep if len(self.collectors) > 1 else ' ',
            sep.join(
                f'{collector.NAME} ({excep.__class__.__qualname__}): {excep}'
                for collector, excep in self.collectors.items()
            )
        )

    @classmethod
    def from_collector(cls, collector, excep):
        return cls({collector: excep})

    @classmethod
    def from_excep_list(cls, exceps):
        return cls(
            collectors={
                collector: excep
                for excep in exceps
                for collector, excep in excep.collectors.items()
            }
        )


class StatsProp:
    """
    Provides a ``stats`` property.
    """
    _STATS_GROUPS = ['board', 'kernel']
    """
    Tag columns commonly used to group plots of WA dataframes.
    """

    _AGG_COLS = ['iteration', 'wa_path']
    """
    Columns that are guaranteed to be found in the dataframes and will always
    be used as aggregation columns, in addition to what the user selects.
    """

    def get_stats(self, ensure_default_groups=True, ref_group=None, agg_cols=None, **kwargs):
        """
        Returns a :class:`lisa.stats.Stats` loaded with the result
        :class:`pandas.DataFrame`.

        :param ensure_default_groups: If ``True``, ensure `ref_group` will contain
            appropriate keys for usual Workload Automation result display.
        :type ensure_default_groups: bool

        :param ref_group: Forwarded to :class:`lisa.stats.Stats`

        :Variable keyword arguments: Forwarded to :class:`lisa.stats.Stats`

        """
        if ensure_default_groups:
            # Make sure the tags to group on are present in the ref_group, even
            # if the user did not specify them
            ref_group = {
                **{
                    k: None
                    for k in self._STATS_GROUPS
                },
                **(ref_group or {}),
            }

        agg_cols = (agg_cols or []) + self._AGG_COLS

        return Stats(
            self.df,
            ref_group=ref_group,
            agg_cols=agg_cols,
            **kwargs
        )<commit_msg>lisa.wa: Avoid passing non-existing column names to Stats()<commit_after># SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2020, Arm Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from collections.abc import Mapping
from collections import defaultdict
import inspect
import os
import abc
import contextlib
import sqlite3
import pathlib

import pandas as pd

from wa import discover_wa_outputs, Status

from lisa.version import VERSION_TOKEN
from lisa.stats import Stats
from lisa.utils import Loggable, memoized, get_subclasses
from lisa.git import find_shortest_symref, get_commit_message
from lisa.trace import Trace

def _df_concat(dfs):
    return pd.concat(dfs, ignore_index=True, copy=False, sort=False)


class WAOutputNotFoundError(Exception):
    def __init__(self, collectors):
        # pylint: disable=super-init-not-called
        self.collectors = collectors

    def __str__(self):
        sep = '\n    '
        return 'Could not find output for collectors{}{}'.format(
            ':' + sep if len(self.collectors) > 1 else ' ',
            sep.join(
                f'{collector.NAME} ({excep.__class__.__qualname__}): {excep}'
                for collector, excep in self.collectors.items()
            )
        )

    @classmethod
    def from_collector(cls, collector, excep):
        return cls({collector: excep})

    @classmethod
    def from_excep_list(cls, exceps):
        return cls(
            collectors={
                collector: excep
                for excep in exceps
                for collector, excep in excep.collectors.items()
            }
        )


class StatsProp:
    """
    Provides a ``stats`` property.
    """
    _STATS_GROUPS = ['board', 'kernel']
    """
    Tag columns commonly used to group plots of WA dataframes.
    """

    _AGG_COLS = ['iteration', 'wa_path']
    """
    Columns that are guaranteed to be found in the dataframes and will always
    be used as aggregation columns, in addition to what the user selects.
    """

    def get_stats(self, ensure_default_groups=True, ref_group=None, agg_cols=None, **kwargs):
        """
        Returns a :class:`lisa.stats.Stats` loaded with the result
        :class:`pandas.DataFrame`.

        :param ensure_default_groups: If ``True``, ensure `ref_group` will contain
            appropriate keys for usual Workload Automation result display.
        :type ensure_default_groups: bool

        :param ref_group: Forwarded to :class:`lisa.stats.Stats`

        :Variable keyword arguments: Forwarded to :class:`lisa.stats.Stats`

        """
        def filter_cols(cols):
            return [
                col
                for col in cols
                if col in df.columns
            ]
        df = self.df

        if ensure_default_groups:
            # Make sure the tags to group on are present in the ref_group, even
            # if the user did not specify them
            ref_group = {
                **dict.fromkeys(filter_cols(self._STATS_GROUPS)),
                **(ref_group or {}),
            }

        agg_cols = (agg_cols or []) + self._AGG_COLS
        agg_cols = filter_cols(agg_cols)

        return Stats(
            df,
            ref_group=ref_group,
            agg_cols=agg_cols,
            **kwargs
        )
{'fraction_non_alphanumeric': 0.08457374830852503, 'fraction_numerical': 0.003247631935047361, 'mean_word_length': 2.4440820130475305, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7263378', 'n_tokens_mistral': 2121, 'n_tokens_neox': 2028, 'n_words': 696}