<commit_before>''' Base Extractor class and associated functionality. '''

from abc import ABCMeta, abstractmethod
from six import with_metaclass
import pandas as pd
import numpy as np
from pliers.transformers import Transformer
from pliers.utils import isgenerator, flatten
from pandas.api.types import is_numeric_dtype


class Extractor(with_metaclass(ABCMeta, Transformer)):

    ''' Base class for all pliers Extractors.'''

    def transform(self, stim, *args, **kwargs):
        result = super(Extractor, self).transform(stim, *args, **kwargs)
        return list(result) if isgenerator(result) else result

    @abstractmethod
    def _extract(self, stim):
        pass

    def _transform(self, stim, *args, **kwargs):
        return self._extract(stim, *args, **kwargs)

    def plot(self, result, stim=None):
        raise NotImplementedError("No plotting method is defined for class "
                                  "%s." % self.__class__.__name__)


class ExtractorResult(object):

    ''' Stores feature data produced by an Extractor.

    Args:
        data (ndarray, iterable): Extracted feature data. Either an ndarray
            (1-d or 2-d), an iterable, or a raw result. If a raw result is
            passed, the source Extractor must implement _to_df().
        stim (Stim): The input Stim object from which features were extracted.
        extractor (Extractor): The Extractor object used in extraction.
        features (list, ndarray): Optional names of extracted features. If
            passed, must have as many elements as there are columns in data.
        onsets (list, ndarray): Optional iterable giving the onsets of the
            rows in data. Length must match the input data.
        durations (list, ndarray): Optional iterable giving the durations
            associated with the rows in data.
        orders (list, ndarray): Optional iterable giving the integer orders
            associated with the rows in data.
        raw: The raw result (net of any containers or overhead) returned by
            the underlying feature extraction tool. Can be an object of any
            type.
    '''

    def __init__(self, data, stim, extractor, features=None, onsets=None,
                 durations=None, orders=None):
        self._data = data
        self.stim = stim
        self.extractor = extractor
        self.features = features
        self._history = None
        self.onset = onsets
        self.duration = durations
        self.order = orders

    @property
    def raw(self):
        ''' Stores raw result of extraction, prior to postprocessing done
        in to_df(). '''
        return self._data if hasattr(self.extractor, '_to_df') else None

    @property
    def data(self):
        ''' Creates a DataFrame with default arguments '''
        return self.to_df()

    def to_df(self, timing=True, metadata=False, format='wide',
              extractor_name=False, object_id=True, **to_df_kwargs):
        ''' Convert current instance to a pandas DatasFrame.

        Args:
            timing (bool): If True, adds columns for event onset and duration.
                Note that these columns will be added even if there are no
                valid values in the current object (NaNs will be inserted).
                If 'auto', timing columns are only inserted if there's at least
                one valid (i.e., non-NaN) onset/order/duration.
            metadata (bool): If True, adds columns for key metadata (including
                the name, filename, class, history, and source file of the
                Stim).
            format (str): Format to return the data in. Can be either 'wide' or
                'long'. In the wide case, every extracted feature is a column,
                and every result object is in a row. In the long case, every
                row contains a single record/feature combination.
            extractor_name (bool): If True, includes the Extractor name as
                a column (in 'long' format) or index level (in 'wide' format).
            object_id (bool): If True, attempts to intelligently add an
                'object_id' column that differentiates between multiple objects
                in the results that may share onsets and durations (and would
                otherwise be impossible to distinguish). This frequently occurs
                for ImageExtractors that identify multiple target objects
                (e.g., faces) within a single ImageStim. In addition to boolean
                values, the special value 'auto' can be passed, in which case
                the object_id column will only be inserted if the resulting
                constant would be non-constant.

        Returns:
            A pandas DataFrame.
        '''

        # Ideally, Extractors should implement their own _to_df() class method
        # that produces a DataFrame in standardized format. Failing that, we
        # assume self._data is already array-like and can be wrapped in a DF.

        if hasattr(self.extractor, '_to_df'):
            df = self.extractor._to_df(self, **to_df_kwargs)
        else:
            features = self.features
            data = np.array(self._data)
            if features is None:
                features = ['feature_%d' % (i + 1)
                            for i in range(data.shape[1])]
            df = pd.DataFrame(data, columns=features)

        index_cols = list(set(df.columns) - set(self.features))

        if hasattr(self, '_onsets'):
            onsets = np.array(self._onsets)
            onsets += 0.0 if self.onset is None else self.onset
        else:
            onsets = np.nan if self.onset is None else self.onset
        durations = getattr(self, '_durations', self.duration)
        durations = np.nan if durations is None else durations
        orders = getattr(self, '_orders', self.order)
        orders = np.nan if orders is None else orders

        # If any features clash with protected keys, append underscore
        protected = ['onset', 'order', 'duration', 'extractor', 'stim_name',
                     'class', 'filename', 'history', 'source_file']
        df = df.rename(columns={k: k + '_' for k in protected})

        # Generally we leave it to Extractors to properly track the number of
        # objects returned in the result DF, using the 'object_id' column.
        # But in cases where the Extractor punt on this and_object_id=True, we
        # take our best guess. The logic is that we increment the object
        # counter for any row in the DF that cannot be uniquely distinguished
        # from other rows by onset and duration.
        if object_id:
            if 'object_id' not in df.columns:<commit_msg>Use features not self.features<commit_after>''' Base Extractor class and associated functionality. '''

from abc import ABCMeta, abstractmethod
from six import with_metaclass
import pandas as pd
import numpy as np
from pliers.transformers import Transformer
from pliers.utils import isgenerator, flatten
from pandas.api.types import is_numeric_dtype


class Extractor(with_metaclass(ABCMeta, Transformer)):

    ''' Base class for all pliers Extractors.'''

    def transform(self, stim, *args, **kwargs):
        result = super(Extractor, self).transform(stim, *args, **kwargs)
        return list(result) if isgenerator(result) else result

    @abstractmethod
    def _extract(self, stim):
        pass

    def _transform(self, stim, *args, **kwargs):
        return self._extract(stim, *args, **kwargs)

    def plot(self, result, stim=None):
        raise NotImplementedError("No plotting method is defined for class "
                                  "%s." % self.__class__.__name__)


class ExtractorResult(object):

    ''' Stores feature data produced by an Extractor.

    Args:
        data (ndarray, iterable): Extracted feature data. Either an ndarray
            (1-d or 2-d), an iterable, or a raw result. If a raw result is
            passed, the source Extractor must implement _to_df().
        stim (Stim): The input Stim object from which features were extracted.
        extractor (Extractor): The Extractor object used in extraction.
        features (list, ndarray): Optional names of extracted features. If
            passed, must have as many elements as there are columns in data.
        onsets (list, ndarray): Optional iterable giving the onsets of the
            rows in data. Length must match the input data.
        durations (list, ndarray): Optional iterable giving the durations
            associated with the rows in data.
        orders (list, ndarray): Optional iterable giving the integer orders
            associated with the rows in data.
        raw: The raw result (net of any containers or overhead) returned by
            the underlying feature extraction tool. Can be an object of any
            type.
    '''

    def __init__(self, data, stim, extractor, features=None, onsets=None,
                 durations=None, orders=None):
        self._data = data
        self.stim = stim
        self.extractor = extractor
        self.features = features
        self._history = None
        self.onset = onsets
        self.duration = durations
        self.order = orders

    @property
    def raw(self):
        ''' Stores raw result of extraction, prior to postprocessing done
        in to_df(). '''
        return self._data if hasattr(self.extractor, '_to_df') else None

    @property
    def data(self):
        ''' Creates a DataFrame with default arguments '''
        return self.to_df()

    def to_df(self, timing=True, metadata=False, format='wide',
              extractor_name=False, object_id=True, **to_df_kwargs):
        ''' Convert current instance to a pandas DatasFrame.

        Args:
            timing (bool): If True, adds columns for event onset and duration.
                Note that these columns will be added even if there are no
                valid values in the current object (NaNs will be inserted).
                If 'auto', timing columns are only inserted if there's at least
                one valid (i.e., non-NaN) onset/order/duration.
            metadata (bool): If True, adds columns for key metadata (including
                the name, filename, class, history, and source file of the
                Stim).
            format (str): Format to return the data in. Can be either 'wide' or
                'long'. In the wide case, every extracted feature is a column,
                and every result object is in a row. In the long case, every
                row contains a single record/feature combination.
            extractor_name (bool): If True, includes the Extractor name as
                a column (in 'long' format) or index level (in 'wide' format).
            object_id (bool): If True, attempts to intelligently add an
                'object_id' column that differentiates between multiple objects
                in the results that may share onsets and durations (and would
                otherwise be impossible to distinguish). This frequently occurs
                for ImageExtractors that identify multiple target objects
                (e.g., faces) within a single ImageStim. In addition to boolean
                values, the special value 'auto' can be passed, in which case
                the object_id column will only be inserted if the resulting
                constant would be non-constant.

        Returns:
            A pandas DataFrame.
        '''

        # Ideally, Extractors should implement their own _to_df() class method
        # that produces a DataFrame in standardized format. Failing that, we
        # assume self._data is already array-like and can be wrapped in a DF.

        if hasattr(self.extractor, '_to_df'):
            df = self.extractor._to_df(self, **to_df_kwargs)
            features = self.features
        else:
            features = self.features
            data = np.array(self._data)
            if features is None:
                features = ['feature_%d' % (i + 1)
                            for i in range(data.shape[1])]
            df = pd.DataFrame(data, columns=features)

        if features is not None:
            index_cols = list(set(df.columns) - set(features))
        else:
            index_cols = list(set(df.columns))

        if hasattr(self, '_onsets'):
            onsets = np.array(self._onsets)
            onsets += 0.0 if self.onset is None else self.onset
        else:
            onsets = np.nan if self.onset is None else self.onset
        durations = getattr(self, '_durations', self.duration)
        durations = np.nan if durations is None else durations
        orders = getattr(self, '_orders', self.order)
        orders = np.nan if orders is None else orders

        # If any features clash with protected keys, append underscore
        protected = ['onset', 'order', 'duration', 'extractor', 'stim_name',
                     'class', 'filename', 'history', 'source_file']
        df = df.rename(columns={k: k + '_' for k in protected})

        # Generally we leave it to Extractors to properly track the number of
        # objects returned in the result DF, using the 'object_id' column.
        # But in cases where the Extractor punt on this and_object_id=True, we
        # take our best guess. The logic is that we increment the object
        # counter for any row in the DF that cannot be uniquely distinguished
        # from other rows by onset and duration.
        if object_id:
            if 'object_id' not in df.columns:
{'fraction_non_alphanumeric': 0.07391592431716115, 'fraction_numerical': 0.0008834572627549142, 'mean_word_length': 2.2327463112803425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '741123', 'n_tokens_mistral': 3509, 'n_tokens_neox': 3328, 'n_words': 1478}