<commit_before>import pandas as pd
from copy import copy
from pandas.api.types import is_numeric_dtype
import warnings
import re
from .variables import (SparseRunVariable, SimpleVariable, DenseRunVariable,
                        merge_variables, BIDSVariable)
from collections import defaultdict
from itertools import chain
from bids.utils import listify, matches_entities


class BIDSVariableCollection(object):

    ''' A container for one or more variables extracted from variable files
    at a single level of analysis.

    Args:
        variables (list): A list of BIDSVariables or SimpleVariables.

    Notes:
        * Variables in the list must all share the same analysis level, which
          must be one of 'session', 'subject', or 'dataset' level. For
          run-level Variables, use the BIDSRunVariableCollection.
    '''

    def __init__(self, variables):

        SOURCE_TO_LEVEL = {
            'events': 'run',
            'physio': 'run',
            'stim': 'run',
            'confounds': 'run',
            'scans': 'session',
            'sessions': 'subject',
            'participants': 'dataset'
        }

        var_levels = set([SOURCE_TO_LEVEL[v.source] for v in variables])

        # TODO: relax this requirement & allow implicit merging between levels
        if len(var_levels) > 1:
            raise ValueError("A Collection cannot be initialized from "
                             "variables at more than one level of analysis. "
                             "Levels found in input variables: %s" %
                             var_levels)

        self.level = list(var_levels)[0]
        variables = self.merge_variables(variables)
        self.variables = {v.name: v for v in variables}
        self._index_entities()

    @staticmethod
    def merge_variables(variables, **kwargs):
        ''' Concatenates Variables along row axis.

        Args:
            variables (list): List of Variables to merge. Variables can have
                different names (and all Variables that share a name will be
                concatenated together).

        Returns:
            A list of Variables.
        '''
        var_dict = defaultdict(list)
        for v in variables:
            var_dict[v.name].append(v)
        return [merge_variables(vars_, **kwargs)
                for vars_ in list(var_dict.values())]

    def to_df(self, variables=None, format='wide', fillna=0, **kwargs):
        ''' Merge variables into a single pandas DataFrame.

        Args:
            variables (list): Optional list of column names to retain; if None,
                all variables are returned.
            format (str): Whether to return a DataFrame in 'wide' or 'long'
                format. In 'wide' format, each row is defined by a unique
                onset/duration, and each variable is in a separate column. In
                'long' format, each row is a unique combination of onset,
                duration, and variable name, and a single 'amplitude' column
                provides the value.
            fillna: Replace missing values with the specified value.
            kwargs: Optional keyword arguments to pass onto each Variable's
                to_df() call (e.g., condition, entities, and timing).

        Returns: A pandas DataFrame.
        '''

        if variables is None:
            variables = list(self.variables.keys())

        # Can receive already-selected Variables from sub-classes
        if not isinstance(variables[0], BIDSVariable):
            variables = [v for v in self.variables.values()
                         if v.name in variables]

        dfs = [v.to_df(**kwargs) for v in variables]
        df = pd.concat(dfs, axis=0)

        if format == 'long':
            return df.reset_index(drop=True).fillna(fillna)

        ind_cols = list(set(df.columns) - {'condition', 'amplitude'})
        df = df.pivot_table(index=ind_cols, columns='condition',
                            values='amplitude', aggfunc='first')
        df = df.reset_index().fillna(fillna)
        df.columns.name = None
        return df

    def clone(self):
        ''' Returns a shallow copy of the current instance, except that all
        variables are deep-cloned.
        '''
        clone = copy(self)
        clone.variables = {k: v.clone() for (k, v) in self.variables.items()}
        return clone

    def matches_entities(self, entities, strict=False):
        ''' Checks whether current Collection's entities match the input. '''
        return matches_entities(self, entities, strict)

    def _index_entities(self):
        ''' Sets current instance's entities based on the existing index.

        Note: Only entity key/value pairs common to all rows in all contained
            Variables are returned. E.g., if a Collection contains Variables
            extracted from runs 1, 2 and 3 from subject '01', the returned dict
            will be {'subject': '01'}; the runs will be excluded as they vary
            across the Collection contents.
        '''
        all_ents = pd.DataFrame.from_records(
            [v.entities for v in self.variables.values()])
        constant = all_ents.apply(lambda x: x.nunique() == 1)
        if constant.empty:
            self.entities = {}
        else:
            keep = all_ents.columns[constant]
            self.entities = {k: all_ents[k].iloc[0] for k in keep}
<commit_msg>add from_df method to construct Collection from DF columns<commit_after>import pandas as pd
from copy import copy
from pandas.api.types import is_numeric_dtype
import warnings
import re
from .variables import (SparseRunVariable, SimpleVariable, DenseRunVariable,
                        merge_variables, BIDSVariable)
from collections import defaultdict
from itertools import chain
from bids.utils import listify, matches_entities


class BIDSVariableCollection(object):

    ''' A container for one or more variables extracted from variable files
    at a single level of analysis.

    Args:
        variables (list): A list of BIDSVariables or SimpleVariables.

    Notes:
        * Variables in the list must all share the same analysis level, which
          must be one of 'session', 'subject', or 'dataset' level. For
          run-level Variables, use the BIDSRunVariableCollection.
    '''

    def __init__(self, variables):

        SOURCE_TO_LEVEL = {
            'events': 'run',
            'physio': 'run',
            'stim': 'run',
            'confounds': 'run',
            'scans': 'session',
            'sessions': 'subject',
            'participants': 'dataset'
        }

        var_levels = set([SOURCE_TO_LEVEL[v.source] for v in variables])

        # TODO: relax this requirement & allow implicit merging between levels
        if len(var_levels) > 1:
            raise ValueError("A Collection cannot be initialized from "
                             "variables at more than one level of analysis. "
                             "Levels found in input variables: %s" %
                             var_levels)

        self.level = list(var_levels)[0]
        variables = self.merge_variables(variables)
        self.variables = {v.name: v for v in variables}
        self._index_entities()

    @staticmethod
    def merge_variables(variables, **kwargs):
        ''' Concatenates Variables along row axis.

        Args:
            variables (list): List of Variables to merge. Variables can have
                different names (and all Variables that share a name will be
                concatenated together).

        Returns:
            A list of Variables.
        '''
        var_dict = defaultdict(list)
        for v in variables:
            var_dict[v.name].append(v)
        return [merge_variables(vars_, **kwargs)
                for vars_ in list(var_dict.values())]

    def to_df(self, variables=None, format='wide', fillna=0, **kwargs):
        ''' Merge variables into a single pandas DataFrame.

        Args:
            variables (list): Optional list of column names to retain; if None,
                all variables are returned.
            format (str): Whether to return a DataFrame in 'wide' or 'long'
                format. In 'wide' format, each row is defined by a unique
                onset/duration, and each variable is in a separate column. In
                'long' format, each row is a unique combination of onset,
                duration, and variable name, and a single 'amplitude' column
                provides the value.
            fillna: Replace missing values with the specified value.
            kwargs: Optional keyword arguments to pass onto each Variable's
                to_df() call (e.g., condition, entities, and timing).

        Returns: A pandas DataFrame.
        '''

        if variables is None:
            variables = list(self.variables.keys())

        # Can receive already-selected Variables from sub-classes
        if not isinstance(variables[0], BIDSVariable):
            variables = [v for v in self.variables.values()
                         if v.name in variables]

        dfs = [v.to_df(**kwargs) for v in variables]
        df = pd.concat(dfs, axis=0)

        if format == 'long':
            return df.reset_index(drop=True).fillna(fillna)

        ind_cols = list(set(df.columns) - {'condition', 'amplitude'})
        df = df.pivot_table(index=ind_cols, columns='condition',
                            values='amplitude', aggfunc='first')
        df = df.reset_index().fillna(fillna)
        df.columns.name = None
        return df

    @classmethod
    def from_df(cls, data, entities=None):
        ''' Create a Collection from a pandas DataFrame.

        Args:
            df (DataFrame): The DataFrame to convert to a Collection. Each
                column will be converted to a SimpleVariable.
            entities (DataFrame): An optional second DataFrame containing
                entity information.

        Returns:
            A BIDSVariableCollection.
        '''
        variables = []
        for col in data.columns:
            _data = pd.DataFrame(data[[col]], columns=['amplitude'])
            if entities is not None:
                _data = pd.concat([_data, entities], axis=1)
            variables.append(SimpleVariable(col, _data, 'contrast'))
        return BIDSVariableCollection(variables)

    def clone(self):
        ''' Returns a shallow copy of the current instance, except that all
        variables are deep-cloned.
        '''
        clone = copy(self)
        clone.variables = {k: v.clone() for (k, v) in self.variables.items()}
        return clone

    def matches_entities(self, entities, strict=False):
        ''' Checks whether current Collection's entities match the input. '''
        return matches_entities(self, entities, strict)

    def _index_entities(self):
        ''' Sets current instance's entities based on the existing index.

        Note: Only entity key/value pairs common to all rows in all contained
            Variables are returned. E.g., if a Collection contains Variables
            extracted from runs 1, 2 and 3 from subject '01', the returned dict
            will be {'subject': '01'}; the runs will be excluded as they vary
            across the Collection contents.
        '''
        all_ents = pd.DataFrame.from_records(
            [v.entities for v in self.variables.values()])
        constant = all_ents.apply(lambda x: x.nunique() == 1)
        if constant.empty:
            self.entities = {}
        else:
            keep = all_ents.columns[constant]
            self.entities = {k: all_ents[k].iloc[0] for k in keep}

{'fraction_non_alphanumeric': 0.08538051881120083, 'fraction_numerical': 0.002490980931111493, 'mean_word_length': 2.0875099443118534, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3686142', 'n_tokens_mistral': 3023, 'n_tokens_neox': 2895, 'n_words': 1096}