<commit_before>"""
AggLimbs for convenient Treant data storage and retrieval.

"""
import six
import pandas as pd

from datreant.core.agglimbs import AggTreeLimb


class AggData(AggTreeLimb):
    """Manipulators for collection data.

    """
    _name = 'data'

    def __repr__(self):
        return "<AggData({})>".format(self.keys(mode='all'))

    def _repr_html_(self):
        data = self.keys(mode='all')
        agg = "Data"
        if not data:
            out = "No Data"
        else:
            out = "<h3>{}</h3>".format(agg)
            out = out + "<ul style='list-style-type:none'>"
            for datum in data:
                out = out + "<li>{}</li>".format(datum)
            out = out + "</ul>"
        return out

    def __getitem__(self, handle):
        """Retrieve aggreggated dataset from all members.

        Returns datasets indexed according to member uuids.
        See :meth:`AggData.retrieve` for more information.

        Raises :exc:`KeyError` if dataset doesn't exist for any members.

        :Arguments:
            *handle*
                name of data to retrieve; may also be a list of names

        :Returns:
            *data*
                aggregated data, indexed by member name; if *handle* was a
                list, will be a list of equal length with the aggregated
                datasets as members

        """
        if isinstance(handle, list):
            out = list()
            for item in handle:
                out.append(self.retrieve(item, by='path'))
        elif isinstance(handle, six.string_types):
            out = self.retrieve(handle, by='path')

        return out

    def keys(self, mode='all'):
        """List available datasets.

        :Arguments:
            *mode*
                'any' returns a list of all handles present in at least one
                member; 'all' returns only handles that are present in all
                members

        :Returns:
            *handles*
                list of handles to available datasets

        """
        datasets = [set(member.data) for member in self._collection
                    if hasattr(member, 'data')]
        if mode == 'any':
            out = set.union(*datasets)
        elif mode == 'all':
            out = set.intersection(*datasets)

        out = list(out)
        out.sort()

        return out

    @property
    def any(self):
        return self.keys('any')<commit_msg>AggData now catches cases where there is no data cleanly.<commit_after>"""
AggLimbs for convenient Treant data storage and retrieval.

"""
import six
import pandas as pd

from datreant.core.agglimbs import AggTreeLimb


class AggData(AggTreeLimb):
    """Manipulators for collection data.

    """
    _name = 'data'

    def __repr__(self):
        return "<AggData({})>".format(self.keys(mode='all'))

    def _repr_html_(self):
        data = self.keys(mode='all')
        agg = "Data"
        if not data:
            out = "No Data"
        else:
            out = "<h3>{}</h3>".format(agg)
            out = out + "<ul style='list-style-type:none'>"
            for datum in data:
                out = out + "<li>{}</li>".format(datum)
            out = out + "</ul>"
        return out

    def __getitem__(self, handle):
        """Retrieve aggreggated dataset from all members.

        Returns datasets indexed according to member uuids.
        See :meth:`AggData.retrieve` for more information.

        Raises :exc:`KeyError` if dataset doesn't exist for any members.

        :Arguments:
            *handle*
                name of data to retrieve; may also be a list of names

        :Returns:
            *data*
                aggregated data, indexed by member name; if *handle* was a
                list, will be a list of equal length with the aggregated
                datasets as members

        """
        if isinstance(handle, list):
            out = list()
            for item in handle:
                out.append(self.retrieve(item, by='path'))
        elif isinstance(handle, six.string_types):
            out = self.retrieve(handle, by='path')

        return out

    def keys(self, mode='all'):
        """List available datasets.

        :Arguments:
            *mode*
                'any' returns a list of all handles present in at least one
                member; 'all' returns only handles that are present in all
                members

        :Returns:
            *handles*
                list of handles to available datasets

        """
        datasets = [set(member.data) for member in self._collection
                    if hasattr(member, 'data')]
        if datasets:
            if mode == 'any':
                out = set.union(*datasets)
            elif mode == 'all':
                out = set.intersection(*datasets)

            out = list(out)
            out.sort()
        else:
            out = list()

        return out

    @property
    def any(self):
        return self.keys('any')
{'fraction_non_alphanumeric': 0.1024, 'fraction_numerical': 0.0008, 'mean_word_length': 1.6988667026443605, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7365456', 'n_tokens_mistral': 1399, 'n_tokens_neox': 1316, 'n_words': 451}