<commit_before>import collections.abc

import declare
from .condition import (
    AttributeExists,
    BeginsWith,
    Between,
    Comparison,
    Contains,
    In
)
from .exceptions import InvalidIndex, InvalidModel
from .tracking import mark
from .util import missing, signal

__all__ = ["BaseModel", "Column", "GlobalSecondaryIndex", "LocalSecondaryIndex", "Index", "model_created"]

# Signals!
model_created = signal("model_created")


INVALID_PROJECTION = ValueError(
    "Index projections must be either 'keys', 'all', or an iterable of model attributes to include.")


def loaded_columns(obj):
    """Yields each (model_name, value) tuple for all columns in an object that aren't missing"""
    for column in sorted(obj.Meta.columns, key=lambda c: c.model_name):
        value = getattr(obj, column.model_name, missing)
        if value is not missing:
            yield (column.model_name, value)


def validate_projection(projection):
    # String check first since it is also an Iterable
    if isinstance(projection, str):
        projection = projection.upper()
        if projection not in ["KEYS", "ALL"]:
            raise INVALID_PROJECTION
    elif isinstance(projection, collections.abc.Iterable):
        projection = list(projection)
        for attribute in projection:
            if not isinstance(attribute, str):
                raise INVALID_PROJECTION
    else:
        raise INVALID_PROJECTION
    return projection


class ModelMetaclass(declare.ModelMetaclass):
    def __new__(mcs, name, bases, attrs):
        model = super().__new__(mcs, name, bases, attrs)
        meta = model.Meta
        meta.model = model
        # new_class will set abstract to true, all other models are assumed
        # to be concrete unless specified
        setdefault(meta, "abstract", False)<commit_msg>more InvalidIndex in models.py #60<commit_after>import collections.abc

import declare
from .condition import (
    AttributeExists,
    BeginsWith,
    Between,
    Comparison,
    Contains,
    In
)
from .exceptions import InvalidIndex, InvalidModel
from .tracking import mark
from .util import missing, signal

__all__ = ["BaseModel", "Column", "GlobalSecondaryIndex", "LocalSecondaryIndex", "Index", "model_created"]

# Signals!
model_created = signal("model_created")


def loaded_columns(obj):
    """Yields each (model_name, value) tuple for all columns in an object that aren't missing"""
    for column in sorted(obj.Meta.columns, key=lambda c: c.model_name):
        value = getattr(obj, column.model_name, missing)
        if value is not missing:
            yield (column.model_name, value)


def validate_projection(projection):
    # String check first since it is also an Iterable
    if isinstance(projection, str):
        projection = projection.upper()
        if projection not in ["KEYS", "ALL"]:
            raise InvalidIndex("{!r} is not a valid Index projection.".format(projection))
    elif isinstance(projection, collections.abc.Iterable):
        projection = list(projection)
        all_names = all(isinstance(item, str) for item in projection)
        if not all_names:
            raise InvalidIndex(
                "Index projection must be a list of strings when selecting specific Columns.")
    else:
        raise InvalidIndex(
            "Index projection must be 'all', 'keys', or a list of Column names.")
    return projection


class ModelMetaclass(declare.ModelMetaclass):
    def __new__(mcs, name, bases, attrs):
        model = super().__new__(mcs, name, bases, attrs)
        meta = model.Meta
        meta.model = model
        # new_class will set abstract to true, all other models are assumed
        # to be concrete unless specified
        setdefault(meta, "abstract", False)
{'fraction_non_alphanumeric': 0.0881961097788436, 'fraction_numerical': 0.0005329070077271516, 'mean_word_length': 3.208520179372197, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5125479', 'n_tokens_mistral': 1028, 'n_tokens_neox': 1008, 'n_words': 354}