<commit_before>"""Serialize

Serialization is essentially the process of converting a model into some other
format.  The current formats supported are a Python dict and a JSON string. One
could use Python's cPickle module or a msgpack too.

The logic includes conversions from Schematics representation into formats
suitable for some serialization method.  For example, Python datetime's will
blow up json.dumps so we first convert the date into ISO8601 format.

Serializing a model instance then looks like this:

    a_dict = to_python(instance)

There is additional logic that allows user's to remove fields from
serialization based on the concept of configured roles.  The idea here is that
it's typical to not want to share every field for some reason.  A whilelist
and blacklist mechanism is provided for those fields.  The actual logic only
requires a function so the behavior of the system is fully customizable.

A class with role access specified looks like this:

    class SomeModel(Model):
        field_a = ...

        class Options:
            roles = {
                'owner': blacklist('field_a'),
                'public': whitelist('field_b', 'field_c'),
            }

Serializing data against a role then looks like this:

    a_dict = make_safe_python(SomeModel, my_data, 'public')

I can convert the data to JSON just as easily too.

    a_str = make_safe_json(SomeModel, my_data, 'public')

In addition to these mechanisms existing, they are fully recursive and will
expand models if they are used as fields.  They will also apply a role
recursively which allows users to embed models with unique role systems inside
container models.

That looks like this in code:

    class SomeModel(Model):
        name = StringType()
        email = EmailType()
        is_active = BooleanType()
        class Options:
            roles = {
                'owner': blacklist('is_active'),
                'public': whitelist('username', 'name'),
            }
    
    class BlogPost(Model):
        title = StringType()    
        content = StringType()
        author = ModelType(Author)
        post_date = DateTimeType(default=datetime.datetime.now)
        comments = ListType(ModelType(Comment))
        deleted = BooleanType()   
        class Options:
            roles = {
                'owner': whotelist(),
                'public': whitelist('author', 'content', 'comments'),
            }        
"""

import copy
import collections

from schematics.types import schematic_types
from schematics.base import json
from schematics.models import BaseModel, Model


###
### Serialization Shaping Functions
###

def _reduce_loop(model, instance_or_dict, field_converter):
    """Each field's name, the field instance and the field's value are
    collected in a truple and yielded, making this a generator.
    """
    for field_name in instance_or_dict:
        field_instance = model._fields[field_name]
        field_value = instance_or_dict[field_name]
        yield (field_name, field_instance, field_value)


def apply_shape(model, instance_or_dict, field_converter, model_converter,
                gottago, allow_none=False, convert_none=False):<commit_msg>Removed unnecessary import of BaseModel, which was removed<commit_after>"""Serialize

Serialization is essentially the process of converting a model into some other
format.  The current formats supported are a Python dict and a JSON string. One
could use Python's cPickle module or a msgpack too.

The logic includes conversions from Schematics representation into formats
suitable for some serialization method.  For example, Python datetime's will
blow up json.dumps so we first convert the date into ISO8601 format.

Serializing a model instance then looks like this:

    a_dict = to_python(instance)

There is additional logic that allows user's to remove fields from
serialization based on the concept of configured roles.  The idea here is that
it's typical to not want to share every field for some reason.  A whilelist
and blacklist mechanism is provided for those fields.  The actual logic only
requires a function so the behavior of the system is fully customizable.

A class with role access specified looks like this:

    class SomeModel(Model):
        field_a = ...

        class Options:
            roles = {
                'owner': blacklist('field_a'),
                'public': whitelist('field_b', 'field_c'),
            }

Serializing data against a role then looks like this:

    a_dict = make_safe_python(SomeModel, my_data, 'public')

I can convert the data to JSON just as easily too.

    a_str = make_safe_json(SomeModel, my_data, 'public')

In addition to these mechanisms existing, they are fully recursive and will
expand models if they are used as fields.  They will also apply a role
recursively which allows users to embed models with unique role systems inside
container models.

That looks like this in code:

    class SomeModel(Model):
        name = StringType()
        email = EmailType()
        is_active = BooleanType()
        class Options:
            roles = {
                'owner': blacklist('is_active'),
                'public': whitelist('username', 'name'),
            }
    
    class BlogPost(Model):
        title = StringType()    
        content = StringType()
        author = ModelType(Author)
        post_date = DateTimeType(default=datetime.datetime.now)
        comments = ListType(ModelType(Comment))
        deleted = BooleanType()   
        class Options:
            roles = {
                'owner': whotelist(),
                'public': whitelist('author', 'content', 'comments'),
            }        
"""

import copy
import collections

from schematics.types import schematic_types
from schematics.base import json
from schematics.models import Model


###
### Serialization Shaping Functions
###

def _reduce_loop(model, instance_or_dict, field_converter):
    """Each field's name, the field instance and the field's value are
    collected in a truple and yielded, making this a generator.
    """
    for field_name in instance_or_dict:
        field_instance = model._fields[field_name]
        field_value = instance_or_dict[field_name]
        yield (field_name, field_instance, field_value)


def apply_shape(model, instance_or_dict, field_converter, model_converter,
                gottago, allow_none=False, convert_none=False):
{'fraction_non_alphanumeric': 0.06731068868806482, 'fraction_numerical': 0.0012464942349641633, 'mean_word_length': 3.0498422712933753, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7284787', 'n_tokens_mistral': 1684, 'n_tokens_neox': 1617, 'n_words': 679}