<commit_before># -*- coding: utf-8 -*-
import inspect
import functools

import marshmallow as ma
from marshmallow import validate, fields
from sqlalchemy.dialects import postgresql, mysql
import sqlalchemy as sa

from .exceptions import ModelConversionError
from .fields import Related

def _is_field(value):
    return (
        isinstance(value, type) and
        issubclass(value, fields.Field)
    )

def _postgres_array_factory(converter, data_type):
    return functools.partial(
        fields.List,
        converter._get_field_class_for_data_type(data_type.item_type),
    )

class ModelConverter(object):
    """Class that converts a SQLAlchemy model into a dictionary of corresponding
    marshmallow `Fields <marshmallow.fields.Field>`.
    """

    SQLA_TYPE_MAPPING = {
        sa.Enum: fields.Field,

        postgresql.BIT: fields.Integer,
        postgresql.UUID: fields.UUID,
        postgresql.MACADDR: fields.String,
        postgresql.INET: fields.String,
        postgresql.JSON: fields.Raw,
        postgresql.JSONB: fields.Raw,
        postgresql.HSTORE: fields.Raw,
        postgresql.ARRAY: _postgres_array_factory,

        mysql.BIT: fields.Integer,
        mysql.YEAR: fields.Integer,
        mysql.SET: fields.List,
        mysql.ENUM: fields.Field,
    }

    DIRECTION_MAPPING = {
        'MANYTOONE': False,
        'MANYTOMANY': True,
        'ONETOMANY': True,
    }

    def fields_for_model(self, model, include_fk=False, fields=None):
        result = {}
        for prop in model.__mapper__.iterate_properties:
            if fields and prop.key not in fields:
                continue
            if hasattr(prop, 'columns'):
                if not include_fk and prop.columns[0].foreign_keys:
                    continue
            field = self.property2field(prop)
            if field:
                result[prop.key] = field
        return result

    def property2field(self, prop, instance=True, **kwargs):
        field_class = self._get_field_class_for_property(prop)
        if not instance:
            return field_class
        field_kwargs = self._get_field_kwargs_for_property(prop)
        field_kwargs.update(kwargs)
        ret = field_class(**field_kwargs)
        if (
            hasattr(prop, 'direction') and<commit_msg>Do not skip overridden columns that are part of the polymorphic hierarchy, they should be included in the serialization<commit_after># -*- coding: utf-8 -*-
import inspect
import functools

import marshmallow as ma
from marshmallow import validate, fields
from sqlalchemy.dialects import postgresql, mysql
import sqlalchemy as sa

from .exceptions import ModelConversionError
from .fields import Related

def _is_field(value):
    return (
        isinstance(value, type) and
        issubclass(value, fields.Field)
    )

def _postgres_array_factory(converter, data_type):
    return functools.partial(
        fields.List,
        converter._get_field_class_for_data_type(data_type.item_type),
    )

class ModelConverter(object):
    """Class that converts a SQLAlchemy model into a dictionary of corresponding
    marshmallow `Fields <marshmallow.fields.Field>`.
    """

    SQLA_TYPE_MAPPING = {
        sa.Enum: fields.Field,

        postgresql.BIT: fields.Integer,
        postgresql.UUID: fields.UUID,
        postgresql.MACADDR: fields.String,
        postgresql.INET: fields.String,
        postgresql.JSON: fields.Raw,
        postgresql.JSONB: fields.Raw,
        postgresql.HSTORE: fields.Raw,
        postgresql.ARRAY: _postgres_array_factory,

        mysql.BIT: fields.Integer,
        mysql.YEAR: fields.Integer,
        mysql.SET: fields.List,
        mysql.ENUM: fields.Field,
    }

    DIRECTION_MAPPING = {
        'MANYTOONE': False,
        'MANYTOMANY': True,
        'ONETOMANY': True,
    }

    def fields_for_model(self, model, include_fk=False, fields=None):
        result = {}
        for prop in model.__mapper__.iterate_properties:
            if fields and prop.key not in fields:
                continue
            if hasattr(prop, 'columns'):
                if not include_fk:
                    # Only skip a column if there is no overriden column
                    # which does not have a Foreign Key.
                    all_fk = True
                    for column in prop.columns:
                        if not column.foreign_keys:
                            all_fk = False
                            break
                    if all_fk is True:
                        continue
            field = self.property2field(prop)
            if field:
                result[prop.key] = field
        return result

    def property2field(self, prop, instance=True, **kwargs):
        field_class = self._get_field_class_for_property(prop)
        if not instance:
            return field_class
        field_kwargs = self._get_field_kwargs_for_property(prop)
        field_kwargs.update(kwargs)
        ret = field_class(**field_kwargs)
        if (
            hasattr(prop, 'direction') and
{'fraction_non_alphanumeric': 0.07937139446986274, 'fraction_numerical': 0.0013924806047344342, 'mean_word_length': 2.229287090558767, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7037995', 'n_tokens_mistral': 1478, 'n_tokens_neox': 1431, 'n_words': 356}