<commit_before>
import logging

from django.db.models import ForeignKey, OneToOneField

from .compat.builtins import StringIO
from .compat.serializers import MultiModelInheritanceSerializer


logger = logging.getLogger(__name__)

class RelatedObjectsCollector(object):
    """
    This class is used to introspect an object, to get every other objects that depend on it, following its
    'relation' fields, i.e. ForeignKey, OneToOneField and ManyToManyField.

    1. We start from given object (of class classA for example), and loop over :

    - Its 'direct' fields, it means the relation fields that are explicitly declared in this django model.
    >>> class classA(models.Model):
    >>>     fkey = models.ForeignKey(classB)
    >>>     o2o = models.OneToOneField(classC)
    >>>     m2m = models.ManyToManyField(classD)

    - Its 'related' fields, so other django model that are related to this object by relation fields.
    >>> class classB(models.Model):
    >>>     fkeyto = models.ForeignKey(classA)
    >>>
    >>> class classC(models.Model):
    >>>     o2oto = models.OneToOneField(classA)
    >>>
    >>> class classD(models.Model):
    >>>     m2mto = models.ManyToManyField(classA)

    2. For every field, we get associated object(s) of objA:

    - If it's a direct field, we get objects by:
    >>> class classA(models.Model):
    >>>     fkey = models.ForeignKey(classB)        # objA.fkey
    >>>     o2o = models.OneToOneField(classC)      # objA.o2o
    >>>     m2m = models.ManyToManyField(classD)    # objA.m2m.all()

    - If it's a related field, we get objects by:
    >>> class classB(models.Model):
    >>>     fkeyto = models.ForeignKey(classA)      # objA.classb_set.all()
    >>>
    >>> class classC(models.Model):
    >>>     o2oto = models.OneToOneField(classA)    # objA.classC (not a manager, because OneToOneField is a unique rel)
    >>>
    >>> class classD(models.Model):
    >>>     m2mto = models.ManyToManyField(classA)  # objA.classd_set.all()

    If we are using related_name attribute, then we access manager with its related_name:
    >>> class classE(models.Model):
    >>>     m2mto = models.ForeignKey(classA, related_name='classE')  # objA.classE.all()

    3. For each associated object, we go back to step 1. and get every field, ...

    ------------------------------------------------------------------------------------------------------------------
    HOWTO use:

    Create a new instance of RelatedObjectsCollector, and launch collector on one object:
    >>> from nested_collector.utils import RelatedObjectsCollector
    >>> from django.contrib.auth.models import User
    >>>
    >>> user = User.objects.all()[0]
    >>> collector = RelatedObjectsCollector()
    >>> collector.collect(user)
    >>> related_objects = collector.get_all_related_objects()

    If you want to save it in a file to be 'django load_data'-like imported, you can use:
    >>> string_buffer = collector.get_json_serialized_objects()

    ------------------------------------------------------------------------------------------------------------------
    PARAMETERS:

    You can customize which model/field is collected.
    By default, every model and field is collected, but you can override some parameters to have custom behaviour:

    - EXCLUDE_MODELS: exclude models (expecting a list of '<app_label>.<module_name>')
    >>> EXCLUDE_MODELS = ['sites.site', 'auth.permission', 'contenttypes.contenttype', 'auth.group']
    Every time we will try to collect an object of this model type, it won't be collected.

    - EXCLUDE_DIRECT_FIELDS: exclude direct fields from specified models
    >>> EXCLUDE_DIRECT_FIELDS = {
            'auth.user': ['groups'],
        }
    On User model, when we will get direct fields, we won't take into account 'groups' field.

    - EXCLUDE_RELATED_FIELDS: Exclude related fields from specified models
    >>> EXCLUDE_RELATED_FIELDS = {
            'auth.user': ['session_set']
        }
    On User model, we don't want to collect sessions that are associated to this user, so we put the exact accessor
    name we have to use to get these session, 'session_set', to exclude it from collecting.

    ------------------------------------------------------------------------------------------------------------------
    MISCELLANEOUS:

    To avoid some recursive collect between 2 objects (if an object has a direct field to another one, it means that
    other object has a related field to this first one), we detect if an object has already been collected before
    trying to collect it.

    We are also avoiding by default to collect objects that have the same type as the root one, to prevent collecting
    too many data. This behaviour can be changed with ALLOWS_SAME_TYPE_AS_ROOT_COLLECT parameter.
    """

    # Models that won't be introspected.
    EXCLUDE_MODELS = []

    # Direct fields that won't be introspected
    EXCLUDE_DIRECT_FIELDS = {}

    # Related fields that won't be introspected
    EXCLUDE_RELATED_FIELDS = {}

    # Allow to recursively collect other objects that have same type as root collected object.
    ALLOWS_SAME_TYPE_AS_ROOT_COLLECT = False

    MAXIMUM_RELATED_INSTANCES = 50
    # We are settings related instances maximum size depending on the model
    MAXIMUM_RELATED_INSTANCES_PER_MODEL = {}

    def clean_by_fields(self, obj, fields, get_field_fn, exclude_list):
        """
        Function used to exclude defined fields from object collect.
        :param obj: the object we are collecting
        :param fields: every field related to this object (direct or reverse one)
        :param get_field_fn: function used to get accessor for each field
        :param exclude_list: model/fields we have defined to be excluded from collect
        :return: fields that are allowed to be collected
        """
        cleaned_list = []
        obj_model = get_model_from_instance(obj)

        for field in fields:
            field_accessor = get_field_fn(field)
            # This field is excluded if:
            # 1/ it's parent model key is in exclude list keys
            # AND
            # 2/ the field has been defined as excluded for this parent model
            is_excluded = obj_model in exclude_list and field_accessor in exclude_list[obj_model]

            if not is_excluded:
                cleaned_list.append(field)

        return cleaned_list

    def get_report(self):
        return {
            'excluded_fields': self.excluded_fields,
            'log': self.saved_log,
        }

    def get_collected_objects(self):
        return self.collected_objs.values()

    def get_json_serialized_objects(self):
        objects = self.get_collected_objects()

        string_buffer = StringIO()

        serializer = MultiModelInheritanceSerializer()
        serializer.serialize(
            objects,
            stream=string_buffer,
            indent=2
        )
        string_buffer.seek(0)

        return string_buffer

    def debug_log(self, msg, *args, **kwargs):
        logger.debug(msg, *args, **kwargs)

        if args:
            msg = msg % args
        elif not isinstance(msg, (str, unicode)):
            msg = str(msg)
        self.saved_log.append(msg)

    def _is_already_collected(self, parent, obj):
        new_key = get_key_from_instance(obj)
        parent_key = get_key_from_instance(parent)
        is_already_collected = new_key in self.collected_objs

        if is_already_collected:
            self.debug_log('-' * 100)
            self.debug_log('Collecting >>> {key} <<< (from {parent_key}) : NOK (already collected)'.format(
                key=new_key, parent_key=parent_key))

        return is_already_collected

    def _is_excluded_model(self, obj):
        obj_model = get_model_from_instance(obj)
        obj_key = get_key_from_instance(obj)
        is_excluded_model = obj_model in self.EXCLUDE_MODELS

        if is_excluded_model:
            self.debug_log('-' * 100)
            self.debug_log('Not Collecting {key} (excluded model)'.format(key=obj_key))

        return is_excluded_model
<commit_msg>Python3 does not know unicode. Anyway, basestring is a more proper and general way to deal with that kind of assertion.<commit_after>
import logging

from django.db.models import ForeignKey, OneToOneField

from .compat.builtins import StringIO
from .compat.serializers import MultiModelInheritanceSerializer


logger = logging.getLogger(__name__)

class RelatedObjectsCollector(object):
    """
    This class is used to introspect an object, to get every other objects that depend on it, following its
    'relation' fields, i.e. ForeignKey, OneToOneField and ManyToManyField.

    1. We start from given object (of class classA for example), and loop over :

    - Its 'direct' fields, it means the relation fields that are explicitly declared in this django model.
    >>> class classA(models.Model):
    >>>     fkey = models.ForeignKey(classB)
    >>>     o2o = models.OneToOneField(classC)
    >>>     m2m = models.ManyToManyField(classD)

    - Its 'related' fields, so other django model that are related to this object by relation fields.
    >>> class classB(models.Model):
    >>>     fkeyto = models.ForeignKey(classA)
    >>>
    >>> class classC(models.Model):
    >>>     o2oto = models.OneToOneField(classA)
    >>>
    >>> class classD(models.Model):
    >>>     m2mto = models.ManyToManyField(classA)

    2. For every field, we get associated object(s) of objA:

    - If it's a direct field, we get objects by:
    >>> class classA(models.Model):
    >>>     fkey = models.ForeignKey(classB)        # objA.fkey
    >>>     o2o = models.OneToOneField(classC)      # objA.o2o
    >>>     m2m = models.ManyToManyField(classD)    # objA.m2m.all()

    - If it's a related field, we get objects by:
    >>> class classB(models.Model):
    >>>     fkeyto = models.ForeignKey(classA)      # objA.classb_set.all()
    >>>
    >>> class classC(models.Model):
    >>>     o2oto = models.OneToOneField(classA)    # objA.classC (not a manager, because OneToOneField is a unique rel)
    >>>
    >>> class classD(models.Model):
    >>>     m2mto = models.ManyToManyField(classA)  # objA.classd_set.all()

    If we are using related_name attribute, then we access manager with its related_name:
    >>> class classE(models.Model):
    >>>     m2mto = models.ForeignKey(classA, related_name='classE')  # objA.classE.all()

    3. For each associated object, we go back to step 1. and get every field, ...

    ------------------------------------------------------------------------------------------------------------------
    HOWTO use:

    Create a new instance of RelatedObjectsCollector, and launch collector on one object:
    >>> from nested_collector.utils import RelatedObjectsCollector
    >>> from django.contrib.auth.models import User
    >>>
    >>> user = User.objects.all()[0]
    >>> collector = RelatedObjectsCollector()
    >>> collector.collect(user)
    >>> related_objects = collector.get_all_related_objects()

    If you want to save it in a file to be 'django load_data'-like imported, you can use:
    >>> string_buffer = collector.get_json_serialized_objects()

    ------------------------------------------------------------------------------------------------------------------
    PARAMETERS:

    You can customize which model/field is collected.
    By default, every model and field is collected, but you can override some parameters to have custom behaviour:

    - EXCLUDE_MODELS: exclude models (expecting a list of '<app_label>.<module_name>')
    >>> EXCLUDE_MODELS = ['sites.site', 'auth.permission', 'contenttypes.contenttype', 'auth.group']
    Every time we will try to collect an object of this model type, it won't be collected.

    - EXCLUDE_DIRECT_FIELDS: exclude direct fields from specified models
    >>> EXCLUDE_DIRECT_FIELDS = {
            'auth.user': ['groups'],
        }
    On User model, when we will get direct fields, we won't take into account 'groups' field.

    - EXCLUDE_RELATED_FIELDS: Exclude related fields from specified models
    >>> EXCLUDE_RELATED_FIELDS = {
            'auth.user': ['session_set']
        }
    On User model, we don't want to collect sessions that are associated to this user, so we put the exact accessor
    name we have to use to get these session, 'session_set', to exclude it from collecting.

    ------------------------------------------------------------------------------------------------------------------
    MISCELLANEOUS:

    To avoid some recursive collect between 2 objects (if an object has a direct field to another one, it means that
    other object has a related field to this first one), we detect if an object has already been collected before
    trying to collect it.

    We are also avoiding by default to collect objects that have the same type as the root one, to prevent collecting
    too many data. This behaviour can be changed with ALLOWS_SAME_TYPE_AS_ROOT_COLLECT parameter.
    """

    # Models that won't be introspected.
    EXCLUDE_MODELS = []

    # Direct fields that won't be introspected
    EXCLUDE_DIRECT_FIELDS = {}

    # Related fields that won't be introspected
    EXCLUDE_RELATED_FIELDS = {}

    # Allow to recursively collect other objects that have same type as root collected object.
    ALLOWS_SAME_TYPE_AS_ROOT_COLLECT = False

    MAXIMUM_RELATED_INSTANCES = 50
    # We are settings related instances maximum size depending on the model
    MAXIMUM_RELATED_INSTANCES_PER_MODEL = {}

    def clean_by_fields(self, obj, fields, get_field_fn, exclude_list):
        """
        Function used to exclude defined fields from object collect.
        :param obj: the object we are collecting
        :param fields: every field related to this object (direct or reverse one)
        :param get_field_fn: function used to get accessor for each field
        :param exclude_list: model/fields we have defined to be excluded from collect
        :return: fields that are allowed to be collected
        """
        cleaned_list = []
        obj_model = get_model_from_instance(obj)

        for field in fields:
            field_accessor = get_field_fn(field)
            # This field is excluded if:
            # 1/ it's parent model key is in exclude list keys
            # AND
            # 2/ the field has been defined as excluded for this parent model
            is_excluded = obj_model in exclude_list and field_accessor in exclude_list[obj_model]

            if not is_excluded:
                cleaned_list.append(field)

        return cleaned_list

    def get_report(self):
        return {
            'excluded_fields': self.excluded_fields,
            'log': self.saved_log,
        }

    def get_collected_objects(self):
        return self.collected_objs.values()

    def get_json_serialized_objects(self):
        objects = self.get_collected_objects()

        string_buffer = StringIO()

        serializer = MultiModelInheritanceSerializer()
        serializer.serialize(
            objects,
            stream=string_buffer,
            indent=2
        )
        string_buffer.seek(0)

        return string_buffer

    def debug_log(self, msg, *args, **kwargs):
        logger.debug(msg, *args, **kwargs)

        if args:
            msg = msg % args
        elif not isinstance(msg, basestring):
            msg = str(msg)
        self.saved_log.append(msg)

    def _is_already_collected(self, parent, obj):
        new_key = get_key_from_instance(obj)
        parent_key = get_key_from_instance(parent)
        is_already_collected = new_key in self.collected_objs

        if is_already_collected:
            self.debug_log('-' * 100)
            self.debug_log('Collecting >>> {key} <<< (from {parent_key}) : NOK (already collected)'.format(
                key=new_key, parent_key=parent_key))

        return is_already_collected

    def _is_excluded_model(self, obj):
        obj_model = get_model_from_instance(obj)
        obj_key = get_key_from_instance(obj)
        is_excluded_model = obj_model in self.EXCLUDE_MODELS

        if is_excluded_model:
            self.debug_log('-' * 100)
            self.debug_log('Not Collecting {key} (excluded model)'.format(key=obj_key))

        return is_excluded_model

{'fraction_non_alphanumeric': 0.1224114691826982, 'fraction_numerical': 0.0036147530939835805, 'mean_word_length': 2.4407672849915683, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 235, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3476783', 'n_tokens_mistral': 4637, 'n_tokens_neox': 4436, 'n_words': 1465}