<commit_before>"""
Contains things to detect changes - either using options passed in on the
commandline, or by using autodetection, etc.
"""

from django.db import models
from django.contrib.contenttypes.generic import GenericRelation
from django.utils.datastructures import SortedDict

from south.creator.freezer import remove_useless_attributes, freeze_apps, model_key
from south.utils import auto_through

class BaseChanges(object):
    """
    Base changes class.
    """
    def suggest_name(self):
        return ''
    
    def split_model_def(self, model, model_def):
        """
        Given a model and its model def (a dict of field: triple), returns three
        items: the real fields dict, the Meta dict, and the M2M fields dict.
        """
        real_fields = SortedDict()
        meta = SortedDict()
        m2m_fields = SortedDict()
        for name, triple in model_def.items():
            if name == "Meta":
                meta = triple
            elif isinstance(model._meta.get_field_by_name(name)[0], models.ManyToManyField):
                m2m_fields[name] = triple
            else:
                real_fields[name] = triple
        return real_fields, meta, m2m_fields
    
    def current_model_from_key(self, key):
        app_label, model_name = key.split(".")
        return models.get_model(app_label, model_name)
    
    def current_field_from_key(self, key, fieldname):
        app_label, model_name = key.split(".")
        # Special, for the magical field from order_with_respect_to
        if fieldname == "_order":
            field = models.IntegerField()
            field.name = "_order"
            field.attname = "_order"
            field.column = "_order"
            field.default = 0
            return field
        # Otherwise, normal.
        return models.get_model(app_label, model_name)._meta.get_field_by_name(fieldname)[0]


class AutoChanges(BaseChanges):
    """
    Detects changes by 'diffing' two sets of frozen model definitions.
    """
    
    # Field types we don't generate add/remove field changes for.
    IGNORED_FIELD_TYPES = [
        GenericRelation,
    ]
    
    def __init__(self, migrations, old_defs, old_orm, new_defs):
        self.migrations = migrations
        self.old_defs = old_defs
        self.old_orm = old_orm
        self.new_defs = new_defs
    
    def suggest_name(self):
        parts = ["auto"]
        for change_name, params in self.get_changes():
            if change_name == "AddModel":
                parts.append("add_%s" % params['model']._meta.object_name.lower())
            elif change_name == "DeleteModel":
                parts.append("del_%s" % params['model']._meta.object_name.lower())
            elif change_name == "AddField":
                parts.append("add_field_%s_%s" % (
                    params['model']._meta.object_name.lower(),
                    params['field'].name,
                ))
            elif change_name == "DeleteField":
                parts.append("del_field_%s_%s" % (
                    params['model']._meta.object_name.lower(),
                    params['field'].name,
                ))
            elif change_name == "ChangeField":
                parts.append("chg_field_%s_%s" % (
                    params['model']._meta.object_name.lower(),
                    params['new_field'].name,
                ))
            elif change_name == "AddUnique":
                parts.append("add_unique_%s_%s" % (
                    params['model']._meta.object_name.lower(),
                    "_".join([x.name for x in params['fields']]),
                ))
            elif change_name == "DeleteUnique":
                parts.append("del_unique_%s_%s" % (
                    params['model']._meta.object_name.lower(),
                    "_".join([x.name for x in params['fields']]),
                ))
        return ("__".join(parts))[:70]
    
    def get_changes(self):
        """
        Returns the difference between the old and new sets of models as a 5-tuple:
        added_models, deleted_models, added_fields, deleted_fields, changed_fields
        """
        
        deleted_models = set()
        
        # See if anything's vanished
        for key in self.old_defs:
            if key not in self.new_defs:
                # We shouldn't delete it if it was managed=False
                old_fields, old_meta, old_m2ms = self.split_model_def(self.old_orm[key], self.old_defs[key])
                if old_meta.get("managed", "True") != "False":
                    # Alright, delete it.
                    yield ("DeleteModel", {
                        "model": self.old_orm[key], 
                        "model_def": old_fields,
                    })
                    # Also make sure we delete any M2Ms it had.
                    for fieldname in old_m2ms:
                        # Only delete its stuff if it wasn't a through=.
                        field = self.old_orm[key + ":" + fieldname]
                        if auto_through(field):
                            yield ("DeleteM2M", {"model": self.old_orm[key], "field": field})
                    # And any unique constraints it had 
                    unique_together = eval(old_meta.get("unique_together", "[]"))
                    if unique_together:
                        # If it's only a single tuple, make it into the longer one
                        if isinstance(unique_together[0], basestring):
                            unique_together = [unique_together]
                        # For each combination, make an action for it
                        for fields in unique_together:
                            yield ("DeleteUnique", {
                                "model": self.old_orm[key],
                                "fields": [self.old_orm[key]._meta.get_field_by_name(x)[0] for x in fields],
                            })
                # We always add it in here so we ignore it later
                deleted_models.add(key)
        
        # Or appeared
        for key in self.new_defs:
            if key not in self.old_defs:
                # We shouldn't add it if it's managed=False
                new_fields, new_meta, new_m2ms = self.split_model_def(self.current_model_from_key(key), self.new_defs[key])
                if new_meta.get("managed", "True") != "False":
                    yield ("AddModel", {
                        "model": self.current_model_from_key(key), 
                        "model_def": new_fields,
                    })
                    # Also make sure we add any M2Ms it has.
                    for fieldname in new_m2ms:
                        # Only create its stuff if it wasn't a through=.
                        field = self.current_field_from_key(key, fieldname)
                        if auto_through(field):
                            yield ("AddM2M", {"model": self.current_model_from_key(key), "field": field})
                    # And any unique constraints it has 
                    unique_together = eval(new_meta.get("unique_together", "[]"))
                    if unique_together:
                        # If it's only a single tuple, make it into the longer one
                        if isinstance(unique_together[0], basestring):
                            unique_together = [unique_together]
                        # For each combination, make an action for it
                        for fields in unique_together:
                            yield ("AddUnique", {
                                "model": self.current_model_from_key(key),
                                "fields": [self.current_model_from_key(key)._meta.get_field_by_name(x)[0] for x in fields],
                            })
        
        # Now, for every model that's stayed the same, check its fields.
        for key in self.old_defs:
            if key not in deleted_models:
                
                old_fields, old_meta, old_m2ms = self.split_model_def(self.old_orm[key], self.old_defs[key])
                new_fields, new_meta, new_m2ms = self.split_model_def(self.current_model_from_key(key), self.new_defs[key])<commit_msg>Make change detection ignore changes in existing non-managed models<commit_after>"""
Contains things to detect changes - either using options passed in on the
commandline, or by using autodetection, etc.
"""

from django.db import models
from django.contrib.contenttypes.generic import GenericRelation
from django.utils.datastructures import SortedDict

from south.creator.freezer import remove_useless_attributes, freeze_apps, model_key
from south.utils import auto_through

class BaseChanges(object):
    """
    Base changes class.
    """
    def suggest_name(self):
        return ''
    
    def split_model_def(self, model, model_def):
        """
        Given a model and its model def (a dict of field: triple), returns three
        items: the real fields dict, the Meta dict, and the M2M fields dict.
        """
        real_fields = SortedDict()
        meta = SortedDict()
        m2m_fields = SortedDict()
        for name, triple in model_def.items():
            if name == "Meta":
                meta = triple
            elif isinstance(model._meta.get_field_by_name(name)[0], models.ManyToManyField):
                m2m_fields[name] = triple
            else:
                real_fields[name] = triple
        return real_fields, meta, m2m_fields
    
    def current_model_from_key(self, key):
        app_label, model_name = key.split(".")
        return models.get_model(app_label, model_name)
    
    def current_field_from_key(self, key, fieldname):
        app_label, model_name = key.split(".")
        # Special, for the magical field from order_with_respect_to
        if fieldname == "_order":
            field = models.IntegerField()
            field.name = "_order"
            field.attname = "_order"
            field.column = "_order"
            field.default = 0
            return field
        # Otherwise, normal.
        return models.get_model(app_label, model_name)._meta.get_field_by_name(fieldname)[0]


class AutoChanges(BaseChanges):
    """
    Detects changes by 'diffing' two sets of frozen model definitions.
    """
    
    # Field types we don't generate add/remove field changes for.
    IGNORED_FIELD_TYPES = [
        GenericRelation,
    ]
    
    def __init__(self, migrations, old_defs, old_orm, new_defs):
        self.migrations = migrations
        self.old_defs = old_defs
        self.old_orm = old_orm
        self.new_defs = new_defs
    
    def suggest_name(self):
        parts = ["auto"]
        for change_name, params in self.get_changes():
            if change_name == "AddModel":
                parts.append("add_%s" % params['model']._meta.object_name.lower())
            elif change_name == "DeleteModel":
                parts.append("del_%s" % params['model']._meta.object_name.lower())
            elif change_name == "AddField":
                parts.append("add_field_%s_%s" % (
                    params['model']._meta.object_name.lower(),
                    params['field'].name,
                ))
            elif change_name == "DeleteField":
                parts.append("del_field_%s_%s" % (
                    params['model']._meta.object_name.lower(),
                    params['field'].name,
                ))
            elif change_name == "ChangeField":
                parts.append("chg_field_%s_%s" % (
                    params['model']._meta.object_name.lower(),
                    params['new_field'].name,
                ))
            elif change_name == "AddUnique":
                parts.append("add_unique_%s_%s" % (
                    params['model']._meta.object_name.lower(),
                    "_".join([x.name for x in params['fields']]),
                ))
            elif change_name == "DeleteUnique":
                parts.append("del_unique_%s_%s" % (
                    params['model']._meta.object_name.lower(),
                    "_".join([x.name for x in params['fields']]),
                ))
        return ("__".join(parts))[:70]
    
    def get_changes(self):
        """
        Returns the difference between the old and new sets of models as a 5-tuple:
        added_models, deleted_models, added_fields, deleted_fields, changed_fields
        """
        
        deleted_models = set()
        
        # See if anything's vanished
        for key in self.old_defs:
            if key not in self.new_defs:
                # We shouldn't delete it if it was managed=False
                old_fields, old_meta, old_m2ms = self.split_model_def(self.old_orm[key], self.old_defs[key])
                if old_meta.get("managed", "True") != "False":
                    # Alright, delete it.
                    yield ("DeleteModel", {
                        "model": self.old_orm[key], 
                        "model_def": old_fields,
                    })
                    # Also make sure we delete any M2Ms it had.
                    for fieldname in old_m2ms:
                        # Only delete its stuff if it wasn't a through=.
                        field = self.old_orm[key + ":" + fieldname]
                        if auto_through(field):
                            yield ("DeleteM2M", {"model": self.old_orm[key], "field": field})
                    # And any unique constraints it had 
                    unique_together = eval(old_meta.get("unique_together", "[]"))
                    if unique_together:
                        # If it's only a single tuple, make it into the longer one
                        if isinstance(unique_together[0], basestring):
                            unique_together = [unique_together]
                        # For each combination, make an action for it
                        for fields in unique_together:
                            yield ("DeleteUnique", {
                                "model": self.old_orm[key],
                                "fields": [self.old_orm[key]._meta.get_field_by_name(x)[0] for x in fields],
                            })
                # We always add it in here so we ignore it later
                deleted_models.add(key)
        
        # Or appeared
        for key in self.new_defs:
            if key not in self.old_defs:
                # We shouldn't add it if it's managed=False
                new_fields, new_meta, new_m2ms = self.split_model_def(self.current_model_from_key(key), self.new_defs[key])
                if new_meta.get("managed", "True") != "False":
                    yield ("AddModel", {
                        "model": self.current_model_from_key(key), 
                        "model_def": new_fields,
                    })
                    # Also make sure we add any M2Ms it has.
                    for fieldname in new_m2ms:
                        # Only create its stuff if it wasn't a through=.
                        field = self.current_field_from_key(key, fieldname)
                        if auto_through(field):
                            yield ("AddM2M", {"model": self.current_model_from_key(key), "field": field})
                    # And any unique constraints it has 
                    unique_together = eval(new_meta.get("unique_together", "[]"))
                    if unique_together:
                        # If it's only a single tuple, make it into the longer one
                        if isinstance(unique_together[0], basestring):
                            unique_together = [unique_together]
                        # For each combination, make an action for it
                        for fields in unique_together:
                            yield ("AddUnique", {
                                "model": self.current_model_from_key(key),
                                "fields": [self.current_model_from_key(key)._meta.get_field_by_name(x)[0] for x in fields],
                            })
        
        # Now, for every model that's stayed the same, check its fields.
        for key in self.old_defs:
            if key not in deleted_models:
                
                old_fields, old_meta, old_m2ms = self.split_model_def(self.old_orm[key], self.old_defs[key])
                new_fields, new_meta, new_m2ms = self.split_model_def(self.current_model_from_key(key), self.new_defs[key])
                
                # Do nothing for models which are now not managed.
                if new_meta.get("managed", "True") == "False":
                    continue
{'fraction_non_alphanumeric': 0.0955651964328754, 'fraction_numerical': 0.0028922631959508315, 'mean_word_length': 1.6108227151171937, 'pattern_counts': {'":': 49, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3481775', 'n_tokens_mistral': 4530, 'n_tokens_neox': 4326, 'n_words': 1117}