<commit_before># -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models
from editorsnotes.main.models import Document

class Migration(DataMigration):

    def forwards(self, orm):
        "Write your forwards methods here."
        # Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..."
        for document in orm['main.Document'].objects.all():
            digest = Document.hash_description(document.description)
            document.description_digest = digest
            document.save()

        duplicates_query = """
        SELECT project_id, description_digest, COUNT(description_digest) as occurences
        FROM main_document
        GROUP BY project_id, description_digest
        HAVING COUNT(description_digest) > 1;
        """

        duplicates = db.execute(duplicates_query);
        for project_id, description_digest, ct in duplicates:
            qs = orm['main.Document'].objects.filter(project_id=project_id,
                                                     description_digest=description_digest)
            for i, document in enumerate(qs[1:], 2):
                description = document.description
                last_el = (list(description.iterdescendants()) or [description])[-1]
                appended_text = u' ({})'.format(i)
                if last_el.tail:
                    last_el.tail += appended_text
                else:
                    last_el.text += appended_text
                document.description = description
                document.description_digest = Document.hash_description(description)
                document.save()
        remaining_duplicates = db.execute(duplicates_query)
        assert not len(remaining_duplicates), "Still duplicates"

    def backwards(self, orm):
        "Write your backwards methods here."

    models = {
        u'auth.group': {
            'Meta': {'object_name': 'Group'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        u'auth.permission': {
            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})<commit_msg>Update descriptions & description digest with SQL<commit_after># -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models
from lxml import etree
from editorsnotes.main.models import Document

class Migration(DataMigration):

    def forwards(self, orm):
        "Write your forwards methods here."
        # Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..."
        digest_update_query = """
        UPDATE main_document
        SET description_digest = %s
        WHERE id = %s
        """
        for document in orm['main.Document'].objects.all():
            db.execute(digest_update_query,
                       params=[Document.hash_description(document.description),
                               document.id])

        duplicates_query = """
        SELECT project_id, description_digest, COUNT(description_digest) as occurences
        FROM main_document
        GROUP BY project_id, description_digest
        HAVING COUNT(description_digest) > 1;
        """

        duplicates = db.execute(duplicates_query);
        for project_id, description_digest, ct in duplicates:
            qs = orm['main.Document'].objects.filter(project_id=project_id,
                                                     description_digest=description_digest)
            for i, document in enumerate(qs[1:], 2):
                description = document.description
                last_el = (list(description.iterdescendants()) or [description])[-1]
                appended_text = u' ({})'.format(i)
                if last_el.tail:
                    last_el.tail += appended_text
                else:
                    last_el.text += appended_text
                update_query = """
                UPDATE main_document
                SET description = %s, description_digest = %s
                WHERE id = %s
                """
                db.execute(update_query, params=[etree.tostring(description),
                                                 Document.hash_description(description),
                                                 document.id])
        remaining_duplicates = db.execute(duplicates_query)
        assert not len(remaining_duplicates), "Still duplicates"

    def backwards(self, orm):
        "Write your backwards methods here."

    models = {
        u'auth.group': {
            'Meta': {'object_name': 'Group'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        u'auth.permission': {
            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
{'fraction_non_alphanumeric': 0.13796892167634595, 'fraction_numerical': 0.00408099199497724, 'mean_word_length': 2.38755980861244, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7389545', 'n_tokens_mistral': 1780, 'n_tokens_neox': 1670, 'n_words': 361}