<commit_before>from __future__ import unicode_literals
import os
import json
import datetime
from django.conf import settings
from django.db import models
from django.db.models import Q
from django.contrib.auth.models import User, Group
from mbox import MboxMessage
from event import emit_event, declare_event
from django.contrib import admin

class Project(models.Model):
    name = models.CharField(max_length=1024, db_index=True)
    mailing_list = models.CharField(max_length=4096, blank=True)
    url = models.CharField(max_length=4096, blank=True)
    git = models.CharField(max_length=4096, blank=True)
    description = models.TextField(blank=True)
    logo = models.ImageField(blank=True, upload_to="logo")
    def __str__(self):
        return self.name

    @classmethod
    def find_message_project(self, m):
        q = []
        for name, addr in m.get_receivers():
            q += self.objects.filter(mailing_list__contains=addr)
        if not q:
            raise Exception("Cannot find project for message: %s" % m)
        if len(q) > 1:
            raise Exception("Found more than one project for message: %s" % m)
        return q[0]

    @classmethod
    def has_project(self, project):
        return self.objects.filter(name=project).exists()

    def get_property(self, prop, default=None):
        a = ProjectProperty.objects.filter(project=self, name=prop).first()
        if a:
            return json.loads(a.value)
        else:
            return default

    def get_properties(self):
        r = {}
        for m in ProjectProperty.objects.filter(project=self):
            r[m.name] = json.loads(m.value)
        return r

    def set_property(self, prop, value):
        if value == None:
            ProjectProperty.objects.filter(project=self, name=prop).delete()
            return
        pp, created = ProjectProperty.objects.get_or_create(project=self,
                                                            name=prop)
        pp.value = json.dumps(value)
        pp.save()

    def total_series_count(self):
        return Message.objects.series_heads(project_name=self.name).count()

    def maintained_by(self, user):
        if user.is_superuser:
            return True
        if user.username in self.get_property("maintainers", []):
            return True
        return False

class ProjectProperty(models.Model):
    project = models.ForeignKey('Project', on_delete=models.CASCADE)
    name = models.CharField(max_length=1024, unique=True, db_index=True)
    value = models.TextField(blank=True)

    class Meta:
        unique_together = ('project', 'name',)

declare_event("SeriesComplete", project="project object",
              series="series instance that is marked complete")

declare_event("MessageAdded", message="message object that is added")

class MessageManager(models.Manager):

    class DuplicateMessageError(Exception):
        pass

    def series_heads(self, project_name=None):
        q = super(MessageManager, self).get_queryset()\
                .filter(is_series_head=True)
        if project_name:
            q = q.filter(project__name=project_name)
        return q

    def find_series(self, message_id, project_name=None):
        return self.series_heads(project_name).filter(message_id=message_id).first()

    def patches(self):
        return super(MessageManager, self).get_queryset().\
                filter(is_patch=True)

    def complete_series(self, project_name=None):
        return self.series_heads(project_name).filter(is_complete=True)

    def update_series(self, msg):
        """Update the series' record to which @msg is replying"""
        s = msg.get_series_head()
        if not s:
            return
        if not s.last_reply_date or s.last_reply_date < msg.date:
            s.last_reply_date = msg.date
            s.save()
        cur, total = s.get_num()
        if cur == total and s.is_patch:
            s.set_complete()
            return
        # TODO: Handle no cover letter case
        find = set(range(1, total + 1))
        for p in s.get_patches():
            assert p.is_patch
            cur, total = p.get_num()
            if cur in find:
                find.remove(cur)
        if not find:
            s.set_complete()

    def delete_subthread(self, msg):
        for r in msg.get_replies():
            self.delete_subthread(r)
        msg.delete()

    def add_message_from_mbox(self, mbox, user, project_name=None):
        m = MboxMessage(mbox)
        msgid = m.get_message_id()
        msg = Message(message_id=msgid,
                      in_reply_to=m.get_in_reply_to() or "",
                      date=m.get_date(),
                      subject=m.get_subject(),
                      stripped_subject=m.get_subject(strip_tags=True),
                      version=m.get_version(),
                      sender=json.dumps(m.get_from()),
                      receivers=json.dumps(m.get_to() + m.get_cc()),
                      prefixes=json.dumps(m.get_prefixes()),
                      is_series_head=m.is_series_head(),
                      is_patch=m.is_patch(),
                      patch_num=m.get_num()[0])
        if project_name:
            p = Project.object.get(name=project_name)
        else:
            p = Project.find_message_project(msg)
        msg.project = p
        if self.filter(message_id=msgid, project__name=p.name).first():
            raise self.DuplicateMessageError(msgid)
        msg.save_mbox(mbox)
        msg.save()
        emit_event("MessageAdded", message=msg)
        self.update_series(msg)

def HeaderFieldModel(**args):
    return models.CharField(max_length=4096, **args)

class Message(models.Model):
    """ Patch email message """

    project = models.ForeignKey('Project', on_delete=models.CASCADE)
    message_id = HeaderFieldModel(db_index=True)
    in_reply_to = HeaderFieldModel(blank=True, db_index=True)
    date = models.DateTimeField(db_index=True)
    last_reply_date = models.DateTimeField(db_index=True, null=True)
    subject = HeaderFieldModel()
    stripped_subject = HeaderFieldModel(db_index=True)
    version = models.PositiveSmallIntegerField(default=0)<commit_msg>models: Duplicate message if it belongs to multiple projects<commit_after>from __future__ import unicode_literals
import os
import json
import datetime
from django.conf import settings
from django.db import models
from django.db.models import Q
from django.contrib.auth.models import User, Group
from mbox import MboxMessage
from event import emit_event, declare_event
from django.contrib import admin

class Project(models.Model):
    name = models.CharField(max_length=1024, db_index=True)
    mailing_list = models.CharField(max_length=4096, blank=True)
    url = models.CharField(max_length=4096, blank=True)
    git = models.CharField(max_length=4096, blank=True)
    description = models.TextField(blank=True)
    logo = models.ImageField(blank=True, upload_to="logo")
    def __str__(self):
        return self.name

    @classmethod
    def has_project(self, project):
        return self.objects.filter(name=project).exists()

    def get_property(self, prop, default=None):
        a = ProjectProperty.objects.filter(project=self, name=prop).first()
        if a:
            return json.loads(a.value)
        else:
            return default

    def get_properties(self):
        r = {}
        for m in ProjectProperty.objects.filter(project=self):
            r[m.name] = json.loads(m.value)
        return r

    def set_property(self, prop, value):
        if value == None:
            ProjectProperty.objects.filter(project=self, name=prop).delete()
            return
        pp, created = ProjectProperty.objects.get_or_create(project=self,
                                                            name=prop)
        pp.value = json.dumps(value)
        pp.save()

    def total_series_count(self):
        return Message.objects.series_heads(project_name=self.name).count()

    def maintained_by(self, user):
        if user.is_superuser:
            return True
        if user.username in self.get_property("maintainers", []):
            return True
        return False

class ProjectProperty(models.Model):
    project = models.ForeignKey('Project', on_delete=models.CASCADE)
    name = models.CharField(max_length=1024, unique=True, db_index=True)
    value = models.TextField(blank=True)

    class Meta:
        unique_together = ('project', 'name',)

declare_event("SeriesComplete", project="project object",
              series="series instance that is marked complete")

declare_event("MessageAdded", message="message object that is added")

class MessageManager(models.Manager):

    class DuplicateMessageError(Exception):
        pass

    def series_heads(self, project_name=None):
        q = super(MessageManager, self).get_queryset()\
                .filter(is_series_head=True)
        if project_name:
            q = q.filter(project__name=project_name)
        return q

    def find_series(self, message_id, project_name=None):
        return self.series_heads(project_name).filter(message_id=message_id).first()

    def patches(self):
        return super(MessageManager, self).get_queryset().\
                filter(is_patch=True)

    def complete_series(self, project_name=None):
        return self.series_heads(project_name).filter(is_complete=True)

    def update_series(self, msg):
        """Update the series' record to which @msg is replying"""
        s = msg.get_series_head()
        if not s:
            return
        if not s.last_reply_date or s.last_reply_date < msg.date:
            s.last_reply_date = msg.date
            s.save()
        cur, total = s.get_num()
        if cur == total and s.is_patch:
            s.set_complete()
            return
        # TODO: Handle no cover letter case
        find = set(range(1, total + 1))
        for p in s.get_patches():
            assert p.is_patch
            cur, total = p.get_num()
            if cur in find:
                find.remove(cur)
        if not find:
            s.set_complete()

    def delete_subthread(self, msg):
        for r in msg.get_replies():
            self.delete_subthread(r)
        msg.delete()

    def add_message_from_mbox(self, mbox, user, project_name=None):

        def find_message_projects(m):
            q = []
            for name, addr in m.get_to() + m.get_cc():
                q += Project.objects.filter(mailing_list__contains=addr)
            if not q:
                raise Exception("Cannot find project for message: %s" % m)
            return q

        m = MboxMessage(mbox)
        msgid = m.get_message_id()
        if project_name:
            projects = [Project.object.get(name=project_name)]
        else:
            projects = find_message_projects(m)
        for p in projects:
            msg = Message(message_id=msgid,
                          in_reply_to=m.get_in_reply_to() or "",
                          date=m.get_date(),
                          subject=m.get_subject(),
                          stripped_subject=m.get_subject(strip_tags=True),
                          version=m.get_version(),
                          sender=json.dumps(m.get_from()),
                          receivers=json.dumps(m.get_to() + m.get_cc()),
                          prefixes=json.dumps(m.get_prefixes()),
                          is_series_head=m.is_series_head(),
                          is_patch=m.is_patch(),
                          patch_num=m.get_num()[0])
            msg.project = p
            if self.filter(message_id=msgid, project__name=p.name).first():
                raise self.DuplicateMessageError(msgid)
            msg.save_mbox(mbox)
            msg.save()
            emit_event("MessageAdded", message=msg)
            self.update_series(msg)

def HeaderFieldModel(**args):
    return models.CharField(max_length=4096, **args)

class Message(models.Model):
    """ Patch email message """

    project = models.ForeignKey('Project', on_delete=models.CASCADE)
    message_id = HeaderFieldModel(db_index=True)
    in_reply_to = HeaderFieldModel(blank=True, db_index=True)
    date = models.DateTimeField(db_index=True)
    last_reply_date = models.DateTimeField(db_index=True, null=True)
    subject = HeaderFieldModel()
    stripped_subject = HeaderFieldModel(db_index=True)
    version = models.PositiveSmallIntegerField(default=0)
{'fraction_non_alphanumeric': 0.1009484005786851, 'fraction_numerical': 0.004661629963028452, 'mean_word_length': 2.3876939831200654, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '598351', 'n_tokens_mistral': 3642, 'n_tokens_neox': 3538, 'n_words': 757}