<commit_before>#coding: utf-8

import gzip, os, re
from collections import defaultdict
import datetime

from django.db import models
from django.conf import settings
from django.core import urlresolvers
from django.core.files.base import ContentFile
from django.template.defaultfilters import slugify
from django.utils.html import strip_tags
from django.utils.safestring import mark_safe

from parliament.core.models import Session, ElectedMember, Politician
from parliament.bills.models import Bill
from parliament.core import parsetools, text_utils
from parliament.core.utils import memoize_property
from parliament.activity import utils as activity

import logging
logger = logging.getLogger(__name__)

class HansardManager(models.Manager):

    def get_query_set(self):
        return super(HansardManager, self).get_query_set().filter(document_type=Document.DEBATE)

class EvidenceManager(models.Manager):

    def get_query_set(self):
        return super(EvidenceManager, self).get_query_set().filter(document_type=Document.EVIDENCE)

class NoStatementManager(models.Manager):
    """Manager restricts to Documents that haven't had statements parsed."""

    def get_query_set(self):
        return super(NoStatementManager, self).get_query_set()\
            .annotate(scount=models.Count('statement'))\
            .exclude(scount__gt=0)

class Document(models.Model):
    
    DEBATE = 'D'
    EVIDENCE = 'E'
    
    document_type = models.CharField(max_length=1, db_index=True, choices=(
        ('D', 'Debate'),
        ('E', 'Committee Evidence'),
    ))
    date = models.DateField(blank=True, null=True)
    number = models.CharField(max_length=6, blank=True) # there exist 'numbers' with letters
    session = models.ForeignKey(Session)
    
    source_id = models.IntegerField(unique=True, db_index=True)
    
    most_frequent_word = models.CharField(max_length=20, blank=True)
    wordcloud = models.ImageField(upload_to='autoimg/wordcloud', blank=True, null=True)

    downloaded = models.BooleanField(default=False)

    public = models.BooleanField("Display on site?", default=False)
    multilingual = models.BooleanField("Content parsed in both languages?", default=False)

    objects = models.Manager()
    hansards = HansardManager()
    evidence = EvidenceManager()
    without_statements = NoStatementManager()
    
    class Meta:
        ordering = ('-date',)
    
    def __unicode__ (self):
        if self.document_type == self.DEBATE:
            return u"Hansard #%s for %s (#%s/#%s)" % (self.number, self.date, self.id, self.source_id)
        else:
            return u"%s evidence for %s (#%s/#%s)" % (
                self.committeemeeting.committee.short_name, self.date, self.id, self.source_id)
    
    @property
    def url_date(self):
        return '%s-%s-%s' % (self.date.year, self.date.month, self.date.day)
        
    def get_absolute_url(self, pretty=False):
        if self.document_type == self.DEBATE:
            return urlresolvers.reverse('debate', kwargs={'hansard_date': self.url_date})
        elif self.document_type == self.EVIDENCE:
            if pretty:
                return self.committeemeeting.get_absolute_url(pretty=True)
            return urlresolvers.reverse('document_redirect', kwargs={'document_id': self.id})

    @property
    def url(self):
        return "http://www.parl.gc.ca/HousePublications/Publication.aspx?DocId=%s&Language=%s&Mode=1" % (
                self.source_id, settings.LANGUAGE_CODE[0].upper()
        )
        
    def _topics(self, l):
        topics = []
        last_topic = ''
        for statement in l:
            if statement[0] and statement[0] != last_topic:
                last_topic = statement[0]
                topics.append((statement[0], statement[1]))
        return topics
        
    def topics(self):
        """Returns a tuple with (topic, statement sequence ID) for every topic mentioned."""
        return self._topics(self.statement_set.all().values_list('h2', 'sequence'))
        
    def headings(self):
        """Returns a tuple with (heading, statement sequence ID) for every heading mentioned."""
        return self._topics(self.statement_set.all().values_list('h1', 'sequence'))
    
    def topics_with_qp(self):
        """Returns the same as topics(), but with a link to Question Period at the start of the list."""
        statements = self.statement_set.all().values_list('h2', 'sequence', 'h1')
        topics = self._topics(statements)
        qp_seq = None
        for s in statements:
            if s[2] == 'Oral Questions':
                qp_seq = s[1]
                break
        if qp_seq is not None:
            topics.insert(0, ('Question Period', qp_seq))
        return topics

    def politicians(self):
        return Politician.objects.filter(statement__document=self).distinct()

    def outside_speakers(self):
        speakers = {}
        for val in self.statement_set.filter(politician__isnull=True, who_hocid__isnull=False)\
          .values_list('who', 'who_context'):
            who = parsetools.r_parens.sub('', val[0])
            who = re.sub('^\s*\S+\s+', '', who).strip() # strip honorific
            speakers[who] = val[1]
        return speakers
    
    def save_activity(self):
        statements = self.statement_set.filter(procedural=False).select_related('member', 'politician')
        politicians = set([s.politician for s in statements if s.politician])
        for pol in politicians:
            topics = {}
            wordcount = 0
            for statement in filter(lambda s: s.politician == pol, statements):
                wordcount += statement.wordcount
                if statement.topic in topics:
                    # If our statement is longer than the previous statement on this topic,
                    # use its text for the excerpt.
                    if len(statement.text_plain()) > len(topics[statement.topic][1]):
                        topics[statement.topic][1] = statement.text_plain()
                        topics[statement.topic][2] = statement.get_absolute_url()
                else:
                    topics[statement.topic] = [statement.sequence, statement.text_plain(), statement.get_absolute_url()]<commit_msg>A few more sequence->slug changes<commit_after>#coding: utf-8

import gzip, os, re
from collections import defaultdict
import datetime

from django.db import models
from django.conf import settings
from django.core import urlresolvers
from django.core.files.base import ContentFile
from django.template.defaultfilters import slugify
from django.utils.html import strip_tags
from django.utils.safestring import mark_safe

from parliament.core.models import Session, ElectedMember, Politician
from parliament.bills.models import Bill
from parliament.core import parsetools, text_utils
from parliament.core.utils import memoize_property
from parliament.activity import utils as activity

import logging
logger = logging.getLogger(__name__)

class HansardManager(models.Manager):

    def get_query_set(self):
        return super(HansardManager, self).get_query_set().filter(document_type=Document.DEBATE)

class EvidenceManager(models.Manager):

    def get_query_set(self):
        return super(EvidenceManager, self).get_query_set().filter(document_type=Document.EVIDENCE)

class NoStatementManager(models.Manager):
    """Manager restricts to Documents that haven't had statements parsed."""

    def get_query_set(self):
        return super(NoStatementManager, self).get_query_set()\
            .annotate(scount=models.Count('statement'))\
            .exclude(scount__gt=0)

class Document(models.Model):
    
    DEBATE = 'D'
    EVIDENCE = 'E'
    
    document_type = models.CharField(max_length=1, db_index=True, choices=(
        ('D', 'Debate'),
        ('E', 'Committee Evidence'),
    ))
    date = models.DateField(blank=True, null=True)
    number = models.CharField(max_length=6, blank=True) # there exist 'numbers' with letters
    session = models.ForeignKey(Session)
    
    source_id = models.IntegerField(unique=True, db_index=True)
    
    most_frequent_word = models.CharField(max_length=20, blank=True)
    wordcloud = models.ImageField(upload_to='autoimg/wordcloud', blank=True, null=True)

    downloaded = models.BooleanField(default=False)

    public = models.BooleanField("Display on site?", default=False)
    multilingual = models.BooleanField("Content parsed in both languages?", default=False)

    objects = models.Manager()
    hansards = HansardManager()
    evidence = EvidenceManager()
    without_statements = NoStatementManager()
    
    class Meta:
        ordering = ('-date',)
    
    def __unicode__ (self):
        if self.document_type == self.DEBATE:
            return u"Hansard #%s for %s (#%s/#%s)" % (self.number, self.date, self.id, self.source_id)
        else:
            return u"%s evidence for %s (#%s/#%s)" % (
                self.committeemeeting.committee.short_name, self.date, self.id, self.source_id)
    
    @property
    def url_date(self):
        return '%s-%s-%s' % (self.date.year, self.date.month, self.date.day)
        
    def get_absolute_url(self, pretty=False):
        if self.document_type == self.DEBATE:
            return urlresolvers.reverse('debate', kwargs={'hansard_date': self.url_date})
        elif self.document_type == self.EVIDENCE:
            if pretty:
                return self.committeemeeting.get_absolute_url(pretty=True)
            return urlresolvers.reverse('document_redirect', kwargs={'document_id': self.id})

    @property
    def url(self):
        return "http://www.parl.gc.ca/HousePublications/Publication.aspx?DocId=%s&Language=%s&Mode=1" % (
                self.source_id, settings.LANGUAGE_CODE[0].upper()
        )
        
    def _topics(self, l):
        topics = []
        last_topic = ''
        for statement in l:
            if statement[0] and statement[0] != last_topic:
                last_topic = statement[0]
                topics.append((statement[0], statement[1]))
        return topics
        
    def topics(self):
        """Returns a tuple with (topic, statement slug) for every topic mentioned."""
        return self._topics(self.statement_set.all().values_list('h2', 'slug'))
        
    def headings(self):
        """Returns a tuple with (heading, statement slug) for every heading mentioned."""
        return self._topics(self.statement_set.all().values_list('h1', 'slug'))
    
    def topics_with_qp(self):
        """Returns the same as topics(), but with a link to Question Period at the start of the list."""
        statements = self.statement_set.all().values_list('h2', 'slug', 'h1')
        topics = self._topics(statements)
        qp_seq = None
        for s in statements:
            if s[2] == 'Oral Questions':
                qp_seq = s[1]
                break
        if qp_seq is not None:
            topics.insert(0, ('Question Period', qp_seq))
        return topics

    def politicians(self):
        return Politician.objects.filter(statement__document=self).distinct()

    def outside_speakers(self):
        speakers = {}
        for val in self.statement_set.filter(politician__isnull=True, who_hocid__isnull=False)\
          .values_list('who', 'who_context'):
            who = parsetools.r_parens.sub('', val[0])
            who = re.sub('^\s*\S+\s+', '', who).strip() # strip honorific
            speakers[who] = val[1]
        return speakers
    
    def save_activity(self):
        statements = self.statement_set.filter(procedural=False).select_related('member', 'politician')
        politicians = set([s.politician for s in statements if s.politician])
        for pol in politicians:
            topics = {}
            wordcount = 0
            for statement in filter(lambda s: s.politician == pol, statements):
                wordcount += statement.wordcount
                if statement.topic in topics:
                    # If our statement is longer than the previous statement on this topic,
                    # use its text for the excerpt.
                    if len(statement.text_plain()) > len(topics[statement.topic][1]):
                        topics[statement.topic][1] = statement.text_plain()
                        topics[statement.topic][2] = statement.get_absolute_url()
                else:
                    topics[statement.topic] = [statement.slug, statement.text_plain(), statement.get_absolute_url()]
{'fraction_non_alphanumeric': 0.11201923076923077, 'fraction_numerical': 0.004166666666666667, 'mean_word_length': 2.8121563836285888, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4527784', 'n_tokens_mistral': 3609, 'n_tokens_neox': 3528, 'n_words': 845}