<commit_before>import time

from django.db import transaction, models
from django.conf import settings

from parliament.politicians import twit
from parliament.politicians import googlenews as gnews
from parliament.imports import parlvotes, legisinfo, parl_document, parl_cmte
from parliament.core.models import Politician, Session
from django.core.mail import mail_admins
from parliament.hansards.models import Document
from parliament.activity import utils as activityutils
from parliament.alerts import utils as alertutils
from parliament.activity.models import Activity

@transaction.commit_on_success
def twitter():
    twit.save_tweets()
    return True
    
def twitter_ids():
    from parliament.imports import politwitter
    politwitter.import_twitter_ids()
    
def googlenews():
    for pol in Politician.objects.current():
        gnews.save_politician_news(pol)
        #time.sleep(1)
        
def votes():
    parlvotes.import_votes()
    
def bills():
    legisinfo.import_bills(Session.objects.current())

@transaction.commit_on_success
def prune_activities():
    for pol in Politician.objects.current():
        activityutils.prune(Activity.public.filter(politician=pol))
    return True

def committee_evidence():
    for document in Document.evidence\
      .annotate(scount=models.Count('statement'))\
      .exclude(scount__gt=0).order_by('date').iterator():
        print document
        parl_document.import_document(document, interactive=False)
        if document.statement_set.all().count():
            document.save_activity()
    
def committees(sess=None):
    from parliament.committees.models import Committee
    if sess is None:
        sess = Session.objects.current()
    parl_cmte.import_committee_list(session=sess)
    parl_cmte.import_committee_documents(sess)
    
@transaction.commit_on_success
def hansards_load():
    parl_document.fetch_latest_debates()
    return True
        
@transaction.commit_manually
def hansards_parse():
    for hansard in Document.objects.filter(document_type=Document.DEBATE)\
      .annotate(scount=models.Count('statement'))\
      .exclude(scount__gt=0).order_by('date').iterator():
        try:
            parl_document.import_document(hansard, interactive=False)
        except Exception, e:
            transaction.rollback()
            mail_admins("Hansard parse failure on #%s" % hansard.id, repr(e))
            continue
        else:
            transaction.commit()
        # now reload the Hansard to get the date
        hansard = Document.objects.get(pk=hansard.id)
        try:
            hansard.save_activity()
        except Exception, e:
            transaction.rollback()
            raise e
        else:
            transaction.commit()
        if getattr(settings, 'PARLIAMENT_SEND_EMAIL', False):
            alertutils.alerts_for_hansard(hansard)
    transaction.commit()
            
def hansards():
    hansards_load()
    hansards_parse()
    
def wordcloud():<commit_msg>jobs: use logger instead of mail_admins<commit_after>import time

from django.db import transaction, models
from django.conf import settings

from parliament.politicians import twit
from parliament.politicians import googlenews as gnews
from parliament.imports import parlvotes, legisinfo, parl_document, parl_cmte
from parliament.core.models import Politician, Session
from parliament.hansards.models import Document
from parliament.activity import utils as activityutils
from parliament.alerts import utils as alertutils
from parliament.activity.models import Activity

import logging
logger = logging.getLogger(__name__)

@transaction.commit_on_success
def twitter():
    twit.save_tweets()
    return True
    
def twitter_ids():
    from parliament.imports import politwitter
    politwitter.import_twitter_ids()
    
def googlenews():
    for pol in Politician.objects.current():
        gnews.save_politician_news(pol)
        #time.sleep(1)
        
def votes():
    parlvotes.import_votes()
    
def bills():
    legisinfo.import_bills(Session.objects.current())

@transaction.commit_on_success
def prune_activities():
    for pol in Politician.objects.current():
        activityutils.prune(Activity.public.filter(politician=pol))
    return True

def committee_evidence():
    for document in Document.evidence\
      .annotate(scount=models.Count('statement'))\
      .exclude(scount__gt=0).order_by('date').iterator():
        print document
        parl_document.import_document(document, interactive=False)
        if document.statement_set.all().count():
            document.save_activity()
    
def committees(sess=None):
    from parliament.committees.models import Committee
    if sess is None:
        sess = Session.objects.current()
    parl_cmte.import_committee_list(session=sess)
    parl_cmte.import_committee_documents(sess)
    
@transaction.commit_on_success
def hansards_load():
    parl_document.fetch_latest_debates()
    return True
        
@transaction.commit_manually
def hansards_parse():
    for hansard in Document.objects.filter(document_type=Document.DEBATE)\
      .annotate(scount=models.Count('statement'))\
      .exclude(scount__gt=0).order_by('date').iterator():
        try:
            parl_document.import_document(hansard, interactive=False)
        except Exception, e:
            transaction.rollback()
            logger.error("Hansard parse failure on #%s: %r" % (hansard.id, e))
            continue
        else:
            transaction.commit()
        # now reload the Hansard to get the date
        hansard = Document.objects.get(pk=hansard.id)
        try:
            hansard.save_activity()
        except Exception, e:
            transaction.rollback()
            raise e
        else:
            transaction.commit()
        if getattr(settings, 'PARLIAMENT_SEND_EMAIL', False):
            alertutils.alerts_for_hansard(hansard)
    transaction.commit()
            
def hansards():
    hansards_load()
    hansards_parse()
    
def wordcloud():
{'fraction_non_alphanumeric': 0.08871102217775555, 'fraction_numerical': 0.0010005002501250625, 'mean_word_length': 3.2751247327156094, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3348614', 'n_tokens_mistral': 1715, 'n_tokens_neox': 1712, 'n_words': 379}