<commit_before>import sys
import logging
import os
from datetime import timedelta
import datetime
from mysite.customs import ohloh
import urllib2
import urllib
import mysite.customs.github
import mysite.profile.models
from mysite.search.models import Project
from celery.decorators import task
from celery.task import Task
import celery.registry
import time
import random
import traceback
import mysite.profile.search_indexes
import mysite.profile.controllers
import shutil
import staticgenerator
import mysite.customs.bitbucket

from django.conf import settings
import django.core.cache

def create_citations_from_ohloh_contributor_facts(dia_id, ohloh_results):
    '''Input: A sequence of Ohloh ContributionFact dicts
    and the id of the DataImport they came from.

    Side-effect: Create matching structures in the DB
    and mark our success in the database.'''
    # {{{
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    for ohloh_contrib_info in ohloh_results:
        (project, _) = Project.objects.get_or_create(
                name=ohloh_contrib_info['project'])
        # FIXME: don't import if blacklisted
        (portfolio_entry, _) = mysite.profile.models.PortfolioEntry.objects.get_or_create(
                person=person, project=project)
        citation = mysite.profile.models.Citation.create_from_ohloh_contrib_info(ohloh_contrib_info)
        citation.portfolio_entry = portfolio_entry
        citation.data_import_attempt = dia 
        citation.save_and_check_for_duplicates()

    person.last_polled = datetime.datetime.now()
    dia.completed = True
    dia.save()
    person.save()
    # }}}

def create_citations_from_bitbucket_results(dia_id, results):
    """
    Given the input of results returned from a query against the 
    bitbucket.org user api, create user related contribution
    records.

    Note that non-authenticated calls to butbuket api returns limited
    project details.
    """
    
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    for repo in results:

        (project, _) = Project.objects.get_or_create(
            name=repo['name'])
        
        description = repo['description']
        if mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project).count() == 0:
            portfolio_entry = mysite.profile.models.PortfolioEntry(person=person,
                                             project=project,
                                             project_description=description)

            portfolio_entry.save()
        portfolio_entry = mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project)[0]
        
        citation = mysite.profile.models.Citation()
        citation.url =  'http://bitbucket.org/%s/%s/' % (dia.query.lower(), repo['name'])
        citation.contributor_role = 'Created a repository on Bitbucket.'
        citation.languages = ''
        
        citation.portfolio_entry = portfolio_entry
        citation.data_import_attempt = dia
        citation.save_and_check_for_duplicates()

    person.last_polled = datetime.datetime.now()
    dia.completed = True
    person.save()
    dia.save()

    # end create_citations_from_bitbucket_results

def rs_action(dia):
    oh = ohloh.get_ohloh()
    data, web_response = oh.get_contribution_info_by_username(
            dia.query, person=dia.person)
    dia.web_response = web_response
    dia.save()
    return data

def ou_action(dia):
    oh = ohloh.get_ohloh()
    data, web_response = oh.get_contribution_info_by_ohloh_username(
            dia.query, person=dia.person)
    dia.web_response = web_response
    dia.save()
    return data

def do_nothing_because_this_functionality_moved_to_twisted(*args):
    return None # This is moved to Twisted now.

def bb_action(dia):
    """
    Given a dia with a username, check to see if there are any 
    source packages maintained by the user.
    """
    return mysite.customs.bitbucket.get_user_repos(dia.query)

source2actual_action = {
        'gh': do_nothing_because_this_functionality_moved_to_twisted,
        'ga': do_nothing_because_this_functionality_moved_to_twisted,
        'db': do_nothing_because_this_functionality_moved_to_twisted,
        'lp': do_nothing_because_this_functionality_moved_to_twisted,
    
        'rs': rs_action,
        'ou': ou_action,
        'lp': lp_action,
        'bb': bb_action
        }

source2result_handler = {
        'db': do_nothing_because_this_functionality_moved_to_twisted,
        'gh': do_nothing_because_this_functionality_moved_to_twisted,
        'ga': do_nothing_because_this_functionality_moved_to_twisted,
        'lp': do_nothing_because_this_functionality_moved_to_twisted,

        'rs': create_citations_from_ohloh_contributor_facts,
        'ou': create_citations_from_ohloh_contributor_facts,
        'bb': create_citations_from_bitbucket_results,
        }

class ReindexPerson(Task):
    def run(self, person_id, **kwargs):
        person = mysite.profile.models.Person.objects.get(id=person_id)
        pi = mysite.profile.search_indexes.PersonIndex(person)
        pi.update_object(person)

class GarbageCollectForwarders(Task):
    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.info("Started garbage collecting profile email forwarders")
        deleted_any = mysite.profile.models.Forwarder.garbage_collect()
        if deleted_any:
            # Well, in that case, we should purge the staticgenerator-generated cache of the people pages.
            clear_people_page_cache()


class RegeneratePostfixAliasesForForwarder(Task):
    def run(self, **kwargs):
        # Generate the table...
        lines = mysite.profile.models.Forwarder.generate_list_of_lines_for_postfix_table()
        # Save it where Postfix expects it...
        fd = open(settings.POSTFIX_FORWARDER_TABLE_PATH, 'w')
        fd.write('\n'.join(lines))
        fd.close()
        # Update the Postfix forwarder database. Note that we do not need
        # to ask Postfix to reload. Yay!
        # FIXME stop using os.system()
        os.system('/usr/sbin/postmap /etc/postfix/virtual_alias_maps') 

class FetchPersonDataFromOhloh(Task):
    name = "profile.FetchPersonDataFromOhloh"

    def run(self, dia_id, **kwargs):
        """"""
        dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
        try:
            logger = self.get_logger(**kwargs)
            logger.info("Starting job for <%s>" % dia)
            if dia.completed:
                logger.info("Bailing out job for <%s>" % dia)
                return
            results = source2actual_action[dia.source](dia)
            source2result_handler[dia.source](dia.id, results)
            logger.info("Results: %s" % repr(results))

        except Exception, e:
            # if the task is in debugging mode, bubble-up the exception
            if getattr(self, 'debugging', None):
                raise
            logger.error("Traceback: ")
            logger.error(traceback.format_exc())

            # else let the exception be logged but not bubble up
            dia.completed = True
            dia.failed = True
            dia.save()
            if hasattr(e, 'code'):
                code = str(e.code)
            else:
                code = 'UNKNOWN'
            if hasattr(e, 'geturl'):
                url = str(e.geturl())
            else:
                raise
            logger.error('Dying: ' + code + ' getting ' + url)
            raise ValueError, {'code': code, 'url': url}

try:
    celery.registry.tasks.register(RegeneratePostfixAliasesForForwarder)
    celery.registry.tasks.register(FetchPersonDataFromOhloh)
    celery.registry.tasks.register(ReindexPerson)
    celery.registry.tasks.register(GarbageCollectForwarders)
except celery.registry.AlreadyRegistered:
    pass

@task
def update_person_tag_cache(person__pk):
    person = mysite.profile.models.Person.objects.get(pk=person__pk)
    cache_key = person.get_tag_texts_cache_key()
    django.core.cache.cache.delete(cache_key)
    
    # This getter will populate the cache
    return person.get_tag_texts_for_map()

@task
def update_someones_pf_cache(person__pk):
    person = mysite.profile.models.Person.objects.get(pk=person__pk)
    cache_key = person.get_cache_key_for_projects()
    django.core.cache.cache.delete(cache_key)
    
    # This getter will populate the cache
    return person.get_names_of_nonarchived_projects()

def fill_recommended_bugs_cache():
    logging.info("Filling recommended bugs cache for all people.")
    for person in mysite.profile.models.Person.objects.all():
        fill_one_person_recommend_bugs_cache.apply(person_id=person.id)
    logging.info("Finished filling recommended bugs cache for all people.")

@task
def fill_one_person_recommend_bugs_cache(person_id):
    p = mysite.profile.models.Person.objects.get(id=person_id)
    logging.info("Recommending bugs for %s" % p)
    suggested_searches = p.get_recommended_search_terms() # expensive?
    recommender = mysite.profile.controllers.RecommendBugs(suggested_searches, n=5) # cache fill prep...
    recommender.recommend() # cache fill do it.

def sync_bug_epoch_from_model_then_fill_recommended_bugs_cache():
    logging.info("Syncing bug epoch...")
    # Find the highest bug object modified date
    from django.db.models import Max
    highest_bug_mtime = mysite.search.models.Bug.all_bugs.all().aggregate(
        Max('modified_date')).values()[0]
    epoch = mysite.search.models.Epoch.get_for_model(
        mysite.search.models.Bug)
    # if the epoch is lower, then set the epoch to that value
    if highest_bug_mtime.timetuple() > epoch:
        mysite.search.models.Epoch.bump_for_model(
            mysite.search.models.Bug)
        logging.info("Whee! Bumped the epoch. Guess I'll fill the cache.")
        fill_recommended_bugs_cache()
    logging.info("Done syncing bug epoch.")

@task
def clear_people_page_cache(*args, **kwargs):
    shutil.rmtree(os.path.join(settings.WEB_ROOT,
                               'people'),
                  ignore_errors=True)

def clear_people_page_cache_task(*args, **kwargs):
    return clear_people_page_cache.delay()

def fill_people_page_cache():
    staticgenerator.quick_publish('/people/')

for model in [mysite.profile.models.PortfolioEntry,
              mysite.profile.models.Person,
              mysite.profile.models.Link_Person_Tag]:
    for signal_to_hook in [
        django.db.models.signals.post_save,
        django.db.models.signals.post_delete]:
        signal_to_hook.connect(
            clear_people_page_cache_task,
            sender=model)
<commit_msg>brown paper bag commit: Fix a NameError in profile.tasks<commit_after>import sys
import logging
import os
from datetime import timedelta
import datetime
from mysite.customs import ohloh
import urllib2
import urllib
import mysite.customs.github
import mysite.profile.models
from mysite.search.models import Project
from celery.decorators import task
from celery.task import Task
import celery.registry
import time
import random
import traceback
import mysite.profile.search_indexes
import mysite.profile.controllers
import shutil
import staticgenerator
import mysite.customs.bitbucket

from django.conf import settings
import django.core.cache

def create_citations_from_ohloh_contributor_facts(dia_id, ohloh_results):
    '''Input: A sequence of Ohloh ContributionFact dicts
    and the id of the DataImport they came from.

    Side-effect: Create matching structures in the DB
    and mark our success in the database.'''
    # {{{
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    for ohloh_contrib_info in ohloh_results:
        (project, _) = Project.objects.get_or_create(
                name=ohloh_contrib_info['project'])
        # FIXME: don't import if blacklisted
        (portfolio_entry, _) = mysite.profile.models.PortfolioEntry.objects.get_or_create(
                person=person, project=project)
        citation = mysite.profile.models.Citation.create_from_ohloh_contrib_info(ohloh_contrib_info)
        citation.portfolio_entry = portfolio_entry
        citation.data_import_attempt = dia 
        citation.save_and_check_for_duplicates()

    person.last_polled = datetime.datetime.now()
    dia.completed = True
    dia.save()
    person.save()
    # }}}

def create_citations_from_bitbucket_results(dia_id, results):
    """
    Given the input of results returned from a query against the 
    bitbucket.org user api, create user related contribution
    records.

    Note that non-authenticated calls to butbuket api returns limited
    project details.
    """
    
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    for repo in results:

        (project, _) = Project.objects.get_or_create(
            name=repo['name'])
        
        description = repo['description']
        if mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project).count() == 0:
            portfolio_entry = mysite.profile.models.PortfolioEntry(person=person,
                                             project=project,
                                             project_description=description)

            portfolio_entry.save()
        portfolio_entry = mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project)[0]
        
        citation = mysite.profile.models.Citation()
        citation.url =  'http://bitbucket.org/%s/%s/' % (dia.query.lower(), repo['name'])
        citation.contributor_role = 'Created a repository on Bitbucket.'
        citation.languages = ''
        
        citation.portfolio_entry = portfolio_entry
        citation.data_import_attempt = dia
        citation.save_and_check_for_duplicates()

    person.last_polled = datetime.datetime.now()
    dia.completed = True
    person.save()
    dia.save()

    # end create_citations_from_bitbucket_results

def rs_action(dia):
    oh = ohloh.get_ohloh()
    data, web_response = oh.get_contribution_info_by_username(
            dia.query, person=dia.person)
    dia.web_response = web_response
    dia.save()
    return data

def ou_action(dia):
    oh = ohloh.get_ohloh()
    data, web_response = oh.get_contribution_info_by_ohloh_username(
            dia.query, person=dia.person)
    dia.web_response = web_response
    dia.save()
    return data

def do_nothing_because_this_functionality_moved_to_twisted(*args):
    return None # This is moved to Twisted now.

def bb_action(dia):
    """
    Given a dia with a username, check to see if there are any 
    source packages maintained by the user.
    """
    return mysite.customs.bitbucket.get_user_repos(dia.query)

source2actual_action = {
        'gh': do_nothing_because_this_functionality_moved_to_twisted,
        'ga': do_nothing_because_this_functionality_moved_to_twisted,
        'db': do_nothing_because_this_functionality_moved_to_twisted,
        'lp': do_nothing_because_this_functionality_moved_to_twisted,
    
        'rs': rs_action,
        'ou': ou_action,
        'bb': bb_action
        }

source2result_handler = {
        'db': do_nothing_because_this_functionality_moved_to_twisted,
        'gh': do_nothing_because_this_functionality_moved_to_twisted,
        'ga': do_nothing_because_this_functionality_moved_to_twisted,
        'lp': do_nothing_because_this_functionality_moved_to_twisted,

        'rs': create_citations_from_ohloh_contributor_facts,
        'ou': create_citations_from_ohloh_contributor_facts,
        'bb': create_citations_from_bitbucket_results,
        }

class ReindexPerson(Task):
    def run(self, person_id, **kwargs):
        person = mysite.profile.models.Person.objects.get(id=person_id)
        pi = mysite.profile.search_indexes.PersonIndex(person)
        pi.update_object(person)

class GarbageCollectForwarders(Task):
    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.info("Started garbage collecting profile email forwarders")
        deleted_any = mysite.profile.models.Forwarder.garbage_collect()
        if deleted_any:
            # Well, in that case, we should purge the staticgenerator-generated cache of the people pages.
            clear_people_page_cache()


class RegeneratePostfixAliasesForForwarder(Task):
    def run(self, **kwargs):
        # Generate the table...
        lines = mysite.profile.models.Forwarder.generate_list_of_lines_for_postfix_table()
        # Save it where Postfix expects it...
        fd = open(settings.POSTFIX_FORWARDER_TABLE_PATH, 'w')
        fd.write('\n'.join(lines))
        fd.close()
        # Update the Postfix forwarder database. Note that we do not need
        # to ask Postfix to reload. Yay!
        # FIXME stop using os.system()
        os.system('/usr/sbin/postmap /etc/postfix/virtual_alias_maps') 

class FetchPersonDataFromOhloh(Task):
    name = "profile.FetchPersonDataFromOhloh"

    def run(self, dia_id, **kwargs):
        """"""
        dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
        try:
            logger = self.get_logger(**kwargs)
            logger.info("Starting job for <%s>" % dia)
            if dia.completed:
                logger.info("Bailing out job for <%s>" % dia)
                return
            results = source2actual_action[dia.source](dia)
            source2result_handler[dia.source](dia.id, results)
            logger.info("Results: %s" % repr(results))

        except Exception, e:
            # if the task is in debugging mode, bubble-up the exception
            if getattr(self, 'debugging', None):
                raise
            logger.error("Traceback: ")
            logger.error(traceback.format_exc())

            # else let the exception be logged but not bubble up
            dia.completed = True
            dia.failed = True
            dia.save()
            if hasattr(e, 'code'):
                code = str(e.code)
            else:
                code = 'UNKNOWN'
            if hasattr(e, 'geturl'):
                url = str(e.geturl())
            else:
                raise
            logger.error('Dying: ' + code + ' getting ' + url)
            raise ValueError, {'code': code, 'url': url}

try:
    celery.registry.tasks.register(RegeneratePostfixAliasesForForwarder)
    celery.registry.tasks.register(FetchPersonDataFromOhloh)
    celery.registry.tasks.register(ReindexPerson)
    celery.registry.tasks.register(GarbageCollectForwarders)
except celery.registry.AlreadyRegistered:
    pass

@task
def update_person_tag_cache(person__pk):
    person = mysite.profile.models.Person.objects.get(pk=person__pk)
    cache_key = person.get_tag_texts_cache_key()
    django.core.cache.cache.delete(cache_key)
    
    # This getter will populate the cache
    return person.get_tag_texts_for_map()

@task
def update_someones_pf_cache(person__pk):
    person = mysite.profile.models.Person.objects.get(pk=person__pk)
    cache_key = person.get_cache_key_for_projects()
    django.core.cache.cache.delete(cache_key)
    
    # This getter will populate the cache
    return person.get_names_of_nonarchived_projects()

def fill_recommended_bugs_cache():
    logging.info("Filling recommended bugs cache for all people.")
    for person in mysite.profile.models.Person.objects.all():
        fill_one_person_recommend_bugs_cache.apply(person_id=person.id)
    logging.info("Finished filling recommended bugs cache for all people.")

@task
def fill_one_person_recommend_bugs_cache(person_id):
    p = mysite.profile.models.Person.objects.get(id=person_id)
    logging.info("Recommending bugs for %s" % p)
    suggested_searches = p.get_recommended_search_terms() # expensive?
    recommender = mysite.profile.controllers.RecommendBugs(suggested_searches, n=5) # cache fill prep...
    recommender.recommend() # cache fill do it.

def sync_bug_epoch_from_model_then_fill_recommended_bugs_cache():
    logging.info("Syncing bug epoch...")
    # Find the highest bug object modified date
    from django.db.models import Max
    highest_bug_mtime = mysite.search.models.Bug.all_bugs.all().aggregate(
        Max('modified_date')).values()[0]
    epoch = mysite.search.models.Epoch.get_for_model(
        mysite.search.models.Bug)
    # if the epoch is lower, then set the epoch to that value
    if highest_bug_mtime.timetuple() > epoch:
        mysite.search.models.Epoch.bump_for_model(
            mysite.search.models.Bug)
        logging.info("Whee! Bumped the epoch. Guess I'll fill the cache.")
        fill_recommended_bugs_cache()
    logging.info("Done syncing bug epoch.")

@task
def clear_people_page_cache(*args, **kwargs):
    shutil.rmtree(os.path.join(settings.WEB_ROOT,
                               'people'),
                  ignore_errors=True)

def clear_people_page_cache_task(*args, **kwargs):
    return clear_people_page_cache.delay()

def fill_people_page_cache():
    staticgenerator.quick_publish('/people/')

for model in [mysite.profile.models.PortfolioEntry,
              mysite.profile.models.Person,
              mysite.profile.models.Link_Person_Tag]:
    for signal_to_hook in [
        django.db.models.signals.post_save,
        django.db.models.signals.post_delete]:
        signal_to_hook.connect(
            clear_people_page_cache_task,
            sender=model)

{'fraction_non_alphanumeric': 0.0908414690841469, 'fraction_numerical': 0.0008368200836820083, 'mean_word_length': 3.020747663551402, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3691628', 'n_tokens_mistral': 6621, 'n_tokens_neox': 6369, 'n_words': 1310}