<commit_before>import sys
import logging
import os
from datetime import timedelta
import datetime
from mysite.customs import ohloh
import urllib2
import urllib
from mysite.customs import lp_grabber
import mysite.customs.github
import mysite.customs.debianqa
import mysite.profile.models
from mysite.search.models import Project
from celery.decorators import task
from celery.task import Task
import celery.registry
import time
import random
import traceback
import mysite.profile.search_indexes
import mysite.profile.controllers
import shutil
import staticgenerator
import mysite.customs.bitbucket

from django.conf import settings
import django.core.cache

def create_citations_from_ohloh_contributor_facts(dia_id, ohloh_results):
    '''Input: A sequence of Ohloh ContributionFact dicts
    and the id of the DataImport they came from.

    Side-effect: Create matching structures in the DB
    and mark our success in the database.'''
    # {{{
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    for ohloh_contrib_info in ohloh_results:
        (project, _) = Project.objects.get_or_create(
                name=ohloh_contrib_info['project'])
        # FIXME: don't import if blacklisted
        (portfolio_entry, _) = mysite.profile.models.PortfolioEntry.objects.get_or_create(
                person=person, project=project)
        citation = mysite.profile.models.Citation.create_from_ohloh_contrib_info(ohloh_contrib_info)
        citation.portfolio_entry = portfolio_entry
        citation.data_import_attempt = dia 
        citation.save_and_check_for_duplicates()

    person.last_polled = datetime.datetime.now()
    dia.completed = True
    dia.save()
    person.save()
    # }}}

def create_citations_from_launchpad_results(dia_id, lp_results):
    "Input: A dictionary that maps from a project name to information "
    "about that project, e.g. "
    """
         {
             'F-Spot': {
                 'url': 'http://launchpad.net/f-spot',
                 'involvement_types': ['Bug Management', 'Bazaar Branches'],
                 'languages': ['python', 'ruby'],
                 'citation_url': "https://launchpad.net/~paulproteus",
             }
         }
    and the id of the DataImportAttempt they came from.

    Side-effect: Create matching structures in the DB
    and mark our success in the database."""
    # {{{
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    for project_name in lp_results:
        result = lp_results[project_name]
        for involvement_type in result['involvement_types']:

            (project, _) = Project.objects.get_or_create(name=project_name)

            # This works like a 'get_first_or_create'.
            # Sometimes there are more than one existing PortfolioEntry
            # with the details in question.
            # FIXME: This is untested.
            if mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project).count() == 0:
                portfolio_entry = mysite.profile.models.PortfolioEntry(person=person, project=project)
                portfolio_entry.save()
            portfolio_entry = mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project)[0]

            citation = mysite.profile.models.Citation()
            citation.languages = ", ".join(result['languages'])
            citation.contributor_role = involvement_type
            citation.portfolio_entry = portfolio_entry
            citation.data_import_attempt = dia
            citation.url = result['citation_url']
            citation.save_and_check_for_duplicates()

    person.last_polled = datetime.datetime.now()
    person.save()

    dia.completed = True
    dia.save()
    # }}}

def create_citations_from_bitbucket_results(dia_id, results):
    """
    Given the input of results returned from a query against the 
    bitbucket.org user api, create user related contribution
    records.

    Note that non-authenticated calls to butbuket api returns limited
    project details.
    """
    
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    for repo in results:

        (project, _) = Project.objects.get_or_create(
            name=repo['name'])
        
        description = repo['description']
        if mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project).count() == 0:
            portfolio_entry = mysite.profile.models.PortfolioEntry(person=person,
                                             project=project,
                                             project_description=description)

            portfolio_entry.save()
        portfolio_entry = mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project)[0]
        
        citation = mysite.profile.models.Citation()
        citation.url =  'http://bitbucket.org/%s/%s/' % (dia.query.lower(), repo['name'])
        citation.contributor_role = 'Created a repository on Bitbucket.'
        citation.languages = ''
        
        citation.portfolio_entry = portfolio_entry
        citation.data_import_attempt = dia
        citation.save_and_check_for_duplicates()

    person.last_polled = datetime.datetime.now()
    dia.completed = True
    person.save()
    dia.save()

    # end create_citations_from_bitbucket_results

def rs_action(dia):
    oh = ohloh.get_ohloh()
    data, web_response = oh.get_contribution_info_by_username(
            dia.query, person=dia.person)
    dia.web_response = web_response
    dia.save()
    return data

def ou_action(dia):
    oh = ohloh.get_ohloh()
    data, web_response = oh.get_contribution_info_by_ohloh_username(
            dia.query, person=dia.person)
    dia.web_response = web_response
    dia.save()
    return data

def do_nothing_because_this_functionality_moved_to_twisted(*args):
    return None # This is moved to Twisted now.

def lp_action(dia):
    # NB: Don't change the way this is called, because calling it this way
    # permits this function to be mocked when we test it.
    return lp_grabber.get_info_for_launchpad_username(dia.query)

def bb_action(dia):
    """
    Given a dia with a username, check to see if there are any 
    source packages maintained by the user.
    """
    return mysite.customs.bitbucket.get_user_repos(dia.query)

source2actual_action = {
        'gh': do_nothing_because_this_functionality_moved_to_twisted,
        'db': do_nothing_because_this_functionality_moved_to_twisted,
    
        'rs': rs_action,
        'ou': ou_action,
        'ga': do_nothing_because_this_functionality_moved_to_twisted,
        'lp': lp_action,
        'bb': bb_action
        }

source2result_handler = {
        'db': do_nothing_because_this_functionality_moved_to_twisted,
        'gh': do_nothing_because_this_functionality_moved_to_twisted,

        'rs': create_citations_from_ohloh_contributor_facts,
        'ou': create_citations_from_ohloh_contributor_facts,
        'ga': do_nothing_because_this_functionality_moved_to_twisted,
        'lp': create_citations_from_launchpad_results,
        'bb': create_citations_from_bitbucket_results,
        }

class ReindexPerson(Task):
    def run(self, person_id, **kwargs):
        person = mysite.profile.models.Person.objects.get(id=person_id)
        pi = mysite.profile.search_indexes.PersonIndex(person)
        pi.update_object(person)
<commit_msg>profile.tasks: Fiddle with the location of removed code.<commit_after>import sys
import logging
import os
from datetime import timedelta
import datetime
from mysite.customs import ohloh
import urllib2
import urllib
from mysite.customs import lp_grabber
import mysite.customs.github
import mysite.customs.debianqa
import mysite.profile.models
from mysite.search.models import Project
from celery.decorators import task
from celery.task import Task
import celery.registry
import time
import random
import traceback
import mysite.profile.search_indexes
import mysite.profile.controllers
import shutil
import staticgenerator
import mysite.customs.bitbucket

from django.conf import settings
import django.core.cache

def create_citations_from_ohloh_contributor_facts(dia_id, ohloh_results):
    '''Input: A sequence of Ohloh ContributionFact dicts
    and the id of the DataImport they came from.

    Side-effect: Create matching structures in the DB
    and mark our success in the database.'''
    # {{{
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    for ohloh_contrib_info in ohloh_results:
        (project, _) = Project.objects.get_or_create(
                name=ohloh_contrib_info['project'])
        # FIXME: don't import if blacklisted
        (portfolio_entry, _) = mysite.profile.models.PortfolioEntry.objects.get_or_create(
                person=person, project=project)
        citation = mysite.profile.models.Citation.create_from_ohloh_contrib_info(ohloh_contrib_info)
        citation.portfolio_entry = portfolio_entry
        citation.data_import_attempt = dia 
        citation.save_and_check_for_duplicates()

    person.last_polled = datetime.datetime.now()
    dia.completed = True
    dia.save()
    person.save()
    # }}}

def create_citations_from_launchpad_results(dia_id, lp_results):
    "Input: A dictionary that maps from a project name to information "
    "about that project, e.g. "
    """
         {
             'F-Spot': {
                 'url': 'http://launchpad.net/f-spot',
                 'involvement_types': ['Bug Management', 'Bazaar Branches'],
                 'languages': ['python', 'ruby'],
                 'citation_url': "https://launchpad.net/~paulproteus",
             }
         }
    and the id of the DataImportAttempt they came from.

    Side-effect: Create matching structures in the DB
    and mark our success in the database."""
    # {{{
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    for project_name in lp_results:
        result = lp_results[project_name]
        for involvement_type in result['involvement_types']:

            (project, _) = Project.objects.get_or_create(name=project_name)

            # This works like a 'get_first_or_create'.
            # Sometimes there are more than one existing PortfolioEntry
            # with the details in question.
            # FIXME: This is untested.
            if mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project).count() == 0:
                portfolio_entry = mysite.profile.models.PortfolioEntry(person=person, project=project)
                portfolio_entry.save()
            portfolio_entry = mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project)[0]

            citation = mysite.profile.models.Citation()
            citation.languages = ", ".join(result['languages'])
            citation.contributor_role = involvement_type
            citation.portfolio_entry = portfolio_entry
            citation.data_import_attempt = dia
            citation.url = result['citation_url']
            citation.save_and_check_for_duplicates()

    person.last_polled = datetime.datetime.now()
    person.save()

    dia.completed = True
    dia.save()
    # }}}

def create_citations_from_bitbucket_results(dia_id, results):
    """
    Given the input of results returned from a query against the 
    bitbucket.org user api, create user related contribution
    records.

    Note that non-authenticated calls to butbuket api returns limited
    project details.
    """
    
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    for repo in results:

        (project, _) = Project.objects.get_or_create(
            name=repo['name'])
        
        description = repo['description']
        if mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project).count() == 0:
            portfolio_entry = mysite.profile.models.PortfolioEntry(person=person,
                                             project=project,
                                             project_description=description)

            portfolio_entry.save()
        portfolio_entry = mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project)[0]
        
        citation = mysite.profile.models.Citation()
        citation.url =  'http://bitbucket.org/%s/%s/' % (dia.query.lower(), repo['name'])
        citation.contributor_role = 'Created a repository on Bitbucket.'
        citation.languages = ''
        
        citation.portfolio_entry = portfolio_entry
        citation.data_import_attempt = dia
        citation.save_and_check_for_duplicates()

    person.last_polled = datetime.datetime.now()
    dia.completed = True
    person.save()
    dia.save()

    # end create_citations_from_bitbucket_results

def rs_action(dia):
    oh = ohloh.get_ohloh()
    data, web_response = oh.get_contribution_info_by_username(
            dia.query, person=dia.person)
    dia.web_response = web_response
    dia.save()
    return data

def ou_action(dia):
    oh = ohloh.get_ohloh()
    data, web_response = oh.get_contribution_info_by_ohloh_username(
            dia.query, person=dia.person)
    dia.web_response = web_response
    dia.save()
    return data

def do_nothing_because_this_functionality_moved_to_twisted(*args):
    return None # This is moved to Twisted now.

def lp_action(dia):
    # NB: Don't change the way this is called, because calling it this way
    # permits this function to be mocked when we test it.
    return lp_grabber.get_info_for_launchpad_username(dia.query)

def bb_action(dia):
    """
    Given a dia with a username, check to see if there are any 
    source packages maintained by the user.
    """
    return mysite.customs.bitbucket.get_user_repos(dia.query)

source2actual_action = {
        'gh': do_nothing_because_this_functionality_moved_to_twisted,
        'ga': do_nothing_because_this_functionality_moved_to_twisted,
        'db': do_nothing_because_this_functionality_moved_to_twisted,
    
        'rs': rs_action,
        'ou': ou_action,
        'lp': lp_action,
        'bb': bb_action
        }

source2result_handler = {
        'db': do_nothing_because_this_functionality_moved_to_twisted,
        'gh': do_nothing_because_this_functionality_moved_to_twisted,
        'ga': do_nothing_because_this_functionality_moved_to_twisted,

        'rs': create_citations_from_ohloh_contributor_facts,
        'ou': create_citations_from_ohloh_contributor_facts,
        'lp': create_citations_from_launchpad_results,
        'bb': create_citations_from_bitbucket_results,
        }

class ReindexPerson(Task):
    def run(self, person_id, **kwargs):
        person = mysite.profile.models.Person.objects.get(id=person_id)
        pi = mysite.profile.search_indexes.PersonIndex(person)
        pi.update_object(person)

{'fraction_non_alphanumeric': 0.08622411611958186, 'fraction_numerical': 0.0009321526066981823, 'mean_word_length': 3.007470651013874, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7535711', 'n_tokens_mistral': 4552, 'n_tokens_neox': 4301, 'n_words': 941}