<commit_before>import sys
import os
from datetime import timedelta
import datetime
from mysite.customs import ohloh
import urllib2
import urllib
from mysite.customs import lp_grabber
import mysite.customs.github
import mysite.customs.debianqa
import mysite.profile.models
from mysite.search.models import Project
from celery.task import Task, PeriodicTask
import celery.registry
import time
import random
import traceback
import mysite.profile.search_indexes

from django.conf import settings
import django.core.cache

def create_citations_from_ohloh_contributor_facts(dia_id, ohloh_results):
    '''Input: A sequence of Ohloh ContributionFact dicts
    and the id of the DataImport they came from.

    Side-effect: Create matching structures in the DB
    and mark our success in the database.'''
    # {{{
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    for ohloh_contrib_info in ohloh_results:
        (project, _) = Project.objects.get_or_create(
                name=ohloh_contrib_info['project'])
        # FIXME: don't import if blacklisted
        (portfolio_entry, _) = mysite.profile.models.PortfolioEntry.objects.get_or_create(
                person=person, project=project)
        citation = mysite.profile.models.Citation.create_from_ohloh_contrib_info(ohloh_contrib_info)
        citation.portfolio_entry = portfolio_entry
        citation.data_import_attempt = dia 
        citation.save_and_check_for_duplicates()

    person.last_polled = datetime.datetime.now()
    dia.completed = True
    dia.save()
    person.save()
    # }}}

def create_citations_from_launchpad_results(dia_id, lp_results):
    "Input: A dictionary that maps from a project name to information "
    "about that project, e.g. "
    """
         {
             'F-Spot': {
                 'url': 'http://launchpad.net/f-spot',
                 'involvement_types': ['Bug Management', 'Bazaar Branches'],
                 'languages': ['python', 'ruby'],
                 'citation_url': "https://launchpad.net/~paulproteus",
             }
         }
    and the id of the DataImportAttempt they came from.

    Side-effect: Create matching structures in the DB
    and mark our success in the database."""
    # {{{
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    for project_name in lp_results:
        result = lp_results[project_name]
        for involvement_type in result['involvement_types']:

            (project, _) = Project.objects.get_or_create(name=project_name)

            # This works like a 'get_first_or_create'.
            # Sometimes there are more than one existing PortfolioEntry
            # with the details in question.
            # FIXME: This is untested.
            if mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project).count() == 0:
                portfolio_entry = mysite.profile.models.PortfolioEntry(person=person, project=project)
                portfolio_entry.save()
            portfolio_entry = mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project)[0]

            citation = mysite.profile.models.Citation()
            citation.languages = ", ".join(result['languages'])
            citation.contributor_role = involvement_type
            citation.portfolio_entry = portfolio_entry
            citation.data_import_attempt = dia
            citation.url = result['citation_url']
            citation.save_and_check_for_duplicates()

    person.last_polled = datetime.datetime.now()
    person.save()

    dia.completed = True
    dia.save()
    # }}}

def create_citations_from_github_activity_feed_results(dia_id, results):
    return create_citations_from_github_results(dia_id, results,
                                                override_contrib='Collaborated on')

def create_citations_from_github_results(dia_id, results,
                                         override_contrib=None):
    repos, dict_mapping_repos_to_languages = results
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    
    for repo in repos:
        (project, _) = Project.objects.get_or_create(name=repo.name)

        # FIXME: Populate project description, name, etc.

        if mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project).count() == 0:
            portfolio_entry = mysite.profile.models.PortfolioEntry(person=person,
                                             project=project,
                                             project_description=repo.description)
            portfolio_entry.save()
        portfolio_entry = mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project)[0]
            
        citation = mysite.profile.models.Citation()
        citation.languages = "" # FIXME ", ".join(result['languages'])
        if repo.fork:
            citation.contributor_role = 'Forked'
        else:
            citation.contributor_role = 'Started'
        if override_contrib:
            citation.contributor_role = override_contrib
        citation.portfolio_entry = portfolio_entry
        citation.data_import_attempt = dia<commit_msg>Make sure the Project model gets a string rather than another false value.<commit_after>import sys
import os
from datetime import timedelta
import datetime
from mysite.customs import ohloh
import urllib2
import urllib
from mysite.customs import lp_grabber
import mysite.customs.github
import mysite.customs.debianqa
import mysite.profile.models
from mysite.search.models import Project
from celery.task import Task, PeriodicTask
import celery.registry
import time
import random
import traceback
import mysite.profile.search_indexes

from django.conf import settings
import django.core.cache

def create_citations_from_ohloh_contributor_facts(dia_id, ohloh_results):
    '''Input: A sequence of Ohloh ContributionFact dicts
    and the id of the DataImport they came from.

    Side-effect: Create matching structures in the DB
    and mark our success in the database.'''
    # {{{
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    for ohloh_contrib_info in ohloh_results:
        (project, _) = Project.objects.get_or_create(
                name=ohloh_contrib_info['project'])
        # FIXME: don't import if blacklisted
        (portfolio_entry, _) = mysite.profile.models.PortfolioEntry.objects.get_or_create(
                person=person, project=project)
        citation = mysite.profile.models.Citation.create_from_ohloh_contrib_info(ohloh_contrib_info)
        citation.portfolio_entry = portfolio_entry
        citation.data_import_attempt = dia 
        citation.save_and_check_for_duplicates()

    person.last_polled = datetime.datetime.now()
    dia.completed = True
    dia.save()
    person.save()
    # }}}

def create_citations_from_launchpad_results(dia_id, lp_results):
    "Input: A dictionary that maps from a project name to information "
    "about that project, e.g. "
    """
         {
             'F-Spot': {
                 'url': 'http://launchpad.net/f-spot',
                 'involvement_types': ['Bug Management', 'Bazaar Branches'],
                 'languages': ['python', 'ruby'],
                 'citation_url': "https://launchpad.net/~paulproteus",
             }
         }
    and the id of the DataImportAttempt they came from.

    Side-effect: Create matching structures in the DB
    and mark our success in the database."""
    # {{{
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    for project_name in lp_results:
        result = lp_results[project_name]
        for involvement_type in result['involvement_types']:

            (project, _) = Project.objects.get_or_create(name=project_name)

            # This works like a 'get_first_or_create'.
            # Sometimes there are more than one existing PortfolioEntry
            # with the details in question.
            # FIXME: This is untested.
            if mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project).count() == 0:
                portfolio_entry = mysite.profile.models.PortfolioEntry(person=person, project=project)
                portfolio_entry.save()
            portfolio_entry = mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project)[0]

            citation = mysite.profile.models.Citation()
            citation.languages = ", ".join(result['languages'])
            citation.contributor_role = involvement_type
            citation.portfolio_entry = portfolio_entry
            citation.data_import_attempt = dia
            citation.url = result['citation_url']
            citation.save_and_check_for_duplicates()

    person.last_polled = datetime.datetime.now()
    person.save()

    dia.completed = True
    dia.save()
    # }}}

def create_citations_from_github_activity_feed_results(dia_id, results):
    return create_citations_from_github_results(dia_id, results,
                                                override_contrib='Collaborated on')

def create_citations_from_github_results(dia_id, results,
                                         override_contrib=None):
    repos, dict_mapping_repos_to_languages = results
    dia = mysite.profile.models.DataImportAttempt.objects.get(id=dia_id)
    person = dia.person
    
    for repo in repos:
        (project, _) = Project.objects.get_or_create(name=repo.name)

        # FIXME: Populate project description, name, etc.

        if mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project).count() == 0:
            portfolio_entry = mysite.profile.models.PortfolioEntry(person=person,
                                             project=project,
                                             project_description=repo.description or '')
            portfolio_entry.save()
        portfolio_entry = mysite.profile.models.PortfolioEntry.objects.filter(person=person, project=project)[0]
            
        citation = mysite.profile.models.Citation()
        citation.languages = "" # FIXME ", ".join(result['languages'])
        if repo.fork:
            citation.contributor_role = 'Forked'
        else:
            citation.contributor_role = 'Started'
        if override_contrib:
            citation.contributor_role = override_contrib
        citation.portfolio_entry = portfolio_entry
        citation.data_import_attempt = dia
{'fraction_non_alphanumeric': 0.08603267541788649, 'fraction_numerical': 0.0009443762394938143, 'mean_word_length': 2.710581639803784, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3081739', 'n_tokens_mistral': 3012, 'n_tokens_neox': 2849, 'n_words': 620}