<commit_before>import json
import os
from urlparse import urlparse
from csv import DictReader, Sniffer
from StringIO import StringIO
from operator import itemgetter
from itertools import groupby
from time import sleep
from json import dumps

from requests import get
from boto.s3.connection import S3Connection
from boto.s3.key import Key

BUCKET = os.environ['S3_BUCKET']
gdocs_url = 'https://docs.google.com/a/codeforamerica.org/spreadsheet/ccc?key=0ArHmv-6U1drqdGNCLWV5Q0d5YmllUzE5WGlUY3hhT2c&output=csv'

if 'GITHUB_TOKEN' in os.environ:
    github_auth = (os.environ['GITHUB_TOKEN'], '')
else:
    github_auth = None

def get_github_api(url):
    '''
    '''
    got = get(url, auth=github_auth)
    
    if github_auth is None:
        sleep(1) # be nice to Github
    
    return got

def get_orgs():
    ''' Get a row for each organization from the Brigade Info spreadsheet.

        Return a list of dictionaries, one for each row past the header.
    '''
    got = get(gdocs_url)
    data = list(DictReader(StringIO(got.text)))
    
    return data

def load_projects(projects_url):
    ''' Load a list of projects from a given URL.
    '''
    got = get(projects_url)

    try:
        data = [dict(code_url=item) for item in got.json()]

    except ValueError:
        projects = got.text.splitlines()
        dialect = Sniffer().sniff(projects[0])
        data = list(DictReader(projects, dialect=dialect))
    
    return [update_project_info(row) for row in data[:2]]

def update_project_info(row):
    ''' Update info from Github, if it's missing.
    
        Modify the row in-place with new info and return it.

        Complete repository project details go into extras, for example
        project details from Github can be found under "github_extras".
    '''
    if 'code_url' not in row:
        return row
    
    _, host, path, _, _, _ = urlparse(row['code_url'])
    
    if host == 'github.com':
        repo_url = 'https://api.github.com/repos' + path
        
        got = get_github_api(repo_url)
        
        if got.status_code in range(400, 499):
            raise IOError('We done got throttled')
        
        repo = got.json()
        
        if 'name' not in row or not row['name']:
            row['name'] = repo['name']
        
        if 'description' not in row or not row['description']:
            row['description'] = repo['description']
        
        if 'link_url' not in row or not row['link_url']:
            row['link_url'] = repo['homepage']
        
        row['github_extras'] = repo
    
    return row

def reformat_project_info(input):
    ''' Return a clone of the project hash, formatted for use by opengovhacknight.org.
    
        The representation here is specifically expected to be used on this page:
        http://opengovhacknight.org/projects.html
    '''
    output = dict()<commit_msg>Clarified behavior of update_project_info()<commit_after>import os
import sys
from urlparse import urlparse
from csv import DictReader, Sniffer
from StringIO import StringIO
from operator import itemgetter
from itertools import groupby
from time import sleep
from json import dumps

from requests import get
from boto.s3.connection import S3Connection
from boto.s3.key import Key

BUCKET = os.environ['S3_BUCKET']
gdocs_url = 'https://docs.google.com/a/codeforamerica.org/spreadsheet/ccc?key=0ArHmv-6U1drqdGNCLWV5Q0d5YmllUzE5WGlUY3hhT2c&output=csv'

if 'GITHUB_TOKEN' in os.environ:
    github_auth = (os.environ['GITHUB_TOKEN'], '')
else:
    github_auth = None

def get_github_api(url):
    '''
    '''
    print >> sys.stderr, 'Asking Github for', url
    
    got = get(url, auth=github_auth)
    
    if github_auth is None:
        sleep(1) # be nice to Github
    
    return got

def get_orgs():
    ''' Get a row for each organization from the Brigade Info spreadsheet.

        Return a list of dictionaries, one for each row past the header.
    '''
    got = get(gdocs_url)
    data = list(DictReader(StringIO(got.text)))
    
    return data

def load_projects(projects_url):
    ''' Load a list of projects from a given URL.
    '''
    got = get(projects_url)

    try:
        data = [dict(code_url=item) for item in got.json()]

    except ValueError:
        projects = got.text.splitlines()
        dialect = Sniffer().sniff(projects[0])
        data = list(DictReader(projects, dialect=dialect))
    
    map(update_project_info, data)
    
    return data

def update_project_info(row):
    ''' Update info from Github, if it's missing.
    
        Modify the row in-place and return nothing.

        Complete repository project details go into extras, for example
        project details from Github can be found under "github_extras".
    '''
    if 'code_url' not in row:
        return row
    
    _, host, path, _, _, _ = urlparse(row['code_url'])
    
    if host == 'github.com':
        repo_url = 'https://api.github.com/repos' + path
        
        got = get_github_api(repo_url)
        
        if got.status_code in range(400, 499):
            raise IOError('We done got throttled')
        
        repo = got.json()
        
        if 'name' not in row or not row['name']:
            row['name'] = repo['name']
        
        if 'description' not in row or not row['description']:
            row['description'] = repo['description']
        
        if 'link_url' not in row or not row['link_url']:
            row['link_url'] = repo['homepage']
        
        row['github_extras'] = repo

def reformat_project_info(input):
    ''' Return a clone of the project hash, formatted for use by opengovhacknight.org.
    
        The representation here is specifically expected to be used on this page:
        http://opengovhacknight.org/projects.html
    '''
    output = dict()
{'fraction_non_alphanumeric': 0.09784296807592753, 'fraction_numerical': 0.007420189818809319, 'mean_word_length': 2.6452830188679246, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1299488', 'n_tokens_mistral': 1818, 'n_tokens_neox': 1792, 'n_words': 541}