<commit_before>import json
from tasks import update_project, get_people_totals, get_org_totals

def update_projects():
    f = open('data/projects.json', 'rb')
    project_list = json.loads(f.read())
    f.close()
    details = []
    for project_url in project_list:
        try:
            pj_details = update_project(project_url)
        except IOError:
            return 'Github is throttling. Just gonna try again after limit is reset.'
        if pj_details:
            details.append(pj_details)
    f = open('data/project_details.json', 'wb')
    f.write(json.dumps(details, indent=4))
    f.close()
    f = open('data/people.json', 'wb')
    f.write(json.dumps(get_people_totals(details), indent=4))
    f.close()
    orgs = [d for d in details if d['owner']['type'] == 'Organization']
    f = open('data/organizations.json', 'wb')
    f.write(json.dumps(get_org_totals(orgs), indent=4))
    f.close()
    return 'Updated'

if __name__ == "__main__":
    update_projects()<commit_msg>Put together example of how one might write out the files to S3 instead of the local file system. Relates to #2<commit_after>import json
import os
from tasks import update_project, get_people_totals, get_org_totals
from boto.s3.connection import S3Connection
from boto.s3.key import Key

BUCKET = os.environ['S3_BUCKET']
AWS_KEY = os.environ['AWS_ACCESS_KEY']
AWS_SECRET = os.environ['AWS_SECRET_KEY']

def update_projects():
    conn = S3Connection(AWS_KEY, AWS_SECRET)
    bucket = conn.get_bucket(BUCKET)
    pj_list = Key(bucket)
    pj_list.key = 'projects.json'
    project_list = json.loads(pj_list.get_contents_as_string())
    k.close()
    details = []
    for project_url in project_list:
        try:
            pj_details = update_project(project_url)
        except IOError:
            return 'Github is throttling. Just gonna try again after limit is reset.'
        if pj_details:
            details.append(pj_details)
    pj_details = Key(bucket)
    pj_details.key = 'project_details.json'
    pj_details.set_contents_from_string(json.dumps(details))
    pj_details.set_metadata('Content-Type', 'application/json')
    pj_details.set_acl('public-read')
    pj_details.close()
    people_list = Key(bucket)
    people_list.key = 'people.json'
    people_list.set_contents_from_string(json.dumps(get_people_totals(details)))
    people_list.set_metadata('Content-Type', 'application/json')
    people_list.set_acl('public-read')
    people_list.close()
    org_list = Key(bucket)
    org_list.key = 'organizations.json'
    org_list.set_contents_from_string(json.dumps(get_org_totals(orgs)))
    org_list.set_metadata('Content-Type', 'application/json')
    org_list.set_acl('public-read')
    org_list.close()
    return 'Updated'

if __name__ == "__main__":
    update_projects()
{'fraction_non_alphanumeric': 0.11552217453505007, 'fraction_numerical': 0.00357653791130186, 'mean_word_length': 3.708754208754209, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5153271', 'n_tokens_mistral': 943, 'n_tokens_neox': 950, 'n_words': 192}