<commit_before>import os
import requests
import json
from aggregation_api import base_directory,AggregationAPI
from csv_output import CsvOut
import yaml
import rollbar
import traceback

def aggregate(project_id, token, href, metadata, environment):
    try:
        panoptes_file = open("/app/config/aggregation.yml","rb")
        api_details = yaml.load(panoptes_file)
        rollbar_token = api_details["default"]["rollbar"]
    except IOError:
        panoptes_file = open(base_directory+"/Databases/aggregation.yml","rb")
        api_details = yaml.load(panoptes_file)
        rollbar_token = api_details["staging"]["rollbar"]

    print rollbar_token
    rollbar.init(rollbar_token,environment)
    rollbar.report_message("step1","info")

    try:
        with AggregationAPI(project_id, environment=environment) as project:
            project.__migrate__()
            project.__aggregate__()
            rollbar.report_message("step2","info")

            with CsvOut(project) as writer:
                rollbar.report_message("step3","info")
                tarpath = writer.__write_out__(compress=True)
                response = send_uploading(metadata, token, href)
                url = response.json()["media"][0]["src"]
                with open(tarpath, 'rb') as tarball:
                    requests.put(url, headers={'Content-Type': 'application/x-gzip'}, data=tarball)
                os.remove(tarpath)
                send_finished(metadata, token, href)
                rollbar.report_message("step4","info")
    except Exception, err:
        print traceback.format_exc()
        rollbar.report_exc_info()



def get_etag(href, token):
    response = requests.get(href, headers=headers(token), params={'admin': True})
    return response.headers['ETag']

def send_uploading(metadata, token, href):
    metadata['state'] = 'uploading'<commit_msg>just trying to get rollbar to report<commit_after>import os
import requests
import json
from aggregation_api import base_directory,AggregationAPI
from csv_output import CsvOut
import yaml
import rollbar
import traceback

def aggregate(project_id, token, href, metadata, environment):
    try:
        panoptes_file = open("/app/config/aggregation.yml","rb")
        api_details = yaml.load(panoptes_file)
        rollbar_token = api_details["default"]["rollbar"]
    except IOError:
        panoptes_file = open(base_directory+"/Databases/aggregation.yml","rb")
        api_details = yaml.load(panoptes_file)
        rollbar_token = api_details["staging"]["rollbar"]

    print rollbar_token
    rollbar.init(rollbar_token,environment)
    print "hello"
    rollbar.report_message("step5","info")

    # try:
    #     with AggregationAPI(project_id, environment=environment) as project:
    #         project.__migrate__()
    #         project.__aggregate__()
    #         rollbar.report_message("step6","info")
    #
    #         with CsvOut(project) as writer:
    #             rollbar.report_message("step3","info")
    #             tarpath = writer.__write_out__(compress=True)
    #             response = send_uploading(metadata, token, href)
    #             url = response.json()["media"][0]["src"]
    #             with open(tarpath, 'rb') as tarball:
    #                 requests.put(url, headers={'Content-Type': 'application/x-gzip'}, data=tarball)
    #             os.remove(tarpath)
    #             send_finished(metadata, token, href)
    #             rollbar.report_message("step4","info")
    # except Exception, err:
    #     print traceback.format_exc()
    #     rollbar.report_exc_info()



def get_etag(href, token):
    response = requests.get(href, headers=headers(token), params={'admin': True})
    return response.headers['ETag']

def send_uploading(metadata, token, href):
    metadata['state'] = 'uploading'
{'fraction_non_alphanumeric': 0.11649214659685864, 'fraction_numerical': 0.002617801047120419, 'mean_word_length': 3.0348468848996832, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1594245', 'n_tokens_mistral': 1129, 'n_tokens_neox': 1106, 'n_words': 213}