<commit_before>import urllib, urllib2
import httplib
try:
    from hashlib import md5
except ImportError:
    from md5 import md5

from pprint import pformat as pp
from util import json, headers


def format_body(message, logger):

    logger.debug('http_emitter: json convert')
    payload = json.dumps(message)

    logger.debug('http_emitter: json converted, hash')
    #logger.debug('http_emitter:\n%s' % pp(message))

    payloadHash = md5(payload).hexdigest()
    postBackData = urllib.urlencode({'payload' : payload, 'hash' : payloadHash})

    logger.debug('http_emitter: hashed')
    return postBackData

def http_emitter(message, logger, agentConfig):
    try: 
        logger.debug('http_emitter: start')    

        # Post back the data
        postBackData = format_body(message, logger)

        logger.debug('http_emitter: attempting postback: ' + agentConfig['ddUrl'])
        
        # Build the request handler
        request = urllib2.Request(agentConfig['ddUrl'] + '/intake/', postBackData, headers(agentConfig))
        
        # Do the request, log any errors
        response = urllib2.urlopen(request)
        
        logger.debug('http_emitter: postback response: ' + str(response.read()))<commit_msg>Include api key in url so we don't have to parse the body to queue it. Don't submit payloads if the api key isn't set<commit_after>import urllib, urllib2
import httplib
try:
    from hashlib import md5
except ImportError:
    from md5 import md5

from pprint import pformat as pp
from util import json, headers


def format_body(message, logger):

    logger.debug('http_emitter: json convert')
    payload = json.dumps(message)

    logger.debug('http_emitter: json converted, hash')
    #logger.debug('http_emitter:\n%s' % pp(message))

    payloadHash = md5(payload).hexdigest()
    postBackData = urllib.urlencode({'payload' : payload, 'hash' : payloadHash})

    logger.debug('http_emitter: hashed')
    return postBackData

def http_emitter(message, logger, agentConfig):
    try: 
        logger.debug('http_emitter: start')    

        # Post back the data
        postBackData = format_body(message, logger)

        logger.debug('http_emitter: attempting postback: ' + agentConfig['ddUrl'])
        
        # Build the request handler
        apiKey = message.get('apiKey', None)
        if apiKey:
            request = urllib2.Request("%s/intake?api_key=%s" % (agentConfig['ddUrl'], apiKey), postBackData, headers(agentConfig))
            # Do the request, log any errors
            response = urllib2.urlopen(request)
            
            logger.debug('http_emitter: postback response: ' + str(response.read()))
        else:
            logger.error("No api key, not sending payload")
{'fraction_non_alphanumeric': 0.10485933503836317, 'fraction_numerical': 0.005115089514066497, 'mean_word_length': 3.212307692307692, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5575561', 'n_tokens_mistral': 825, 'n_tokens_neox': 787, 'n_words': 218}