<commit_before>import logging
import urlparse
import json
import itertools
from StringIO import StringIO
from pprint import pformat
from twisted.internet import reactor
from twisted.internet.defer import Deferred, DeferredQueue, setDebugging, inlineCallbacks
from twisted.web.client import Agent, ProxyAgent, FileBodyProducer
from twisted.protocols.basic import LineReceiver
from twisted.internet.endpoints import UNIXClientEndpoint, TCP4ClientEndpoint
from twisted.web.http_headers import Headers
from twisted.internet.error import ConnectionRefusedError
from config import CALLBACK_URL, DOCKER_HOST
from json_producer import JsonProducer
from events import DockerEventsProtocol
from utils import getBody, waitFor

log = logging.getLogger(__name__)
queued_events = DeferredQueue()
http_agent = Agent(reactor)


def callback(response):
    """GET /events

    Called after received response from Docker.
    """
    log.debug('Callback')
    log.debug('Response version: {}'.format(response.version))
    log.debug('Response code: {}'.format(response.code))
    log.debug('Response phrase: {}'.format(response.phrase))
    log.debug('Response headers: {}'.format(
        pformat(list(response.headers.getAllRawHeaders()))))
    dfd = Deferred()
    protocol = DockerEventsProtocol(queued_events, dfd)
    response.deliverBody(protocol)
    return dfd


def errback(e):
    log.error('Error: {0}'.format(e))


@inlineCallbacks
def consumeEvents():
    for event_number in itertools.count(1):
        log.info('Waiting for event... {}'.format(event_number))
        event = yield queued_events.get()
        log.debug('Event: {!r}'.format(event))
        while True:
            try:
                response = yield http_agent.request(
                    'POST',
                    CALLBACK_URL,
                    Headers({
                        'Content-Type': ['application/javascript']
                    }),
                    FileBodyProducer(StringIO(json.dumps(event))))
                response_body = yield getBody(response)       
                log.debug('Event transmitted')
                log.debug('Transmit response version: {}'.format(response.version))<commit_msg>Better structure of payload.<commit_after>import logging
import urlparse
import json
import itertools
from StringIO import StringIO
from pprint import pformat
from twisted.internet import reactor
from twisted.internet.defer import Deferred, DeferredQueue, setDebugging, inlineCallbacks
from twisted.web.client import Agent, ProxyAgent, FileBodyProducer
from twisted.protocols.basic import LineReceiver
from twisted.internet.endpoints import UNIXClientEndpoint, TCP4ClientEndpoint
from twisted.web.http_headers import Headers
from twisted.internet.error import ConnectionRefusedError
from config import CALLBACK_URL, DOCKER_HOST
from json_producer import JsonProducer
from events import DockerEventsProtocol
from utils import getBody, waitFor

log = logging.getLogger(__name__)
queued_events = DeferredQueue()
http_agent = Agent(reactor)


def callback(response):
    """GET /events

    Called after received response from Docker.
    """
    log.debug('Callback')
    log.debug('Response version: {}'.format(response.version))
    log.debug('Response code: {}'.format(response.code))
    log.debug('Response phrase: {}'.format(response.phrase))
    log.debug('Response headers: {}'.format(
        pformat(list(response.headers.getAllRawHeaders()))))
    dfd = Deferred()
    protocol = DockerEventsProtocol(queued_events, dfd)
    response.deliverBody(protocol)
    return dfd


def errback(e):
    log.error('Error: {0}'.format(e))


@inlineCallbacks
def consumeEvents():
    for event_number in itertools.count(1):
        log.info('Waiting for event... {}'.format(event_number))
        event = yield queued_events.get()
        log.debug('Event: {!r}'.format(event))
        while True:
            try:
                payload = {
                    'docker_host': DOCKER_HOST,
                    'event': event
                }
                response = yield http_agent.request(
                    'POST',
                    CALLBACK_URL,
                    Headers({
                        'Content-Type': ['application/javascript']
                    }),
                    FileBodyProducer(StringIO(json.dumps(payload))))
                response_body = yield getBody(response)       
                log.debug('Event transmitted')
                log.debug('Transmit response version: {}'.format(response.version))
{'fraction_non_alphanumeric': 0.0982773851590106, 'fraction_numerical': 0.001325088339222615, 'mean_word_length': 3.0473637176050046, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6967709', 'n_tokens_mistral': 1221, 'n_tokens_neox': 1176, 'n_words': 301}