<commit_before>import copy
import os
import random
import string
import sys
from Queue import Queue
from contextlib import contextmanager
from threading import Thread

import time

_script_dir = os.path.dirname(os.path.realpath(__file__))

# Define variables for use in tests
project_dir = os.path.dirname(os.path.dirname(_script_dir))
appmock_dir = os.path.join(project_dir, 'appmock')
docker_dir = os.path.join(project_dir, 'bamboos', 'docker')
annotations_dir = os.path.join(project_dir, 'test', 'annotations')

# Append useful modules to the path
sys.path = [appmock_dir, docker_dir, annotations_dir] + sys.path

from proto import messages_pb2


def random_int():
    return random.randint(1, 100)


def random_str(size=random_int()):
    return ''.join(random.choice(string.ascii_uppercase + string.digits) for
                   _ in xrange(size))


def msg_num_param(num):
    return Parameter('msg_num', 'Number of sent messages.', num)


def msg_size_param(size, unit):
    return Parameter('msg_size', 'Size of each message.', size, unit)


def send_time_param(value, unit='ms'):
    return Parameter('send_time', 'Summary time since first messages sent '
                                  'till last message received.', value, unit)


def recv_time_param(value, unit='ms'):
    return Parameter('recv_time', 'Summary time since first reply sent '
                                  'till last reply received.', value, unit)


def communicate_time_param(value, unit='ms'):
    return Parameter('communicate_time', 'Summary time since first message '
                                         'sent till last reply received.',
                     value, unit)


def mbps_param(msg_num, msg_size, us):
    return Parameter('mbps', 'Transfer speed.',
                     (1000000. * msg_num * msg_size) / (1048576 * us), 'MB/s')


def translate_unit(unit):
    if unit == 'kB':
        return 1024
    elif unit == 'MB':
        return 1048576
    else:
        return 1


def _with_reply_process(endpoint, responses, queue):
    for response in responses:
        [received_msg] = endpoint.wait_for_any_messages(return_history=True)
        endpoint.client.reset_tcp_history()

        client_message = messages_pb2.ClientMessage()
        client_message.ParseFromString(received_msg)

        response.message_id = client_message.message_id.encode('utf-8')
        endpoint.send(response.SerializeToString())

        queue.put(client_message)


@contextmanager
def reply(endpoint, responses):
    if not isinstance(responses, list):
        responses = [responses]

    queue = Queue()
    p = Thread(target=_with_reply_process, args=(endpoint, responses, queue))
    p.start()

    try:
        yield queue
    finally:
        p.join()


class PerformanceResult(object):
    def set(self, value):
        self.value = value<commit_msg>VFS-1371 Reply only if message_id is set in integration tests helpers.<commit_after>import copy
import os
import random
import string
import sys
from Queue import Queue
from contextlib import contextmanager
from threading import Thread

import time

_script_dir = os.path.dirname(os.path.realpath(__file__))

# Define variables for use in tests
project_dir = os.path.dirname(os.path.dirname(_script_dir))
appmock_dir = os.path.join(project_dir, 'appmock')
docker_dir = os.path.join(project_dir, 'bamboos', 'docker')
annotations_dir = os.path.join(project_dir, 'test', 'annotations')

# Append useful modules to the path
sys.path = [appmock_dir, docker_dir, annotations_dir] + sys.path

from proto import messages_pb2


def random_int():
    return random.randint(1, 100)


def random_str(size=random_int()):
    return ''.join(random.choice(string.ascii_uppercase + string.digits) for
                   _ in xrange(size))


def msg_num_param(num):
    return Parameter('msg_num', 'Number of sent messages.', num)


def msg_size_param(size, unit):
    return Parameter('msg_size', 'Size of each message.', size, unit)


def send_time_param(value, unit='ms'):
    return Parameter('send_time', 'Summary time since first messages sent '
                                  'till last message received.', value, unit)


def recv_time_param(value, unit='ms'):
    return Parameter('recv_time', 'Summary time since first reply sent '
                                  'till last reply received.', value, unit)


def communicate_time_param(value, unit='ms'):
    return Parameter('communicate_time', 'Summary time since first message '
                                         'sent till last reply received.',
                     value, unit)


def mbps_param(msg_num, msg_size, us):
    return Parameter('mbps', 'Transfer speed.',
                     (1000000. * msg_num * msg_size) / (1048576 * us), 'MB/s')


def translate_unit(unit):
    if unit == 'kB':
        return 1024
    elif unit == 'MB':
        return 1048576
    else:
        return 1


def _with_reply_process(endpoint, responses, queue):
    while responses:
        [received_msg] = endpoint.wait_for_any_messages(return_history=True)
        endpoint.client.reset_tcp_history()

        client_message = messages_pb2.ClientMessage()
        client_message.ParseFromString(received_msg)

        if hasattr(client_message, 'message_id'):
            response = responses.pop(0)
            response.message_id = client_message.message_id.encode('utf-8')
            endpoint.send(response.SerializeToString())

        queue.put(client_message)


@contextmanager
def reply(endpoint, responses):
    if not isinstance(responses, list):
        responses = [responses]

    queue = Queue()
    p = Thread(target=_with_reply_process, args=(endpoint, responses, queue))
    p.start()

    try:
        yield queue
    finally:
        p.join()


class PerformanceResult(object):
    def set(self, value):
        self.value = value
{'fraction_non_alphanumeric': 0.10309984586401781, 'fraction_numerical': 0.012159616372666552, 'mean_word_length': 3.13012729844413, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7226052', 'n_tokens_mistral': 1768, 'n_tokens_neox': 1661, 'n_words': 454}