<commit_before># Copyright (c) 2015, CodiLime Inc.

import time

from code_executor import CodeExecutor
from gateway_resolver import GatewayResolver
from py4j.java_gateway import JavaGateway, GatewayClient, java_import
from pyspark import SparkContext, SQLContext, SparkConf

# TODO: get the address from env, or hardcode a sensible one
RABBIT_MQ_ADDRESS = ("localhost", 5672)


class PyExecutor(object):
    def __init__(self, mq_address):
        self.gateway_resolver = GatewayResolver(mq_address)

    def run(self):
        gateway = self._initialize_gateway(
            gateway_address=self.gateway_resolver.get_gateway_address())

        # noinspection PyProtectedMember
        callback_server_port = gateway._callback_server.port
        gateway.entry_point.reportCallbackServerPort(callback_server_port)

        spark_context, sql_context = self._initialize_spark_contexts(gateway)

        code_executor = CodeExecutor(spark_context, sql_context, gateway.entry_point)
        gateway.entry_point.registerCodeExecutor(code_executor)

        # Wait for the end of the world
        while True:
            time.sleep(1)<commit_msg>Make PyExecutor listen on random port<commit_after># Copyright (c) 2015, CodiLime Inc.

import time

from code_executor import CodeExecutor
from gateway_resolver import GatewayResolver
from py4j.java_gateway import JavaGateway, GatewayClient, java_import
from pyspark import SparkContext, SQLContext, SparkConf

# TODO: get the address from env, or hardcode a sensible one
RABBIT_MQ_ADDRESS = ("localhost", 5672)


class PyExecutor(object):
    def __init__(self, mq_address):
        self.gateway_resolver = GatewayResolver(mq_address)

    def run(self):
        gateway = self._initialize_gateway(
            gateway_address=self.gateway_resolver.get_gateway_address())

        # noinspection PyProtectedMember
        callback_server_port = gateway._callback_server.server_socket.getsockname()[1]
        gateway.entry_point.reportCallbackServerPort(callback_server_port)

        spark_context, sql_context = self._initialize_spark_contexts(gateway)

        code_executor = CodeExecutor(spark_context, sql_context, gateway.entry_point)
        gateway.entry_point.registerCodeExecutor(code_executor)

        # Wait for the end of the world
        while True:
            time.sleep(1)
{'fraction_non_alphanumeric': 0.06458511548331908, 'fraction_numerical': 0.008982035928143712, 'mean_word_length': 4.1862527716186255, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3844488', 'n_tokens_mistral': 706, 'n_tokens_neox': 658, 'n_words': 158}