<commit_before>#!/bin/bash

# $TOPICS is on env variable that needs to be passed into the docker container
# Optional ENV variables:
# * ADVERTISED_HOST: the external ip for the container, e.g. `docker-machine ip \`docker-machine active\``
# * ADVERTISED_PORT: the external port for Kafka, e.g. 9092
# * ZK_CHROOT: the zookeeper chroot that's used by Kafka (without / prefix), e.g. "kafka"
# * LOG_RETENTION_HOURS: the minimum age of a log file in hours to be eligible for deletion (default is 168, for 1 week)
# * LOG_RETENTION_BYTES: configure the size at which segments are pruned from the log, (default is 1073741824, for 1GB)
# * NUM_PARTITIONS: configure the default number of log partitions per topic

# For reference: http://kafka.apache.org/documentation.html#brokerconfigs

# Set sensible defaults for environmental variables where possible
HOST_NAME=${HOST_NAME:-0.0.0.0}
PORT=${PORT:-9092}
# ADVERTISED_HOST=${ADVERTISED_HOST:-192.168.1.1}
ADVERTISED_PORT=${ADVERTISED_PORT:-9092}
BROKER_ID=${BROKER_ID:-1}
ZK_CONNECTION_STRING=${ZK_CONNECTION_STRING:-127.0.0.1:2181}
#LOG_DIRS=${LOG_DIRS:-/tmp/kafka-logs}
#LOG_RETENTION_HOURS=${LOG_RETENTION_HOURS:-168}
#LOG_RETENTION_BYTES=${LOG_RETENTION_BYTES:-1073741824}
NUM_PARTITIONS=${NUM_PARTITIONS:-1}
AUTO_CREATE_TOPICS=${AUTO_CREATE_TOPICS:-true}

readonly INITIALIZATION_TIMEOUT_SECONDS=30
readonly TOPIC_PARTITIONS=${NUM_PARTITIONS}
readonly TOPIC_REPLICATION=1

function update_kafka_config {

    # Set the internal host and port
    if [ ! -z "$HOST_NAME" ]; then
        echo "host name: $HOST_NAME"
        sed -r -i "s/#(host.name)=(.*)/\1=$HOST_NAME/g" $KAFKA_HOME/config/server.properties
    fi
    if [ ! -z "$PORT" ]; then
        echo "port: $PORT"
        sed -r -i "s/#(port)=(.*)/\1=$PORT/g" $KAFKA_HOME/config/server.properties
    fi

    # Set the advertised host and port
    if [ ! -z "$ADVERTISED_HOST" ]; then
        echo "advertised host: $ADVERTISED_HOST"
        sed -r -i "s/#(advertised.host.name)=(.*)/\1=$ADVERTISED_HOST/g" $KAFKA_HOME/config/server.properties
    fi
    if [ ! -z "$ADVERTISED_PORT" ]; then
        echo "advertised port: $ADVERTISED_PORT"
        sed -r -i "s/#(advertised.port)=(.*)/\1=$ADVERTISED_PORT/g" $KAFKA_HOME/config/server.properties
    fi<commit_msg>Don't provide default Zookeeper connection string<commit_after>#!/bin/bash

# $TOPICS is on env variable that needs to be passed into the docker container
# Optional ENV variables:
# * ADVERTISED_HOST: the external ip for the container, e.g. `docker-machine ip \`docker-machine active\``
# * ADVERTISED_PORT: the external port for Kafka, e.g. 9092
# * ZK_CHROOT: the zookeeper chroot that's used by Kafka (without / prefix), e.g. "kafka"
# * LOG_RETENTION_HOURS: the minimum age of a log file in hours to be eligible for deletion (default is 168, for 1 week)
# * LOG_RETENTION_BYTES: configure the size at which segments are pruned from the log, (default is 1073741824, for 1GB)
# * NUM_PARTITIONS: configure the default number of log partitions per topic

# For reference: http://kafka.apache.org/documentation.html#brokerconfigs

# Set sensible defaults for environmental variables where possible
HOST_NAME=${HOST_NAME:-0.0.0.0}
PORT=${PORT:-9092}
# ADVERTISED_HOST=${ADVERTISED_HOST:-192.168.1.1}
ADVERTISED_PORT=${ADVERTISED_PORT:-9092}
BROKER_ID=${BROKER_ID:-1}
#ZK_CONNECTION_STRING=${ZK_CONNECTION_STRING:-127.0.0.1:2181}
#LOG_DIRS=${LOG_DIRS:-/tmp/kafka-logs}
#LOG_RETENTION_HOURS=${LOG_RETENTION_HOURS:-168}
#LOG_RETENTION_BYTES=${LOG_RETENTION_BYTES:-1073741824}
NUM_PARTITIONS=${NUM_PARTITIONS:-1}
AUTO_CREATE_TOPICS=${AUTO_CREATE_TOPICS:-true}

readonly INITIALIZATION_TIMEOUT_SECONDS=30
readonly TOPIC_PARTITIONS=${NUM_PARTITIONS}
readonly TOPIC_REPLICATION=1

function update_kafka_config {

    # Set the internal host and port
    if [ ! -z "$HOST_NAME" ]; then
        echo "host name: $HOST_NAME"
        sed -r -i "s/#(host.name)=(.*)/\1=$HOST_NAME/g" $KAFKA_HOME/config/server.properties
    fi
    if [ ! -z "$PORT" ]; then
        echo "port: $PORT"
        sed -r -i "s/#(port)=(.*)/\1=$PORT/g" $KAFKA_HOME/config/server.properties
    fi

    # Set the advertised host and port
    if [ ! -z "$ADVERTISED_HOST" ]; then
        echo "advertised host: $ADVERTISED_HOST"
        sed -r -i "s/#(advertised.host.name)=(.*)/\1=$ADVERTISED_HOST/g" $KAFKA_HOME/config/server.properties
    fi
    if [ ! -z "$ADVERTISED_PORT" ]; then
        echo "advertised port: $ADVERTISED_PORT"
        sed -r -i "s/#(advertised.port)=(.*)/\1=$ADVERTISED_PORT/g" $KAFKA_HOME/config/server.properties
    fi
{'fraction_non_alphanumeric': 0.1322277983853371, 'fraction_numerical': 0.030984071568841372, 'mean_word_length': 4.317865429234339, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1965620', 'n_tokens_mistral': 1786, 'n_tokens_neox': 1619, 'n_words': 394}