<commit_before>#!/bin/bash

set -e

if [ -z "$MC_SOLR_SHARD_COUNT" ]; then
    echo "MC_SOLR_SHARD_COUNT (total shard count) is not set."
    exit 1
fi

set -u

MC_SOLR_ZOOKEEPER_HOST="solr-zookeeper"
MC_SOLR_ZOOKEEPER_PORT=2181
MC_SOLR_PORT=8983

# Timeout in milliseconds at which Solr shard disconnects from ZooKeeper
MC_SOLR_ZOOKEEPER_TIMEOUT=300000

# <luceneMatchVersion> value
MC_SOLR_LUCENEMATCHVERSION="6.5.0"

# Make Solr use 90% of available RAM allotted to the container
MC_RAM_SIZE=$(/container_memory_limit.sh)
MC_SOLR_MX=$((MC_RAM_SIZE / 10 * 9))

# Run Solr
java_args=(
    -server
    "-Xmx${MC_SOLR_MX}m"
    -Djava.util.logging.config.file=file:///var/lib/solr/resources/log4j.properties
    -Djetty.base=/var/lib/solr<commit_msg>Wait for ZooKeeper before starting Solr shard<commit_after>#!/bin/bash

set -e

if [ -z "$MC_SOLR_SHARD_COUNT" ]; then
    echo "MC_SOLR_SHARD_COUNT (total shard count) is not set."
    exit 1
fi

set -u

MC_SOLR_ZOOKEEPER_HOST="solr-zookeeper"
MC_SOLR_ZOOKEEPER_PORT=2181
MC_SOLR_PORT=8983

# Timeout in milliseconds at which Solr shard disconnects from ZooKeeper
MC_SOLR_ZOOKEEPER_TIMEOUT=300000

# <luceneMatchVersion> value
MC_SOLR_LUCENEMATCHVERSION="6.5.0"

# Make Solr use 90% of available RAM allotted to the container
MC_RAM_SIZE=$(/container_memory_limit.sh)
MC_SOLR_MX=$((MC_RAM_SIZE / 10 * 9))

# Wait for ZooKeeper container to show up
while true; do
    echo "Waiting for ZooKeeper to start..."
    if nc -z -w 10 solr-zookeeper 2181; then
        break
    else
        sleep 1
    fi
done

# Run Solr
java_args=(
    -server
    "-Xmx${MC_SOLR_MX}m"
    -Djava.util.logging.config.file=file:///var/lib/solr/resources/log4j.properties
    -Djetty.base=/var/lib/solr
{'fraction_non_alphanumeric': 0.10352805089647195, 'fraction_numerical': 0.031810294968189705, 'mean_word_length': 4.1335311572700295, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4648102', 'n_tokens_mistral': 776, 'n_tokens_neox': 693, 'n_words': 153}