<commit_before>#!/bin/bash
#*******************************************************************************
#
#    Copyright (C) 2015-2018 the BBoxDB project
#  
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#  
#      http://www.apache.org/licenses/LICENSE-2.0
#  
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License. 
#    
#*******************************************************************************
#
# Manage the local BBoxDB instance
#
#
#########################################

# Is the environment configured?
if [ -z "$BBOXDB_HOME" ]; then
   echo "Your environment variable \$(BBOXDB_HOME) is empty. Please check your .bboxdbrc"
   exit -1
fi

# Load all required functions and variables
. $BBOXDB_HOME/bin/bootstrap.sh


# Discover zookeeper nodes
zookeeper_nodes=$(read_nodes_file $zookeeper_node_file)
if [ -z "$zookeeper_nodes" ]; then
   echo -e "Your Zookeeper nodes ($zookeeper_node_file) are empty, please check your configuration $failed" 
   exit -1
fi

###
# Update and build the bboxdb
###
bboxdb_update() {
   echo "Update the BBoxDB"
   
   git pull
   
   # Remove old jars
   if [ -d target/lib/ ]; then
      rm -r target/lib/
   fi
   
   rm -f target/*.jar
   
   mvn package -DskipTests -Dassembly.skipAssembly=true
   
   echo -e "BBoxDB data is successfully updated $done"
}

###
# Start the bboxdb
###
bboxdb_start() {
    echo "Starting the BBoxDB"
    cd $BBOXDB_HOME

    # Is already running?
    if [ -f $bboxdb_pid ]; then
        pid=$(cat $bboxdb_pid)
        if [ -d /proc/$pid ]; then
            echo "BBoxDB is already running PID ($pid)"
            return
        fi
    fi
    
    # Create log dir if needed
    if [ ! -d $logdir ]; then
        mkdir $logdir
    fi

    debug_args=""

    # Logging
    if [ ! -z "$BBOXDB_LOG" ]; then
	    if [ "$BBOXDB_LOG" == "debug" ]; then
	         echo "Debug startup......"
	         debug_args+="-Dlog4j.configuration=log4j_debug.properties"
	    fi
	    
	    if [ "$BBOXDB_LOG" == "trace" ]; then
	         echo "Trace startup......"
	         debug_args+="-Dlog4j.configuration=log4j_trace.properties"
	    fi
    fi

    # Remote debug
    if [ ! -z "$BBOXDB_REMOTE_DEBUG" ]; then
        echo "Enabling remote debuging on port $jvm_debug_port"
        debug_args+="-Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=$jvm_debug_port,suspend=n"
    fi
    
    config="$BBOXDB_HOME/conf/bboxdb.yaml"

    if [ ! -f $config ]; then
        echo -e "Unable to locate bboxdb config $config $failed"
        exit -1
    fi

    # Configure zookeeper contact points
    zookeeper_connect="" 
    for i in $zookeeper_nodes; do
        if [[ -z "$zookeeper_connect" ]]; then
            zookeeper_connect="['$i:$zookeeper_clientport'"
        else
            zookeeper_connect="$zookeeper_connect, '$i:$zookeeper_clientport'"
        fi
    done
    zookeeper_connect="$zookeeper_connect]"

    sed -i "s/zookeepernodes: .*/zookeepernodes: $zookeeper_connect/" $config

    if [ -f $logdir/bboxdb.out.log ]; then
         rm $logdir/bboxdb.out.log
    fi

    cd $BBOXDB_HOME/misc

    echo "`ulimit -a`" >> $logdir/bboxdb.out.log 2>&1
    
    if [ ! -z "$BBOXDB_FOREGROUND" ]; then
       # Start BBoxDB server in foreground (e.g., needed for docker)
       java $debug_args $jvm_ops -cp $classpath -Dbboxdb.log.dir="$logdir" org.bboxdb.BBoxDBMain
    else 
       # Start BBoxDB server as daemon
       nohup java $debug_args $jvm_ops -cp $classpath -Dbboxdb.log.dir="$logdir" org.bboxdb.BBoxDBMain > $logdir/bboxdb.out.log 2>&1 < /dev/null &
    <commit_msg>Fixed ulimit log output<commit_after>#!/bin/bash
#*******************************************************************************
#
#    Copyright (C) 2015-2018 the BBoxDB project
#  
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#  
#      http://www.apache.org/licenses/LICENSE-2.0
#  
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License. 
#    
#*******************************************************************************
#
# Manage the local BBoxDB instance
#
#
#########################################

# Is the environment configured?
if [ -z "$BBOXDB_HOME" ]; then
   echo "Your environment variable \$(BBOXDB_HOME) is empty. Please check your .bboxdbrc"
   exit -1
fi

# Load all required functions and variables
. $BBOXDB_HOME/bin/bootstrap.sh


# Discover zookeeper nodes
zookeeper_nodes=$(read_nodes_file $zookeeper_node_file)
if [ -z "$zookeeper_nodes" ]; then
   echo -e "Your Zookeeper nodes ($zookeeper_node_file) are empty, please check your configuration $failed" 
   exit -1
fi

###
# Update and build the bboxdb
###
bboxdb_update() {
   echo "Update the BBoxDB"
   
   git pull
   
   # Remove old jars
   if [ -d target/lib/ ]; then
      rm -r target/lib/
   fi
   
   rm -f target/*.jar
   
   mvn package -DskipTests -Dassembly.skipAssembly=true
   
   echo -e "BBoxDB data is successfully updated $done"
}

###
# Start the bboxdb
###
bboxdb_start() {
    echo "Starting the BBoxDB"
    cd $BBOXDB_HOME

    # Is already running?
    if [ -f $bboxdb_pid ]; then
        pid=$(cat $bboxdb_pid)
        if [ -d /proc/$pid ]; then
            echo "BBoxDB is already running PID ($pid)"
            return
        fi
    fi
    
    # Create log dir if needed
    if [ ! -d $logdir ]; then
        mkdir $logdir
    fi

    debug_args=""

    # Logging
    if [ ! -z "$BBOXDB_LOG" ]; then
	    if [ "$BBOXDB_LOG" == "debug" ]; then
	         echo "Debug startup......"
	         debug_args+="-Dlog4j.configuration=log4j_debug.properties"
	    fi
	    
	    if [ "$BBOXDB_LOG" == "trace" ]; then
	         echo "Trace startup......"
	         debug_args+="-Dlog4j.configuration=log4j_trace.properties"
	    fi
    fi

    # Remote debug
    if [ ! -z "$BBOXDB_REMOTE_DEBUG" ]; then
        echo "Enabling remote debuging on port $jvm_debug_port"
        debug_args+="-Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=$jvm_debug_port,suspend=n"
    fi
    
    config="$BBOXDB_HOME/conf/bboxdb.yaml"

    if [ ! -f $config ]; then
        echo -e "Unable to locate bboxdb config $config $failed"
        exit -1
    fi

    # Configure zookeeper contact points
    zookeeper_connect="" 
    for i in $zookeeper_nodes; do
        if [[ -z "$zookeeper_connect" ]]; then
            zookeeper_connect="['$i:$zookeeper_clientport'"
        else
            zookeeper_connect="$zookeeper_connect, '$i:$zookeeper_clientport'"
        fi
    done
    zookeeper_connect="$zookeeper_connect]"

    sed -i "s/zookeepernodes: .*/zookeepernodes: $zookeeper_connect/" $config

    if [ -f $logdir/bboxdb.out.log ]; then
         rm $logdir/bboxdb.out.log
    fi

    cd $BBOXDB_HOME/misc

    echo "`ulimit -a`" >> $logdir/bboxdb.out.log 2>&1
    
    if [ ! -z "$BBOXDB_FOREGROUND" ]; then
       # Start BBoxDB server in foreground (e.g., needed for docker)
       java $debug_args $jvm_ops -cp $classpath -Dbboxdb.log.dir="$logdir" org.bboxdb.BBoxDBMain
    else 
       # Start BBoxDB server as daemon
       nohup java $debug_args $jvm_ops -cp $classpath -Dbboxdb.log.dir="$logdir" org.bboxdb.BBoxDBMain >> $logdir/bboxdb.out.log 2>&1 < /dev/null &
    
{'fraction_non_alphanumeric': 0.15576754110706664, 'fraction_numerical': 0.005773816995104807, 'mean_word_length': 2.758490566037736, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2844657', 'n_tokens_mistral': 2667, 'n_tokens_neox': 2548, 'n_words': 729}