<commit_before>#!/bin/bash

GREEN="\\033[1;32m"
DEFAULT="\\033[0;39m"
RED="\\033[1;31m"
ROSE="\\033[1;35m"
BLUE="\\033[1;34m"
WHITE="\\033[0;02m"
YELLOW="\\033[1;33m"
CYAN="\\033[1;36m"

#Modify these PATH
export PATH=$(pwd):$PATH
export PATH=/opt/redis-2.8.12/src/:$PATH
export PATH=/opt/redis-leveldb/:$PATH

function helptext {
    echo -e $YELLOW"

              .o.            ooooo      ooooo
             .888.           \`888'      \`888'
            .8\"888.           888        888
           .8' \`888.          888        888
          .88ooo8888.         888        888
         .8'     \`888.        888        888       o
        o88o     o8888o   o  o888o   o  o888ooooood8

         Analysis Information Leak framework
    "$DEFAULT"
    This script launch:
    "$CYAN"
    - All the ZMQ queuing modules.
    - All the ZMQ processing modules.
    - All Redis in memory servers.
    - All Level-DB on disk servers.
    "$DEFAULT"
    (Inside screen Daemons)
    "$RED"
    But first of all you'll need to edit few path where you installed
    your redis & leveldb servers.
    "$DEFAULT"
    Usage:
    -----
    "
}

function launching_redis {
    conf_dir='/home/user/AIL/configs/'

    screen -dmS "Redis"
    sleep 0.1
    echo -e $GREEN"\t* Launching Redis servers"$DEFAULT
    screen -S "Redis" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x'
    sleep 0.1
    screen -S "Redis" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x'
    sleep 0.1
    screen -S "Redis" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x'
}

function launching_lvldb {
    #Want to launch more level_db?
    lvdbhost='127.0.0.1'
    lvdbdir='/home/user/LEVEL_DB_DATA/'
    db1_y='2013'
    db2_y='2014'
    nb_db=13

    screen -dmS "LevelDB"
    sleep 0.1
    echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT
    #Add lines here with appropriates options.
    screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
    sleep 0.1
    screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
}

function launching_logs {
    screen -dmS "Logging"
    sleep 0.1
    echo -e $GREEN"\t* Launching logging process"$DEFAULT
    screen -S "Logging" -X screen -t "LogQueue" bash -c './log_subscriber -p 6380 -c Queuing -l ../logs/; read x'
    sleep 0.1
    screen -S "Logging" -X screen -t "LogScript" bash -c './log_subscriber -p 6380 -c Script -l ../logs/; read x'
}

function launching_queues {
    screen -dmS "Queue"
    sleep 0.1

    echo -e $GREEN"\t* Launching redis ZMQ queues"$DEFAULT
    screen -S "Queue" -X screen -t "QFeed" bash -c './ZMQ_Feed_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "QDuplicate" bash -c './ZMQ_Sub_Duplicate_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "QAttributes" bash -c './ZMQ_Sub_Attributes_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "Qlines" bash -c './ZMQ_PubSub_Lines_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "QCateg" bash -c './ZMQ_PubSub_Categ_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "QTokenize" bash -c './ZMQ_PubSub_Tokenize_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "Qcreditcard" bash -c './ZMQ_Sub_CreditCards_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "QOnion" bash -c './ZMQ_Sub_Onion_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "Qmails" bash -c './ZMQ_Sub_Mails_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "Qurls" bash -c './ZMQ_Sub_Urls_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "QCurve" bash -c './ZMQ_Sub_Curve_Q.py; read x'
}

function launching_scripts {
    screen -dmS "Script"
    sleep 0.1

    echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT

    screen -S "Script" -X screen -t "Feed" bash -c './ZMQ_Feed.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Duplicate" bash -c './ZMQ_Sub_Duplicate.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Attributes" bash -c './ZMQ_Sub_Attributes.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Lines" bash -c './ZMQ_PubSub_Lines.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Categ" bash -c './ZMQ_PubSub_Categ.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Tokenize" bash -c './ZMQ_PubSub_Tokenize.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Creditcard" bash -c './ZMQ_Sub_CreditCards.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Onion" bash -c './ZMQ_Sub_Onion.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Mails" bash -c './ZMQ_Sub_Mails.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Urls" bash -c './ZMQ_Sub_Urls.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Curve" bash -c './ZMQ_Sub_Curve.py; read x'
}

#If no params, display the help
#[[ $@ ]] || { helptext; exit 1;}

helptext;

############### TESTS ###################
isredis=`screen -ls | awk '/\.Redis\t/ {print strtonum($1)}'`
islvldb=`screen -ls | awk '/\.LevelDB\t/ {print strtonum($1)}'`
islogged=`screen -ls | awk '/\.Logging\t/ {print strtonum($1)}'`
isqueued=`screen -ls | awk '/\.Queue\t/ {print strtonum($1)}'`
isscripted=`screen -ls | awk '/\.Script\t/ {print strtonum($1)}'`<commit_msg>Indexer queue and script added to "BBS-like" LAUNCH script<commit_after>#!/bin/bash

GREEN="\\033[1;32m"
DEFAULT="\\033[0;39m"
RED="\\033[1;31m"
ROSE="\\033[1;35m"
BLUE="\\033[1;34m"
WHITE="\\033[0;02m"
YELLOW="\\033[1;33m"
CYAN="\\033[1;36m"

#Modify these PATH
export PATH=$(pwd):$PATH
export PATH=/opt/redis-2.8.12/src/:$PATH
export PATH=/opt/redis-leveldb/:$PATH

function helptext {
    echo -e $YELLOW"

              .o.            ooooo      ooooo
             .888.           \`888'      \`888'
            .8\"888.           888        888
           .8' \`888.          888        888
          .88ooo8888.         888        888
         .8'     \`888.        888        888       o
        o88o     o8888o   o  o888o   o  o888ooooood8

         Analysis Information Leak framework
    "$DEFAULT"
    This script launch:
    "$CYAN"
    - All the ZMQ queuing modules.
    - All the ZMQ processing modules.
    - All Redis in memory servers.
    - All Level-DB on disk servers.
    "$DEFAULT"
    (Inside screen Daemons)
    "$RED"
    But first of all you'll need to edit few path where you installed
    your redis & leveldb servers.
    "$DEFAULT"
    Usage:
    -----
    "
}

function launching_redis {
    conf_dir='/home/adulau/AIL-framework/configs/'

    screen -dmS "Redis"
    sleep 0.1
    echo -e $GREEN"\t* Launching Redis servers"$DEFAULT
    screen -S "Redis" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x'
    sleep 0.1
    screen -S "Redis" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x'
    sleep 0.1
    screen -S "Redis" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x'
}

function launching_lvldb {
    #Want to launch more level_db?
    lvdbhost='127.0.0.1'
    lvdbdir='/home/adulau//AIL-framework/LEVEL_DB_DATA/'
    db1_y='2013'
    db2_y='2014'
    nb_db=13

    screen -dmS "LevelDB"
    sleep 0.1
    echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT
    #Add lines here with appropriates options.
    screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
    sleep 0.1
    screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
}

function launching_logs {
    screen -dmS "Logging"
    sleep 0.1
    echo -e $GREEN"\t* Launching logging process"$DEFAULT
    screen -S "Logging" -X screen -t "LogQueue" bash -c './log_subscriber -p 6380 -c Queuing -l ../logs/; read x'
    sleep 0.1
    screen -S "Logging" -X screen -t "LogScript" bash -c './log_subscriber -p 6380 -c Script -l ../logs/; read x'
}

function launching_queues {
    screen -dmS "Queue"
    sleep 0.1

    echo -e $GREEN"\t* Launching redis ZMQ queues"$DEFAULT
    screen -S "Queue" -X screen -t "QFeed" bash -c './ZMQ_Feed_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "QDuplicate" bash -c './ZMQ_Sub_Duplicate_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "QAttributes" bash -c './ZMQ_Sub_Attributes_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "Qlines" bash -c './ZMQ_PubSub_Lines_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "QCateg" bash -c './ZMQ_PubSub_Categ_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "QTokenize" bash -c './ZMQ_PubSub_Tokenize_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "Qcreditcard" bash -c './ZMQ_Sub_CreditCards_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "QOnion" bash -c './ZMQ_Sub_Onion_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "Qmails" bash -c './ZMQ_Sub_Mails_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "Qurls" bash -c './ZMQ_Sub_Urls_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "QCurve" bash -c './ZMQ_Sub_Curve_Q.py; read x'
    sleep 0.1
    screen -S "Queue" -X screen -t "QIndexer" bash -c './ZMQ_Sub_Indexer_Q.py; read x'
}

function launching_scripts {
    screen -dmS "Script"
    sleep 0.1

    echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT

    screen -S "Script" -X screen -t "Feed" bash -c './ZMQ_Feed.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Duplicate" bash -c './ZMQ_Sub_Duplicate.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Attributes" bash -c './ZMQ_Sub_Attributes.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Lines" bash -c './ZMQ_PubSub_Lines.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Categ" bash -c './ZMQ_PubSub_Categ.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Tokenize" bash -c './ZMQ_PubSub_Tokenize.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Creditcard" bash -c './ZMQ_Sub_CreditCards.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Onion" bash -c './ZMQ_Sub_Onion.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Mails" bash -c './ZMQ_Sub_Mails.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Urls" bash -c './ZMQ_Sub_Urls.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Curve" bash -c './ZMQ_Sub_Curve.py; read x'
    sleep 0.1
    screen -S "Script" -X screen -t "Indexer" bash -c './ZMQ_Sub_Indexer.py; read x'
}

#If no params, display the help
#[[ $@ ]] || { helptext; exit 1;}

helptext;

############### TESTS ###################
isredis=`screen -ls | awk '/\.Redis\t/ {print strtonum($1)}'`
islvldb=`screen -ls | awk '/\.LevelDB\t/ {print strtonum($1)}'`
islogged=`screen -ls | awk '/\.Logging\t/ {print strtonum($1)}'`
isqueued=`screen -ls | awk '/\.Queue\t/ {print strtonum($1)}'`
isscripted=`screen -ls | awk '/\.Script\t/ {print strtonum($1)}'`
{'fraction_non_alphanumeric': 0.15810665712240515, 'fraction_numerical': 0.044738725841088046, 'mean_word_length': 2.3245092207019633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5644594', 'n_tokens_mistral': 4848, 'n_tokens_neox': 4395, 'n_words': 1345}