<commit_before>#!/bin/bash

# Download Zeppelin from Google Drive
# Zeppelin distribution by doanduyhai http://www.doanduyhai.com/blog/?p=2325

# If we are on node0 install Zeppelin, copy Notebooks and start it up.
if [ `hostname` == 'node0' ]
then
  curl -o zeppelin-0.7.1.tar.gz 'https://s3.amazonaws.com/dse-sketch-examples/zeppelin-0.7.1-dse-5.1.1.tar.gz' -L
  tar -zxvf zeppelin-0.7.1.tar.gz

  #Zeppelin Notebook API.
  zeppelin-0.7.1/bin/zeppelin-daemon.sh start

  #Although start is synchronous, I'll give it some time anyway
  sleep 5

  #XML Parser
  sudo apt-get install jq

  #Cassandra interpreter id
  CASSANDRA_INTERP_ID=$(curl localhost:8080/api/interpreter/setting | jq '.body|.[]|select(.name=="cassandra")|.id' -r)

  #Cassandra interpreter settings
  #Modify localhost
  curl localhost:8080/api/interpreter/setting | jq '.body|.[]|select(.name=="cassandra")|setpath(["properties","cassandra.cluster"]; "Cluster 1")|setpath(["properties","cassandra.hosts"]; "node0")|del(.id)' > cassandra-settings.json

  #Update Interpreter Settings
  curl -vX PUT "http://node0:8080/api/interpreter/setting/$CASSANDRA_INTERP_ID" -d @cassandra-settings.json \--header "Content-Type: application/json"

  #Spark interpreter id
  SPARK_INTERP_ID=$(curl localhost:8080/api/interpreter/setting | jq '.body|.[]|select(.name=="spark")|.id' -r)

  #Spark interpreter settings
  #Add cassandra host setting
  curl localhost:8080/api/interpreter/setting | jq '.body|.[]|select(.name=="spark")|setpath(["properties","spark.cassandra.connection.host"]; "node0")|del(.id)' > spark-settings.json

  #Update Interpreter Settings
  curl -vX PUT "http://node0:8080/api/interpreter/setting/$SPARK_INTERP_ID" -d @spark-settings.json \--header "Content-Type: application/json"

  sleep 5

  curl -vX POST http://localhost:8080/api/notebook/import -d @notebook/CountMinSketch/note.json \--header "Content-Type: application/json"
  curl -vX POST http://localhost:8080/api/notebook/import -d @notebook/HyperLogLog/note.json \--header "Content-Type: application/json"

  sleep 5
  
  zeppelin-0.7.1/bin/zeppelin-daemon.sh restart

  #Determine the localhost
  #MYIP=$(ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1'|grep -v "172.17.0.1")
  #node0
fi
<commit_msg>strtup scripts initialize zeppelin<commit_after>#!/bin/bash

# Download Zeppelin from Google Drive
# Zeppelin distribution by doanduyhai http://www.doanduyhai.com/blog/?p=2325

# If we are on node0 install Zeppelin, copy Notebooks and start it up.
if [ `hostname` == 'node0' ]
then
  curl -o zeppelin-0.7.1.tar.gz 'https://s3.amazonaws.com/dse-sketch-examples/zeppelin-0.7.1-dse-5.1.1.tar.gz' -L

  tar -zxvf zeppelin-0.7.1.tar.gz

  #Zeppelin Notebook API.
  zeppelin-0.7.1/bin/zeppelin-daemon.sh start

  #Although start is synchronous, I'll give it some time anyway
  sleep 5

  #XML Parser
  sudo apt-get install jq

  #Initialize Zeppelin.
  curl -o zepplinhome.out "http://172.16.17.32:8080/#/" -L
  curl -o zepplininterp.out "http://34.214.105.123:8080/#/interpreter" -L

  #Cassandra interpreter id
  CASSANDRA_INTERP_ID=$(curl localhost:8080/api/interpreter/setting | jq '.body|.[]|select(.name=="cassandra")|.id' -r)

  #Cassandra interpreter settings
  #Modify localhost
  curl localhost:8080/api/interpreter/setting | jq '.body|.[]|select(.name=="cassandra")|setpath(["properties","cassandra.cluster"]; "Cluster 1")|setpath(["properties","cassandra.hosts"]; "node0")|del(.id)' > cassandra-settings.json

  #Update Interpreter Settings
  curl -vX PUT "http://node0:8080/api/interpreter/setting/$CASSANDRA_INTERP_ID" -d @cassandra-settings.json \--header "Content-Type: application/json"

  #Spark interpreter id
  SPARK_INTERP_ID=$(curl localhost:8080/api/interpreter/setting | jq '.body|.[]|select(.name=="spark")|.id' -r)

  #Spark interpreter settings
  #Add cassandra host setting
  curl localhost:8080/api/interpreter/setting | jq '.body|.[]|select(.name=="spark")|setpath(["properties","spark.cassandra.connection.host"]; "node0")|del(.id)' > spark-settings.json

  #Update Interpreter Settings
  curl -vX PUT "http://node0:8080/api/interpreter/setting/$SPARK_INTERP_ID" -d @spark-settings.json \--header "Content-Type: application/json"

  sleep 5

  curl -vX POST http://localhost:8080/api/notebook/import -d @notebook/CountMinSketch/note.json \--header "Content-Type: application/json"
  curl -vX POST http://localhost:8080/api/notebook/import -d @notebook/HyperLogLog/note.json \--header "Content-Type: application/json"

  sleep 5

  zeppelin-0.7.1/bin/zeppelin-daemon.sh restart

  #Determine the localhost
  #MYIP=$(ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1'|grep -v "172.17.0.1")
  #node0
fi

{'fraction_non_alphanumeric': 0.17118997912317327, 'fraction_numerical': 0.043006263048016705, 'mean_word_length': 4.282249173098125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3190815', 'n_tokens_mistral': 1979, 'n_tokens_neox': 1783, 'n_words': 349}