<jupyter_start><jupyter_text>Setup spark<jupyter_code>!apt-get install openjdk-8-jdk-headless -qq > /dev/null
!java -version

!wget http://www-eu.apache.org/dist/spark/spark-2.3.3/spark-2.3.3-bin-hadoop2.6.tgz
!tar -xvf spark-2.3.3-bin-hadoop2.6.tgz
!pip install findspark
!pip install systemml
import os
os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
os.environ["SPARK_HOME"] = "/content/spark-2.3.3-bin-hadoop2.6"<jupyter_output><empty_output><jupyter_text>DL4j code<jupyter_code>!wget https://raw.githubusercontent.com/maxpumperla/dl4j_coursera/master/iris.txt
import numpy
import pandas
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier
from keras.utils import np_utils
from sklearn.preprocessing import LabelEncoder

seed = 10
numpy.random.seed(seed)


dataframe = pandas.read_csv('iris.txt',header=None)
dataset = dataframe.values
x = dataset[:,0:4].astype(float)
y = dataset[:,4]

encoder = LabelEncoder()
encoder.fit(y)
encoded_Y = encoder.transform(y)
dummy_y = np_utils.to_categorical(encoded_Y)

model = Sequential()
model.add(Dense(4,input_dim = 4, kernel_initializer = 'normal', activation = 'relu'))
model.add(Dense(3, kernel_initializer = 'normal', activation = 'sigmoid'))

model.compile(loss = 'categorical_crossentropy',optimizer = 'adam', metrics = ['accuracy'])

model.fit(x, dummy_y, epochs = 20, batch_size = 5)

model.save('my_model.h5')

!wget https://github.com/maxpumperla/dl4j_coursera/releases/download/v0.4/dl4j-snapshot.jar
    
#case "META-INF/services/org.nd4j.linalg.factory.Nd4jBackend" => MergeStrategy.first
#case "META-INF/services/org.nd4j.linalg.compression.NDArrayCompressor" => MergeStrategy.first
!ls $SPARK_HOME

#/content/spark-2.3.3-bin-hadoop2.6.tgz \

!$SPARK_HOME/bin/spark-submit \
--class skymind.dsx.KerasImportCSVSparkRunner \
--files iris.txt,my_model.h5 \
--master local[*] \
dl4j-snapshot.jar \
-batchSizePerWorker 15 \
-indexLabel 4 \
-train false \
-numClasses 3 \
-modelFileName my_model.h5 \
-dataFileName iris.txt \
-useSparkLocal true

#--master $MASTER \<jupyter_output>SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/opt/ibm/spark/jars/slf4j-log4j12-1.7.16.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/opt/ibm/image-libs/spark2/tika-app-1.14.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
19/07/20 10:38:59 [WARN] o.a.h.u.NativeCodeLoader - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
19/07/20 10:39:01 [INFO] o.s.j.u.log - Logging initialized @3219ms
19/07/20 10:39:01 [INFO] o.s.j.s.Server - jetty-9.3.z-SNAPSHOT, build timestamp: unknown, git hash: unknown
19/07/20 10:39:01 [INFO] o.s.j.s.Server - Started @3306ms
19/07/20 10:39:01 [WARN] o.a.s.u.Utils - Service 'SparkUI' could not bind on port 4040. Attempting port 4041.
19/07/20 10:39:01 [INFO] o.s.j.s.AbstractConnector - Starte[...]
{'fraction_non_alphanumeric': 0.14752, 'fraction_numerical': 0.05376, 'mean_word_length': 4.842990654205607, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '24613', 'license': 'MIT', 'n_tokens_mistral': 1341, 'n_tokens_neox': 1245, 'n_words': 220, 'path': 'Applied AI Study Group #2 - February 2020/week3/4-DL4J.ipynb', 'repo_name': 'cerob/Applied-AI-Study-Group', 'size': '3125'}