<jupyter_start><jupyter_text>Sequence classification model for IMDB Sentiment Analysis(c) <NAME>, 2018 * Objectives: Learn the structure of the IMDB dataset and train a simple RNN model.* Prerequisites: RNN models (06.rnn.ipynb), param, GRU, nll, minibatch, accuracy, Adam, train!* Knet: dir (used by imdb.jl)<jupyter_code>using Pkg
for p in ("Knet","ProgressMeter")
    haskey(Pkg.installed(),p) || Pkg.add(p)
end
EPOCHS=3          # Number of training epochs
BATCHSIZE=64      # Number of instances in a minibatch
EMBEDSIZE=125     # Word embedding size
NUMHIDDEN=100     # Hidden layer size
MAXLEN=150        # maximum size of the word sequence, pad shorter sequences, truncate longer ones
VOCABSIZE=30000   # maximum vocabulary size, keep the most frequent 30K, map the rest to UNK token
NUMCLASS=2        # number of output classes
DROPOUT=0.0       # Dropout rate
LR=0.001          # Learning rate
BETA_1=0.9        # Adam optimization parameter
BETA_2=0.999      # Adam optimization parameter
EPS=1e-08         # Adam optimization parameter<jupyter_output><empty_output><jupyter_text>Load and view data<jupyter_code>using Knet: Knet
ENV["COLUMNS"]=92                     # column width for array printing
include(Knet.dir("data","imdb.jl"))   # defines imdb loader
@doc imdb
@time (xtrn,ytrn,xtst,ytst,imdbdict)=imdb(maxlen=MAXLEN,maxval=VOCABSIZE);
summary.((xtrn,ytrn,xtst,ytst,imdbdict))
# Words are encoded with integers
rand(xtrn)'
# Each word sequence is padded or truncated to length 150
length.(xtrn)'
# Define a function that can print the actual words:
imdbvocab = Array{String}(undef,length(imdbdict))
for (k,v) in imdbdict; imdbvocab[v]=k; end
imdbvocab[VOCABSIZE-2:VOCABSIZE] = ["<unk>","<s>","<pad>"]
function reviewstring(x,y=0)
    x = x[x.!=VOCABSIZE] # remove pads
    """$(("Sample","Negative","Positive")[y+1]) review:\n$(join(imdbvocab[x]," "))"""
end
# Hit Ctrl-Enter to see random reviews:
r = rand(1:length(xtrn))
println(reviewstring(xtrn[r],ytrn[r]))
# Here are the labels: 1=negative, 2=positive
ytrn'<jupyter_output><empty_output><jupyter_text>Define the model<jupyter_code>using Knet: param, dropout, RNN
struct SequenceClassifier; input; rnn; output; end
SequenceClassifier(input::Int, embed::Int, hidden::Int, output::Int) =
    SequenceClassifier(param(embed,input), RNN(embed,hidden,rnnType=:gru), param(output,hidden))
function (sc::SequenceClassifier)(input; pdrop=0)
    embed = sc.input[:, permutedims(hcat(input...))]
    embed = dropout(embed,pdrop)
    hidden = sc.rnn(embed)
    hidden = dropout(hidden,pdrop)
    return sc.output * hidden[:,:,end]
end<jupyter_output><empty_output><jupyter_text>Experiment<jupyter_code>using Knet: minibatch
dtrn = minibatch(xtrn,ytrn,BATCHSIZE;shuffle=true)
dtst = minibatch(xtst,ytst,BATCHSIZE)
length.((dtrn,dtst))
# For running experiments
using Knet: train!, Adam
import ProgressMeter

function trainresults(file,model)
    if (print("Train from scratch? ");readline()[1]=='y')
        updates = 0; prog = ProgressMeter.Progress(EPOCHS * length(dtrn))
        function callback(J)
            ProgressMeter.update!(prog, updates)
            return (updates += 1) <= prog.n
        end
        opt = Adam(lr=LR, beta1=BETA_1, beta2=BETA_2, eps=EPS)
        train!(model, dtrn; callback=callback, optimizer=opt, pdrop=DROPOUT)
        Knet.gc()
        Knet.save(file,"model",model)
    else
        isfile(file) || download("http://people.csail.mit.edu/deniz/models/tutorial/$file",file)
        model = Knet.load(file,"model")
    end
    return model
end
using Knet: nll, accuracy
model = SequenceClassifier(VOCABSIZE,EMBEDSIZE,NUMHIDDEN,NUMCLASS)
nll(model,dtrn), nll(model,dtst), accuracy(model,dtrn), accuracy(model,dtst)
model = trainresults("imdbmodel.jld2",model);
# 33s (0.059155148f0, 0.3877507f0, 0.9846153846153847, 0.8583733974358975)
nll(model,dtrn), nll(model,dtst), accuracy(model,dtrn), accuracy(model,dtst)<jupyter_output><empty_output><jupyter_text>Playground<jupyter_code>predictstring(x)="\nPrediction: " * ("Negative","Positive")[argmax(Array(vec(model([x]))))]
UNK = VOCABSIZE-2
str2ids(s::String)=[(i=get(imdbdict,w,UNK); i>=UNK ? UNK : i) for w in split(lowercase(s))]
# Here we can see predictions for random reviews from the test set; hit Ctrl-Enter to sample:
r = rand(1:length(xtst))
println(reviewstring(xtst[r],ytst[r]))
println(predictstring(xtst[r]))
# Here the user can enter their own reviews and classify them:
println(predictstring(str2ids(readline(stdin))))<jupyter_output>stdin> I can't recommend this movie .

Prediction: Negative
{'fraction_non_alphanumeric': 0.13715163484748738, 'fraction_numerical': 0.026991441737985518, 'mean_word_length': 5.383753501400561, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '450575', 'license': 'MIT', 'n_tokens_mistral': 1724, 'n_tokens_neox': 1623, 'n_words': 363, 'path': 'tutorial/07.imdb.ipynb', 'repo_name': 'JuliaDocsForks/Knet.jl', 'size': '4562'}