<commit_before>using Dates
msg(x)=(println("$(now()) $x"); flush(STDOUT))
msg("Setting up workers"); 
ncpu=12
nbatch=2000
(nworkers() < ncpu) && (addprocs(ncpu - nprocs() + 1))
require("CUDArt")
@everywhere CUDArt.device((myid()-1) % CUDArt.devcount())
require("CUBLAS")
require("KUnet")
require("KUparser")

msg("Loading conllWSJToken_wikipedia2MUNK-100.jld")
using HDF5, JLD
@time @load "conllWSJToken_wikipedia2MUNK-100.jld"

evalheads(h,c)=mean(vcat(h...) .== vcat(map(s->s.head,c)...))
feats=KUparser.Flist.fv021a

if true
    msg("Parsing trn")          # still initializing with oparser?
    @time (h,x,y)=KUparser.oparse(trn, feats, ncpu)
    @show evalheads(h,trn)
else
    msg("Parsing dev")          # for debugging
    @time (h,x,y)=KUparser.oparse(dev, feats, ncpu)
    @show evalheads(h,dev)
end

msg("Setting up net")
net=KUnet.newnet(KUnet.relu, 1326, 20000, 3; learningRate=2f-2, adagrad=1f-8, dropout=7f-1)
KUnet.setparam!(net[1]; dropout=2f-1)
net[end].f=KUnet.logp
@show net

for epoch=1:256
    msg("epoch=$epoch KUnet.train(net, x, y; batch=128, loss=KUnet.logploss)")
    @time KUnet.train(net, x, y; batch=128, loss=KUnet.logploss)
    x=y=nothing
    @everywhere gc()
    run(`nvidia-smi`); run(`free`);
    msg("KUparser.gparse(trn, net, feats, nbatch, ncpu)")
    @time (htrn,x,y) = KUparser.gparse(trn, net, feats, nbatch, ncpu)
    e1 = evalheads(htrn,trn)
    msg("KUparser.gparse(dev, net, feats, nbatch, ncpu)")
    @time (hdev,) = KUparser.gparse(dev, net, feats, nbatch, ncpu)
    e2 = evalheads(hdev,dev)
    msg("KUparser.gparse(tst, net, feats, nbatch, ncpu)")
    @time (htst,) = KUparser.gparse(tst, net, feats, nbatch, ncpu)
    e3 = evalheads(htst,tst)
    println("DATA:\t$epoch\t$e1\t$e2\t$e3"); flush(STDOUT)
end

msg("done")<commit_msg>Fixed memory issues by not merging multithreaded parse results.<commit_after>using HDF5, JLD, Dates, KUparser, KUnet

macro date(_x) :(println("$(now()) "*$(string(_x)));flush(STDOUT);@time $(esc(_x))) end
meminfo()=(@everywhere gc(); whos(); run(`nvidia-smi`); run(`ps auxww`|>`grep julia`); run(`free`))
evalheads(p,c)=mean(vcat(vcat(map(q->q[1],p)...)...) .== vcat(map(s->s.head,c)...))

function initworkers(ncpu)
    (nworkers() < ncpu) && (addprocs(ncpu - nprocs() + 1))
    require("CUDArt")
    @everywhere CUDArt.device((myid()-1) % CUDArt.devcount())
    require("CUBLAS")
    require("KUnet")
    require("KUparser")
end

function main()
    ncpu=12
    nbatch=2000
    feats=KUparser.Flist.fv021a

    net=KUnet.newnet(KUnet.relu, 1326, 20000, 3; learningRate=2f-2, adagrad=1f-8, dropout=7f-1)
    KUnet.setparam!(net[1]; dropout=2f-1)
    net[end].f=KUnet.logp
    @show net

    @date @load "conllWSJToken_wikipedia2MUNK-100.jld"
    @date initworkers(ncpu)

    @date p1=KUparser.oparse(trn, feats, ncpu)
    @show evalheads(p1,trn)
    @date p2=KUparser.oparse(dev, feats, ncpu)
    @show evalheads(p2,dev)
    @date p3=KUparser.oparse(tst, feats, ncpu)
    @show evalheads(p3,tst)

    for epoch=1:256
        @show epoch
        for q in p1
            @date KUnet.train(net, q[2], q[3]; batch=128, loss=KUnet.logploss)
        end
        p1=p2=p3=nothing
        meminfo()
        @date p1 = KUparser.gparse(trn, net, feats, nbatch, ncpu)
        @show e1 = evalheads(p1,trn)
        @date p2 = KUparser.gparse(dev, net, feats, nbatch, ncpu)
        @show e2 = evalheads(p2,dev)
        @date p3 = KUparser.gparse(tst, net, feats, nbatch, ncpu)
        @show e3 = evalheads(p3,tst)
        println("DATA:\t$epoch\t$e1\t$e2\t$e3"); flush(STDOUT)
    end
end

main()
{'fraction_non_alphanumeric': 0.18316970546984573, 'fraction_numerical': 0.03422159887798037, 'mean_word_length': 4.043847241867044, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1723541', 'n_tokens_mistral': 1610, 'n_tokens_neox': 1545, 'n_words': 275}