<commit_before>#!/usr/bin/env th

require 'convnet'
require 'convnet_io'
require 'postprocess'

----------------------------------------------------------------
-- parse arguments
----------------------------------------------------------------
cmd = torch.CmdLine()
cmd:text()
cmd:text('DNA ConvNet testing')
cmd:text()
cmd:text('Arguments')
cmd:argument('model_file')
cmd:argument('data_file')
cmd:argument('out_file')
cmd:text()
cmd:text('Options:')
cmd:option('-norm', false, 'Normalize all targets to a level plane')
cmd:text()
opt = cmd:parse(arg)

----------------------------------------------------------------
-- load data
----------------------------------------------------------------
local test_seqs = load_test_seqs(opt.data_file)

----------------------------------------------------------------
-- construct model
----------------------------------------------------------------
-- initialize
local convnet = ConvNet:__init()

-- load from saved parameters
local convnet_params = torch.load(opt.model_file)
convnet:load(convnet_params)

----------------------------------------------------------------
-- predict and test
----------------------------------------------------------------
-- guarantee evaluate mode
convnet.model:evaluate()

-- measure accuracy on a test set
local preds = convnet:predict(test_seqs)

if opt.norm then
    -- TEMP! TMP!
    if convnet.pred_means == nil then
        convnet.pred_means = preds:mean(1):squeeze()
    end

    preds = troy_norm(preds, convnet.pred_means)
end

----------------------------------------------------------------
-- dump to file
----------------------------------------------------------------
local predict_out = io.open(opt.out_file, 'w')

-- print labels
if convnet.target_labels ~= nil do   -- TEMP! TMP!
    predict_out:write(convnet.target_labels[1])
    for ti=2,(#convnet.target_labels) do
        predict_out:write(string.format("\t%s",convnet.target_labels[ti]))
    predict_out:write("\n")
end

-- print predictions
for si=1,(#preds)[1] do
    predict_out:write(preds[{si,1}])
    for ti=2,(#preds)[2] do
        predict_out:write(string.format("\t%s",preds[{si,ti}]))
    end
    predict_out:write("\n")
end

predict_out:close()

-- local hdf_out = hdf5.open(opt.out_file, 'w')
-- hdf_out:write("preds", preds)
-- hdf_out:close()<commit_msg>end bug<commit_after>#!/usr/bin/env th

require 'convnet'
require 'convnet_io'
require 'postprocess'

----------------------------------------------------------------
-- parse arguments
----------------------------------------------------------------
cmd = torch.CmdLine()
cmd:text()
cmd:text('DNA ConvNet testing')
cmd:text()
cmd:text('Arguments')
cmd:argument('model_file')
cmd:argument('data_file')
cmd:argument('out_file')
cmd:text()
cmd:text('Options:')
cmd:option('-norm', false, 'Normalize all targets to a level plane')
cmd:text()
opt = cmd:parse(arg)

----------------------------------------------------------------
-- load data
----------------------------------------------------------------
local test_seqs = load_test_seqs(opt.data_file)

----------------------------------------------------------------
-- construct model
----------------------------------------------------------------
-- initialize
local convnet = ConvNet:__init()

-- load from saved parameters
local convnet_params = torch.load(opt.model_file)
convnet:load(convnet_params)

----------------------------------------------------------------
-- predict and test
----------------------------------------------------------------
-- guarantee evaluate mode
convnet.model:evaluate()

-- measure accuracy on a test set
local preds = convnet:predict(test_seqs)

if opt.norm then
    -- TEMP! TMP!
    if convnet.pred_means == nil then
        convnet.pred_means = preds:mean(1):squeeze()
    end

    preds = troy_norm(preds, convnet.pred_means)
end

----------------------------------------------------------------
-- dump to file
----------------------------------------------------------------
local predict_out = io.open(opt.out_file, 'w')

-- print labels
if convnet.target_labels ~= nil do   -- TEMP! TMP!
    predict_out:write(convnet.target_labels[1])
    for ti=2,(#convnet.target_labels) do
        predict_out:write(string.format("\t%s",convnet.target_labels[ti]))
    end
    predict_out:write("\n")
end

-- print predictions
for si=1,(#preds)[1] do
    predict_out:write(preds[{si,1}])
    for ti=2,(#preds)[2] do
        predict_out:write(string.format("\t%s",preds[{si,ti}]))
    end
    predict_out:write("\n")
end

predict_out:close()

-- local hdf_out = hdf5.open(opt.out_file, 'w')
-- hdf_out:write("preds", preds)
-- hdf_out:close()
{'fraction_non_alphanumeric': 0.3861683848797251, 'fraction_numerical': 0.003865979381443299, 'mean_word_length': 1.4446194225721785, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1513644', 'n_tokens_mistral': 1283, 'n_tokens_neox': 1262, 'n_words': 259}