<commit_before>require 'pl'
local __FILE__ = (function() return string.gsub(debug.getinfo(2, 'S').source, "^@", "") end)()
package.path = path.join(path.dirname(__FILE__), "lib", "?.lua;") .. package.path
require 'sys'
require 'w2nn'
local iproc = require 'iproc'
local reconstruct = require 'reconstruct'
local image_loader = require 'image_loader'

torch.setdefaulttensortype('torch.FloatTensor')

local function convert_image(opt)
   local x, alpha = image_loader.load_float(opt.i)
   local new_x = nil
   local t = sys.clock()
   local scale_f, image_f
   if opt.tta == 1 then
      scale_f = reconstruct.scale_tta
      image_f = reconstruct.image_tta
   else
      scale_f = reconstruct.scale
      image_f = reconstruct.image
   end
   if opt.o == "(auto)" then
      local name = path.basename(opt.i)
      local e = path.extension(name)
      local base = name:sub(0, name:len() - e:len())
      opt.o = path.join(path.dirname(opt.i), string.format("%s_%s.png", base, opt.m))
   end
   if opt.m == "noise" then
      local model_path = path.join(opt.model_dir, ("noise%d_model.t7"):format(opt.noise_level))
      local model = torch.load(model_path, "ascii")
      if not model then
	 error("Load Error: " .. model_path)
      end
      new_x = image_f(model, x, opt.crop_size)
   elseif opt.m == "scale" then
      local model_path = path.join(opt.model_dir, ("scale%.1fx_model.t7"):format(opt.scale))
      local model = torch.load(model_path, "ascii")
      if not model then
	 error("Load Error: " .. model_path)
      end
      new_x = scale_f(model, opt.scale, x, opt.crop_size)
   elseif opt.m == "noise_scale" then
      local noise_model_path = path.join(opt.model_dir, ("noise%d_model.t7"):format(opt.noise_level))
      local noise_model = torch.load(noise_model_path, "ascii")
      local scale_model_path = path.join(opt.model_dir, ("scale%.1fx_model.t7"):format(opt.scale))
      local scale_model = torch.load(scale_model_path, "ascii")
      
      if not noise_model then
	 error("Load Error: " .. noise_model_path)
      end
      if not scale_model then
	 error("Load Error: " .. scale_model_path)
      end
      x = image_f(noise_model, x)
      new_x = scale_f(scale_model, opt.scale, x, opt.crop_size)
   else
      error("undefined method:" .. opt.method)
   end
   image_loader.save_png(opt.o, new_x, alpha, opt.depth)
   print(opt.o .. ": " .. (sys.clock() - t) .. " sec")
end
local function convert_frames(opt)
   local model_path, noise1_model, noise2_model, scale_model
   local scale_f, image_f
   if opt.tta == 1 then
      scale_f = reconstruct.scale_tta
      image_f = reconstruct.image_tta
   else
      scale_f = reconstruct.scale
      image_f = reconstruct.image
   end
   if opt.m == "scale" then
      model_path = path.join(opt.model_dir, ("scale%.1fx_model.t7"):format(opt.scale))
      scale_model = torch.load(model_path, "ascii")
      if not scale_model then
	 error("Load Error: " .. model_path)
      end
   elseif opt.m == "noise" and opt.noise_level == 1 then
      model_path = path.join(opt.model_dir, "noise1_model.t7")
      noise1_model = torch.load(model_path, "ascii")
      if not noise1_model then
	 error("Load Error: " .. model_path)
      end
   elseif opt.m == "noise" and opt.noise_level == 2 then
      model_path = path.join(opt.model_dir, "noise2_model.t7")
      noise2_model = torch.load(model_path, "ascii")
      if not noise2_model then
	 error("Load Error: " .. model_path)
      end
   elseif opt.m == "noise_scale" then
      model_path = path.join(opt.model_dir, ("scale%.1fx_model.t7"):format(opt.scale))
      scale_model = torch.load(model_path, "ascii")
      if not scale_model then
	 error("Load Error: " .. model_path)
      end
      if opt.noise_level == 1 then
	 model_path = path.join(opt.model_dir, "noise1_model.t7")
	 noise1_model = torch.load(model_path, "ascii")
	 if not noise1_model then
	    error("Load Error: " .. model_path)
	 end
      elseif opt.noise_level == 2 then
	 model_path = path.join(opt.model_dir, "noise2_model.t7")
	 noise2_model = torch.load(model_path, "ascii")
	 if not noise2_model then
	    error("Load Error: " .. model_path)
	 end
      end
   end
   local fp = io.open(opt.l)
   if not fp then
      error("Open Error: " .. opt.l)
   end
   local count = 0
   local lines = {}
   for line in fp:lines() do
      table.insert(lines, line)
   end
   fp:close()
   for i = 1, #lines do
      if opt.resume == 0 or path.exists(string.format(opt.o, i)) == false then
	 local x, alpha = image_loader.load_float(lines[i])
	 local new_x = nil
	 if opt.m == "noise" and opt.noise_level == 1 then
	    new_x = image_f(noise1_model, x, opt.crop_size)
	 elseif opt.m == "noise" and opt.noise_level == 2 then
	    new_x = image_func(noise2_model, x)
	 elseif opt.m == "scale" then
	    new_x = scale_f(scale_model, opt.scale, x, opt.crop_size)
	 elseif opt.m == "noise_scale" and opt.noise_level == 1 then
	    x = image_f(noise1_model, x)<commit_msg>Fix unused crop_size option<commit_after>require 'pl'
local __FILE__ = (function() return string.gsub(debug.getinfo(2, 'S').source, "^@", "") end)()
package.path = path.join(path.dirname(__FILE__), "lib", "?.lua;") .. package.path
require 'sys'
require 'w2nn'
local iproc = require 'iproc'
local reconstruct = require 'reconstruct'
local image_loader = require 'image_loader'

torch.setdefaulttensortype('torch.FloatTensor')

local function convert_image(opt)
   local x, alpha = image_loader.load_float(opt.i)
   local new_x = nil
   local t = sys.clock()
   local scale_f, image_f
   if opt.tta == 1 then
      scale_f = reconstruct.scale_tta
      image_f = reconstruct.image_tta
   else
      scale_f = reconstruct.scale
      image_f = reconstruct.image
   end
   if opt.o == "(auto)" then
      local name = path.basename(opt.i)
      local e = path.extension(name)
      local base = name:sub(0, name:len() - e:len())
      opt.o = path.join(path.dirname(opt.i), string.format("%s_%s.png", base, opt.m))
   end
   if opt.m == "noise" then
      local model_path = path.join(opt.model_dir, ("noise%d_model.t7"):format(opt.noise_level))
      local model = torch.load(model_path, "ascii")
      if not model then
	 error("Load Error: " .. model_path)
      end
      new_x = image_f(model, x, opt.crop_size)
   elseif opt.m == "scale" then
      local model_path = path.join(opt.model_dir, ("scale%.1fx_model.t7"):format(opt.scale))
      local model = torch.load(model_path, "ascii")
      if not model then
	 error("Load Error: " .. model_path)
      end
      new_x = scale_f(model, opt.scale, x, opt.crop_size)
   elseif opt.m == "noise_scale" then
      local noise_model_path = path.join(opt.model_dir, ("noise%d_model.t7"):format(opt.noise_level))
      local noise_model = torch.load(noise_model_path, "ascii")
      local scale_model_path = path.join(opt.model_dir, ("scale%.1fx_model.t7"):format(opt.scale))
      local scale_model = torch.load(scale_model_path, "ascii")
      
      if not noise_model then
	 error("Load Error: " .. noise_model_path)
      end
      if not scale_model then
	 error("Load Error: " .. scale_model_path)
      end
      x = image_f(noise_model, x, opt.crop_size)
      new_x = scale_f(scale_model, opt.scale, x, opt.crop_size)
   else
      error("undefined method:" .. opt.method)
   end
   image_loader.save_png(opt.o, new_x, alpha, opt.depth)
   print(opt.o .. ": " .. (sys.clock() - t) .. " sec")
end
local function convert_frames(opt)
   local model_path, noise1_model, noise2_model, scale_model
   local scale_f, image_f
   if opt.tta == 1 then
      scale_f = reconstruct.scale_tta
      image_f = reconstruct.image_tta
   else
      scale_f = reconstruct.scale
      image_f = reconstruct.image
   end
   if opt.m == "scale" then
      model_path = path.join(opt.model_dir, ("scale%.1fx_model.t7"):format(opt.scale))
      scale_model = torch.load(model_path, "ascii")
      if not scale_model then
	 error("Load Error: " .. model_path)
      end
   elseif opt.m == "noise" and opt.noise_level == 1 then
      model_path = path.join(opt.model_dir, "noise1_model.t7")
      noise1_model = torch.load(model_path, "ascii")
      if not noise1_model then
	 error("Load Error: " .. model_path)
      end
   elseif opt.m == "noise" and opt.noise_level == 2 then
      model_path = path.join(opt.model_dir, "noise2_model.t7")
      noise2_model = torch.load(model_path, "ascii")
      if not noise2_model then
	 error("Load Error: " .. model_path)
      end
   elseif opt.m == "noise_scale" then
      model_path = path.join(opt.model_dir, ("scale%.1fx_model.t7"):format(opt.scale))
      scale_model = torch.load(model_path, "ascii")
      if not scale_model then
	 error("Load Error: " .. model_path)
      end
      if opt.noise_level == 1 then
	 model_path = path.join(opt.model_dir, "noise1_model.t7")
	 noise1_model = torch.load(model_path, "ascii")
	 if not noise1_model then
	    error("Load Error: " .. model_path)
	 end
      elseif opt.noise_level == 2 then
	 model_path = path.join(opt.model_dir, "noise2_model.t7")
	 noise2_model = torch.load(model_path, "ascii")
	 if not noise2_model then
	    error("Load Error: " .. model_path)
	 end
      end
   end
   local fp = io.open(opt.l)
   if not fp then
      error("Open Error: " .. opt.l)
   end
   local count = 0
   local lines = {}
   for line in fp:lines() do
      table.insert(lines, line)
   end
   fp:close()
   for i = 1, #lines do
      if opt.resume == 0 or path.exists(string.format(opt.o, i)) == false then
	 local x, alpha = image_loader.load_float(lines[i])
	 local new_x = nil
	 if opt.m == "noise" and opt.noise_level == 1 then
	    new_x = image_f(noise1_model, x, opt.crop_size)
	 elseif opt.m == "noise" and opt.noise_level == 2 then
	    new_x = image_func(noise2_model, x, opt.crop_size)
	 elseif opt.m == "scale" then
	    new_x = scale_f(scale_model, opt.scale, x, opt.crop_size)
	 elseif opt.m == "noise_scale" and opt.noise_level == 1 then
	    x = image_f(noise1_model, x, opt.crop_size)
{'fraction_non_alphanumeric': 0.13009106374462123, 'fraction_numerical': 0.00920644451115781, 'mean_word_length': 3.13658940397351, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '445543', 'n_tokens_mistral': 3629, 'n_tokens_neox': 3499, 'n_words': 835}