<commit_before>local ffi = require 'ffi'

local batchNumber, nImgs = 0

torch.setdefaulttensortype('torch.FloatTensor')

function batchRepresent()
   local loadSize   = {3, opt.imgDim, opt.imgDim}
   print(opt.data)
   local cacheFile = paths.concat(opt.data, 'cache.t7')
   print('cache lotation: ', cacheFile)
   local dumpLoader = nil
   if paths.filep(cacheFile) then
      print('Loading metadata from cache.')
      print('If your dataset has changed, delete the cache file.')
      dumpLoader = torch.load(cacheFile)
   else
      print('Creating metadata for cache.')
      dumpLoader = dataLoader{
         paths = {opt.data},
         loadSize = loadSize,
         sampleSize = loadSize,
         split = 0,
         verbose = true
      }
      torch.save(cacheFile, dumpLoader)
   end
   collectgarbage()
   nImgs = dumpLoader:sizeTest()
   print('nImgs: ', nImgs)
   assert(nImgs > 0, "Failed to get nImgs")

   batchNumber = 0

   for i=1,math.ceil(nImgs/opt.batchSize) do
      local indexStart = (i-1) * opt.batchSize + 1
      local indexEnd = math.min(nImgs, indexStart + opt.batchSize - 1)
      local inputs, labels = dumpLoader:get(indexStart, indexEnd)
      local paths = {}
      for j=indexStart,indexEnd do
         table.insert(paths, ffi.string(dumpLoader.imagePath[j]:data()))<commit_msg>Fix luacheck warning.<commit_after>local ffi = require 'ffi'

local batchNumber, nImgs = 0

torch.setdefaulttensortype('torch.FloatTensor')

function batchRepresent()
   local loadSize   = {3, opt.imgDim, opt.imgDim}
   print(opt.data)
   local cacheFile = paths.concat(opt.data, 'cache.t7')
   print('cache lotation: ', cacheFile)
   local dumpLoader
   if paths.filep(cacheFile) then
      print('Loading metadata from cache.')
      print('If your dataset has changed, delete the cache file.')
      dumpLoader = torch.load(cacheFile)
   else
      print('Creating metadata for cache.')
      dumpLoader = dataLoader{
         paths = {opt.data},
         loadSize = loadSize,
         sampleSize = loadSize,
         split = 0,
         verbose = true
      }
      torch.save(cacheFile, dumpLoader)
   end
   collectgarbage()
   nImgs = dumpLoader:sizeTest()
   print('nImgs: ', nImgs)
   assert(nImgs > 0, "Failed to get nImgs")

   batchNumber = 0

   for i=1,math.ceil(nImgs/opt.batchSize) do
      local indexStart = (i-1) * opt.batchSize + 1
      local indexEnd = math.min(nImgs, indexStart + opt.batchSize - 1)
      local inputs, labels = dumpLoader:get(indexStart, indexEnd)
      local paths = {}
      for j=indexStart,indexEnd do
         table.insert(paths, ffi.string(dumpLoader.imagePath[j]:data()))
{'fraction_non_alphanumeric': 0.11305007587253414, 'fraction_numerical': 0.007587253414264037, 'mean_word_length': 2.9713855421686746, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4873487', 'n_tokens_mistral': 861, 'n_tokens_neox': 844, 'n_words': 208}