<commit_before>-- freak.lua: task management utility
freak = {}
local cfile = "freakdata.lua"
local minheap = 15000 -- if we've dropped below this after a task, restart
local function getconf() return pcall(dofile, cfile) or {} end
local function persist(t)
  file.open(cfile, "w"); file.write("return "..t2str(t)); file.close()
end
local function t2str(t)
  buf = "{ "
  for k, v in pairs(t) do
    if type(v) == "table" then v = t2str(v) end
    buf = buf .. string.format(' %s="%s",', k, v)
  end
  return buf .. " }\n"
end
local function run(continuation, nexttask) -- main "loop"
  taskname = continuation.tasks[nexttask]
  if type(taskname) == "number" then node.deepsleep(taskname * 1000)
  else
    preconfunc = continuation.precons[taskname] -- check preconditions
    if not preconfunc or pcall(preconfunc, continuation) then
      taskchunk = require(taskname)
      pcall(taskchunk.run, continuation)
    end
  end
  nexttask = nexttask + 1
  if(nexttask > #continuation.tasks) then nexttask = 1 end -- start over
  continuation.taskdata.nexttask = nexttask
  if node.getheap() < minheap then
    persist(continuation.taskdata)
    node.restart() -- reset the chip and start over
  end
  return run(continuation, nexttask) -- run the next task (tail call)
end
function freak.begin(tasks, precons)
  continuation = { tasks=tasks, precons=precons }
  _, continuation.taskdata = getconf()
  return run(continuation, continuation.taskdata.nexttask or 1)<commit_msg>bedugging<commit_after>-- freak.lua: task management utility
freak = {}
local cfile = "freakdata.lua"
local minheap = 15000 -- if we've dropped below this after a task, restart
local function getconf() return pcall(dofile, cfile) or {} end
local function t2str(t)
  buf = "{ "
  for k, v in pairs(t) do
    if type(v) == "table" then v = t2str(v) end
    buf = buf .. string.format(' %s="%s",', k, v)
  end
  return buf .. " }\n"
end
local function persist(t)
  file.open(cfile, "w"); file.write("return "..t2str(t)); file.close()
end
local function run(continuation, nexttask) -- main "loop"
  taskname = continuation.tasks[nexttask]
  if type(taskname) == "number" then node.deepsleep(taskname * 1000)
  else
    preconfunc = continuation.precons[taskname] -- check preconditions
    if not preconfunc or pcall(preconfunc, continuation) then
      taskchunk = require(taskname)
      pcall(taskchunk.run, continuation)
    end
  end
  nexttask = nexttask + 1
  if(nexttask > #continuation.tasks) then nexttask = 1 end -- start over
  continuation.taskdata.nexttask = nexttask
  if node.heap() < minheap then
    persist(continuation.taskdata)
    node.restart() -- reset the chip and start over
  end
  return run(continuation, nexttask) -- run the next task (tail call)
end
function freak.begin(tasks, precons)
  continuation = { tasks=tasks, precons=precons }
  _, results = getconf()
  continuation.taskdata = results or { }
  return run(continuation, continuation.taskdata.nexttask or 1)
{'fraction_non_alphanumeric': 0.12036412677006068, 'fraction_numerical': 0.010114632501685773, 'mean_word_length': 3.8879736408566723, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3062404', 'n_tokens_mistral': 996, 'n_tokens_neox': 953, 'n_words': 296}