<commit_before>local DEFAULT_RECORDER = "luatrace.trace_file"


-- Check if the ffi is available, and get a handle on the c library's clock.
-- LuaJIT doesn't compile traces containing os.clock yet.
local ffi
if jit and jit.status and jit.status() then
  local ok
  ok, ffi = pcall(require, "ffi")
  if ok then
    ffi.cdef("unsigned long clock(void);") 
  else
    ffi = nil
  end
end

-- See if the c hook is available
local c_hook
do
  local ok
  ok, c_hook = pcall(require, "luatrace.c_hook")
  if not ok then
    c_hook = nil
  end
end


-- Trace recording -------------------------------------------------------------

local recorder                          -- The thing that's recording traces
local current_line                      -- The line we currently think is active
local accumulated_us                    -- The microseconds we've accumulated for that line


-- Emit a trace if the current line has changed
-- and reset the current line and accumulated time
local function set_current_line(l)
  if l ~= current_line then
    -- if the current line *is* -1 then we're in a series of tail calls,
    -- and we'll throw the accumulated time away - most of it's probably
    -- trace overhead anyway
    if current_line ~= -1 then
      recorder.record(current_line, accumulated_us)
    end
    accumulated_us = 0
    current_line = l
  end
end


-- We only trace Lua functions
local function should_trace(f)
  return f and f.source:sub(1,1) == "@"
end


local CALLEE_INDEX, CALLER_INDEX

local watch_thread

-- Record an action reported to the hook.
local function record(action, line, time)
  accumulated_us = accumulated_us + time

  if watch_thread then
    if action == "call" then
      -- Whether the thread changed (and this is the *first* resume on the thread)
      -- or the thread was dead, and we're still in the same thread doesn't matter
      -- we're going to do what we want - report a call.
      watch_thread = nil
    elseif action == "line" then
      if watch_thread ~= (coroutine.running() or 1) then
        -- This was a resume into a running thread.  We make it look like a call
        local current = debug.getinfo(CALLEE_INDEX, "Sl")
        recorder.record(">", current.short_src, current.linedefined, current.lastlinedefined)
      end
      watch_thread = nil
    end
  end

  if action == "line" then
    set_current_line(line)
  elseif action == "call" or action == "return" then
    local callee = debug.getinfo(CALLEE_INDEX, "Sln")
    local caller = debug.getinfo(CALLER_INDEX, "Sl")
    
    if action == "call" then
      if should_trace(caller) then
        set_current_line(caller.currentline)
      end
      if should_trace(callee) then
        set_current_line(callee.currentline)
        recorder.record(">", callee.short_src, callee.linedefined, callee.lastlinedefined)
      elseif callee and callee.source == "=[C]" then
        if callee.name == "yield" then
          -- Treat it like a return, but we don't know where we're headed yet
          -- some time is lost (if yield gets renamed, all bets are off)
          set_current_line(-1)
          recorder.record("<")
        else
          -- Because of coroutine.wrap, any c function could resume a different
          -- thread.  Watch the current thread and catch it if it changes.
          watch_thread = coroutine.running() or 1
        end
      end
    else
      if should_trace(callee) then
        set_current_line(callee.currentline)
      end
      if should_trace(caller) then
        set_current_line(caller.currentline)
      elseif not caller                         -- final return from a coroutine
        or caller.source == "=(tail call)" then -- about to get tail-returned
        -- In both cases, there's no point recording time until we're
        -- back on our feet
        set_current_line(-1)
      end
      if should_trace(callee) then
        recorder.record("<")
      end
    end
  elseif action == "tail return" then
    local caller = debug.getinfo(CALLER_INDEX, "Sl")
    -- If we've got a real caller, we're heading back to non-tail-call land
    if should_trace(caller) then
      set_current_line(caller.currentline)
    end
    recorder.record("<")
  end
end<commit_msg>added more comments to luatrace.record<commit_after>local DEFAULT_RECORDER = "luatrace.trace_file"


-- Check if the ffi is available, and get a handle on the c library's clock.
-- LuaJIT doesn't compile traces containing os.clock yet.
local ffi
if jit and jit.status and jit.status() then
  local ok
  ok, ffi = pcall(require, "ffi")
  if ok then
    ffi.cdef("unsigned long clock(void);") 
  else
    ffi = nil
  end
end

-- See if the c hook is available
local c_hook
do
  local ok
  ok, c_hook = pcall(require, "luatrace.c_hook")
  if not ok then
    c_hook = nil
  end
end


-- Trace recording -------------------------------------------------------------

local recorder                          -- The thing that's recording traces
local current_line                      -- The line we currently think is active
local accumulated_us                    -- The microseconds we've accumulated for that line


-- Emit a trace if the current line has changed
-- and reset the current line and accumulated time
local function set_current_line(l)
  if l ~= current_line then
    -- if the current line *is* -1 then we're in a series of tail calls,
    -- and we'll throw the accumulated time away - most of it's probably
    -- trace overhead anyway
    if current_line ~= -1 then
      recorder.record(current_line, accumulated_us)
    end
    accumulated_us = 0
    current_line = l
  end
end


-- We only trace Lua functions
local function should_trace(f)
  return f and f.source:sub(1,1) == "@"
end


local CALLEE_INDEX, CALLER_INDEX

local watch_thread

-- Record an action reported to the hook.
local function record(action, line, time)
  accumulated_us = accumulated_us + time

  if watch_thread then
    if action == "call" then
      -- Whether the thread changed (and this is the *first* resume on the thread)
      -- or the thread was dead, and we're still in the same thread doesn't matter
      -- we're going to do what we want - report a call.
      watch_thread = nil
    elseif action == "line" then
      if watch_thread ~= (coroutine.running() or 1) then
        -- This was a resume into a running thread.  We make it look like a call
        local current = debug.getinfo(CALLEE_INDEX, "Sl")
        recorder.record(">", current.short_src, current.linedefined, current.lastlinedefined)
      end
      watch_thread = nil
    end
  end

  if action == "line" then
    set_current_line(line)

  elseif action == "call" or action == "return" then
    local callee = debug.getinfo(CALLEE_INDEX, "Sln")
    local caller = debug.getinfo(CALLER_INDEX, "Sl")
    
    if action == "call" then
      if should_trace(caller) then
        -- square up the caller's time to the last line executed
        set_current_line(caller.currentline)
      end
      if should_trace(callee) then
        -- start charging the callee for time, and record where we're going
        set_current_line(callee.currentline)
        recorder.record(">", callee.short_src, callee.linedefined, callee.lastlinedefined)
      elseif callee and callee.source == "=[C]" then
        if callee.name == "yield" then
          -- We don't know where we're headed yet so some time is lost (and if
          -- yield gets renamed, all bets are off)
          set_current_line(-1)
          recorder.record("<")
        else
          -- Because of coroutine.wrap, any c function could resume a different
          -- thread.  Watch the current thread and catch it if it changes.
          watch_thread = coroutine.running() or 1
        end
      end

    else -- action == "return"
      if should_trace(callee) then
        -- square up the callee's time to the last line executed
        set_current_line(callee.currentline)
      end
      if not caller                             -- final return from a coroutine
        or caller.source == "=(tail call)" then -- about to get tail-returned
        -- In both cases, there's no point recording time until we're
        -- back on our feet
        set_current_line(-1)
      elseif should_trace(caller) then
        -- The caller gets charged for time from here on
        set_current_line(caller.currentline)
      end
      if should_trace(callee) then
        recorder.record("<")
      end
    end

  elseif action == "tail return" then
    local caller = debug.getinfo(CALLER_INDEX, "Sl")
    -- If we've got a real caller, we're heading back to non-tail-call land
    -- start charging the caller for time
    if should_trace(caller) then
      set_current_line(caller.currentline)
    end
    recorder.record("<")
  end
end
{'fraction_non_alphanumeric': 0.09252102750625142, 'fraction_numerical': 0.0020459195271652648, 'mean_word_length': 2.3430851063829787, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '757936', 'n_tokens_mistral': 2573, 'n_tokens_neox': 2511, 'n_words': 975}