<commit_before>local trace_file = require("luatrace.trace_file")

local source_files                              -- Map of source files we've seen
local functions                                 -- Map of functions

local threads                                   -- All the running threads
local thread_stack                              -- and the order they're running in

local stack                                     -- Call stack

local total_time                                -- Running total of all the time we've recorded

local trace_count                               -- How many traces we've seen (for reporting errors)
local error_count                               -- How many errors we've seen

local profile = {}


--------------------------------------------------------------------------------

function profile.open()
  source_files, functions = {}, {}
  local main_thread = { top=0 }
  threads = { main_thread }
  thread_stack = { main_thread, top=1 }
  stack = { top=0 }
  total_time = 0
  trace_count, error_count = 0, 0
end


local function get_source_file(filename)
  local source_file = source_files[filename]
  if not source_file then
    source_file = { filename=filename, lines = {} }
    source_files[filename] = source_file
  end
  return source_file
end


local function get_function(filename, line_defined, last_line_defined)
  local name = filename..":"..tostring(line_defined).."-"..tostring(last_line_defined)
  local f = functions[name]
  if not f then
    f = { name=name, source_file=get_source_file(filename), line_defined=line_defined, last_line_defined=last_line_defined }
    functions[name] = f
  end
  return f
end


local function get_thread(thread_id)
  local thread = threads[thread_id]
  if not thread then
    thread = { top=0 }
    threads[thread_id] = thread
  end
  return thread
end


local function replay_push(frame)
  stack.top = stack.top + 1
  stack[stack.top] = frame
end


local function push(frame)
  replay_push(frame)
  local thread = thread_stack[thread_stack.top]
  thread.top = thread.top + 1
  thread[thread.top] = { func=frame.func }
end


local function push_thread(thread)
  thread_stack.top = thread_stack.top + 1
  thread_stack[thread_stack.top] = thread
end


local function replay_pop()
  local frame = stack[stack.top]
  stack[stack.top] = nil
  stack.top = stack.top - 1
  return frame
end


local function pop()
  local frame = replay_pop()
  local thread = thread_stack[thread_stack.top]
  thread[thread.top] = nil
  thread.top = thread.top - 1
  return frame
end


local function pop_thread()
  local thread = thread_stack[thread_stack.top]
  thread_stack[thread_stack.top] = nil
  thread_stack.top = thread_stack.top - 1
  return thread
end


local function get_top()
  return stack[stack.top]
end


local function thread_top()
  return thread_stack[thread_stack.top]
end


local function get_line(line_number)
  local top = get_top()
  local line = top.source_file.lines[line_number]
  if not line then
    line = { visits = 0, self_time = 0, child_time = 0 }
    top.source_file.lines[line_number] = line
  end
  return line
end


local function call(filename, line_defined, last_line_defined)
  push{ source_file=source_file, func=func, frame_time=0 }
end


function profile.record(a, b, c, d)
  trace_count = trace_count + 1

  if a == "S" or a == ">" then                  -- Call or start
    local filename, line_defined, last_line_defined = b, c, d
    local source_file = get_source_file(filename)
    local func = get_function(filename, line_defined, last_line_defined)
    push{ source_file=source_file, func=func, frame_time=0 }
<commit_msg>removed unused function call from profile<commit_after>local trace_file = require("luatrace.trace_file")

local source_files                              -- Map of source files we've seen
local functions                                 -- Map of functions

local threads                                   -- All the running threads
local thread_stack                              -- and the order they're running in

local stack                                     -- Call stack

local total_time                                -- Running total of all the time we've recorded

local trace_count                               -- How many traces we've seen (for reporting errors)
local error_count                               -- How many errors we've seen

local profile = {}


--------------------------------------------------------------------------------

function profile.open()
  source_files, functions = {}, {}
  local main_thread = { top=0 }
  threads = { main_thread }
  thread_stack = { main_thread, top=1 }
  stack = { top=0 }
  total_time = 0
  trace_count, error_count = 0, 0
end


local function get_source_file(filename)
  local source_file = source_files[filename]
  if not source_file then
    source_file = { filename=filename, lines = {} }
    source_files[filename] = source_file
  end
  return source_file
end


local function get_function(filename, line_defined, last_line_defined)
  local name = filename..":"..tostring(line_defined).."-"..tostring(last_line_defined)
  local f = functions[name]
  if not f then
    f = { name=name, source_file=get_source_file(filename), line_defined=line_defined, last_line_defined=last_line_defined }
    functions[name] = f
  end
  return f
end


local function get_thread(thread_id)
  local thread = threads[thread_id]
  if not thread then
    thread = { top=0 }
    threads[thread_id] = thread
  end
  return thread
end


local function replay_push(frame)
  stack.top = stack.top + 1
  stack[stack.top] = frame
end


local function push(frame)
  replay_push(frame)
  local thread = thread_stack[thread_stack.top]
  thread.top = thread.top + 1
  thread[thread.top] = { func=frame.func }
end


local function push_thread(thread)
  thread_stack.top = thread_stack.top + 1
  thread_stack[thread_stack.top] = thread
end


local function replay_pop()
  local frame = stack[stack.top]
  stack[stack.top] = nil
  stack.top = stack.top - 1
  return frame
end


local function pop()
  local frame = replay_pop()
  local thread = thread_stack[thread_stack.top]
  thread[thread.top] = nil
  thread.top = thread.top - 1
  return frame
end


local function pop_thread()
  local thread = thread_stack[thread_stack.top]
  thread_stack[thread_stack.top] = nil
  thread_stack.top = thread_stack.top - 1
  return thread
end


local function get_top()
  return stack[stack.top]
end


local function thread_top()
  return thread_stack[thread_stack.top]
end


local function get_line(line_number)
  local top = get_top()
  local line = top.source_file.lines[line_number]
  if not line then
    line = { visits = 0, self_time = 0, child_time = 0 }
    top.source_file.lines[line_number] = line
  end
  return line
end


function profile.record(a, b, c, d)
  trace_count = trace_count + 1

  if a == "S" or a == ">" then                  -- Call or start
    local filename, line_defined, last_line_defined = b, c, d
    local source_file = get_source_file(filename)
    local func = get_function(filename, line_defined, last_line_defined)
    push{ source_file=source_file, func=func, frame_time=0 }

{'fraction_non_alphanumeric': 0.10487465181058496, 'fraction_numerical': 0.0051532033426183845, 'mean_word_length': 2.617632241813602, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4592084', 'n_tokens_mistral': 2195, 'n_tokens_neox': 2110, 'n_words': 609}