<commit_before>-- module for initial setup

local config = require "config"
local categoryList = config.categories

local dbmodule = require "dbmodule"
-- scriptdb = config.scriptdb
local setup = {}

function setup.install(lines)
  dbmodule.InitSetup("wc")
  local t ={}
  local id_script = 0
  for k,v in ipairs(lines) do
    v = v:gsub('%Entry { filename = "',""):gsub('", categories = { "',',"'):gsub('", } }','"'):gsub('", "','","')
    for i,c in ipairs(categoryList) do
      v = v:gsub('"'..c..'"',i)
    end
    for a in string.gmatch(v,"([^,]+)") do
      table.insert(t,a)
    end
    for key,value in ipairs(t) do
      if t[1] == value then
        local val = {value}
        id_script = dbmodule.InsertScript(val,"scripts")
      else
        dbmodule.InsertCategory(id_script,value)
      end
    end
    t = {}
  end
end
return setup
<commit_msg>Add banner to the initial setup process<commit_after>-- module for initial setup

local config = require "config"
local categoryList = config.categories
local dbmodule = require "dbmodule"
local helper = require "helper"
-- scriptdb = config.scriptdb
local setup = {}

function setup.install(lines)
  print('\27[1m \27[36m'..helper.banner()..'\27[21m \27[0m')
  dbmodule.InitSetup("wc")
  local t ={}
  local id_script = 0
  for k,v in ipairs(lines) do
    v = v:gsub('%Entry { filename = "',""):gsub('", categories = { "',',"'):gsub('", } }','"'):gsub('", "','","')
    for i,c in ipairs(categoryList) do
      v = v:gsub('"'..c..'"',i)
    end
    for a in string.gmatch(v,"([^,]+)") do
      table.insert(t,a)
    end
    for key,value in ipairs(t) do
      if t[1] == value then
        local val = {value}
        id_script = dbmodule.InsertScript(val,"scripts")
      else
        dbmodule.InsertCategory(id_script,value)
      end
    end
    t = {}
  end
end
return setup

{'fraction_non_alphanumeric': 0.1748099891422367, 'fraction_numerical': 0.009771986970684038, 'mean_word_length': 3.0505494505494504, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3125141', 'n_tokens_mistral': 653, 'n_tokens_neox': 640, 'n_words': 169}