<commit_before>#!/usr/bin/env lua
--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
--
--  file:     glew_parse.lua
--  author:   <NAME> (aka: trepan)
--  date:     Jan 15, 2009
--  license:  LGPL 2.1
--  desc:     generates glew.lua from glew.h and the constants
--            being used by the source files in the current directory
--
--------------------------------------------------------------------------------
--------------------------------------------------------------------------------

local wantAll = false

local wantSpaces = false

local sortNumeric = false

local desired = {}

local glew_h = '/usr/include/GL/glew.h'

local outputName


--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
--
--  parse the command line arguments
--

local function PrintHelp()
  print('usage:  ' .. arg[-1] .. ' ' .. arg[0] .. ' [options]')
  print('  -h         : this help')
  print('  -g <path>  : the path to glew.h')
  print('  -o <path>  : the output filename')
  print('  -a         : extract all header definitions')
  print('  -s         : align the output text using spaces')
  print('  -n         : numeric sort')
end


local i = 1

while (i <= #arg) do
  local a = arg[i]
  if ((a == '-h') or (a == '-help') or (a == '--help')) then
    PrintHelp()
    os.exit(1)
  elseif (a == '-g') then
    i = i + 1
    if (arg[i]) then
      glew_h = arg[i]
    else
      print('missing ' .. a .. ' parameter')
      os.exit(1)
    end
  elseif (a == '-o') then
    i = i + 1
    if (arg[i]) then
      outputName = arg[i]
    else
      print('missing ' .. a .. ' parameter')
      os.exit(1)
    end
  elseif (a == '-a') then
    wantAll = true
  elseif (a == '-s') then
    wantSpaces = true
  elseif (a == '-n') then
    sortNumeric = true
  else
    print('unknown option: "' .. a .. '"')
    print()
    PrintHelp()
    os.exit(1)
  end
  
  i = i + 1
end


--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
--
--  find the GL.xxx tags in the code, with recursion
--

if (not wantAll) then

  require('lfs')

  local function DesiredFile(file)
    local f = io.open(file, 'r')
    if (not f) then
      return
    end
    for line in f:lines() do
      for enum in line:gmatch('[^%w]GL%.([%w_]+)') do
        desired[enum] = true
      end
    end
  end

  local function lfs_safedir(dir)
    local success, func, state, var = pcall(lfs.dir, dir)
    if (not success) then
      return function() end
    end
    return func, state, var
  end

  local function DesiredDir(dir)
    for file in lfs_safedir(dir) do
      if (not file:match('^%.')) then
        local fullPath = dir .. file
        local mode = lfs.attributes(fullPath, 'mode')
        if (mode == 'directory') then
          DesiredDir(fullPath .. '/')
        elseif( mode == 'file') then
          local lower = file:lower()
          if (lower:match('.+%.lua$') and (lower ~= 'glew.lua')) then
            DesiredFile(fullPath)
          end
        end
      end
    end
  end

  DesiredDir('./')
end


--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
--
--  open the header file and the output file
--

local input, err = io.open(glew_h, 'rt')
if (not input) then
  print(err)
  print()
  PrintHelp()
  os.exit(1)
end

local output = io.stdout
if (type(outputName) == 'string') then
  local err
  output, err = io.open(outputName, 'wt')
  if (not output) then
    print(err)
    print()
    PrintHelp()
    os.exit(1)
  end
end


--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
--
--  parse glew.h, and copy the header
--

local GL = {}
local headers = true

output:write('--[[\n\n')
for line in input:lines() do
  if (headers and not line:match('^#')) then
    output:write(line:gsub('\r','') .. '\n')
  else
    if (headers) then
      output:write('--]]\n')
      headers = false
    end
    -- this excludes the extension presence definitions
    local k, v = line:match('^#define%s*GL_([A-Z0-9_]*)%s*(%S*)\r?$')
    if (k and v and (wantAll or desired[k])) then
      GL[k] = tonumber(v)
    end
  end
end

input:close()


--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
--
--  sort by name, and find the longest name
--

local count = 0
local sorted = {}
local maxLen = 0
for k, v in pairs(GL) do
  count = count + 1
  sorted[count] = { k, v }
  local len = #k<commit_msg>* added the -d option  (for setting the source directory to scan)<commit_after>#!/usr/bin/env lua
--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
--
--  file:     glew_parse.lua
--  author:   <NAME> (aka: trepan)
--  date:     Jan 15, 2009
--  license:  LGPL 2.1
--  desc:     generates glew.lua from glew.h and the constants
--            being used by the source files in the current directory
--
--------------------------------------------------------------------------------
--------------------------------------------------------------------------------

local wantAll = false

local wantSpaces = false

local sortNumeric = false

local desired = {}

local scanDir = '.'

local glew_h = '/usr/include/GL/glew.h'

local outputName


--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
--
--  parse the command line arguments
--

local function PrintHelp()
  print('usage:  ' .. arg[-1] .. ' ' .. arg[0] .. ' [options]')
  print('  -h         : this help')
  print('  -g <path>  : the path to glew.h')
  print('  -o <path>  : the output filename')
  print('  -d <path>  : the scan directory')
  print('  -a         : extract all header definitions')
  print('  -s         : align the output text using spaces')
  print('  -n         : numeric sort')
end


local i = 1

while (i <= #arg) do
  local a = arg[i]
  if ((a == '-h') or (a == '-help') or (a == '--help')) then
    PrintHelp()
    os.exit(1)
  elseif (a == '-g') then
    i = i + 1
    if (arg[i]) then
      glew_h = arg[i]
    else
      print('missing ' .. a .. ' parameter')
      os.exit(1)
    end
  elseif (a == '-o') then
    i = i + 1
    if (arg[i]) then
      outputName = arg[i]
    else
      print('missing ' .. a .. ' parameter')
      os.exit(1)
    end
  elseif (a == '-d') then
    i = i + 1
    if (arg[i]) then
      scanDir = arg[i]
      if (not scanDir:match('/$')) then
        scanDir = scanDir .. '/'
      end
    else
      print('missing ' .. a .. ' parameter')
      os.exit(1)
    end
  elseif (a == '-a') then
    wantAll = true
  elseif (a == '-s') then
    wantSpaces = true
  elseif (a == '-n') then
    sortNumeric = true
  else
    print('unknown option: "' .. a .. '"')
    print()
    PrintHelp()
    os.exit(1)
  end
  
  i = i + 1
end


--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
--
--  find the GL.xxx tags in the code, with recursion
--

if (not wantAll) then

  require('lfs')

  local function DesiredFile(file)
    local f = io.open(file, 'r')
    if (not f) then
      return
    end
    for line in f:lines() do
      for enum in line:gmatch('[^%w]GL%.([%w_]+)') do
        desired[enum] = true
      end
    end
  end

  local function lfs_safedir(dir)
    local success, func, state, var = pcall(lfs.dir, dir)
    if (not success) then
      return function() end
    end
    return func, state, var
  end

  local function DesiredDir(dir)
    for file in lfs_safedir(dir) do
      if (not file:match('^%.')) then
        local fullPath = dir .. file
        local mode = lfs.attributes(fullPath, 'mode')
        if (mode == 'directory') then
          DesiredDir(fullPath .. '/')
        elseif( mode == 'file') then
          local lower = file:lower()
          if (lower:match('.+%.lua$') and (lower ~= 'glew.lua')) then
            DesiredFile(fullPath)
          end
        end
      end
    end
  end

  if (not scanDir:match('/$')) then
    scanDir = scanDir .. '/'
  end

  DesiredDir(scanDir)
end


--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
--
--  open the header file and the output file
--

local input, err = io.open(glew_h, 'rt')
if (not input) then
  print(err)
  print()
  PrintHelp()
  os.exit(1)
end

local output = io.stdout
if (type(outputName) == 'string') then
  local err
  output, err = io.open(outputName, 'wt')
  if (not output) then
    print(err)
    print()
    PrintHelp()
    os.exit(1)
  end
end


--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
--
--  parse glew.h, and copy the header
--

local GL = {}
local headers = true

output:write('--[[\n\n')
for line in input:lines() do
  if (headers and not line:match('^#')) then
    output:write(line:gsub('\r','') .. '\n')
  else
    if (headers) then
      output:write('--]]\n')
      headers = false
    end
    -- this excludes the extension presence definitions
    local k, v = line:match('^#define%s+GL_([%u%d_]*)%s+(%S+)\r?$')
    if (k and v and (wantAll or desired[k])) then
      GL[k] = tonumber(v)
    end
  end
end

input:close()


--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
--
--  sort by name, and find the longest name
--

local count = 0
local sorted = {}
local maxLen = 0
for k, v in pairs(GL) do
  count = count + 1
  sorted[count] = { k, v }
  local len = #k
{'fraction_non_alphanumeric': 0.33941570416383576, 'fraction_numerical': 0.00485295544986897, 'mean_word_length': 1.1258510418815761, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6772078', 'n_tokens_mistral': 2967, 'n_tokens_neox': 2922, 'n_words': 910}