<commit_before>-- Archlinux PKGBUILD LPeg Lexer

local l = lexer
local token, style, color, word_match = l.token, l.style, l.color, l.word_match
local P, R, S = l.lpeg.P, l.lpeg.R, l.lpeg.S

module(...)

local ws = token(l.WHITESPACE, l.space^1)

-- comments
local comment = token(l.COMMENT, '#' * l.nonnewline^0)

-- strings
local sq_str = l.delimited_range("'", nil, true)
local dq_str = l.delimited_range('"', '\\', true)
local ex_str = l.delimited_range('`', '\\', true)
local heredoc = '<<' * P(function(input, index)
  local s, e, _, delimiter =
    input:find('(["\']?)([%a_][%w_]*)%1[\n\r\f;]+', index)
  if s == index and delimiter then
    local _, e = input:find('[\n\r\f]+'..delimiter, e)
    return e and e + 1 or #input + 1
  end
end)
local string = token(l.STRING, sq_str + dq_str + ex_str + heredoc)

-- numbers
local number = token(l.NUMBER, l.float + l.integer)

-- keywords
local keyword = token(l.KEYWORD, word_match({
  'patch', 'cd', 'make', 'patch', 'mkdir', 'cp', 'sed', 'install', 'rm',
  'if', 'then', 'elif', 'else', 'fi', 'case', 'in', 'esac', 'while', 'for',
  'do', 'done', 'continue', 'local', 'return', 'git', 'svn', 'co', 'clone',
  'gconf-merge-schema', 'msg', 'echo', 'ln',
  -- operators
  '-a', '-b', '-c', '-d', '-e', '-f', '-g', '-h', '-k', '-p', '-r', '-s', '-t',
  '-u', '-w', '-x', '-O', '-G', '-L', '-S', '-N', '-nt', '-ot', '-ef', '-o',
  '-z', '-n', '-eq', '-ne', '-lt', '-le', '-gt', '-ge', '-Np', '-i'
}, '-'))

-- functions
local func = token(l.FUNCTION, word_match { 'build' })

local constant = token(l.CONSTANT, word_match {
  'pkgname', 'pkgver', 'pkgrel', 'pkgdesc', 'arch', 'url',
  'license', 'optdepends', 'depends', 'makedepends', 'provides',
  'conflicts', 'replaces', 'install', 'source', 'md5sums',
  'pkgdir', 'srcdir'
})

-- identifiers
local identifier = token(l.IDENTIFIER, l.word)

-- variables
local variable = token(l.VARIABLE, '$' * (S('!#?*@$') +
                       l.delimited_range('()', nil, true, false, '\n') +
                       l.delimited_range('[]', nil, true, false, '\n') +
                       l.delimited_range('{}', nil, true, false, '\n') +
                       l.delimited_range('`', nil, true, false, '\n') +
                       l.digit^1 + l.word))

-- operators
local operator = token(l.OPERATOR, S('=!<>+-/*^~.,:;?()[]{}'))

_rules = {
  { 'whitespace', ws },
  { 'comment', comment },
  { 'string', string },
  { 'number', number },
  { 'keyword', keyword },
  { 'function', func },
  { 'constant', constant },
  { 'identifier', identifier },
  { 'variable', variable },
  { 'operator', operator },
  { 'any_char', l.any_char },
}
<commit_msg>Added folding for pkgbuild; lexers/pkgbuild.lua<commit_after>-- Archlinux PKGBUILD LPeg Lexer

local l = lexer
local token, style, color, word_match = l.token, l.style, l.color, l.word_match
local P, R, S = l.lpeg.P, l.lpeg.R, l.lpeg.S

module(...)

local ws = token(l.WHITESPACE, l.space^1)

-- comments
local comment = token(l.COMMENT, '#' * l.nonnewline^0)

-- strings
local sq_str = l.delimited_range("'", nil, true)
local dq_str = l.delimited_range('"', '\\', true)
local ex_str = l.delimited_range('`', '\\', true)
local heredoc = '<<' * P(function(input, index)
  local s, e, _, delimiter =
    input:find('(["\']?)([%a_][%w_]*)%1[\n\r\f;]+', index)
  if s == index and delimiter then
    local _, e = input:find('[\n\r\f]+'..delimiter, e)
    return e and e + 1 or #input + 1
  end
end)
local string = token(l.STRING, sq_str + dq_str + ex_str + heredoc)

-- numbers
local number = token(l.NUMBER, l.float + l.integer)

-- keywords
local keyword = token(l.KEYWORD, word_match({
  'patch', 'cd', 'make', 'patch', 'mkdir', 'cp', 'sed', 'install', 'rm',
  'if', 'then', 'elif', 'else', 'fi', 'case', 'in', 'esac', 'while', 'for',
  'do', 'done', 'continue', 'local', 'return', 'git', 'svn', 'co', 'clone',
  'gconf-merge-schema', 'msg', 'echo', 'ln',
  -- operators
  '-a', '-b', '-c', '-d', '-e', '-f', '-g', '-h', '-k', '-p', '-r', '-s', '-t',
  '-u', '-w', '-x', '-O', '-G', '-L', '-S', '-N', '-nt', '-ot', '-ef', '-o',
  '-z', '-n', '-eq', '-ne', '-lt', '-le', '-gt', '-ge', '-Np', '-i'
}, '-'))

-- functions
local func = token(l.FUNCTION, word_match { 'build' })

local constant = token(l.CONSTANT, word_match {
  'pkgname', 'pkgver', 'pkgrel', 'pkgdesc', 'arch', 'url',
  'license', 'optdepends', 'depends', 'makedepends', 'provides',
  'conflicts', 'replaces', 'install', 'source', 'md5sums',
  'pkgdir', 'srcdir'
})

-- identifiers
local identifier = token(l.IDENTIFIER, l.word)

-- variables
local variable = token(l.VARIABLE, '$' * (S('!#?*@$') +
                       l.delimited_range('()', nil, true, false, '\n') +
                       l.delimited_range('[]', nil, true, false, '\n') +
                       l.delimited_range('{}', nil, true, false, '\n') +
                       l.delimited_range('`', nil, true, false, '\n') +
                       l.digit^1 + l.word))

-- operators
local operator = token(l.OPERATOR, S('=!<>+-/*^~.,:;?()[]{}'))

_rules = {
  { 'whitespace', ws },
  { 'comment', comment },
  { 'string', string },
  { 'number', number },
  { 'keyword', keyword },
  { 'function', func },
  { 'constant', constant },
  { 'identifier', identifier },
  { 'variable', variable },
  { 'operator', operator },
  { 'any_char', l.any_char },
}

_foldsymbols = {
  _patterns = { '[%(%){}]' },
  operator = { ['('] = 1, [')'] = -1, ['{'] = 1, ['}'] = -1 }
}

{'fraction_non_alphanumeric': 0.2657111356119074, 'fraction_numerical': 0.0033076074972436605, 'mean_word_length': 3.1141345427059712, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3350386', 'n_tokens_mistral': 2150, 'n_tokens_neox': 2097, 'n_words': 515}