<commit_before>-- Copyright 2006-2011 Mitchell mitchell<att>caladbolg.net. See LICENSE.
-- Java LPeg Lexer

-- Modified by <NAME>

local l = lexer
local token, style, color, word_match = l.token, l.style, l.color, l.word_match
local P, R, S = l.lpeg.P, l.lpeg.R, l.lpeg.S

module(...)

local ws = token(l.WHITESPACE, l.space^1)

-- comments
local line_comment = '//' * l.nonnewline_esc^0
local block_comment = '/*' * (l.any - '*/')^0 * P('*/')^-1
local comment = token(l.COMMENT, line_comment + block_comment)

-- strings
local sq_str = l.delimited_range("'", '\\', true, false, '\n')
local dq_str = l.delimited_range('"', '\\', true, false, '\n')
local string = token(l.STRING, sq_str + dq_str)

-- numbers
local number = token(l.NUMBER, (l.float + l.integer) * S('LlFfDd')^-1)

-- keywords
local keyword = token(l.KEYWORD, word_match {
  'abstract', 'assert', 'break', 'case', 'catch', 'class', 'const', 'continue',
  'default', 'do', 'else', 'enum', 'extends', 'final', 'finally', 'for',
  'future', 'generic', 'goto', 'if', 'implements', 'import', 'inner',
  'instanceof', 'interface', 'native', 'new', 'null', 'outer', 'package',
  'private', 'protected', 'public', 'rest', 'return', 'static', 'super',
  'switch', 'synchronized', 'this', 'throw', 'throws', 'transient', 'try',
  'var', 'while', 'volatile', 'true', 'false'
})

-- types
local type = token(l.TYPE, word_match {
  'boolean', 'byte', 'char', 'double', 'float', 'int', 'long', 'short', 'void'
})

-- identifiers
local identifier = token(l.IDENTIFIER, l.word)

-- annotations
local annotation = token('annotation', '@' * l.word)

local func = token(l.FUNCTION, l.word) * #P('(')

local class_sequence = token(l.KEYWORD, P('class')) * ws^1 *
                       token(l.CLASS, l.word)
<commit_msg>Add primitive classes as types; lexers/java.lua<commit_after>-- Copyright 2006-2011 Mitchell mitchell<att>caladbolg.net. See LICENSE.
-- Java LPeg Lexer

-- Modified by <NAME>

local l = lexer
local token, style, color, word_match = l.token, l.style, l.color, l.word_match
local P, R, S = l.lpeg.P, l.lpeg.R, l.lpeg.S

module(...)

local ws = token(l.WHITESPACE, l.space^1)

-- comments
local line_comment = '//' * l.nonnewline_esc^0
local block_comment = '/*' * (l.any - '*/')^0 * P('*/')^-1
local comment = token(l.COMMENT, line_comment + block_comment)

-- strings
local sq_str = l.delimited_range("'", '\\', true, false, '\n')
local dq_str = l.delimited_range('"', '\\', true, false, '\n')
local string = token(l.STRING, sq_str + dq_str)

-- numbers
local number = token(l.NUMBER, (l.float + l.integer) * S('LlFfDd')^-1)

-- keywords
local keyword = token(l.KEYWORD, word_match {
  'abstract', 'assert', 'break', 'case', 'catch', 'class', 'const', 'continue',
  'default', 'do', 'else', 'enum', 'extends', 'final', 'finally', 'for',
  'future', 'generic', 'goto', 'if', 'implements', 'import', 'inner',
  'instanceof', 'interface', 'native', 'new', 'null', 'outer', 'package',
  'private', 'protected', 'public', 'rest', 'return', 'static', 'super',
  'switch', 'synchronized', 'this', 'throw', 'throws', 'transient', 'try',
  'var', 'while', 'volatile', 'true', 'false'
})

-- types
local type = token(l.TYPE, word_match {
  'boolean', 'byte', 'char', 'double', 'float', 'int', 'long', 'short', 'void',
  'Boolean', 'Byte', 'Character', 'Double', 'Float', 'Integer', 'Long', 'Short',
  'String'
})

-- identifiers
local identifier = token(l.IDENTIFIER, l.word)

-- annotations
local annotation = token('annotation', '@' * l.word)

local func = token(l.FUNCTION, l.word) * #P('(')

local class_sequence = token(l.KEYWORD, P('class')) * ws^1 *
                       token(l.CLASS, l.word)

{'fraction_non_alphanumeric': 0.22465154413774255, 'fraction_numerical': 0.007652364033889041, 'mean_word_length': 4.266187050359712, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1016534', 'n_tokens_mistral': 1372, 'n_tokens_neox': 1336, 'n_words': 346}