<commit_before>-- Copyright 2006-2017 Mitchell mitchell.att.foicica.com. See LICENSE.
-- RHTML LPeg lexer.

local l = require('lexer')
local token, word_match = l.token, l.word_match
local P, R, S = lpeg.P, lpeg.R, lpeg.S

local M = {_NAME = 'rhtml'}

-- Embedded in HTML.
local html = l.load('html')

-- Embedded Ruby.
local ruby = l.load('rails')
local ruby_start_rule = token('rhtml_tag', '<%' * P('=')^-1)
local ruby_end_rule = token('rhtml_tag', '%>')
l.embed_lexer(html, ruby, ruby_start_rule, ruby_end_rule)

M._tokenstyles = {
  rhtml_tag = l.STYLE_EMBEDDED
}

local _foldsymbols = html._foldsymbols
_foldsymbols._patterns[#_foldsymbols._patterns + 1] = '<%%'
_foldsymbols._patterns[#_foldsymbols._patterns + 1] = '%%>'
_foldsymbols.rhtml_tag = {['<%'] = 1, ['%>'] = -1}
M._foldsymbols = _foldsymbols

return M<commit_msg>Refactored RHTML lexer to be more object-oriented.<commit_after>-- Copyright 2006-2017 Mitchell mitchell.att.foicica.com. See LICENSE.
-- RHTML LPeg lexer.

local lexer = require('lexer')
local token, word_match = lexer.token, lexer.word_match
local P, R, S = lpeg.P, lpeg.R, lpeg.S

local lex = lexer.new('rhtml', {inherit = lexer.load('html')})

-- Embedded Ruby.
local ruby = lexer.load('rails')
local ruby_start_rule = token('rhtml_tag', '<%' * P('=')^-1)
local ruby_end_rule = token('rhtml_tag', '%>')
lex:embed(ruby, ruby_start_rule, ruby_end_rule)
lex:add_style('rhtml_tag', lexer.STYLE_EMBEDDED)

-- Fold points.
lex:add_fold_point('rhtml_tag', '<%', '%>')

return lex
{'fraction_non_alphanumeric': 0.17264276228419656, 'fraction_numerical': 0.014608233731739707, 'mean_word_length': 4.98015873015873, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2435236', 'n_tokens_mistral': 649, 'n_tokens_neox': 594, 'n_words': 125}