<commit_before>-- See LICENSE file for copyright and license details

local Misc = require('misc')
local LexerHelp = require('lexerHelp')

-- shortcuts
local isLetterOrDigit = LexerHelp.isLetterOrDigit
local isNameOrNumber = LexerHelp.isNameOrNumber
local isDigit = LexerHelp.isDigit
local isLetter = LexerHelp.isLetter
local isNumber = LexerHelp.isNumber
local isName = LexerHelp.isName
local nthChar = LexerHelp.nthChar

-- lexer:lexem() -- get current lexem
-- lexer:peek(number) -- peek following lexems
-- lexer:next() -- go to next lexems
-- lexer:eat() -- сожрать определенную лексему

local M = {}
M.__index = M

M.new = function()
  local self = {}
  self._lexems = {}
  self._currentLexem = ''
  setmetatable(self, M)
  return self
end

-- разбивает строку на прелексемы
M.parseString = function(source)
  local lexems = {}
  local appendCharToLastLexem = function(char)
    lexems[#lexems] = lexems[#lexems] .. char
  end
  for i = 1, #source do
    local char = nthChar(source, i)
    local lastLexem = lexems[#lexems]
    if i == 1 then
      table.insert(lexems, char)
    elseif isNumber(lastLexem) and isDigit(char) then
      appendCharToLastLexem(char)
    elseif isName(lastLexem) and isLetterOrDigit(char) then
      appendCharToLastLexem(char)
    else
      table.insert(lexems, char)
    end
  end
  return lexems
end

M.incDecIndent = function(prelexems)
  local newLexemsList = {}
  local prevIndentLevel = 0
  local i = 1
  while i <= #prelexems do
    if prelexems[i] == '\n' then
      table.insert(newLexemsList, {tag = 'endOfLine'})
      i = i + 1
      local spacesCount = 0
      while prelexems[i] == ' ' do
        spacesCount = spacesCount + 1
        i = i + 1
      end
      -- print('prevIndentLevel = '.. prevIndentLevel)
      -- print('spacesCount = '.. spacesCount)
      indentLevel = spacesCount / 2
      -- print('indentLevel = '.. indentLevel)
      local diff = prevIndentLevel - indentLevel
      prevIndentLevel = indentLevel
      -- print('diff = '.. diff)
      if diff == 0 then
        table.insert(newLexemsList, {tag = 'samIndent'})
      else
        while diff < 0 do
          table.insert(newLexemsList, {tag = 'incIndent'})
          diff = diff + 1
        end
        while diff > 0 do
          table.insert(newLexemsList, {tag = 'decIndent'})
          diff = diff - 1
        end
      end
    else
      -- local keywords = {
      --   'procedure',
      --   'if',
      --   'else',
      --   'while',
      --   'case',
      --   'of',
      --   'do',
      -- }
      -- for _, keyword in pairs(keywords) do
      --   if prelexems[i] == keyword then
      --     table.insert(newLexemsList, {tag = keyword})
      --   end
      -- end<commit_msg>lexer: Removed commented-out debug prints<commit_after>-- See LICENSE file for copyright and license details

local Misc = require('misc')
local LexerHelp = require('lexerHelp')

-- shortcuts
local isLetterOrDigit = LexerHelp.isLetterOrDigit
local isNameOrNumber = LexerHelp.isNameOrNumber
local isDigit = LexerHelp.isDigit
local isLetter = LexerHelp.isLetter
local isNumber = LexerHelp.isNumber
local isName = LexerHelp.isName
local nthChar = LexerHelp.nthChar

-- lexer:lexem() -- get current lexem
-- lexer:peek(number) -- peek following lexems
-- lexer:next() -- go to next lexems
-- lexer:eat() -- сожрать определенную лексему

local M = {}
M.__index = M

M.new = function()
  local self = {}
  self._lexems = {}
  self._currentLexem = ''
  setmetatable(self, M)
  return self
end

-- разбивает строку на прелексемы
M.parseString = function(source)
  local lexems = {}
  local appendCharToLastLexem = function(char)
    lexems[#lexems] = lexems[#lexems] .. char
  end
  for i = 1, #source do
    local char = nthChar(source, i)
    local lastLexem = lexems[#lexems]
    if i == 1 then
      table.insert(lexems, char)
    elseif isNumber(lastLexem) and isDigit(char) then
      appendCharToLastLexem(char)
    elseif isName(lastLexem) and isLetterOrDigit(char) then
      appendCharToLastLexem(char)
    else
      table.insert(lexems, char)
    end
  end
  return lexems
end

M.incDecIndent = function(prelexems)
  local newLexemsList = {}
  local prevIndentLevel = 0
  local i = 1
  while i <= #prelexems do
    if prelexems[i] == '\n' then
      table.insert(newLexemsList, {tag = 'endOfLine'})
      i = i + 1
      local spacesCount = 0
      while prelexems[i] == ' ' do
        spacesCount = spacesCount + 1
        i = i + 1
      end
      indentLevel = spacesCount / 2
      local diff = prevIndentLevel - indentLevel
      prevIndentLevel = indentLevel
      if diff == 0 then
        table.insert(newLexemsList, {tag = 'samIndent'})
      else
        while diff < 0 do
          table.insert(newLexemsList, {tag = 'incIndent'})
          diff = diff + 1
        end
        while diff > 0 do
          table.insert(newLexemsList, {tag = 'decIndent'})
          diff = diff - 1
        end
      end
    else
      -- local keywords = {
      --   'procedure',
      --   'if',
      --   'else',
      --   'while',
      --   'case',
      --   'of',
      --   'do',
      -- }
      -- for _, keyword in pairs(keywords) do
      --   if prelexems[i] == keyword then
      --     table.insert(newLexemsList, {tag = keyword})
      --   end
      -- end
{'fraction_non_alphanumeric': 0.11400528501321253, 'fraction_numerical': 0.005285013212533031, 'mean_word_length': 2.345328282828283, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '279237', 'n_tokens_mistral': 1861, 'n_tokens_neox': 1777, 'n_words': 457}