<commit_before>-- Use of this source code is governed by the Apache 2.0 license; see COPYING.

-- This module implements a YANG parser which will parse a YANG module to a
-- Lua table. This is used by instantiating the Parser with the module and
-- optionally the filename. You then should call Parser:parse_module
--
-- The parser uses the same termanology as the specification (rfc6020). The
-- following is an example YANG module and lua representation:
-- YANG:
--
-- leaf port {
--     type inet:port-number;
--     default;
--     description "The port to which the SSH server listens"
--  }
--
-- Lua table:
-- {
--     {argument="port", keyword="leaf", statements={
--         {keyword="type", argument="inet:port-number"},
--         {keyword="default"},
--         {
--             keyword="description",
--             argument="The port to which the SSH server listens"
--         }}
--     }
-- }

module(..., package.seeall)

local lib = require('core.lib')

Parser = {}
function Parser.new(str, filename)
   local ret = {pos=1, str=str, filename=filename, line=1, column=0, line_pos=1}
   ret = setmetatable(ret, {__index = Parser})
   ret.peek_char = ret:read_char()
   return ret
end

function Parser:loc()
   return string.format('%s:%d:%d', self.name or '<unknown>', self.line,
                        self.column)
end

function Parser:error(msg, ...)
   print(self.str:match("[^\n]*", self.line_pos))
   print(string.rep(" ", self.column).."^")
   error(('%s: error: '..msg):format(self:loc(), ...))
end

function Parser:read_char()
   if self.pos <= #self.str then
      local ret = self.str:sub(self.pos,self.pos)
      self.pos = self.pos + 1
      return ret
   end
end

function Parser:peek() return self.peek_char end
function Parser:is_eof() return not self:peek() end

function Parser:next()
   local chr = self.peek_char
   if chr == '\n' then
      self.line_pos = self.pos + 1
      self.column = 0
      self.line = self.line + 1
   elseif chr == "\t" then
      self.column = self.column + 8
      self.column = 8 * math.floor(self.column / 8)
   elseif chr then
      self.column = self.column + 1
   end
   self.peek_char = self:read_char()
   return chr
end

function Parser:peek_n(n)
   local end_index = self.pos + n - 1
   if end_index < #self.str then
      return self:peek() .. self.str:sub(self.pos, (end_index - 1))
   end
   return self:peek() .. self.str:sub(self.pos)
end

function Parser:check(expected)
   if self:peek() == expected then
      if expected then self:next() end
      return true
   end
   return false
end

function Parser:check_pattern(pattern)
   if not self:is_eof() then
      return self:peek():match(pattern)
   end
   return false
end

function Parser:consume(expected)
   if not self:check(expected) then
      local ch = self:peek()
      if ch == nil then
         self:error("while looking for '%s', got EOF", expected)
      elseif expected then
         self:error("expected '%s', got '%s'", expected, ch)
      else
         self:error("expected EOF, got '%s'", ch)
      end
   end
end

function Parser:take_while(pattern)
   local res = {}
   while not self:is_eof() and self:peek():match(pattern) do
      table.insert(res, self:next())
   end
   return table.concat(res)
end

function Parser:skip_c_comment()
   repeat
      self:take_while("[^*]")
      self:consume("*")
   until self:check("/")
end

-- Returns true if has consumed any whitespace
function Parser:skip_whitespace()
   local result = false
   if self:take_while('%s') ~= "" then result = true end
   -- Skip comments, which start with # and continue to the end of line.
   while self:check('/') do
      result = true
      if self:check("*") then
         self:skip_c_comment()
      else
         self:consume("/")
         self:take_while('[^\n]')
      end
      self:take_while('%s')
   end
   return result
end

function Parser:consume_whitespace()
   if not self:skip_whitespace() then
      self:error("Missing whitespace")
   end
end

function Parser:consume_token(pattern, expected)
   local tok = self:take_while(pattern)
   if tok:lower() ~= expected then
      self:error("expected '%s', got '%s'", expected, tok)
   end
end

function Parser:parse_qstring(quote)
   local start_column = self.column
   local terminators = "\n"..quote
   if quote == '"' then terminators = terminators.."\\" end

   local result = ""
   while true do
      result = result..self:take_while("[^"..terminators.."]")
      if self:check(quote) then break end
      if self:check("\n") then
         while self.column < start_column do
            if not self:check(" ") and not self:check("\t") then break end
         end
         result = result.."\n"
         if self.column > start_column then
            result = result..string.rep(" ", self.column-start_column)
         end
      elseif self:check("\\") then
         if self:check("n") then result = result.."\n"
         elseif self:check("t") then result = result.."\t"
         elseif self:check('"') then result = result..'"'
         elseif self:check("\\") then result = result.."\\"
         else
            result = result.."\\"
         end
      end
   end
   self:check(quote)
   self:skip_whitespace()

   if not self:check("+") then return result end
   self:skip_whitespace()

   -- Strings can be concatenated together with a +
   if self:check("'") then
      return result..self:parse_qstring("'")
   elseif self:check('"') then
      return result..self:parse_qstring('"')
   else
      self:error("Expected quote character")
   end
end

function Parser:parse_string()
   if self:check("'") then return self:parse_qstring("'")
   elseif self:check('"') then return self:parse_qstring('"')
   else
      local ret = {}
      repeat
         table.insert(ret, self:take_while("[^%s;{}\"'/]"))
         if self:is_eof() then break end
         if self:peek_n(2) == "/*" then break end
         if self:peek_n(2) == "//" then break end
         if self:check("/") then
            table.insert(ret, "/")<commit_msg>Improve speed of yang qstring parser.<commit_after>-- Use of this source code is governed by the Apache 2.0 license; see COPYING.

-- This module implements a YANG parser which will parse a YANG module to a
-- Lua table. This is used by instantiating the Parser with the module and
-- optionally the filename. You then should call Parser:parse_module
--
-- The parser uses the same termanology as the specification (rfc6020). The
-- following is an example YANG module and lua representation:
-- YANG:
--
-- leaf port {
--     type inet:port-number;
--     default;
--     description "The port to which the SSH server listens"
--  }
--
-- Lua table:
-- {
--     {argument="port", keyword="leaf", statements={
--         {keyword="type", argument="inet:port-number"},
--         {keyword="default"},
--         {
--             keyword="description",
--             argument="The port to which the SSH server listens"
--         }}
--     }
-- }

module(..., package.seeall)

local lib = require('core.lib')

Parser = {}
function Parser.new(str, filename)
   local ret = {pos=1, str=str, filename=filename, line=1, column=0, line_pos=1}
   ret = setmetatable(ret, {__index = Parser})
   ret.peek_char = ret:read_char()
   return ret
end

function Parser:loc()
   return string.format('%s:%d:%d', self.name or '<unknown>', self.line,
                        self.column)
end

function Parser:error(msg, ...)
   print(self.str:match("[^\n]*", self.line_pos))
   print(string.rep(" ", self.column).."^")
   error(('%s: error: '..msg):format(self:loc(), ...))
end

function Parser:read_char()
   if self.pos <= #self.str then
      local ret = self.str:sub(self.pos,self.pos)
      self.pos = self.pos + 1
      return ret
   end
end

function Parser:peek() return self.peek_char end
function Parser:is_eof() return not self:peek() end

function Parser:next()
   local chr = self.peek_char
   if chr == '\n' then
      self.line_pos = self.pos + 1
      self.column = 0
      self.line = self.line + 1
   elseif chr == "\t" then
      self.column = self.column + 8
      self.column = 8 * math.floor(self.column / 8)
   elseif chr then
      self.column = self.column + 1
   end
   self.peek_char = self:read_char()
   return chr
end

function Parser:peek_n(n)
   local end_index = self.pos + n - 1
   if end_index < #self.str then
      return self:peek() .. self.str:sub(self.pos, (end_index - 1))
   end
   return self:peek() .. self.str:sub(self.pos)
end

function Parser:check(expected)
   if self:peek() == expected then
      if expected then self:next() end
      return true
   end
   return false
end

function Parser:check_pattern(pattern)
   if not self:is_eof() then
      return self:peek():match(pattern)
   end
   return false
end

function Parser:consume(expected)
   if not self:check(expected) then
      local ch = self:peek()
      if ch == nil then
         self:error("while looking for '%s', got EOF", expected)
      elseif expected then
         self:error("expected '%s', got '%s'", expected, ch)
      else
         self:error("expected EOF, got '%s'", ch)
      end
   end
end

function Parser:take_while(pattern)
   local res = {}
   while not self:is_eof() and self:peek():match(pattern) do
      table.insert(res, self:next())
   end
   return table.concat(res)
end

function Parser:skip_c_comment()
   repeat
      self:take_while("[^*]")
      self:consume("*")
   until self:check("/")
end

-- Returns true if has consumed any whitespace
function Parser:skip_whitespace()
   local result = false
   if self:take_while('%s') ~= "" then result = true end
   -- Skip comments, which start with # and continue to the end of line.
   while self:check('/') do
      result = true
      if self:check("*") then
         self:skip_c_comment()
      else
         self:consume("/")
         self:take_while('[^\n]')
      end
      self:take_while('%s')
   end
   return result
end

function Parser:consume_whitespace()
   if not self:skip_whitespace() then
      self:error("Missing whitespace")
   end
end

function Parser:consume_token(pattern, expected)
   local tok = self:take_while(pattern)
   if tok:lower() ~= expected then
      self:error("expected '%s', got '%s'", expected, tok)
   end
end

function Parser:parse_qstring(quote)
   local start_column = self.column
   local terminators = "\n"..quote
   if quote == '"' then terminators = terminators.."\\" end

   local result = {}
   while true do
      table.insert(result, self:take_while("[^"..terminators.."]"))
      if self:check(quote) then break end
      if self:check("\n") then
         while self.column < start_column do
            if not self:check(" ") and not self:check("\t") then break end
         end
         table.insert(result, "\n")
         if self.column > start_column then
            table.insert(result, string.rep(" ", self.column-start_column))
         end
      elseif self:check("\\") then
         if self:check("n")      then table.insert(result, "\n")
         elseif self:check("t")  then table.insert(result, "\t")
         elseif self:check('"')  then table.insert(result, '"')
         elseif self:check("\\") then table.insert(result, "\\")
         else                         table.insert(result, "\\") end
      end
   end
   result = table.concat(result)
   self:check(quote)
   self:skip_whitespace()

   if not self:check("+") then return result end
   self:skip_whitespace()

   -- Strings can be concatenated together with a +
   if self:check("'") then
      return result..self:parse_qstring("'")
   elseif self:check('"') then
      return result..self:parse_qstring('"')
   else
      self:error("Expected quote character")
   end
end

function Parser:parse_string()
   if self:check("'") then return self:parse_qstring("'")
   elseif self:check('"') then return self:parse_qstring('"')
   else
      local ret = {}
      repeat
         table.insert(ret, self:take_while("[^%s;{}\"'/]"))
         if self:is_eof() then break end
         if self:peek_n(2) == "/*" then break end
         if self:peek_n(2) == "//" then break end
         if self:check("/") then
            table.insert(ret, "/")
{'fraction_non_alphanumeric': 0.1405800673732643, 'fraction_numerical': 0.003615150768219538, 'mean_word_length': 2.6795646916565903, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '777603', 'n_tokens_mistral': 3895, 'n_tokens_neox': 3863, 'n_words': 1114}