<commit_before>-- Load Specl spec-files into native Lua tables.
--
-- Copyright (c) 2013 Free Software Foundation, Inc.
-- Written by <NAME>, 2013
--
-- This program is free software; you can redistribute it and/or modify
-- it under the terms of the GNU General Public License as published by
-- the Free Software Foundation; either version 3, or (at your option)
-- any later version.
--
-- This program is distributed in the hope that it will be useful, but
-- WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-- General Public License for more details.
--
-- You should have received a copy of the GNU General Public License
-- along with this program; see the file COPYING.  If not, write to the
-- Free Software Foundation, Fifth Floor, 51 Franklin Street, Boston,
-- MA 02111-1301, USA.


local macro = require "macro"
local util  = require "specl.util"
local yaml  = require "yaml"


local TAG_PREFIX = "tag:yaml.org,2002:"
local null       = { type = "LYAML null" }


-- Capture errors thrown by expectations.
macro.define 'expect(expr)  _expect (pcall (function () return expr end))'


-- Metatable for Parser objects.
local parser_mt = {
  __index = {
    -- Return the type of the current event.
    type = function (self)
      return tostring (self.event.type)
    end,

    -- Raise a parse error.
    error = function (self, errmsg)
      return error (self.filename .. ":" .. self.mark.line .. ":" ..
                    self.mark.column .. ": " .. errmsg, 0)
    end,

    -- Compile S into a callable function.
    compile = function (self, s)
      -- Wrap the fragment into a function that we can call later.
      local f, errmsg = loadstring ("return function () " ..
                          macro.substitute_tostring (s) .. "\nend", self.filename)
      if f == nil then
        local line, msg = errmsg:match ('%[string "[^"]*"%]:([1-9][0-9]*): (.*)$')
        if msg ~= nil then
          line = line + self.mark.line
	  if self.mark.style == "PLAIN" then line = line - 1 end
          errmsg = self.filename .. ":" .. tostring (line) .. ": " .. msg
        end
      else
        -- Execute the function from 'loadstring' or report an error right away.
        f, errmsg = f ()
      end
      if errmsg ~= nil then
        io.stderr:write (errmsg .. "\n")
        os.exit (1)
      end
      return f
    end,

    -- Save node in the anchor table for reference in future ALIASes.
    add_anchor = function (self, node)
      if self.event.anchor ~= nil then
        self.anchors[self.event.anchor] = node
      end
    end,

    -- Fetch the next event.
    parse = function (self)
      self.event = self.next ()
      self.mark  = {<commit_msg>refactor: simplify a redundant function wrap and execute.<commit_after>-- Load Specl spec-files into native Lua tables.
--
-- Copyright (c) 2013 Free Software Foundation, Inc.
-- Written by <NAME>, 2013
--
-- This program is free software; you can redistribute it and/or modify
-- it under the terms of the GNU General Public License as published by
-- the Free Software Foundation; either version 3, or (at your option)
-- any later version.
--
-- This program is distributed in the hope that it will be useful, but
-- WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-- General Public License for more details.
--
-- You should have received a copy of the GNU General Public License
-- along with this program; see the file COPYING.  If not, write to the
-- Free Software Foundation, Fifth Floor, 51 Franklin Street, Boston,
-- MA 02111-1301, USA.


local macro = require "macro"
local util  = require "specl.util"
local yaml  = require "yaml"


local TAG_PREFIX = "tag:yaml.org,2002:"
local null       = { type = "LYAML null" }


-- Capture errors thrown by expectations.
macro.define 'expect(expr)  _expect (pcall (function () return expr end))'


-- Metatable for Parser objects.
local parser_mt = {
  __index = {
    -- Return the type of the current event.
    type = function (self)
      return tostring (self.event.type)
    end,

    -- Raise a parse error.
    error = function (self, errmsg)
      return error (self.filename .. ":" .. self.mark.line .. ":" ..
                    self.mark.column .. ": " .. errmsg, 0)
    end,

    -- Compile S into a callable function.
    compile = function (self, s)
      local f, errmsg = loadstring ((macro.substitute_tostring (s)))
      if f == nil then
        local line, msg = errmsg:match ('%[string "[^"]*"%]:([1-9][0-9]*): (.*)$')
        if msg ~= nil then
          line = line + self.mark.line
	  if self.mark.style == "PLAIN" then line = line - 1 end
          errmsg = self.filename .. ":" .. tostring (line) .. ": " .. msg
        end
      end
      if errmsg ~= nil then
        io.stderr:write (errmsg .. "\n")
        os.exit (1)
      end
      return f
    end,

    -- Save node in the anchor table for reference in future ALIASes.
    add_anchor = function (self, node)
      if self.event.anchor ~= nil then
        self.anchors[self.event.anchor] = node
      end
    end,

    -- Fetch the next event.
    parse = function (self)
      self.event = self.next ()
      self.mark  = {
{'fraction_non_alphanumeric': 0.11572588257504247, 'fraction_numerical': 0.0117047385312441, 'mean_word_length': 2.321630094043887, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '311876', 'n_tokens_mistral': 1592, 'n_tokens_neox': 1513, 'n_words': 629}