<commit_before>
----------------------------------------------------------------------------
--  msg_spec.lua - Message specification wrapper
--
--  Created: Mon Jul 26 16:59:59 2010 (at Intel Research, Pittsburgh)
--  License: BSD, cf. LICENSE file of roslua
--  Copyright  2010  <NAME> [www.niemueller.de]
--             2010  Carnegie Mellon University
--             2010  Intel Research Pittsburgh
----------------------------------------------------------------------------

--- Message specification.
-- This module contains the MsgSpec class to read and represent ROS message
-- specification (YAML files). Message specifications should be obtained by
-- using the <code>get_msgspec()</code> function, which is aliased for
-- convenience as <code>roslua.get_msgspec()</code>.
-- <br /><br />
-- The message files are read on the fly, no offline code generation is
-- necessary. This avoids the need to write yet another code generator. After
-- reading the table field <code>fields</code> contains key value pairs, where
-- the keys are the names of the data fields and the value is the type of the
-- data field.
-- @copyright <NAME>, Carnegie Mellon University, Intel Research Pittsburgh
-- @release Released under BSD license
module("roslua.msg_spec", package.seeall)

require("md5")
require("roslua.message")
require("roslua.utils")

BUILTIN_TYPES = { "int8","uint8","int16","uint16","int32","uint32",
		  "int64","uint64","float32","float64","string","bool",
		  "byte", "char"}
for _,v in ipairs(BUILTIN_TYPES) do
   BUILTIN_TYPES[v] = v
end
EXTENDED_TYPES = { time={"uint32", "uint32"}, duration={"int32", "int32"} }

local msgspec_cache = {}

--- Get the base version of type, i.e. the non-array type.
-- @param type type to get base type for
-- @return base type, for array types returns the non-array type, for non-array
-- type returns the given value.
function base_type(type)
   return type:match("^([^%[]+)") or type
end

--- Check if type is an array type.
-- @param type type to check
-- @return true if given type is an array type, false otherwise
function is_array_type(type)
   return type:find("%[") ~= nil
end

--- Check if given type is a built-in type.
-- @param type type to check
-- @return true if type is a built-in type, false otherwise
function is_builtin_type(type)
   local t = base_type(type)
   return BUILTIN_TYPES[t] ~= nil or  EXTENDED_TYPES[t] ~= nil
end

--- Resolve the given type.
-- @param type to resolve
-- @param package to which the type should be resolve relatively
-- @return the given value if it is either a base type or contains a slash,
-- otherwise returns package/type.
function resolve_type(type, package)
   if is_builtin_type(type) or type:find("/") then
      return type
   else
      return string.format("%s/%s", package, type)
   end
end

--- Get message specification.
-- It is recommended to use the aliased version <code>roslua.get_msgspec()</code>.
-- @param msg_type message type (e.g. std_msgs/String). The name must include
-- the package.
function get_msgspec(msg_type, specstr)
   roslua.utils.assert_rospack()

   if not msgspec_cache[msg_type] then
      msgspec_cache[msg_type] = MsgSpec:new{type=msg_type, specstr=specstr}
   end

   return msgspec_cache[msg_type]
end

--- Check if the given object is a message spec.
-- @param testobj object to test
-- @return true if testobj is a message spec, false otherwise
function is_msgspec(testobj)
   if type(testobj) == "table" then
      if getmetatable(testobj) == MsgSpec then
	 return true
      end
   end
   return false
end




MsgSpec = { md5sum = nil }


--- Contstructor.
-- @param o Object initializer, must contain a field type with the string
-- representation of the type name. Optionally can contain a specstr field,
-- in which case the string will be parsed as message specification and no
-- attempt will be made to read the message specification file.
function MsgSpec:new(o)
   setmetatable(o, self)
   self.__index = self

   assert(o.type, "Message type is missing")

   local slashpos = o.type:find("/")
   o.package    = "roslib"
   o.short_type = o.type
   if slashpos then
      o.package    = o.type:sub(1, slashpos - 1)
      o.short_type = o.type:sub(slashpos + 1)
   end

   if o.specstr then
      o:load_from_string(o.specstr)
   else
      o:load()
   end

   o.base_format, o.base_farray = o:generate_base_format()

   return o
end


--- Resolve a type relative to this message spec.
-- This is similar to the global resolve_type function, but the package parameter
-- can be omitted as it is taken from the message spec instance.
-- @param type type to resolve
-- @return resolved type
-- @see resolve_type()
function MsgSpec:resolve_type(type)
   return resolve_type(type, self.package)
end

-- (internal) load from iterator
-- @param iterator iterator that returns one line of the specification at a time
function MsgSpec:load_from_iterator(iterator)
   self.fields = {}
   self.constants = {}

   local field_i = 1

   for line in iterator do
      line = line:match("^([^#]*)") or ""   -- strip comment
      line = line:match("(.-)%s+$") or line -- strip trailing whitespace

      if line ~= "" then -- else comment or empty
	 local ftype, fname = string.match(line, "^([%w_/%[%]]+) ([%w_%[%]]+)$")
	 if ftype and fname then
	    if ftype == "Header" then ftype = "roslib/Header" end
	    ftype = self:resolve_type(ftype)
	    local msgspec = nil
	    if not is_builtin_type(ftype) then
	       -- load sub-spec
	       msgspec = get_msgspec(base_type(ftype))
	    end
	    local typeinfo = {ftype, fname, msgspec, type=ftype, name=fname,
			      base_type=base_type(ftype), spec=msgspec,
			      is_array=is_array_type(ftype), is_builtin=is_builtin_type(ftype),
			      value_index=field_i}
	    self.fields[fname] = typeinfo
	    table.insert(self.fields, typeinfo)
	    field_i = field_i + 1
	 else -- check for constant
	    local ctype, cname, cvalue = line:match("^([%w_]+) ([%w_]+)[%s]*=[%s]*([-%w]+)$")
	    if ctype and cname and cvalue then
	       local nv = tonumber(cvalue)
	       if nv ~= nil then cvalue = nv end
	       self.constants[cname] = { ctype, cvalue, type=ctype, value=cvalue }
	       table.insert(self.constants, {ctype, cname, cvalue})
	    else
	       error("Unparsable line: " .. line)
	    end
	 end
      end
   end
end

--- Load specification from string.
-- @param s string containing the message specification
function MsgSpec:load_from_string(s)
   return self:load_from_iterator(s:gmatch("(.-)\n"))
end

--- Load message specification from file.
-- Will search for the appropriate message specification file (using rospack)
-- and will then read and parse the file.
function MsgSpec:load()
   local package_path = roslua.utils.find_rospack(self.package)
   self.file = package_path .. "/msg/" .. self.short_type .. ".msg"

   return self:load_from_iterator(io.lines(self.file))
end
<commit_msg>MsgSpec parser: allow arbitrary space<commit_after>
----------------------------------------------------------------------------
--  msg_spec.lua - Message specification wrapper
--
--  Created: Mon Jul 26 16:59:59 2010 (at Intel Research, Pittsburgh)
--  License: BSD, cf. LICENSE file of roslua
--  Copyright  2010  <NAME> [www.niemueller.de]
--             2010  Carnegie Mellon University
--             2010  Intel Research Pittsburgh
----------------------------------------------------------------------------

--- Message specification.
-- This module contains the MsgSpec class to read and represent ROS message
-- specification (YAML files). Message specifications should be obtained by
-- using the <code>get_msgspec()</code> function, which is aliased for
-- convenience as <code>roslua.get_msgspec()</code>.
-- <br /><br />
-- The message files are read on the fly, no offline code generation is
-- necessary. This avoids the need to write yet another code generator. After
-- reading the table field <code>fields</code> contains key value pairs, where
-- the keys are the names of the data fields and the value is the type of the
-- data field.
-- @copyright <NAME>, Carnegie Mellon University, Intel Research Pittsburgh
-- @release Released under BSD license
module("roslua.msg_spec", package.seeall)

require("md5")
require("roslua.message")
require("roslua.utils")

BUILTIN_TYPES = { "int8","uint8","int16","uint16","int32","uint32",
		  "int64","uint64","float32","float64","string","bool",
		  "byte", "char"}
for _,v in ipairs(BUILTIN_TYPES) do
   BUILTIN_TYPES[v] = v
end
EXTENDED_TYPES = { time={"uint32", "uint32"}, duration={"int32", "int32"} }

local msgspec_cache = {}

--- Get the base version of type, i.e. the non-array type.
-- @param type type to get base type for
-- @return base type, for array types returns the non-array type, for non-array
-- type returns the given value.
function base_type(type)
   return type:match("^([^%[]+)") or type
end

--- Check if type is an array type.
-- @param type type to check
-- @return true if given type is an array type, false otherwise
function is_array_type(type)
   return type:find("%[") ~= nil
end

--- Check if given type is a built-in type.
-- @param type type to check
-- @return true if type is a built-in type, false otherwise
function is_builtin_type(type)
   local t = base_type(type)
   return BUILTIN_TYPES[t] ~= nil or  EXTENDED_TYPES[t] ~= nil
end

--- Resolve the given type.
-- @param type to resolve
-- @param package to which the type should be resolve relatively
-- @return the given value if it is either a base type or contains a slash,
-- otherwise returns package/type.
function resolve_type(type, package)
   if is_builtin_type(type) or type:find("/") then
      return type
   else
      return string.format("%s/%s", package, type)
   end
end

--- Get message specification.
-- It is recommended to use the aliased version <code>roslua.get_msgspec()</code>.
-- @param msg_type message type (e.g. std_msgs/String). The name must include
-- the package.
function get_msgspec(msg_type, specstr)
   roslua.utils.assert_rospack()

   if not msgspec_cache[msg_type] then
      msgspec_cache[msg_type] = MsgSpec:new{type=msg_type, specstr=specstr}
   end

   return msgspec_cache[msg_type]
end

--- Check if the given object is a message spec.
-- @param testobj object to test
-- @return true if testobj is a message spec, false otherwise
function is_msgspec(testobj)
   if type(testobj) == "table" then
      if getmetatable(testobj) == MsgSpec then
	 return true
      end
   end
   return false
end




MsgSpec = { md5sum = nil }


--- Contstructor.
-- @param o Object initializer, must contain a field type with the string
-- representation of the type name. Optionally can contain a specstr field,
-- in which case the string will be parsed as message specification and no
-- attempt will be made to read the message specification file.
function MsgSpec:new(o)
   setmetatable(o, self)
   self.__index = self

   assert(o.type, "Message type is missing")

   local slashpos = o.type:find("/")
   o.package    = "roslib"
   o.short_type = o.type
   if slashpos then
      o.package    = o.type:sub(1, slashpos - 1)
      o.short_type = o.type:sub(slashpos + 1)
   end

   if o.specstr then
      o:load_from_string(o.specstr)
   else
      o:load()
   end

   o.base_format, o.base_farray = o:generate_base_format()

   return o
end


--- Resolve a type relative to this message spec.
-- This is similar to the global resolve_type function, but the package parameter
-- can be omitted as it is taken from the message spec instance.
-- @param type type to resolve
-- @return resolved type
-- @see resolve_type()
function MsgSpec:resolve_type(type)
   return resolve_type(type, self.package)
end

-- (internal) load from iterator
-- @param iterator iterator that returns one line of the specification at a time
function MsgSpec:load_from_iterator(iterator)
   self.fields = {}
   self.constants = {}

   local field_i = 1

   for line in iterator do
      line = line:match("^([^#]*)") or ""   -- strip comment
      line = line:match("(.-)%s+$") or line -- strip trailing whitespace

      if line ~= "" then -- else comment or empty
	 local ftype, fname = string.match(line, "^([%w_/%[%]]+)[%s]+([%w_%[%]]+)$")
	 if ftype and fname then
	    if ftype == "Header" then ftype = "roslib/Header" end
	    ftype = self:resolve_type(ftype)
	    local msgspec = nil
	    if not is_builtin_type(ftype) then
	       -- load sub-spec
	       msgspec = get_msgspec(base_type(ftype))
	    end
	    local typeinfo = {ftype, fname, msgspec, type=ftype, name=fname,
			      base_type=base_type(ftype), spec=msgspec,
			      is_array=is_array_type(ftype), is_builtin=is_builtin_type(ftype),
			      value_index=field_i}
	    self.fields[fname] = typeinfo
	    table.insert(self.fields, typeinfo)
	    field_i = field_i + 1
	 else -- check for constant
	    local ctype, cname, cvalue = line:match("^([%w_]+)[%s]+([%w_]+)[%s]*=[%s]*([-%w]+)$")
	    if ctype and cname and cvalue then
	       local nv = tonumber(cvalue)
	       if nv ~= nil then cvalue = nv end
	       self.constants[cname] = { ctype, cvalue, type=ctype, value=cvalue }
	       table.insert(self.constants, {ctype, cname, cvalue})
	    else
	       error("Unparsable line: " .. line)
	    end
	 end
      end
   end
end

--- Load specification from string.
-- @param s string containing the message specification
function MsgSpec:load_from_string(s)
   return self:load_from_iterator(s:gmatch("(.-)\n"))
end

--- Load message specification from file.
-- Will search for the appropriate message specification file (using rospack)
-- and will then read and parse the file.
function MsgSpec:load()
   local package_path = roslua.utils.find_rospack(self.package)
   self.file = package_path .. "/msg/" .. self.short_type .. ".msg"

   return self:load_from_iterator(io.lines(self.file))
end

{'fraction_non_alphanumeric': 0.13630453626119618, 'fraction_numerical': 0.008234614273331407, 'mean_word_length': 3.0613083015547082, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6165847', 'n_tokens_mistral': 4471, 'n_tokens_neox': 4213, 'n_words': 1476}