<commit_before>#    This file is part of Metasm, the Ruby assembly manipulation suite
#    Copyright (C) 2007 <NAME>
#
#    Licence is LGPL, see LICENCE in the top-level directory


require 'metasm/main'
require 'metasm/preprocessor'

module Metasm
# c parser
# inspired from http://www.math.grin.edu/~stone/courses/languages/C-syntax.xhtml
module C
	Keyword = %w[struct union enum  if else for while do switch goto
			register extern auto static typedef  const volatile
			void int float double char  signed unsigned long short
			case continue break return default  __attribute__
			asm __asm __asm__ sizeof typeof
			__declspec __cdecl __stdcall __fastcall
			inline __inline __inline__ __volatile__
			__int8 __int16 __int32 __int64
			__builtin_offsetof
	].inject({}) { |h, w| h.update w => true }

	class Statement
	end

	class Block < Statement
		attr_accessor :symbol	# hash name => Type/Variable/enum value
		attr_accessor :struct	# hash name => Struct/Union/Enum
		attr_accessor :outer	# parent block
		attr_accessor :statements	# array of Statement/Declaration
		attr_accessor :anonymous_enums	# array of anonymous Enum

		def initialize(outer)
			@symbol, @struct = {}, {}
			@statements = []
			@outer = outer
		end

		def struct_ancestors
			@outer ? @outer.struct_ancestors.merge(@struct) : @struct
		end

		def symbol_ancestors
			@outer ? @outer.symbol_ancestors.merge(@symbol) : @symbol
		end
	end

	module Attributes
		attr_accessor :attributes

		PREFIXED = %w[cdecl stdcall fastcall inline]

		# parses a sequence of __attribute__((anything)) into self.attributes (array of string)
		def parse_attributes(parser, allow_declspec = false)
			while tok = parser.skipspaces and tok.type == :string
			    case keyword = tok.raw
			    when '__attribute__', '__declspec'	# synonymous: __attribute__((foo)) == __declspec(foo)
				raise tok || parser if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '('
				raise tok || parser if keyword == '__attribute__' and (not tok = parser.skipspaces or tok.type != :punct or tok.raw != '(')
				nest = 0
				attrib = ''
				loop do
					raise parser if not tok = parser.skipspaces
					if tok.type == :punct and tok.raw == ')'
						if nest == 0
							raise tok || parser if keyword == '__attribute__' and (not tok = parser.skipspaces or tok.type != :punct or tok.raw != ')')
							break
						else
							nest -= 1
						end
					elsif tok.type == :punct and tok.raw == '('
						nest += 1
					elsif nest == 0 and tok.type == :punct and tok.raw == ','
						raise tok || parser if not allow_declspec and %w[inline stdcall fastcall cdecl naked].include? attrib
						(@attributes ||= []) << attrib
						attrib = ''
						next
					end
					attrib << tok.raw
				end
				raise tok || parser if not allow_declspec and %w[inline stdcall fastcall cdecl naked].include? attrib
			    when 'inline', '__inline', '__inline__', '__stdcall', '__fastcall', '__cdecl'
				break if not allow_declspec
				attrib = keyword.delete '_'
			    else break
			    end
			    (@attributes ||= []) << attrib
			end
			parser.unreadtok tok
		end
	end

	class Type
		include Attributes
		attr_accessor :qualifier	# const volatile

		def pointer? ;    false end
		def arithmetic? ; false end
		def integral? ;   false end
		def float? ;      false end
		def base ;        self  end
		def untypedef ;   self  end

		def parse_initializer(parser, scope)
			raise parser, 'expr expected' if not ret = CExpression.parse(parser, scope, false)
			p, i = pointer?, integral?
			r = ret.reduce(parser) if p or i
			if (not p and not i) or (i and not r.kind_of? ::Integer) or (p and r != 0)
				parser.check_compatible_type(parser, ret.type, self)
			end
			ret
		end

		def parse_initializer_designator(parser, scope, value, idx, root=true)
			if not root and (not nt = parser.skipspaces or nt.type != :punct or nt.raw != '=')
				raise nt || parser, '"=" expected'
			end
			value[idx] = parse_initializer(parser, scope)
			idx + 1
		end
	end
	class BaseType < Type
		attr_accessor :name		# :int :long :longlong :short :double :longdouble :float :char :void :__int8/16/32/64
		attr_accessor :specifier	# sign specifier only

		def arithmetic? ; @name != :void end
		def integral? ; [:char, :short, :int, :long, :longlong, :ptr,
			:__int8, :__int16, :__int32, :__int64].include? @name end
		def float? ; [:float, :double, :longdouble].include? @name end
		def align(parser) @name == :double ? 4 : parser.typesize[@name] end

		def initialize(name, *specs)
			@name = name
			specs.each { |s|
				case s
				when :const, :volatile; (@qualifier ||= []) << s
				when :signed, :unsigned; @specifier = s
				when nil
				else raise "internal error, got #{name.inspect} #{specs.inspect}"
				end
			}
		end

		def ==(o)
			o.class == self.class and o.name == self.name and o.specifier == self.specifier and o.attributes == self.attributes
		end
	end
	class TypeDef < Type
		attr_accessor :name
		attr_accessor :type
		attr_accessor :backtrace

		def initialize(name, type, backtrace)
			@name, @type, @backtrace = name, type, backtrace
		end

		def parse_initializer(parser, scope)
			@type.parse_initializer(parser, scope)
		end

		def pointer? ;    @type.pointer?      end
		def arithmetic? ; @type.arithmetic?   end
		def integral? ;   @type.integral?     end
		def float? ;      @type.float?        end
		def untypedef ;   @type.untypedef     end
		def align(parser) @type.align(parser) end	# XXX __attribute__ ?
	end
	class Function < Type
		attr_accessor :type		# return type
		attr_accessor :args		# [name, Variable]
		attr_accessor :varargs		# true/false

		def initialize(type=nil)
			@type = type
		end

		def base ; @type.base ; end
	end
	class Union < Type
		attr_accessor :members		# [Variable]
		attr_accessor :bits		# [bits] or nil
		attr_accessor :name
		attr_accessor :backtrace

		def align(parser) @members.map { |m| m.type.align(parser) }.max end

		def findmember(name)
			if m = @members.find { |m| m.name == name }
				return m
			else
				@members.each { |m|
					if t = m.type.untypedef and t.kind_of? Union and mm = t.findmember(name)
						return mm
					end
				}
			end
			nil
		end

		def parse_members(parser, scope)
			@members = []
			# parse struct/union members in definition
			loop do
				raise parser if not tok = parser.skipspaces
				break if tok.type == :punct and tok.raw == '}'
				parser.unreadtok tok

				raise tok, 'invalid struct member type' if not basetype = Variable.parse_type(parser, scope)
				loop do
					member = basetype.dup
					member.parse_declarator(parser, scope)
					# raise parser if not member.name	# can be useful while hacking: struct foo {int; int*; int iwant;};
					raise member.backtrace, 'member redefinition' if member.name and @members.find { |m| m.name == member.name }
					@members << member

					raise tok || parser if not tok = parser.skipspaces or tok.type != :punct

					if tok.raw == ':'	# bits
						raise tok, 'bad type for bitslice' if not member.type.integral?
						bits = nil
						raise tok, "bad bit count #{bits.inspect}" if not bits = CExpression.parse(parser, scope, false) or
							not bits.constant? or not (bits = bits.reduce(parser)).kind_of? ::Integer
						#raise tok, 'need more bits' if bits > 8*parser.sizeof(member)
						# WORD wReserved:17; => yay windows.h
						(@bits ||= [])[@members.length-1] = bits
						raise tok || parser, '"," or ";" expected' if not tok = parser.skipspaces or tok.type != :punct
					end

					case tok.raw
					when ';'; break
					when ','
					else raise tok, '"," or ";" expected'
					end
				end
			end
			parse_attributes(parser)
		end

		def parse_initializer(parser, scope)
			if tok = parser.skipspaces and tok.type == :punct and tok.raw == '{'
				# struct x toto = { 1, .4, .member[0][6].bla = 12 };
				raise tok, 'undefined struct' if not @members
				ret = []
				if tok = parser.skipspaces and (tok.type != :punct or tok.raw != '}')
					parser.unreadtok tok
					idx = 0
					loop do
						idx = parse_initializer_designator(parser, scope, ret, idx, true)
						raise tok || parser, '"," or "}" expected' if not tok = parser.skipspaces or tok.type != :punct or (tok.raw != '}' and tok.raw != ',')
						break if tok.raw == '}'
					end
				end
				ret
			else
				parser.unreadtok tok
				super
			end
		end

		# parses a designator+initializer eg '.toto = 4' or '.tutu[42][12].bla = 16' or (root ? '4' : '=4')
		def parse_initializer_designator(parser, scope, value, idx, root=true)
			if nt = parser.skipspaces and nt.type == :punct and nt.raw == '.' and
					nnt = parser.skipspaces and nnt.type == :string and
					m = findmember(nnt.raw)
				raise nnt, 'unhandled initializer' if not nidx = @members.index(@members.find { |m| m.name == nnt.raw })	# TODO
				value = value[idx] ||= [] if not root
				idx = nidx
				@members[idx].type.parse_initializer_designator(parser, scope, value, idx, false)
			else
				parser.unreadtok nnt
				if root
					parser.unreadtok nt
					value[idx] = @members[idx].type.parse_initializer(parser, scope)
				else
					raise nt || parser, '"=" expected' if not nt or nt.type != :punct or nt.raw != '='
					value[idx] = parse_initializer(parser, scope)
				end
			end
			idx + 1
		end
	end
	class Struct < Union
		attr_accessor :pack

		def align(parser) [@members.map { |m| m.type.align(parser) }.max, (pack || 8)].min end

		def offsetof(parser, name)
			raise parser, 'undefined structure' if not @members
			raise parser, 'unknown structure member' if not findmember(name)
			raise parser, 'unhandled indirect offsetof' if not @members.find { |m| m.name == name }	# TODO
			al = align(parser)
			off = 0
			bit_off = 0
			@members.each_with_index { |m, i|
				break if m.name == name
				if bits and b = @bits[i]
					isz = parser.typesize[:int]
					if bit_off + b > 8*isz
						bit_off = 0
						off = (off + isz - 1) / isz * isz + isz
					else
						bit_off += b
					end
				else
					mal = [m.type.align(parser), al].min
					off = (off + mal - 1) / mal * mal
					off += parser.sizeof(m)
				end
			}
			off
		end

		def parse_members(parser, scope)
			super
			if defined? @attributes and @attributes
				if @attributes.include? 'packed'
					@pack = 1
				elsif p = @attributes.grep(/^pack\(\d+\)$/).first
					@pack = p[/\d+/].to_i
				end
			end
		end
	end
	class Enum < Type
		# name => value
		attr_accessor :members
		attr_accessor :name
		attr_accessor :backtrace

		def align(parser) BaseType.new(:int).align(parser) end

		def parse_members(parser, scope)
			val = -1
			@members = {}
			loop do
				raise parser if not tok = parser.skipspaces
				break if tok.type == :punct and tok.raw == '}'

				name = tok.raw
				raise tok, 'bad enum name' if tok.type != :string or Keyword[name] or (?0..?9).include?(name[0])

				raise parser if not tok = parser.skipspaces
				if tok.type == :punct and tok.raw == '='
					raise tok || parser if not val = CExpression.parse(parser, scope, false) or not val = val.reduce(parser) or not tok = parser.skipspaces
				else
					val += 1
				end
				raise tok, "enum value #{name} redefinition" if scope.symbol[name] and scope.symbol[name] != val
				@members[name] = val
				scope.symbol[name] = val

				if tok.type == :punct and tok.raw == '}'
					break
				elsif tok.type == :punct and tok.raw == ','
				else raise tok, '"," or "}" expected'
				end
			end
			parse_attributes(parser)
		end

	end
	class Pointer < Type
		attr_accessor :type

		def initialize(type=nil)
			@type = type
		end

		def pointer? ; true ; end
		def arithmetic? ; true ; end
		def base ; @type.base ; end
		def align(parser) BaseType.new(:ptr).align(parser) end

		def ==(o)
			o.class == self.class and o.type == self.type
		end
	end
	class Array < Pointer
		attr_accessor :length

		def align(parser) @type.align(parser) end

		def parse_initializer(parser, scope)
			raise parser, 'cannot initialize dynamic array' if @length.kind_of? CExpression
			if tok = parser.skipspaces and tok.type == :punct and tok.raw == '{'
				# struct x foo[] = { { 4 }, [12].tutu = 2 };
				ret = []
				if tok = parser.skipspaces and (tok.type != :punct or tok.raw != '}')
					parser.unreadtok tok
					idx = 0
					loop do
						idx = parse_initializer_designator(parser, scope, ret, idx, true)
						raise tok || parser, '"," or "}" expected' if not tok = parser.skipspaces or tok.type != :punct or (tok.raw != '}' and tok.raw != ',')
						break if tok.raw == '}'
						# allow int x[] = {1, 2, 3, };
						break if tok = parser.skipspaces and tok.type == :punct and tok.raw == '}'
						parser.unreadtok tok
					end
				end
				ret
			else
				parser.unreadtok tok
				super
			end
		end

		# parses a designator+initializer eg '[12] = 4' or '[42].bla = 16' or (root ? '4' : '=4')
		def parse_initializer_designator(parser, scope, value, idx, root=true)
			if nt = parser.skipspaces and nt.type == :punct and nt.raw == '['
				value = value[idx] ||= [] if not root
				raise nt, 'const expected' if not idx = CExpression.parse(parser, scope) or not idx.constant? or not idx = idx.reduce(parser) or not idx.kind_of? ::Integer
				raise nt || parser, '"]" expected' if not nt = parser.skipspaces or nt.type != :punct or nt.raw != ']'
				@type.parse_initializer_designator(parser, scope, value, idx, false)
			else
				if root
					parser.unreadtok nt
					value[idx] = @type.parse_initializer(parser, scope)
				else
					raise nt || parser, '"=" expected' if not nt or nt.type != :punct or nt.raw != '='
					value[idx] = parse_initializer(parser, scope)
				end
			end
			idx + 1
		end
	end

	class Variable
		include Attributes
		attr_accessor :type
		attr_accessor :initializer	# CExpr	/ Block (for Functions)
		attr_accessor :name
		attr_accessor :storage		# auto register static extern typedef
		attr_accessor :backtrace	# definition backtrace info (the name token)
	end

	# found in a block's Statements, used to know the initialization order
	# eg { int i; i = 4; struct foo { int k; } toto = {i}; }
	class Declaration
		attr_accessor :var
		def initialize(var)
			@var = var
		end
	end

	class If < Statement
		attr_accessor :test		# expression
		attr_accessor :bthen, :belse	# statements
		def initialize(test, bthen, belse=nil)
			@test = test
			@bthen = bthen
			@belse = belse if belse
		end

		def self.parse(parser, scope, nest)
			tok = nil
			raise tok || self, '"(" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '('
			raise tok, 'expr expected' if not expr = CExpression.parse(parser, scope) or not expr.type.arithmetic?
			raise tok || self, '")" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ')'
			bthen = parser.parse_statement scope, nest
			if tok = parser.skipspaces and tok.type == :string and tok.raw == 'else'
				belse = parser.parse_statement scope, nest
			else
				parser.unreadtok tok
			end

			new expr, bthen, belse
		end
	end
	class For < Statement
		attr_accessor :init, :test, :iter	# CExpressions, init may be Block
		attr_accessor :body
		def initialize(init, test, iter, body)
			@init, @test, @iter, @body = init, test, iter, body
		end

		def self.parse(parser, scope, nest)
			tok = nil
			raise tok || parser, '"(" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '('
			init = forscope = Block.new(scope)
			if not parser.parse_definition(forscope)
				forscope = scope
				init = CExpression.parse(parser, forscope)
				raise tok || parser, '";" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ';'
			end
			test = CExpression.parse(parser, forscope)
			raise tok || parser, '";" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ';'
			raise tok, 'bad test expression in for loop' if test and not test.type.arithmetic?
			iter = CExpression.parse(parser, forscope)
			raise tok || parser, '")" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ')'

			new init, test, iter, parser.parse_statement(forscope, nest + [:loop])
		end
	end
	class While < Statement
		attr_accessor :test
		attr_accessor :body

		def initialize(test, body)
			@test = test
			@body = body
		end

		def self.parse(parser, scope, nest)
			tok = nil
			raise tok || parser, '"(" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '('
			raise tok, 'expr expected' if not expr = CExpression.parse(parser, scope) or not expr.type.arithmetic?
			raise tok || parser, '")" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ')'

			new expr, parser.parse_statement(scope, nest + [:loop])
		end
	end
	class DoWhile < While
		def self.parse(parser, scope, nest)
			body = parser.parse_statement(scope, nest + [:loop])
			tok = nil
			raise tok || parser, '"while" expected' if not tok = parser.skipspaces or tok.type != :string or tok.raw != 'while'
			raise tok || parser, '"(" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '('
			raise tok, 'expr expected' if not expr = CExpression.parse(parser, scope) or not expr.type.arithmetic?
			raise tok || parser, '")" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ')'
			raise tok || parser, '";" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ';'

			new expr, body
		end
	end
	class Switch < Statement
		attr_accessor :test, :body

		def initialize(test, body)
			@test = test
			@body = body
		end

		def self.parse(parser, scope, nest)
			raise tok || parser, '"(" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '('
			raise tok, 'expr expected' if not expr = CExpression.parse(parser, scope) or not expr.type.integral?
			raise tok || parser, '")" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ')'

			new expr, parser.parse_statement(scope, nest + [:switch])
		end
	end

	class Continue < Statement
	end
	class Break < Statement
	end
	class Goto < Statement
		attr_accessor :target
		def initialize(target)
			@target = target
		end
	end
	class Return < Statement
		attr_accessor :value
		def initialize(value)
			@value = value
		end
	end
	class Label < Statement
		attr_accessor :name
		attr_accessor :statement
		def initialize(name, statement=nil)
			@name, @statement = name, statement
		end
	end
	class Case < Label
		attr_accessor :expr, :exprup	# exprup if range, expr may be 'default'
		def initialize(expr, exprup, statement)
			@expr, @statement = expr, statement
			@exprup = exprup if exprup
		end

		def self.parse(parser, scope, nest)
			raise parser, 'invalid case' if not expr = CExpression.parse(parser, scope) or not expr.constant? or not expr.type.integral?
			raise tok || parser, '":" or "..." expected' if not tok = parser.skipspaces or tok.type != :punct or (tok.raw != ':' and tok.raw != '.')
			if tok.raw == '.'
				raise tok || parser, '".." expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '.'
				raise tok || parser,  '"." expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '.'
				raise tok, 'invalid case range' if not exprup = CExpression.parse(parser, scope) or not exprup.constant? or not exprup.type.integral?
				raise tok || parser, '":" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ':'
			end
			body = parser.parse_statement scope, nest
			new expr, exprup, body
		end
	end

	# inline asm statement
	class Asm < Statement
		attr_accessor :body		# asm source (::String)
		attr_accessor :output, :input, :clobber	# I/O, gcc-style (::Array)
		attr_accessor :backtrace	# body Token
		attr_accessor :volatile

		def initialize(body, backtrace, output, input, clobber, volatile)
			@body, @backtrace, @output, @input, @clobber, @volatile = body, backtrace, output, input, clobber, volatile
		end

		def self.parse(parser, scope)
			if tok = parser.skipspaces and tok.type == :string and (tok.raw == 'volatile' or tok.raw == '__volatile__')
				volatile = true
				tok = parser.skipspaces
			end
			if not tok or tok.type != :punct or tok.raw != '('
				# detect MS-style inline asm: "__asm .* __asm .*" or "asm { [\s.]* }"
				ftok = tok
				body = ''
				if tok.type == :punct and tok.raw == '{'
					loop do
						raise ftok, 'unterminated asm block' if not tok = parser.lexer.readtok
						break if tok.type == :punct and tok.raw == '}'
						case tok.type
						when :space; body << ' '
						when :eol; body << "\n"
						when :punct; body << tok.raw
						when :quoted; body << tok.value.inspect	# concat adjacent c strings
						when :string
							body << \
							case tok.raw
							when 'asm', '__asm', '__asm__'; "\n"
							when '_emit'; 'db'
							else tok.raw
							end
						end
					end
				else
					parser.lexer.unreadtok tok
					loop do
						break if not tok = parser.lexer.readtok or tok.type == :eol
						case tok.type
						when :space; body << ' '
						when :punct
							case tok.raw
							when '}'
								parser.lexer.unreadtok tok
								break
							else body << tok.raw
							end
						when :quoted; body << tok.value.inspect
						when :string
							body << \
							case tok.raw
							when 'asm', '__asm', '__asm__'; "\n"
							when '_emit'; 'db'
							else tok.raw
							end
						end
					end
				end
				return new(body, ftok, nil, nil, nil, volatile)
			end
			raise tok || parser, '"(" expected' if not tok or tok.type != :punct or tok.raw != '('
			raise tok || parser, 'qstring expected' if not tok = parser.skipspaces or tok.type != :quoted
			body = tok
			tok = parser.skipspaces
			raise tok || parser, '":" or ")" expected' if not tok or tok.type != :punct or (tok.raw != ':' and tok.raw != ')')

			if tok.raw == ':'
				output = []
				raise parser if not tok = parser.skipspaces
				while tok.type == :quoted
					type = tok.value
					raise tok, 'expr expected' if not var = CExpression.parse_value(parser, scope)
					output << [type, var]
					raise tok || parser, '":" or "," or ")" expected' if not tok = parser.skipspaces or tok.type != :punct or (tok.raw != ',' and tok.raw != ')' and tok.raw != ':')
					break if tok.raw == ':' or tok.raw == ')'
					raise tok || parser, 'qstring expected' if not tok = parser.skipspaces or tok.type != :quoted
				end
			end
			if tok.raw == ':'
				input = []
				raise parser if not tok = parser.skipspaces
				while tok.type == :quoted
					type = tok.value
					raise tok, 'expr expected' if not var = CExpression.parse_value(parser, scope)
					input << [type, var]
					raise tok || parser, '":" or "," or ")" expected' if not tok = parser.skipspaces or tok.type != :punct or (tok.raw != ',' and tok.raw != ')' and tok.raw != ':')
					break if tok.raw == ':' or tok.raw == ')'
					raise tok || parser, 'qstring expected' if not tok = parser.skipspaces or tok.type != :quoted
				end
			end
			if tok.raw == ':'
				clobber = []
				raise parser if not tok = parser.skipspaces
				while tok.type == :quoted
					clobber << tok.value
					raise tok || parser, '"," or ")" expected' if not tok = parser.skipspaces or tok.type != :punct or (tok.raw != ',' and tok.raw != ')')
					break if tok.raw == ')'
					raise tok || parser, 'qstring expected' if not tok = parser.skipspaces or tok.type != :quoted
				end
			end
			raise tok || parser, '")" expected' if not tok or tok.type != :punct or tok.raw != ')'
			raise tok || parser, '";" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ';'

			new body.value, body, output, input, clobber, volatile
		end
	end

	class CExpression < Statement
		# may be :,, :., :'->', :funcall (function, [arglist]), :[] (array indexing), nil (cast)
		attr_accessor :op
		# nil/CExpr/Variable/Label/::String( = :quoted/struct member name)/::Integer/::Float/Block
		attr_accessor :lexpr, :rexpr
		# a Type
		attr_accessor :type
		def initialize(l, o, r, t)
			@lexpr, @op, @rexpr, @type = l, o, r, t
		end
	end


	class Parser
		# creates a new CParser, parses all top-level statements
		def self.parse(text)
			c = new
	                c.parse text
			raise c.lexer.readtok || c, 'invalid definition' if not c.lexer.eos?
			c
		end

		# parses the current lexer content (or the text arg) for toplevel definitions
		def parse(text=nil, filename='<unk>', lineno=1)
			@lexer.feed text, filename, lineno if text
			nil while not @lexer.eos? and (parse_definition(@toplevel) or parse_toplevel_statement(@toplevel))
			sanity_checks
			self
		end

		# parses a C file
		def parse_file(file)
			parse(File.read(file), file)
		end

		attr_accessor :lexer, :toplevel, :typesize, :pragma_pack
		def initialize(lexer = nil, model=:ilp32)
			@lexer = lexer || Preprocessor.new
			@prev_pragma_callback = @lexer.pragma_callback
			@lexer.pragma_callback = proc { |tok| parse_pragma_callback(tok) }
			@toplevel = Block.new(nil)
			@unreadtoks = []
			@typesize = { :void => 0, :__int8 => 1, :__int16 => 2, :__int32 => 4, :__int64 => 8,
				:char => 1, :float => 4, :double => 8, :longdouble => 12 }
			send model
		end

		def ilp16
			# XXX check this
			@typesize.update :short => 2, :ptr => 2,
				:int => 2, :long => 4, :longlong => 4
		end
		def lp32
			@typesize.update :short => 2, :ptr => 4,
				:int => 2, :long => 4, :longlong => 8
		end
		def ilp32
			@typesize.update :short => 2, :ptr => 4,
				:int => 4, :long => 4, :longlong => 8
		end
		def llp64
			# longlong should only exist here
			@typesize.update :short => 2, :ptr => 8,
				:int => 4, :long => 4, :longlong => 8
		end
		def ilp64
			@typesize.update :short => 2, :ptr => 8,
				:int => 8, :long => 8, :longlong => 8
		end
		def lp64
			@typesize.update :short => 2, :ptr => 8,
				:int => 4, :long => 8, :longlong => 8
		end

		attr_accessor :auto_predeclare_unknown_structs
		def parse_pragma_callback(otok)
			case otok.raw
			when 'pack'
				nil while lp = @lexer.readtok and lp.type == :space
				nil while rp = @lexer.readtok and rp.type == :space
				if not rp or rp.type != :punct or rp.raw != ')'
					v1 = rp
					nil while rp = @lexer.readtok and rp.type == :space
				end
				if rp and rp.type == :punct and rp.raw == ','
					nil while v2 = @lexer.readtok and v2.type == :space
					nil while rp = @lexer.readtok and rp.type == :space
				end
				raise cmd if not rp or lp.type != :punct or rp.type != :punct or lp.raw != '(' or rp.raw != ')'
				raise cmd if (v1 and v1.type != :string) or (v2 and (v2.type != :string or v2.raw =~ /[^\d]/))
				if not v1
					@pragma_pack = nil
				elsif v1.raw == 'push'
					@pragma_pack_stack ||= []
					@pragma_pack_stack << pragma_pack
					@pragma_pack = v2.raw.to_i if v2
					raise v2, 'bad pack value' if pragma_pack == 0
				elsif v1.raw == 'pop'
					@pragma_pack_stack ||= []
					raise v1, 'pack stack empty' if @pragma_pack_stack.empty?
					@pragma_pack = @pragma_pack_stack.pop
					@pragma_pack = v2.raw.to_i if v2 and v2.raw	# #pragma pack(pop, 4) => pop stack, but use 4 as pack value (imho)
					raise v2, 'bad pack value' if pragma_pack == 0
				elsif v1.raw =~ /^\d+$/
					raise v2, '2nd arg unexpected' if v2
					@pragma_pack = v1.raw.to_i
					raise v1, 'bad pack value' if @pragma_pack == 0
				else raise otok
				end
				# the caller checks for :eol
			when 'auto_predeclare_unknown_structs'
				@auto_predeclare_unknown_structs = true
			when 'warning'
				if $DEBUG
					@prev_pragma_callback[otok]
				else
					# silent discard
					nil while tok = @lexer.readtok_nopp and tok.type != :eol
					@lexer.unreadtok tok
				end
			when 'prepare_visualstudio'
				prepare_visualstudio
			when 'prepare_gcc'
				prepare_gcc
			else @prev_pragma_callback[otok]
			end
		end

		def prepare_visualstudio
			@auto_predeclare_unknown_structs = true
			@lexer.define_weak('_WIN32')
			@lexer.define_weak('_WIN32_WINNT', 0x500)
			@lexer.define_weak('_INTEGRAL_MAX_BITS', 64)
			@lexer.define_weak('__w64')
			@lexer.define_weak('_cdecl', '__cdecl')	# typo ? seen in winreg.h
			@lexer.define_weak('_fastcall', '__fastcall')	# typo ? seen in ntddk.h
			@lexer.define_weak('_MSC_VER', 1300)	# handle '#pragma once' and _declspec(noreturn)
			@lexer.define_weak('__forceinline', '__inline')
			@lexer.define_weak('__ptr32')	# needed with msc_ver 1300, don't understand their use
			@lexer.define_weak('__ptr64')
		end

		def prepare_gcc
			@lexer.define_weak('__GNUC__', 2)	# otherwise __attribute__ is defined to void..
			@lexer.define_weak('__STDC__')
			@lexer.define_weak('__const', 'const')
			@lexer.define_weak('__signed', 'signed')
			@lexer.define_weak('__volatile', 'volatile')
			@lexer.nodefine_strong('__REDIRECT_NTH')	# booh gnu
			@lexer.hooked_include['stddef.h'] = <<EOH
/* simplified, define all at first invocation. may break things... */
#undef __need_ptrdiff_t
#undef __need_size_t
#undef __need_wint_t
#undef __need_wchar_t
#undef __need_NULL
#undef NULL
#if !defined (_STDDEF_H)
#define _STDDEF_H
#define __PTRDIFF_TYPE__ long int
typedef __PTRDIFF_TYPE__ ptrdiff_t;
#define __SIZE_TYPE__ long unsigned int
typedef __SIZE_TYPE__ size_t;
#define __WINT_TYPE__ unsigned int
typedef __WINT_TYPE__ wint_t;
#define __WCHAR_TYPE__ int
typedef __WCHAR_TYPE__ wchar_t;
#define NULL 0
#define offsetof(TYPE, MEMBER) __builtin_offsetof (TYPE, MEMBER)
#endif
EOH
			# TODO va_args
			@lexer.hooked_include['stdarg.h'] = <<EOH
// TODO
typedef void* __gnuc_va_list;
/*
typedef void* va_list;
#define va_start(v, l)
#define va_end(v)
#define va_arg(v, l)
#define va_copy(d, s)
*/
EOH
		end

		# C sanity checks
		def sanity_checks
			return if not $VERBOSE
			#  TODO
		end

		# checks that the types are compatible (variable predeclaration, function argument..)
		# strict = false for func call/assignment (eg char compatible with int -- but int is incompatible with char)
		def check_compatible_type(tok, oldtype, newtype, strict = false, checked = [])
			oldtype = oldtype.untypedef
			newtype = newtype.untypedef
			oldtype = BaseType.new(:int) if oldtype.kind_of? Enum
			newtype = BaseType.new(:int) if newtype.kind_of? Enum

			puts tok.exception('type qualifier mismatch').message if $VERBOSE and oldtype.qualifier.to_a.uniq.length > newtype.qualifier.to_a.uniq.length

			# avoid infinite recursion
			return if checked.include? oldtype
			checked = checked + [oldtype]

		    begin
			case newtype
			when Function
				raise tok if not oldtype.kind_of? Function
				check_compatible_type tok, oldtype.type, newtype.type, strict, checked
				if oldtype.args and newtype.args
					if oldtype.args.length != newtype.args.length or
							oldtype.varargs != newtype.varargs
						raise tok
					end
					oldtype.args.zip(newtype.args) { |oa, na|
						# begin ; rescue ParseError: raise $!.message + "in parameter #{oa.name}" end
						check_compatible_type tok, oa.type, na.type, strict, checked
					}
				end
			when Pointer
				if oldtype.kind_of? BaseType and oldtype.integral?
					puts tok.exception('making pointer from integer without a cast').message if $VERBOSE
					return
				end
				raise tok if not oldtype.kind_of? Pointer
				hasvoid = true if (t = newtype.type.untypedef).kind_of? BaseType and t.name == :void
				hasvoid = true if (t = oldtype.type.untypedef).kind_of? BaseType and t.name == :void	# struct foo *f = NULL;
				if strict or not hasvoid
					check_compatible_type tok, oldtype.type, newtype.type, strict, checked<commit_msg>parse_c: fix ptr to struct to ptr to struct<commit_after>#    This file is part of Metasm, the Ruby assembly manipulation suite
#    Copyright (C) 2007 <NAME>
#
#    Licence is LGPL, see LICENCE in the top-level directory


require 'metasm/main'
require 'metasm/preprocessor'

module Metasm
# c parser
# inspired from http://www.math.grin.edu/~stone/courses/languages/C-syntax.xhtml
module C
	Keyword = %w[struct union enum  if else for while do switch goto
			register extern auto static typedef  const volatile
			void int float double char  signed unsigned long short
			case continue break return default  __attribute__
			asm __asm __asm__ sizeof typeof
			__declspec __cdecl __stdcall __fastcall
			inline __inline __inline__ __volatile__
			__int8 __int16 __int32 __int64
			__builtin_offsetof
	].inject({}) { |h, w| h.update w => true }

	class Statement
	end

	class Block < Statement
		attr_accessor :symbol	# hash name => Type/Variable/enum value
		attr_accessor :struct	# hash name => Struct/Union/Enum
		attr_accessor :outer	# parent block
		attr_accessor :statements	# array of Statement/Declaration
		attr_accessor :anonymous_enums	# array of anonymous Enum

		def initialize(outer)
			@symbol, @struct = {}, {}
			@statements = []
			@outer = outer
		end

		def struct_ancestors
			@outer ? @outer.struct_ancestors.merge(@struct) : @struct
		end

		def symbol_ancestors
			@outer ? @outer.symbol_ancestors.merge(@symbol) : @symbol
		end
	end

	module Attributes
		attr_accessor :attributes

		PREFIXED = %w[cdecl stdcall fastcall inline]

		# parses a sequence of __attribute__((anything)) into self.attributes (array of string)
		def parse_attributes(parser, allow_declspec = false)
			while tok = parser.skipspaces and tok.type == :string
			    case keyword = tok.raw
			    when '__attribute__', '__declspec'	# synonymous: __attribute__((foo)) == __declspec(foo)
				raise tok || parser if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '('
				raise tok || parser if keyword == '__attribute__' and (not tok = parser.skipspaces or tok.type != :punct or tok.raw != '(')
				nest = 0
				attrib = ''
				loop do
					raise parser if not tok = parser.skipspaces
					if tok.type == :punct and tok.raw == ')'
						if nest == 0
							raise tok || parser if keyword == '__attribute__' and (not tok = parser.skipspaces or tok.type != :punct or tok.raw != ')')
							break
						else
							nest -= 1
						end
					elsif tok.type == :punct and tok.raw == '('
						nest += 1
					elsif nest == 0 and tok.type == :punct and tok.raw == ','
						raise tok || parser if not allow_declspec and %w[inline stdcall fastcall cdecl naked].include? attrib
						(@attributes ||= []) << attrib
						attrib = ''
						next
					end
					attrib << tok.raw
				end
				raise tok || parser if not allow_declspec and %w[inline stdcall fastcall cdecl naked].include? attrib
			    when 'inline', '__inline', '__inline__', '__stdcall', '__fastcall', '__cdecl'
				break if not allow_declspec
				attrib = keyword.delete '_'
			    else break
			    end
			    (@attributes ||= []) << attrib
			end
			parser.unreadtok tok
		end
	end

	class Type
		include Attributes
		attr_accessor :qualifier	# const volatile

		def pointer? ;    false end
		def arithmetic? ; false end
		def integral? ;   false end
		def float? ;      false end
		def base ;        self  end
		def untypedef ;   self  end

		def parse_initializer(parser, scope)
			raise parser, 'expr expected' if not ret = CExpression.parse(parser, scope, false)
			p, i = pointer?, integral?
			r = ret.reduce(parser) if p or i
			if (not p and not i) or (i and not r.kind_of? ::Integer) or (p and r != 0)
				parser.check_compatible_type(parser, ret.type, self)
			end
			ret
		end

		def parse_initializer_designator(parser, scope, value, idx, root=true)
			if not root and (not nt = parser.skipspaces or nt.type != :punct or nt.raw != '=')
				raise nt || parser, '"=" expected'
			end
			value[idx] = parse_initializer(parser, scope)
			idx + 1
		end
	end
	class BaseType < Type
		attr_accessor :name		# :int :long :longlong :short :double :longdouble :float :char :void :__int8/16/32/64
		attr_accessor :specifier	# sign specifier only

		def arithmetic? ; @name != :void end
		def integral? ; [:char, :short, :int, :long, :longlong, :ptr,
			:__int8, :__int16, :__int32, :__int64].include? @name end
		def float? ; [:float, :double, :longdouble].include? @name end
		def align(parser) @name == :double ? 4 : parser.typesize[@name] end

		def initialize(name, *specs)
			@name = name
			specs.each { |s|
				case s
				when :const, :volatile; (@qualifier ||= []) << s
				when :signed, :unsigned; @specifier = s
				when nil
				else raise "internal error, got #{name.inspect} #{specs.inspect}"
				end
			}
		end

		def ==(o)
			o.class == self.class and o.name == self.name and o.specifier == self.specifier and o.attributes == self.attributes
		end
	end
	class TypeDef < Type
		attr_accessor :name
		attr_accessor :type
		attr_accessor :backtrace

		def initialize(name, type, backtrace)
			@name, @type, @backtrace = name, type, backtrace
		end

		def parse_initializer(parser, scope)
			@type.parse_initializer(parser, scope)
		end

		def pointer? ;    @type.pointer?      end
		def arithmetic? ; @type.arithmetic?   end
		def integral? ;   @type.integral?     end
		def float? ;      @type.float?        end
		def untypedef ;   @type.untypedef     end
		def align(parser) @type.align(parser) end	# XXX __attribute__ ?
	end
	class Function < Type
		attr_accessor :type		# return type
		attr_accessor :args		# [name, Variable]
		attr_accessor :varargs		# true/false

		def initialize(type=nil)
			@type = type
		end

		def base ; @type.base ; end
	end
	class Union < Type
		attr_accessor :members		# [Variable]
		attr_accessor :bits		# [bits] or nil
		attr_accessor :name
		attr_accessor :backtrace

		def align(parser) @members.map { |m| m.type.align(parser) }.max end

		def findmember(name)
			if m = @members.find { |m| m.name == name }
				return m
			else
				@members.each { |m|
					if t = m.type.untypedef and t.kind_of? Union and mm = t.findmember(name)
						return mm
					end
				}
			end
			nil
		end

		def parse_members(parser, scope)
			@members = []
			# parse struct/union members in definition
			loop do
				raise parser if not tok = parser.skipspaces
				break if tok.type == :punct and tok.raw == '}'
				parser.unreadtok tok

				raise tok, 'invalid struct member type' if not basetype = Variable.parse_type(parser, scope)
				loop do
					member = basetype.dup
					member.parse_declarator(parser, scope)
					# raise parser if not member.name	# can be useful while hacking: struct foo {int; int*; int iwant;};
					raise member.backtrace, 'member redefinition' if member.name and @members.find { |m| m.name == member.name }
					@members << member

					raise tok || parser if not tok = parser.skipspaces or tok.type != :punct

					if tok.raw == ':'	# bits
						raise tok, 'bad type for bitslice' if not member.type.integral?
						bits = nil
						raise tok, "bad bit count #{bits.inspect}" if not bits = CExpression.parse(parser, scope, false) or
							not bits.constant? or not (bits = bits.reduce(parser)).kind_of? ::Integer
						#raise tok, 'need more bits' if bits > 8*parser.sizeof(member)
						# WORD wReserved:17; => yay windows.h
						(@bits ||= [])[@members.length-1] = bits
						raise tok || parser, '"," or ";" expected' if not tok = parser.skipspaces or tok.type != :punct
					end

					case tok.raw
					when ';'; break
					when ','
					else raise tok, '"," or ";" expected'
					end
				end
			end
			parse_attributes(parser)
		end

		def parse_initializer(parser, scope)
			if tok = parser.skipspaces and tok.type == :punct and tok.raw == '{'
				# struct x toto = { 1, .4, .member[0][6].bla = 12 };
				raise tok, 'undefined struct' if not @members
				ret = []
				if tok = parser.skipspaces and (tok.type != :punct or tok.raw != '}')
					parser.unreadtok tok
					idx = 0
					loop do
						idx = parse_initializer_designator(parser, scope, ret, idx, true)
						raise tok || parser, '"," or "}" expected' if not tok = parser.skipspaces or tok.type != :punct or (tok.raw != '}' and tok.raw != ',')
						break if tok.raw == '}'
					end
				end
				ret
			else
				parser.unreadtok tok
				super
			end
		end

		# parses a designator+initializer eg '.toto = 4' or '.tutu[42][12].bla = 16' or (root ? '4' : '=4')
		def parse_initializer_designator(parser, scope, value, idx, root=true)
			if nt = parser.skipspaces and nt.type == :punct and nt.raw == '.' and
					nnt = parser.skipspaces and nnt.type == :string and
					m = findmember(nnt.raw)
				raise nnt, 'unhandled initializer' if not nidx = @members.index(@members.find { |m| m.name == nnt.raw })	# TODO
				value = value[idx] ||= [] if not root
				idx = nidx
				@members[idx].type.parse_initializer_designator(parser, scope, value, idx, false)
			else
				parser.unreadtok nnt
				if root
					parser.unreadtok nt
					value[idx] = @members[idx].type.parse_initializer(parser, scope)
				else
					raise nt || parser, '"=" expected' if not nt or nt.type != :punct or nt.raw != '='
					value[idx] = parse_initializer(parser, scope)
				end
			end
			idx + 1
		end
	end
	class Struct < Union
		attr_accessor :pack

		def align(parser) [@members.map { |m| m.type.align(parser) }.max, (pack || 8)].min end

		def offsetof(parser, name)
			raise parser, 'undefined structure' if not @members
			raise parser, 'unknown structure member' if not findmember(name)
			raise parser, 'unhandled indirect offsetof' if not @members.find { |m| m.name == name }	# TODO
			al = align(parser)
			off = 0
			bit_off = 0
			@members.each_with_index { |m, i|
				break if m.name == name
				if bits and b = @bits[i]
					isz = parser.typesize[:int]
					if bit_off + b > 8*isz
						bit_off = 0
						off = (off + isz - 1) / isz * isz + isz
					else
						bit_off += b
					end
				else
					mal = [m.type.align(parser), al].min
					off = (off + mal - 1) / mal * mal
					off += parser.sizeof(m)
				end
			}
			off
		end

		def parse_members(parser, scope)
			super
			if defined? @attributes and @attributes
				if @attributes.include? 'packed'
					@pack = 1
				elsif p = @attributes.grep(/^pack\(\d+\)$/).first
					@pack = p[/\d+/].to_i
				end
			end
		end
	end
	class Enum < Type
		# name => value
		attr_accessor :members
		attr_accessor :name
		attr_accessor :backtrace

		def align(parser) BaseType.new(:int).align(parser) end

		def parse_members(parser, scope)
			val = -1
			@members = {}
			loop do
				raise parser if not tok = parser.skipspaces
				break if tok.type == :punct and tok.raw == '}'

				name = tok.raw
				raise tok, 'bad enum name' if tok.type != :string or Keyword[name] or (?0..?9).include?(name[0])

				raise parser if not tok = parser.skipspaces
				if tok.type == :punct and tok.raw == '='
					raise tok || parser if not val = CExpression.parse(parser, scope, false) or not val = val.reduce(parser) or not tok = parser.skipspaces
				else
					val += 1
				end
				raise tok, "enum value #{name} redefinition" if scope.symbol[name] and scope.symbol[name] != val
				@members[name] = val
				scope.symbol[name] = val

				if tok.type == :punct and tok.raw == '}'
					break
				elsif tok.type == :punct and tok.raw == ','
				else raise tok, '"," or "}" expected'
				end
			end
			parse_attributes(parser)
		end

	end
	class Pointer < Type
		attr_accessor :type

		def initialize(type=nil)
			@type = type
		end

		def pointer? ; true ; end
		def arithmetic? ; true ; end
		def base ; @type.base ; end
		def align(parser) BaseType.new(:ptr).align(parser) end

		def ==(o)
			o.class == self.class and o.type == self.type
		end
	end
	class Array < Pointer
		attr_accessor :length

		def align(parser) @type.align(parser) end

		def parse_initializer(parser, scope)
			raise parser, 'cannot initialize dynamic array' if @length.kind_of? CExpression
			if tok = parser.skipspaces and tok.type == :punct and tok.raw == '{'
				# struct x foo[] = { { 4 }, [12].tutu = 2 };
				ret = []
				if tok = parser.skipspaces and (tok.type != :punct or tok.raw != '}')
					parser.unreadtok tok
					idx = 0
					loop do
						idx = parse_initializer_designator(parser, scope, ret, idx, true)
						raise tok || parser, '"," or "}" expected' if not tok = parser.skipspaces or tok.type != :punct or (tok.raw != '}' and tok.raw != ',')
						break if tok.raw == '}'
						# allow int x[] = {1, 2, 3, };
						break if tok = parser.skipspaces and tok.type == :punct and tok.raw == '}'
						parser.unreadtok tok
					end
				end
				ret
			else
				parser.unreadtok tok
				super
			end
		end

		# parses a designator+initializer eg '[12] = 4' or '[42].bla = 16' or (root ? '4' : '=4')
		def parse_initializer_designator(parser, scope, value, idx, root=true)
			if nt = parser.skipspaces and nt.type == :punct and nt.raw == '['
				value = value[idx] ||= [] if not root
				raise nt, 'const expected' if not idx = CExpression.parse(parser, scope) or not idx.constant? or not idx = idx.reduce(parser) or not idx.kind_of? ::Integer
				raise nt || parser, '"]" expected' if not nt = parser.skipspaces or nt.type != :punct or nt.raw != ']'
				@type.parse_initializer_designator(parser, scope, value, idx, false)
			else
				if root
					parser.unreadtok nt
					value[idx] = @type.parse_initializer(parser, scope)
				else
					raise nt || parser, '"=" expected' if not nt or nt.type != :punct or nt.raw != '='
					value[idx] = parse_initializer(parser, scope)
				end
			end
			idx + 1
		end
	end

	class Variable
		include Attributes
		attr_accessor :type
		attr_accessor :initializer	# CExpr	/ Block (for Functions)
		attr_accessor :name
		attr_accessor :storage		# auto register static extern typedef
		attr_accessor :backtrace	# definition backtrace info (the name token)
	end

	# found in a block's Statements, used to know the initialization order
	# eg { int i; i = 4; struct foo { int k; } toto = {i}; }
	class Declaration
		attr_accessor :var
		def initialize(var)
			@var = var
		end
	end

	class If < Statement
		attr_accessor :test		# expression
		attr_accessor :bthen, :belse	# statements
		def initialize(test, bthen, belse=nil)
			@test = test
			@bthen = bthen
			@belse = belse if belse
		end

		def self.parse(parser, scope, nest)
			tok = nil
			raise tok || self, '"(" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '('
			raise tok, 'expr expected' if not expr = CExpression.parse(parser, scope) or not expr.type.arithmetic?
			raise tok || self, '")" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ')'
			bthen = parser.parse_statement scope, nest
			if tok = parser.skipspaces and tok.type == :string and tok.raw == 'else'
				belse = parser.parse_statement scope, nest
			else
				parser.unreadtok tok
			end

			new expr, bthen, belse
		end
	end
	class For < Statement
		attr_accessor :init, :test, :iter	# CExpressions, init may be Block
		attr_accessor :body
		def initialize(init, test, iter, body)
			@init, @test, @iter, @body = init, test, iter, body
		end

		def self.parse(parser, scope, nest)
			tok = nil
			raise tok || parser, '"(" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '('
			init = forscope = Block.new(scope)
			if not parser.parse_definition(forscope)
				forscope = scope
				init = CExpression.parse(parser, forscope)
				raise tok || parser, '";" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ';'
			end
			test = CExpression.parse(parser, forscope)
			raise tok || parser, '";" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ';'
			raise tok, 'bad test expression in for loop' if test and not test.type.arithmetic?
			iter = CExpression.parse(parser, forscope)
			raise tok || parser, '")" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ')'

			new init, test, iter, parser.parse_statement(forscope, nest + [:loop])
		end
	end
	class While < Statement
		attr_accessor :test
		attr_accessor :body

		def initialize(test, body)
			@test = test
			@body = body
		end

		def self.parse(parser, scope, nest)
			tok = nil
			raise tok || parser, '"(" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '('
			raise tok, 'expr expected' if not expr = CExpression.parse(parser, scope) or not expr.type.arithmetic?
			raise tok || parser, '")" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ')'

			new expr, parser.parse_statement(scope, nest + [:loop])
		end
	end
	class DoWhile < While
		def self.parse(parser, scope, nest)
			body = parser.parse_statement(scope, nest + [:loop])
			tok = nil
			raise tok || parser, '"while" expected' if not tok = parser.skipspaces or tok.type != :string or tok.raw != 'while'
			raise tok || parser, '"(" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '('
			raise tok, 'expr expected' if not expr = CExpression.parse(parser, scope) or not expr.type.arithmetic?
			raise tok || parser, '")" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ')'
			raise tok || parser, '";" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ';'

			new expr, body
		end
	end
	class Switch < Statement
		attr_accessor :test, :body

		def initialize(test, body)
			@test = test
			@body = body
		end

		def self.parse(parser, scope, nest)
			raise tok || parser, '"(" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '('
			raise tok, 'expr expected' if not expr = CExpression.parse(parser, scope) or not expr.type.integral?
			raise tok || parser, '")" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ')'

			new expr, parser.parse_statement(scope, nest + [:switch])
		end
	end

	class Continue < Statement
	end
	class Break < Statement
	end
	class Goto < Statement
		attr_accessor :target
		def initialize(target)
			@target = target
		end
	end
	class Return < Statement
		attr_accessor :value
		def initialize(value)
			@value = value
		end
	end
	class Label < Statement
		attr_accessor :name
		attr_accessor :statement
		def initialize(name, statement=nil)
			@name, @statement = name, statement
		end
	end
	class Case < Label
		attr_accessor :expr, :exprup	# exprup if range, expr may be 'default'
		def initialize(expr, exprup, statement)
			@expr, @statement = expr, statement
			@exprup = exprup if exprup
		end

		def self.parse(parser, scope, nest)
			raise parser, 'invalid case' if not expr = CExpression.parse(parser, scope) or not expr.constant? or not expr.type.integral?
			raise tok || parser, '":" or "..." expected' if not tok = parser.skipspaces or tok.type != :punct or (tok.raw != ':' and tok.raw != '.')
			if tok.raw == '.'
				raise tok || parser, '".." expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '.'
				raise tok || parser,  '"." expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != '.'
				raise tok, 'invalid case range' if not exprup = CExpression.parse(parser, scope) or not exprup.constant? or not exprup.type.integral?
				raise tok || parser, '":" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ':'
			end
			body = parser.parse_statement scope, nest
			new expr, exprup, body
		end
	end

	# inline asm statement
	class Asm < Statement
		attr_accessor :body		# asm source (::String)
		attr_accessor :output, :input, :clobber	# I/O, gcc-style (::Array)
		attr_accessor :backtrace	# body Token
		attr_accessor :volatile

		def initialize(body, backtrace, output, input, clobber, volatile)
			@body, @backtrace, @output, @input, @clobber, @volatile = body, backtrace, output, input, clobber, volatile
		end

		def self.parse(parser, scope)
			if tok = parser.skipspaces and tok.type == :string and (tok.raw == 'volatile' or tok.raw == '__volatile__')
				volatile = true
				tok = parser.skipspaces
			end
			if not tok or tok.type != :punct or tok.raw != '('
				# detect MS-style inline asm: "__asm .* __asm .*" or "asm { [\s.]* }"
				ftok = tok
				body = ''
				if tok.type == :punct and tok.raw == '{'
					loop do
						raise ftok, 'unterminated asm block' if not tok = parser.lexer.readtok
						break if tok.type == :punct and tok.raw == '}'
						case tok.type
						when :space; body << ' '
						when :eol; body << "\n"
						when :punct; body << tok.raw
						when :quoted; body << tok.value.inspect	# concat adjacent c strings
						when :string
							body << \
							case tok.raw
							when 'asm', '__asm', '__asm__'; "\n"
							when '_emit'; 'db'
							else tok.raw
							end
						end
					end
				else
					parser.lexer.unreadtok tok
					loop do
						break if not tok = parser.lexer.readtok or tok.type == :eol
						case tok.type
						when :space; body << ' '
						when :punct
							case tok.raw
							when '}'
								parser.lexer.unreadtok tok
								break
							else body << tok.raw
							end
						when :quoted; body << tok.value.inspect
						when :string
							body << \
							case tok.raw
							when 'asm', '__asm', '__asm__'; "\n"
							when '_emit'; 'db'
							else tok.raw
							end
						end
					end
				end
				return new(body, ftok, nil, nil, nil, volatile)
			end
			raise tok || parser, '"(" expected' if not tok or tok.type != :punct or tok.raw != '('
			raise tok || parser, 'qstring expected' if not tok = parser.skipspaces or tok.type != :quoted
			body = tok
			tok = parser.skipspaces
			raise tok || parser, '":" or ")" expected' if not tok or tok.type != :punct or (tok.raw != ':' and tok.raw != ')')

			if tok.raw == ':'
				output = []
				raise parser if not tok = parser.skipspaces
				while tok.type == :quoted
					type = tok.value
					raise tok, 'expr expected' if not var = CExpression.parse_value(parser, scope)
					output << [type, var]
					raise tok || parser, '":" or "," or ")" expected' if not tok = parser.skipspaces or tok.type != :punct or (tok.raw != ',' and tok.raw != ')' and tok.raw != ':')
					break if tok.raw == ':' or tok.raw == ')'
					raise tok || parser, 'qstring expected' if not tok = parser.skipspaces or tok.type != :quoted
				end
			end
			if tok.raw == ':'
				input = []
				raise parser if not tok = parser.skipspaces
				while tok.type == :quoted
					type = tok.value
					raise tok, 'expr expected' if not var = CExpression.parse_value(parser, scope)
					input << [type, var]
					raise tok || parser, '":" or "," or ")" expected' if not tok = parser.skipspaces or tok.type != :punct or (tok.raw != ',' and tok.raw != ')' and tok.raw != ':')
					break if tok.raw == ':' or tok.raw == ')'
					raise tok || parser, 'qstring expected' if not tok = parser.skipspaces or tok.type != :quoted
				end
			end
			if tok.raw == ':'
				clobber = []
				raise parser if not tok = parser.skipspaces
				while tok.type == :quoted
					clobber << tok.value
					raise tok || parser, '"," or ")" expected' if not tok = parser.skipspaces or tok.type != :punct or (tok.raw != ',' and tok.raw != ')')
					break if tok.raw == ')'
					raise tok || parser, 'qstring expected' if not tok = parser.skipspaces or tok.type != :quoted
				end
			end
			raise tok || parser, '")" expected' if not tok or tok.type != :punct or tok.raw != ')'
			raise tok || parser, '";" expected' if not tok = parser.skipspaces or tok.type != :punct or tok.raw != ';'

			new body.value, body, output, input, clobber, volatile
		end
	end

	class CExpression < Statement
		# may be :,, :., :'->', :funcall (function, [arglist]), :[] (array indexing), nil (cast)
		attr_accessor :op
		# nil/CExpr/Variable/Label/::String( = :quoted/struct member name)/::Integer/::Float/Block
		attr_accessor :lexpr, :rexpr
		# a Type
		attr_accessor :type
		def initialize(l, o, r, t)
			@lexpr, @op, @rexpr, @type = l, o, r, t
		end
	end


	class Parser
		# creates a new CParser, parses all top-level statements
		def self.parse(text)
			c = new
	                c.parse text
			raise c.lexer.readtok || c, 'invalid definition' if not c.lexer.eos?
			c
		end

		# parses the current lexer content (or the text arg) for toplevel definitions
		def parse(text=nil, filename='<unk>', lineno=1)
			@lexer.feed text, filename, lineno if text
			nil while not @lexer.eos? and (parse_definition(@toplevel) or parse_toplevel_statement(@toplevel))
			sanity_checks
			self
		end

		# parses a C file
		def parse_file(file)
			parse(File.read(file), file)
		end

		attr_accessor :lexer, :toplevel, :typesize, :pragma_pack
		def initialize(lexer = nil, model=:ilp32)
			@lexer = lexer || Preprocessor.new
			@prev_pragma_callback = @lexer.pragma_callback
			@lexer.pragma_callback = proc { |tok| parse_pragma_callback(tok) }
			@toplevel = Block.new(nil)
			@unreadtoks = []
			@typesize = { :void => 0, :__int8 => 1, :__int16 => 2, :__int32 => 4, :__int64 => 8,
				:char => 1, :float => 4, :double => 8, :longdouble => 12 }
			send model
		end

		def ilp16
			# XXX check this
			@typesize.update :short => 2, :ptr => 2,
				:int => 2, :long => 4, :longlong => 4
		end
		def lp32
			@typesize.update :short => 2, :ptr => 4,
				:int => 2, :long => 4, :longlong => 8
		end
		def ilp32
			@typesize.update :short => 2, :ptr => 4,
				:int => 4, :long => 4, :longlong => 8
		end
		def llp64
			# longlong should only exist here
			@typesize.update :short => 2, :ptr => 8,
				:int => 4, :long => 4, :longlong => 8
		end
		def ilp64
			@typesize.update :short => 2, :ptr => 8,
				:int => 8, :long => 8, :longlong => 8
		end
		def lp64
			@typesize.update :short => 2, :ptr => 8,
				:int => 4, :long => 8, :longlong => 8
		end

		attr_accessor :auto_predeclare_unknown_structs
		def parse_pragma_callback(otok)
			case otok.raw
			when 'pack'
				nil while lp = @lexer.readtok and lp.type == :space
				nil while rp = @lexer.readtok and rp.type == :space
				if not rp or rp.type != :punct or rp.raw != ')'
					v1 = rp
					nil while rp = @lexer.readtok and rp.type == :space
				end
				if rp and rp.type == :punct and rp.raw == ','
					nil while v2 = @lexer.readtok and v2.type == :space
					nil while rp = @lexer.readtok and rp.type == :space
				end
				raise cmd if not rp or lp.type != :punct or rp.type != :punct or lp.raw != '(' or rp.raw != ')'
				raise cmd if (v1 and v1.type != :string) or (v2 and (v2.type != :string or v2.raw =~ /[^\d]/))
				if not v1
					@pragma_pack = nil
				elsif v1.raw == 'push'
					@pragma_pack_stack ||= []
					@pragma_pack_stack << pragma_pack
					@pragma_pack = v2.raw.to_i if v2
					raise v2, 'bad pack value' if pragma_pack == 0
				elsif v1.raw == 'pop'
					@pragma_pack_stack ||= []
					raise v1, 'pack stack empty' if @pragma_pack_stack.empty?
					@pragma_pack = @pragma_pack_stack.pop
					@pragma_pack = v2.raw.to_i if v2 and v2.raw	# #pragma pack(pop, 4) => pop stack, but use 4 as pack value (imho)
					raise v2, 'bad pack value' if pragma_pack == 0
				elsif v1.raw =~ /^\d+$/
					raise v2, '2nd arg unexpected' if v2
					@pragma_pack = v1.raw.to_i
					raise v1, 'bad pack value' if @pragma_pack == 0
				else raise otok
				end
				# the caller checks for :eol
			when 'auto_predeclare_unknown_structs'
				@auto_predeclare_unknown_structs = true
			when 'warning'
				if $DEBUG
					@prev_pragma_callback[otok]
				else
					# silent discard
					nil while tok = @lexer.readtok_nopp and tok.type != :eol
					@lexer.unreadtok tok
				end
			when 'prepare_visualstudio'
				prepare_visualstudio
			when 'prepare_gcc'
				prepare_gcc
			else @prev_pragma_callback[otok]
			end
		end

		def prepare_visualstudio
			@auto_predeclare_unknown_structs = true
			@lexer.define_weak('_WIN32')
			@lexer.define_weak('_WIN32_WINNT', 0x500)
			@lexer.define_weak('_INTEGRAL_MAX_BITS', 64)
			@lexer.define_weak('__w64')
			@lexer.define_weak('_cdecl', '__cdecl')	# typo ? seen in winreg.h
			@lexer.define_weak('_fastcall', '__fastcall')	# typo ? seen in ntddk.h
			@lexer.define_weak('_MSC_VER', 1300)	# handle '#pragma once' and _declspec(noreturn)
			@lexer.define_weak('__forceinline', '__inline')
			@lexer.define_weak('__ptr32')	# needed with msc_ver 1300, don't understand their use
			@lexer.define_weak('__ptr64')
		end

		def prepare_gcc
			@lexer.define_weak('__GNUC__', 2)	# otherwise __attribute__ is defined to void..
			@lexer.define_weak('__STDC__')
			@lexer.define_weak('__const', 'const')
			@lexer.define_weak('__signed', 'signed')
			@lexer.define_weak('__volatile', 'volatile')
			@lexer.nodefine_strong('__REDIRECT_NTH')	# booh gnu
			@lexer.hooked_include['stddef.h'] = <<EOH
/* simplified, define all at first invocation. may break things... */
#undef __need_ptrdiff_t
#undef __need_size_t
#undef __need_wint_t
#undef __need_wchar_t
#undef __need_NULL
#undef NULL
#if !defined (_STDDEF_H)
#define _STDDEF_H
#define __PTRDIFF_TYPE__ long int
typedef __PTRDIFF_TYPE__ ptrdiff_t;
#define __SIZE_TYPE__ long unsigned int
typedef __SIZE_TYPE__ size_t;
#define __WINT_TYPE__ unsigned int
typedef __WINT_TYPE__ wint_t;
#define __WCHAR_TYPE__ int
typedef __WCHAR_TYPE__ wchar_t;
#define NULL 0
#define offsetof(TYPE, MEMBER) __builtin_offsetof (TYPE, MEMBER)
#endif
EOH
			# TODO va_args
			@lexer.hooked_include['stdarg.h'] = <<EOH
// TODO
typedef void* __gnuc_va_list;
/*
typedef void* va_list;
#define va_start(v, l)
#define va_end(v)
#define va_arg(v, l)
#define va_copy(d, s)
*/
EOH
		end

		# C sanity checks
		def sanity_checks
			return if not $VERBOSE
			#  TODO
		end

		# checks that the types are compatible (variable predeclaration, function argument..)
		# strict = false for func call/assignment (eg char compatible with int -- but int is incompatible with char)
		def check_compatible_type(tok, oldtype, newtype, strict = false, checked = [])
			oldtype = oldtype.untypedef
			newtype = newtype.untypedef
			oldtype = BaseType.new(:int) if oldtype.kind_of? Enum
			newtype = BaseType.new(:int) if newtype.kind_of? Enum

			puts tok.exception('type qualifier mismatch').message if $VERBOSE and oldtype.qualifier.to_a.uniq.length > newtype.qualifier.to_a.uniq.length

			# avoid infinite recursion
			return if checked.include? oldtype
			checked = checked + [oldtype]

		    begin
			case newtype
			when Function
				raise tok if not oldtype.kind_of? Function
				check_compatible_type tok, oldtype.type, newtype.type, strict, checked
				if oldtype.args and newtype.args
					if oldtype.args.length != newtype.args.length or
							oldtype.varargs != newtype.varargs
						raise tok
					end
					oldtype.args.zip(newtype.args) { |oa, na|
						# begin ; rescue ParseError: raise $!.message + "in parameter #{oa.name}" end
						check_compatible_type tok, oa.type, na.type, strict, checked
					}
				end
			when Pointer
				if oldtype.kind_of? BaseType and oldtype.integral?
					puts tok.exception('making pointer from integer without a cast').message if $VERBOSE
					return
				end
				raise tok if not oldtype.kind_of? Pointer
				hasvoid = true if (t = newtype.type.untypedef).kind_of? BaseType and t.name == :void
				hasvoid = true if (t = oldtype.type.untypedef).kind_of? BaseType and t.name == :void	# struct foo *f = NULL;
				if strict and not hasvoid
					check_compatible_type tok, oldtype.type, newtype.type, strict, checked
{'fraction_non_alphanumeric': 0.12469867015756957, 'fraction_numerical': 0.006513513944986351, 'mean_word_length': 3.0275188066610945, 'pattern_counts': {'":': 10, '<': 131, '<?xml version=': 0, '>': 105, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1409654', 'n_tokens_mistral': 25536, 'n_tokens_neox': 21003, 'n_words': 6761}