<commit_before>require 'metasm/main'
require 'metasm/encode'
require 'metasm/preprocessor'

module Metasm
class Data
	# keywords for data definition (used to recognize label names)
	DataSpec = %w[db dw dd dq]
end

class CPU
	# parses prefix/name/arguments
	# returns an +Instruction+ or raise a ParseError
	def parse_instruction(lexer)
		i = Instruction.new self

		# find prefixes, break on opcode name
		while tok = lexer.readtok and parse_prefix(i, tok.raw)
			lexer.skip_space_eol
		end
	
		# allow '.' in opcode name
		while ntok = lexer.nexttok and ntok.type == :punct and ntok.raw == '.'
			tok.raw << lexer.readtok.raw
			ntok = lexer.readtok
			raise tok, 'invalid opcode name' if not ntok or ntok.type != :string
			tok.raw << ntok.raw
		end

		raise tok, 'invalid opcode' if not opcode_list_byname[tok.raw]

		i.opname = tok.raw
		i.backtrace = tok.backtrace.dup
		lexer.skip_space

		# find arguments list
		loop do
			if not ntok = lexer.nexttok or opcode_list_byname[ntok.raw] or not arg = parse_argument(lexer)
				break if i.args.empty?
				raise tok, 'invalid argument'
			end
			i.args << arg
			lexer.skip_space
			break if not ntok = lexer.nexttok or ntok.type != :punct or ntok.raw != ','
			lexer.readtok
			lexer.skip_space_eol
		end

		opcode_list_byname[i.opname].to_a.find { |o|
			o.args.length == i.args.length and o.args.zip(i.args).all? { |f, a| parse_arg_valid?(o, f, a) }
		} or raise tok, "invalid opcode arguments #{i.to_s.inspect}"

		parse_instruction_fixup(i)

		i
	end

	# called after the instruction is fully parsed
	def parse_instruction_fixup(i)
	end

	# return false if not a prefix
	def parse_prefix(i, word)
	end

	# returns a parsed argument
	# add your own arguments parser here (registers, memory references..)
	def parse_argument(lexer)
		Expression.parse(lexer)
	end

	# handles .instructions
	# XXX handle HLA here ?
	def parse_parser_instruction(lexer, instr)
		raise instr, 'unknown parser instruction'
	end
end

# asm-specific preprocessor
# handles asm arguments (; ... eol)
# asm macros (name macro args\nbody endm, name equ val)
# initializes token.value (reads integers in hex etc)
# merges consecutive space/eol
class AsmPreprocessor
	# an assembler macro, similar to preprocessor macro
	# handles local labels
	class Macro
		attr_accessor :name, :args, :body, :labels
		
		def initialize(name)
			@name = name
			@args, @body, @labels = [], [], []
		end

		# returns the array of token resulting from the application of the macro
		# parses arguments if needed, handles macro-local labels
		def apply(macro, lexer, program)
			# read arguments
			args = []
			lexer.skip_space
			if not @args.empty? and tok = lexer.nexttok and tok.type == :punct and tok.raw == '('
				lexer.readtok
				loop do
					lexer.skip_space_eol
					args << Expression.parse_toks(lexer)
					lexer.skip_space_eol
					raise @name, 'invalid arg list' if not tok = lexer.readtok or tok.type != :punct or (tok.raw != ')' and tok.raw != ',')
					break if tok.raw == ')'
				end
			end
			raise @name, 'invalid argument count' if args.length != @args.length

			labels = @labels.inject({}) { |h, l| h.update l => program.new_label(l) }
			args = @args.zip(args).inject({}) { |h, (fa, a)| h.update fa.raw => a }

			# apply macro
			@body.map { |t|
				t = t.dup
				t.backtrace += macro.backtrace[-2..-1] if not macro.backtrace.empty?
				if labels[t.raw]
					t.raw = labels[t.raw]
				elsif args[t.raw]
					args[t.raw].map { |a|
						tt = t.dup
						tt.raw = a.raw
						tt
					}
				else
					t
				end
			}.flatten
		end

		# parses the argument list and the body from lexer
		# recognize the local labels
		# XXX add eax,
		#      toto db 42	; zomg h4x
		def parse_definition(lexer)
			lexer.skip_space
			while tok = lexer.nexttok and tok.type != :eol
				# no preprocess argument list
				raise @name, 'invalid arg definition' if not tok = lexer.readtok_nopp or tok.type != :string
				@args << tok
				lexer.skip_space
				raise @name, 'invalid arg separator' if not tok = lexer.readtok_nopp or ((tok.type != :punct or tok != ',') and tok != :eol)
				break if tok.type == :eol
				lexer.skip_space
			end

			lexer.skip_space_eol
			while tok = lexer.nexttok and (tok.type != :string or tok.raw != 'endm')
				@body << lexer.readtok
				if @body[-2] and @body[-2].type == :string and @body[-1].raw == ':' and (not @body[-3] or @body[-3].type == :eol)
					@labels[@body[-2].raw] = true
				elsif @body[-3] and @body[-3].type == :string and @body[-2].type == :space and Data::DataSpec.include?(@body[-1].raw) and (not @body[-4] or @body[-4].type == :eol)
					@labels[@body[-3].raw] = true
				end
			end
		end
	end

	# the program (used to create new label names)
	attr_accessor :program
	# hash macro name => Macro
	attr_accessor :macro

	def initialize(program)
		@program = program
		@macro = {}
		super()
	end

	def skip_space_eol
		readtok while t = nexttok and (t.type == :space or t.type == :eol)
	end

	def skip_space
		readtok while t = nexttok and t.type == :space
	end

	def nexttok
		t = readtok
		unreadtok t
		t
	end

	# reads a token, handles macros/comments/integers/etc
	# argument is for internal use
	def readtok_asmpp(rec = false)
		tok = readtok_cpp

		# handle ; comments
		if tok and tok.type == :punct and tok.raw == ';'
			tok.type = :eol
			while ntok = readtok_nopp and ntok.type != :eol
				tok.raw << ntok.raw
			end
			tok.raw << ntok.raw if ntok
		end

		# aggregate space/eol
		if tok and (tok.type == :space or tok.type == :eol)
			if ntok = readtok and ntok.type == :space
				tok.raw << ntok.raw<commit_msg>allow unterminated strings in asm comments<commit_after>require 'metasm/main'
require 'metasm/encode'
require 'metasm/preprocessor'

module Metasm
class Data
	# keywords for data definition (used to recognize label names)
	DataSpec = %w[db dw dd dq]
end

class CPU
	# parses prefix/name/arguments
	# returns an +Instruction+ or raise a ParseError
	def parse_instruction(lexer)
		i = Instruction.new self

		# find prefixes, break on opcode name
		while tok = lexer.readtok and parse_prefix(i, tok.raw)
			lexer.skip_space_eol
		end
	
		# allow '.' in opcode name
		while ntok = lexer.nexttok and ntok.type == :punct and ntok.raw == '.'
			tok.raw << lexer.readtok.raw
			ntok = lexer.readtok
			raise tok, 'invalid opcode name' if not ntok or ntok.type != :string
			tok.raw << ntok.raw
		end

		raise tok, 'invalid opcode' if not opcode_list_byname[tok.raw]

		i.opname = tok.raw
		i.backtrace = tok.backtrace.dup
		lexer.skip_space

		# find arguments list
		loop do
			if not ntok = lexer.nexttok or opcode_list_byname[ntok.raw] or not arg = parse_argument(lexer)
				break if i.args.empty?
				raise tok, 'invalid argument'
			end
			i.args << arg
			lexer.skip_space
			break if not ntok = lexer.nexttok or ntok.type != :punct or ntok.raw != ','
			lexer.readtok
			lexer.skip_space_eol
		end

		opcode_list_byname[i.opname].to_a.find { |o|
			o.args.length == i.args.length and o.args.zip(i.args).all? { |f, a| parse_arg_valid?(o, f, a) }
		} or raise tok, "invalid opcode arguments #{i.to_s.inspect}"

		parse_instruction_fixup(i)

		i
	end

	# called after the instruction is fully parsed
	def parse_instruction_fixup(i)
	end

	# return false if not a prefix
	def parse_prefix(i, word)
	end

	# returns a parsed argument
	# add your own arguments parser here (registers, memory references..)
	def parse_argument(lexer)
		Expression.parse(lexer)
	end

	# handles .instructions
	# XXX handle HLA here ?
	def parse_parser_instruction(lexer, instr)
		raise instr, 'unknown parser instruction'
	end
end

# asm-specific preprocessor
# handles asm arguments (; ... eol)
# asm macros (name macro args\nbody endm, name equ val)
# initializes token.value (reads integers in hex etc)
# merges consecutive space/eol
class AsmPreprocessor
	# an assembler macro, similar to preprocessor macro
	# handles local labels
	class Macro
		attr_accessor :name, :args, :body, :labels
		
		def initialize(name)
			@name = name
			@args, @body, @labels = [], [], []
		end

		# returns the array of token resulting from the application of the macro
		# parses arguments if needed, handles macro-local labels
		def apply(macro, lexer, program)
			# read arguments
			args = []
			lexer.skip_space
			if not @args.empty? and tok = lexer.nexttok and tok.type == :punct and tok.raw == '('
				lexer.readtok
				loop do
					lexer.skip_space_eol
					args << Expression.parse_toks(lexer)
					lexer.skip_space_eol
					raise @name, 'invalid arg list' if not tok = lexer.readtok or tok.type != :punct or (tok.raw != ')' and tok.raw != ',')
					break if tok.raw == ')'
				end
			end
			raise @name, 'invalid argument count' if args.length != @args.length

			labels = @labels.inject({}) { |h, l| h.update l => program.new_label(l) }
			args = @args.zip(args).inject({}) { |h, (fa, a)| h.update fa.raw => a }

			# apply macro
			@body.map { |t|
				t = t.dup
				t.backtrace += macro.backtrace[-2..-1] if not macro.backtrace.empty?
				if labels[t.raw]
					t.raw = labels[t.raw]
				elsif args[t.raw]
					args[t.raw].map { |a|
						tt = t.dup
						tt.raw = a.raw
						tt
					}
				else
					t
				end
			}.flatten
		end

		# parses the argument list and the body from lexer
		# recognize the local labels
		# XXX add eax,
		#      toto db 42	; zomg h4x
		def parse_definition(lexer)
			lexer.skip_space
			while tok = lexer.nexttok and tok.type != :eol
				# no preprocess argument list
				raise @name, 'invalid arg definition' if not tok = lexer.readtok_nopp or tok.type != :string
				@args << tok
				lexer.skip_space
				raise @name, 'invalid arg separator' if not tok = lexer.readtok_nopp or ((tok.type != :punct or tok != ',') and tok != :eol)
				break if tok.type == :eol
				lexer.skip_space
			end

			lexer.skip_space_eol
			while tok = lexer.nexttok and (tok.type != :string or tok.raw != 'endm')
				@body << lexer.readtok
				if @body[-2] and @body[-2].type == :string and @body[-1].raw == ':' and (not @body[-3] or @body[-3].type == :eol)
					@labels[@body[-2].raw] = true
				elsif @body[-3] and @body[-3].type == :string and @body[-2].type == :space and Data::DataSpec.include?(@body[-1].raw) and (not @body[-4] or @body[-4].type == :eol)
					@labels[@body[-3].raw] = true
				end
			end
		end
	end

	# the program (used to create new label names)
	attr_accessor :program
	# hash macro name => Macro
	attr_accessor :macro

	def initialize(program)
		@program = program
		@macro = {}
		super()
	end

	def skip_space_eol
		readtok while t = nexttok and (t.type == :space or t.type == :eol)
	end

	def skip_space
		readtok while t = nexttok and t.type == :space
	end

	def nexttok
		t = readtok
		unreadtok t
		t
	end

	# reads a token, handles macros/comments/integers/etc
	# argument is for internal use
	def readtok_asmpp(rec = false)
		tok = readtok_cpp

		# handle ; comments
		if tok and tok.type == :punct and tok.raw == ';'
			tok.type = :eol
			begin
				while ntok = readtok_nopp and ntok.type != :eol
					tok.raw << ntok.raw
				end
				tok.raw << ntok.raw if ntok
			rescue ParseError
				# unterminated string
			end
		end

		# aggregate space/eol
		if tok and (tok.type == :space or tok.type == :eol)
			if ntok = readtok and ntok.type == :space
				tok.raw << ntok.raw
{'fraction_non_alphanumeric': 0.1186124911284599, 'fraction_numerical': 0.00319375443577005, 'mean_word_length': 3.046302943287868, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5766474', 'n_tokens_mistral': 4619, 'n_tokens_neox': 3989, 'n_words': 1236}