<commit_before># function for parsing the data
import docker
import sys
import os
from redomatfunctions import Redomat

def data_parser(docker_line, redo):
	docker_command = docker_line.split(" ")

	if not hasattr(redo, docker_command[0]):
		raise Exception("unknown command <%s>"%docker_command[0])
	callback = getattr(redo, docker_command[0])

	callback(" ".join(docker_command[1:]).strip())

for dockerfile in sys.argv[1:]:
	if os.path.exists(dockerfile) is False:
		raise Exception("No Dockerfile found")
		sys.exit(1)

	inputfile = open(dockerfile)

	redo = Redomat(docker.Client(base_url='unix://var/run/docker.sock',version='0.6.0'))
	for line in inputfile:
		if line.strip() == "":
			continue
		if line.strip().startswith("#"):
			continue
		data_parser(line.strip(), redo)
	inputfile.close()<commit_msg>integrating the xml parser into the redomat<commit_after># function for parsing the data
import docker
import sys
import os
import xml.etree.ElementTree as XML
from redomatfunctions import Redomat

def xml_parser(xml_file):
	manifest_root = XML.parse(xml_file).getroot()
	stages = []

	for buildstage in manifest_root.iter('buildstage'):
		stage = {'id': buildstage.get('id'),
			'dockerlines' : [],
			'bitbake_target' : {'cmd' : [], 'target' : [] }}
		stages.append(stage)

		for stage_command in buildstage.iter():

			if stage_command.tag == 'dockerline':
				stage['dockerlines'].append(stage_command.text)

			if stage_command.tag == 'bitbake_target':
				stage['dockerlines'].append('RUN bitbake ')
				if stage_command.get('command'):
					stage['dockerlines'].append(stage['dockerlines'].pop() + '-c ' + stage_command.get('command'))
					stage['dockerlines'].append(stage['dockerlines'].pop() + " " + stage_command.text)
				else:
					stage['dockerlines'].append(stage['dockerlines'].pop() + stage_command.text)
	return stages

def data_parser(docker_line, redo):
	docker_command = docker_line.split(" ")

	if not hasattr(redo, docker_command[0]):
		raise Exception("unknown command <%s>"%docker_command[0])
	callback = getattr(redo, docker_command[0])

	callback(" ".join(docker_command[1:]).strip())

for dockerfile in sys.argv[1:]:
	if os.path.exists(dockerfile) is False:
		raise Exception("No Dockerfile found")
		sys.exit(1)

	stages=xml_parser(dockerfile)

	redo = Redomat(docker.Client(base_url='unix://var/run/docker.sock',version='0.6.0'))

	for stage in stages:
		print(stage['id'])
		for line in stage['dockerlines']:
			print(line)
			data_parser(line, redo)
		print()
{'fraction_non_alphanumeric': 0.1532934131736527, 'fraction_numerical': 0.00718562874251497, 'mean_word_length': 5.3443037974683545, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7263498', 'n_tokens_mistral': 942, 'n_tokens_neox': 823, 'n_words': 178}