<commit_before>import re

class Editor(object):
	def match_line(self, line_index, content):
		return False
	
	def insert_line(self, line_index, content):
		return False
	
	def delete_line(self, line_index, content):
		return False

class ListEditor(Editor):
	lst = []
	def __init__(self, lst):
		self.lst = lst
	
	def match_line(self, line_index, content):
		if line_index < len(self.lst):
			return self.lst[line_index] == content
		return False
	
	def insert_line(self, line_index, content):
		if line_index <= len(self.lst):
			self.lst.insert(line_index, content)
			return True
		return False
	
	def delete_line(self, line_index):
		if line_index < len(self.lst):
			del self.lst[line_index]
			return True
		return False
	
	def __repr__(self):
		return '\n'.join(self.lst)

px = re.compile(r'^([+][+][+]|[-][-][-]|[ ]|[@][@]|[+]|[-])(.*)$', re.MULTILINE)
def patch(editor, diff):
	patch = px.findall(diff)
	ep = len(patch)
	if ep > 2 and patch[0][0] == '---' and patch[1][0] == '+++':
		sp = 2
		while sp < ep:
			p, ln = patch[sp]
			if p == '@@':
				ln = ln.strip('@ ').split(' ')
				if len(ln) == 2 and ln[0][0] == '-' and ln[1][0] == '+':
					ed_index = ln[1][1:].split(',')
					ed_index = int(ed_index[0]) - 1
					sp += 1
					while sp < ep:
						p, ln = patch[sp]
						if p == ' ':
							if not editor.match_line(ed_index, ln):
								return 'Contextual match failed'
							ed_index += 1
						elif p == '+':
							if not editor.insert_line(ed_index, ln):
								return 'Failed to insert ``%s"' % ln
							ed_index += 1
						elif p == '-':
							if not editor.delete_line(ed_index):
								return 'Failed to delete line %d' % (ed_index+1)
						else:
							# we expect a new hunk
							break
						sp += 1
				else:
					return 'Invalid range'
			else:
				return 'Expected hunk, got ``%s"' % ln
		return ''
	return 'Invalid diff'<commit_msg>- account for junk(trailing text inserted by git-diff, etc.) in the hunk headers<commit_after>import re

class Editor(object):
	def match_line(self, line_index, content):
		return False
	
	def insert_line(self, line_index, content):
		return False
	
	def delete_line(self, line_index, content):
		return False

class ListEditor(Editor):
	lst = []
	def __init__(self, lst):
		self.lst = lst
	
	def match_line(self, line_index, content):
		if line_index < len(self.lst):
			return self.lst[line_index] == content
		return False
	
	def insert_line(self, line_index, content):
		if line_index <= len(self.lst):
			self.lst.insert(line_index, content)
			return True
		return False
	
	def delete_line(self, line_index):
		if line_index < len(self.lst):
			del self.lst[line_index]
			return True
		return False
	
	def __repr__(self):
		return '\n'.join(self.lst)

px = re.compile(r'^([+][+][+]|[-][-][-]|[ ]|[@][@]|[+]|[-])(.*)$', re.MULTILINE)
def patch(editor, diff):
	patch = px.findall(diff)
	ep = len(patch)
	if ep > 2 and patch[0][0] == '---' and patch[1][0] == '+++':
		sp = 2
		while sp < ep:
			p, ln = patch[sp]
			if p == '@@':
				ln = ln.strip('').split(' ')
				if len(ln) > 2 and ln[0][0] == '-' and ln[1][0] == '+' and ln[2].strip('') == '@@':
					ed_index = ln[1][1:].split(',')
					ed_index = int(ed_index[0]) - 1
					sp += 1
					while sp < ep:
						p, ln = patch[sp]
						if p == ' ':
							if not editor.match_line(ed_index, ln):
								return 'Contextual match failed'
							ed_index += 1
						elif p == '+':
							if not editor.insert_line(ed_index, ln):
								return 'Failed to insert ``%s"' % ln
							ed_index += 1
						elif p == '-':
							if not editor.delete_line(ed_index):
								return 'Failed to delete line %d' % (ed_index+1)
						else:
							# we expect a new hunk
							break
						sp += 1
				else:
					return 'Invalid range'
			else:
				return 'Expected hunk, got ``%s"' % ln
		return ''
	return 'Invalid diff'
{'fraction_non_alphanumeric': 0.17127215849843588, 'fraction_numerical': 0.010688216892596455, 'mean_word_length': 2.8217131474103585, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6145235', 'n_tokens_mistral': 1783, 'n_tokens_neox': 1397, 'n_words': 378}