<commit_before>#!/usr/bin/env python
from __future__ import print_function
import re
import subprocess


class DiffTransformer(object):

	def __init__(self, revision1, revision2):
		self.revision1 = revision1
		self.revision2 = revision2

	def transform_line(self, line):
		if line.startswith('diff --git '):
			m = re.search(r'^diff --git (.+) \1$', line)
			self.filename = m.group(1)
			return 'Index: ' + self.filename
		elif line.startswith('index '):
			return 67 * '='
		elif line.startswith('--- '):
			if line == '--- /dev/null':
				rev = 0
			else:
				rev = self.revision1
			return '--- %s\t(revision %d)' % (self.filename, rev)
		elif line.startswith('+++ '):
			if self.revision2 is None:
				rev_str = 'working copy'
			else:
				rev_str = 'revision %d' % self.revision2
			return '+++ %s\t(%s)' % (self.filename, rev_str)
		elif line.startswith('new file mode ') or line.startswith('deleted file mode '):
			return None
		else:
			return line


def main():
	import sys
	
	args_parser = ArgumentsParser()
	version1, version2, verbose = args_parser.parse(sys.argv[1:])
	
	versions_solver = VersionsSolver()
	versions_solver.solve_version1(version1)
	versions_solver.solve_version2(version2)
	
	commits = [version1.commit]
	if version2.commit is not None:
		commits.append(version2.commit)
	
	cmd = ['git', 'diff', '--no-prefix'] + commits
	if verbose:
		print('Executing %r' % cmd, file=sys.stderr)
	output = subprocess.check_output(cmd)

	lines = output.splitlines()
	transformer = DiffTransformer(version1.assumed_revision, version2.assumed_revision)
	for line in lines:
		line = transformer.transform_line(line)
		if line is not None:
			print(line)


class Version(object):<commit_msg>Refactor: move lines loop to the new generator method DiffTransformer.transform_lines<commit_after>#!/usr/bin/env python
from __future__ import print_function
import re
import subprocess


class DiffTransformer(object):

	def __init__(self, revision1, revision2):
		self.revision1 = revision1
		self.revision2 = revision2
	
	def transform_lines(self, lines):
		for line in lines:
			line = line.rstrip('\r\n')
			line = self.transform_line(line)
			if line is not None:
				yield line

	def transform_line(self, line):
		if line.startswith('diff --git '):
			m = re.search(r'^diff --git (.+) \1$', line)
			self.filename = m.group(1)
			return 'Index: ' + self.filename
		elif line.startswith('index '):
			return 67 * '='
		elif line.startswith('--- '):
			if line == '--- /dev/null':
				rev = 0
			else:
				rev = self.revision1
			return '--- %s\t(revision %d)' % (self.filename, rev)
		elif line.startswith('+++ '):
			if self.revision2 is None:
				rev_str = 'working copy'
			else:
				rev_str = 'revision %d' % self.revision2
			return '+++ %s\t(%s)' % (self.filename, rev_str)
		elif line.startswith('new file mode ') or line.startswith('deleted file mode '):
			return None
		else:
			return line


def main():
	import sys
	
	args_parser = ArgumentsParser()
	version1, version2, verbose = args_parser.parse(sys.argv[1:])
	
	versions_solver = VersionsSolver()
	versions_solver.solve_version1(version1)
	versions_solver.solve_version2(version2)
	
	commits = [version1.commit]
	if version2.commit is not None:
		commits.append(version2.commit)
	
	cmd = ['git', 'diff', '--no-prefix'] + commits
	if verbose:
		print('Executing %r' % cmd, file=sys.stderr)
	process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
	process.poll()
	if process.returncode not in (None, 0):
		raise NotImplementedError()
	
	transformer = DiffTransformer(version1.assumed_revision, version2.assumed_revision)
	for line in transformer.transform_lines(process.stdout):
		print(line)


class Version(object):
{'fraction_non_alphanumeric': 0.13705308775731312, 'fraction_numerical': 0.014355362946912243, 'mean_word_length': 3.977088948787062, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3462298', 'n_tokens_mistral': 1439, 'n_tokens_neox': 1278, 'n_words': 307}