<commit_before>#!python
from __future__ import print_function
import sys
import re
import inspect
import optparse

def get_options():
	parser = optparse.OptionParser()
	options, args = parser.parse_args()
	try:
		options.filename = args.pop()
	except IndexError:
		parser.error("Filename required")
	return options

# each of the replacement functions should have a docstring
#  which is a regular expression to be matched.

def replace_external_link(matcher):
	r"\[(?P<href>(?P<scheme>\w+)\://.+?) (?P<name>.+?)\]"
	return '`{name} <{href}>`_'.format(**matcher.groupdict())

def replace_wiki_link(matcher):
	r"\[wiki\:(?P<ref>.+?) (?P<name>.+?)\]"
	return '`{name} <TODO-fix wiki target {ref}>`_'.format(**matcher.groupdict())

replacements = [
	replace_external_link,
	replace_wiki_link,
	]

def convert_file(filename):
	text = open(filename).read()
	new_text = text
	for repl in replacements:
		pattern = re.compile(inspect.getdoc(repl))
		new_text = pattern.sub(repl, text)

	open(filename, 'w').write(new_text)
	print("done")


def handle_command_line():
	options = get_options()
	convert_file(options.filename)

if __name__ == '__main__':
	handle_command_line()<commit_msg>Fixed bug in convert_file<commit_after>#!python
from __future__ import print_function
import sys
import re
import inspect
import optparse

def get_options():
	parser = optparse.OptionParser()
	options, args = parser.parse_args()
	try:
		options.filename = args.pop()
	except IndexError:
		parser.error("Filename required")
	return options

# each of the replacement functions should have a docstring
#  which is a regular expression to be matched.

def replace_external_link(matcher):
	r"\[(?P<href>(?P<scheme>\w+)\://.+?) (?P<name>.+?)\]"
	return '`{name} <{href}>`_'.format(**matcher.groupdict())

def replace_wiki_link(matcher):
	r"\[wiki\:(?P<ref>.+?) (?P<name>.+?)\]"
	return '`{name} <TODO-fix wiki target {ref}>`_'.format(**matcher.groupdict())

replacements = [
	replace_external_link,
	replace_wiki_link,
	]

def convert_file(filename):
	text = open(filename).read()
	new_text = text
	for repl in replacements:
		pattern = re.compile(inspect.getdoc(repl))
		new_text = pattern.sub(repl, new_text)

	open(filename, 'w').write(new_text)
	print("done")


def handle_command_line():
	options = get_options()
	convert_file(options.filename)

if __name__ == '__main__':
	handle_command_line()
{'fraction_non_alphanumeric': 0.15838247683235046, 'fraction_numerical': 0.0, 'mean_word_length': 5.709039548022599, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2947512', 'n_tokens_mistral': 860, 'n_tokens_neox': 829, 'n_words': 188}