<commit_before>from os.path import isdir, join
from glob import glob 
import argparse

import avro.schema
from avro.datafile import DataFileReader, DataFileWriter
from avro.io import DatumReader, DatumWriter



SCHEMA = \
 """
 {
 	"type": "record",
 	"name": "TextFile",
 	"fields": [
     	{"name": "filename", "type": "string"},
     	{"name": "contents",  "type": "string"},
 	]
}
 """


parser = argparse.ArgumentParser(description='Flatten directory of text files into an AVRO file')
parser.add_argument('input_path', type = str, required = True, help='Path to source files')

parser.add_argument('--output', type = str, default = 'output.avro', help='AVRO file')

def collect_filenames(path):
	wildcard_paths = path.split(',')
	filenames = []
	for wildcard_path in wildcard_paths:
		if isdir(wildcard_path):
			wildcard_path = join(wildcard_path, "*")
		filenames.extend(glob(wildcard_path))
	return filenames 

if __name __ == '__main__':
	args = parser.parse_args()
	with open(args.output, 'w') as output_file:
		writer = DataFileWriter(output_file, DatumWriter(), SCHEMA)
		for i, filename in enumerate(collect_filenames(args.input_path)):
			print "Processing #%d %s" % (i, filename)
			with open(filename, 'r') as input_filename:
				contents = input_filename.read()
			writer.append({"filename":filename, "contents": contents})
		writer.close()
<commit_msg>decode UTF8 and then handle escaped characters<commit_after>
from os import listdir
from os.path import isdir, join, expanduser
import cgi
from glob import glob 
import argparse

import avro.schema
from avro.datafile import DataFileReader, DataFileWriter
from avro.io import DatumReader, DatumWriter


parser = argparse.ArgumentParser(description='Flatten directory of text files into an AVRO file')
parser.add_argument('input_path', type = str, help='Path to source files')

parser.add_argument('--output', type = str, default = 'output.avro', help='AVRO file')


SCHEMA_TEXT = \
"""
{
 	"type": "record",
 	"name": "TextFile",
 	"fields": [
     	{"name": "filename", "type": "string"},
     	{"name": "contents",  "type": "string"}
 	]
}
 """

SCHEMA = avro.schema.parse(SCHEMA_TEXT) 


def collect_filenames(path):
	wildcard_paths = path.split(',')
	filenames = []
	for wildcard_path in wildcard_paths:
		wildcard_path = expanduser(wildcard_path)
		if isdir(wildcard_path):
			curr_filenames = [join(wildcard_path, filename) for filename in listdir(wildcard_path)]
		else:
			curr_filenames = glob(wildcard_path)
		# drop hidden files 
		curr_filenames = [filename for filename in curr_filenames if not filename.startswith('.')]
		filenames.extend(curr_filenames)
	return filenames 

if __name__ == '__main__':
	args = parser.parse_args()
	with open(args.output, 'w') as output_file:
		writer = DataFileWriter(output_file, DatumWriter(), SCHEMA)
		for i, filename in enumerate(collect_filenames(args.input_path)):
			print "Processing #%d %s" % (i, filename)
			with open(filename, 'r') as input_filename:
				raw = input_filename.read()
				utf8 = raw.decode('utf8')
				ascii = utf8.encode('ascii', 'xmlcharrefreplace')
				escaped = cgi.escape(ascii)
				
			writer.append({"filename":filename, "contents": escaped})
		writer.close()

{'fraction_non_alphanumeric': 0.1339146683276238, 'fraction_numerical': 0.0012457178449081284, 'mean_word_length': 5.003738317757009, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '281242', 'n_tokens_mistral': 1151, 'n_tokens_neox': 1063, 'n_words': 273}