<commit_before>import sys, os, datetime, configparser
import numpy as np

def maptext(char):
	if char.isdigit(): return 'NUM'
	elif char.isalpha(): return char.lower()
	return char

def count(filename, vocab = dict()):
	for line in open(filename):
		for word in map(maptext, line.split()):
			if word in vocab: vocab[word] += 1
			else: vocab[word] = 1
	return vocab

def writetofile(filenamein, filenameout, vocab):
	with open(filenameout, 'w') as fileout:
		for line in open(filenamein):
			for word in map(maptext, line.split()):
				fileout.write(str(vocab.index(word) + 1) + ' ')
			fileout.write('\n')

def readindexfile(filename):
	maximum, vocabulary, reverse = 0, dict(), dict()
	for line in open(filename):
		index, token = line.split('\t')
		vocabulary[token.strip()] = int(index)
		reverse[int(index)] = token.strip()
		maximum = max(maximum, index)
	return vocabulary, reverse, maximum

def readalignfile(file1, file2, lengths):
	alignments1, alignments2 = list(), list()
	alignments = list()
	for line1, line2, length in zip(open(file1), open(file2), lengths):
		alignment1, alignment2 = [list() for _ in xrange(length)], [list() for _ in xrange(length)]
		for align in line1.strip().split(' '):
			left, right = align.split('-')
			alignment1[int(right)].append(left)
		for align in line2.strip().split(' '):
			left, right = align.split('-')
			alignment2[int(right)].append(left)
		alignments.append(';'.join([' '.join(align1) + ',' + ' '.join(align2) for align1, align2 in zip(alignment1, alignment2)]))
	return alignments

def readword2vec(filename, vocabulary = dict()):<commit_msg>adding 0 lookup to vocabulary<commit_after>import sys, os, datetime, configparser
import numpy as np

def maptext(char):
	if char.isdigit(): return 'NUM'
	elif char.isalpha(): return char.lower()
	return char

def count(filename, vocab = dict()):
	for line in open(filename):
		for word in map(maptext, line.split()):
			if word in vocab: vocab[word] += 1
			else: vocab[word] = 1
	return vocab

def writetofile(filenamein, filenameout, vocab):
	with open(filenameout, 'w') as fileout:
		for line in open(filenamein):
			for word in map(maptext, line.split()):
				fileout.write(str(vocab.index(word) + 1) + ' ')
			fileout.write('\n')

def readindexfile(filename):
	maximum, vocabulary, reverse = 0, {'BLANK': 0}, {0: 'BLANK'}
	for line in open(filename):
		index, token = line.split('\t')
		vocabulary[token.strip()] = int(index)
		reverse[int(index)] = token.strip()
		maximum = max(maximum, index)
	return vocabulary, reverse, maximum

def readalignfile(file1, file2, lengths):
	alignments1, alignments2 = list(), list()
	alignments = list()
	for line1, line2, length in zip(open(file1), open(file2), lengths):
		alignment1, alignment2 = [list() for _ in xrange(length)], [list() for _ in xrange(length)]
		for align in line1.strip().split(' '):
			left, right = align.split('-')
			alignment1[int(right)].append(left)
		for align in line2.strip().split(' '):
			left, right = align.split('-')
			alignment2[int(right)].append(left)
		alignments.append(';'.join([' '.join(align1) + ',' + ' '.join(align2) for align1, align2 in zip(alignment1, alignment2)]))
	return alignments

def readword2vec(filename, vocabulary = dict()):
{'fraction_non_alphanumeric': 0.16150990099009901, 'fraction_numerical': 0.01639851485148515, 'mean_word_length': 4.910420475319927, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1278323', 'n_tokens_mistral': 1189, 'n_tokens_neox': 1101, 'n_words': 307}