<commit_before>import sys
import csv
import os
import re

from itertools import combinations
from optparse import OptionParser

""" return a regular expression representing the encoding scheme """

def read_encoding ( filename ) :
	f= open(filename, 'r')
	scheme= f.read()
	scheme= re.sub('[\s\t\n\r]','',scheme)
	f.close()

	regex= ""
	nextpos= idx= rnd= 0
	for m in re.finditer('[\{\(]\d+[\}\)]', scheme):
		n= re.search('\d+', m.group(0)).group(0) # string
		if m.group(0).startswith('{') :
			idx+= 1
			var= "(?P<bb%d>.{%s})" % (idx, n)
		else:
			rnd= 1
			var= "(?P<rnd>.{%s})" % n
		regex += scheme[nextpos:m.start()] + var
		nextpos= m.end()
	regex += scheme[nextpos:]
	return regex, idx, rnd


""" read a building block sequence file and return its map object """

def read_building_block_sequence ( filename ) :
	mapobj= {}
  	f= open(filename, 'r')
	linenumber= 0
	for line in f :
		linenumber += 1
		line= line.strip()
		if line.startswith("#") :
			continue
		
		col= line.split()
		
		try:
			(seq,mol,cyc) = (col[0],col[1],col[2])
		except:
			print "[Error] expecting 3 columns for each line: <sequence> <building-block-id> <cycle-number>"
			print "[Error] line %d: %s", linenumber, line
			sys.exit(1)
			
		if not cyc in mapobj :
			mapobj[cyc]= {}
			
		if seq in mapobj[cyc] :
			print "[Error] identical sequence already assigned to a building block %s for cycle %s", mapobj[cyc][seq], cyc
			print "[Error] line %d: %s", linenumber, line<commit_msg>Create decode.py<commit_after>import sys
import csv
import os
import re

from itertools import combinations
from optparse import OptionParser

""" return a regular expression representing the encoding scheme """

def read_encoding ( filename ) :
	f= open(filename, 'r')
	scheme= f.read()
	scheme= re.sub('[\s\t\n\r]','',scheme)
	f.close()

	regex= ""
	nextpos= idx= rnd= 0
	for m in re.finditer('[\{\(]\d+[\}\)]', scheme):
		n= re.search('\d+', m.group(0)).group(0) # string
		if m.group(0).startswith('{') :
			idx+= 1
			var= "(?P<bb%d>.{%s})" % (idx, n)
		else:
			rnd= 1
			var= "(?P<rnd>.{%s})" % n
		regex += scheme[nextpos:m.start()] + var
		nextpos= m.end()
	regex += scheme[nextpos:]
	return regex, idx, rnd


""" read a building block sequence file and return its map object """

def read_building_block_sequence ( filename ) :
	mapobj= {}
  	f= open(filename, 'r')
	linenumber= 0
	for line in f :
		linenumber += 1
		line= line.strip()
		if line.startswith("#") :
			continue
		
		col= line.split()
		
		try:
			(seq,mol,cyc) = (col[0],col[1],col[2])
		except:
			print "[Error] expecting 3 columns for each line: <sequence> <building-block-id> <cycle-number>"
			print "[Error] line %d: %s" % (linenumber, line)
			sys.exit(1)
			
		if not cyc in mapobj :
			mapobj[cyc]= {}
			
		if seq in mapobj[cyc] :
			print "[Error] identical sequence already assigned to a building block %s for cycle %s" % (mapobj[cyc][seq], cyc)
			print "[Error] line %d: %s" % (linenumber, line)
{'fraction_non_alphanumeric': 0.17680865449628128, 'fraction_numerical': 0.00878972278566599, 'mean_word_length': 3.6379310344827585, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2159457', 'n_tokens_mistral': 1217, 'n_tokens_neox': 1096, 'n_words': 326}