<commit_before>#!/usr/bin/python

import sys
import gzip
import re

#base=sys.argv[1]+'/'
#datatype=sys.argv[2]
infile_path=sys.argv[1]
outfile_path=sys.argv[2]
datatype=sys.argv[3]

print 'process_master.py: Processing', infile_path, 'into', outfile_path

# what to expect in the master file?
if datatype=='FP':
    times=['0','2','5','12.5','25','50']
elif datatype=='TRP':
    times=['DMSO','12.5','25','50']

n=0
#master=gzip.open(base+datatype+'_dREG_regions.bed.gz','r')
infile=gzip.open(infile_path,'r')
outfile=gzip.open(outfile_path,'w')

header='chr\tstart\tend\tdREG_id\t'+'\t'.join(times)+'\tinflation\twhen_smallest'+'\n'
outfile.write(header)
for line in infile:
    time_pattern=[]
    n=n+1
    chro=line.split()[0]
    uniq_ID='dREG_'+datatype+'_'+str(n)
    uniq_start=line.split()[1]
    uniq_end=line.split()[2]
    # gonna look for the smallest region which contributed to the final region
    smallest = int(uniq_end)-int(uniq_start)
    when_smallest=re.sub('_\d+[|]+[chr]*[1-9MXY]*[1-9]*','',line.split()[3])
#    when_smallest=re.sub('[|]+[chr]+[1-9MXY][1-9]*','',line.split()[3])
    for time in times:
        if datatype+'_'+time+'_' in line:
            # yep, this region appeared at this timepoint... mark it down!
            time_pattern.append('1')
            # split the line into timepoint-data (by |), ask which one has the timepoint we want, then isolate that segment
            where=[s.find(datatype+'_'+time+'_')>0 for s in line.split('|')].index(True)
            segment=line.split('|')[where]
            start=segment.split()[1]
            # expand the boundaries
            if int(start)<int(uniq_start):
                uniq_start = start
            end=segment.split()[2]
            if int(end)>int(uniq_end):
                uniq_end = end
            size=int(end)-int(start)
            if size<smallest:
                smallest = size
                when_smallest=re.sub('_\d$','',segment.split()[3])<commit_msg>now we have preprocess_master, so things are slightly different<commit_after>#!/usr/bin/python

import sys
import gzip
import re

#base=sys.argv[1]+'/'
#datatype=sys.argv[2]
infile_path=sys.argv[1]
outfile_path=sys.argv[2]
datatype=sys.argv[3]

print 'process_master.py: Processing', infile_path, 'into', outfile_path

# what to expect in the master file?
if datatype=='FP':
    times=['0','2','5','12.5','25','50']
elif datatype=='TRP':
    times=['DMSO','12.5','25','50']

n=0
#master=gzip.open(base+datatype+'_dREG_regions.bed.gz','r')
infile=gzip.open(infile_path,'r')
outfile=gzip.open(outfile_path,'w')

header='chr\tstart\tend\tdREG_id\t'+'\t'.join(times)+'\tinflation\twhen_smallest'+'\n'
outfile.write(header)
for line in infile:
    time_pattern=[]
    n=n+1
    chro=line.split()[0]
    uniq_ID='dREG_'+datatype+'_'+str(n)
    uniq_start=line.split()[1]
    uniq_end=line.split()[2]
    # gonna look for the smallest region which contributed to the final region
    # start with this claim...
    smallest = int(uniq_end)-int(uniq_start)
    when_smallest=re.sub('_\d+[|]+[chr]*[1-9MXY]*[1-9]*','',line.split()[3])
    for time in times:
        if datatype+'_'+time+'_' in line:
            # yep, this region appeared at this timepoint... mark it down!
            time_pattern.append('1')
            # split the line into timepoint-data (by |), ask which one has the timepoint we want, then isolate that segment
            where=[s.find(datatype+'_'+time+'_')>0 for s in line.split('|')].index(True)
            segment=line.split('|')[where]
            start=segment.split()[1]
            # expand the boundaries (this shouldn't be necessary given preprocess_master)
            if int(start)<int(uniq_start):
                uniq_start = start
            end=segment.split()[2]
            if int(end)>int(uniq_end):
                uniq_end = end
            size=int(end)-int(start)
            if size<smallest:
                smallest = size
                when_smallest=re.sub('_\d$','',segment.split()[3])
{'fraction_non_alphanumeric': 0.17473789316025962, 'fraction_numerical': 0.019720419370943584, 'mean_word_length': 3.2447033898305087, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4008611', 'n_tokens_mistral': 1453, 'n_tokens_neox': 1417, 'n_words': 268}