<commit_before>#!/usr/bin/python
# This will take a bedgraph and expand it into a weird sort of bed file, where we have
# CHR LOC DNASE_1 ... DNASE_N
# N is the size of the window around LOC, and DNASE_i are the DNASE values corresponding the window locations
# The output of this script shall be the input to the feature extraction part of the larger code
import gzip
import sys

# arbitrary!
WIN_SIZE = 25
WIN_EDGE = WIN_SIZE/2

def overlap(A1,A2,B1,B2):
    if A1 <= B1 and B1 <= A2:
        return True
    elif B1 <= A1 and A1 <= B2:
        return True
    else:
        return False

def in_range(x,A,B):
    if x < A or x > B:
        return False
    else:
        return True

def get_peak_range(pre_line):
    line = pre_line.strip()
    colon = line.find(':')
    hyphen = line.find('-')
    start = int(line[(colon+1):hyphen])
    end = int(line[(hyphen+1):])
    return (start,end)

def get_flanks(i,buff,WIN_EDGE):
# OUTSTANDING QUESTION: what to do when there is insufficient buffer to do this?
# Also WHY is that happening?
    flank_list = [element.split()[2] for element in buff[(i-WIN_EDGE):(i+WIN_EDGE+1)]]
#    if not len(flank_list)==(2*WIN_EDGE+1):
#        print buff[(i-WIN_EDGE):(i+WIN_EDGE+1)]
#        print WIN_EDGE
#        print i-WIN_EDGE
#        print buff[i+WIN_EDGE+1]
#        print buff[i-WIN_EDGE]
#        print len(flank_list)
#        sys.exit()
    flanks = ' '.join(flank_list)
    return flanks

def process_peak(peak_num,peak_start,peak_end,buff,outfile,new_peaks_file):
    # what we have: one line per location (good!), with CHR LOC VAL PEAK_NUM
    # peak_end is the last location in the peak, not the upper bound
    peak_length = peak_end-peak_start+1
    buff_length = len(buff)
    # what if there isn't enough buffer? NOTE: THIS MODIFIES THE PEAK CALLS, SO WE NEED TO USE THE OUTPUT OF THIS SCRIPT TO DEFINE OUR PEAKS FROM NOW ON... (the new_peaks_file)..
    buff_start = int(buff[0].split()[1])
    buff_end = int(buff[-1].split()[1])
    if (peak_start-buff_start)<WIN_EDGE:
        # move up the start of the peak
        peak_start = buff_start+WIN_EDGE
    if (buff_end-peak_end)<WIN_EDGE:
        # move in the end of the peak
        peak_end = buff_end-WIN_EDGE
    # this will be the total signal in the peak... can rewrite the dnase_peaks file to include this, i guess
    peak_total = 0
    for i in range(buff_length):
        element = buff[i]
        [chr,loc,val] = element.split()
        if in_range(int(loc),peak_start,peak_end):
            flanks = get_flanks(i,buff,WIN_EDGE)
            peak_total = peak_total + float(val)
            loc_line = chr+' '+str(loc)+' '+str(peak_num)+' '+flanks+'\n'
            outfile.write(loc_line)
    new_peaks_file.write(chr+'\t'+str(peak_start)+'\t'+str(peak_end)+'\t'+str(peak_total)+'\n') 

if len(sys.argv)<3:
    sys.exit('Requires .bedGraph and peaklist file!')

# each line of this file contains peak range
peaks_file = open(sys.argv[2],'r')<commit_msg>still getting issues<commit_after>#!/usr/bin/python
# This will take a bedgraph and expand it into a weird sort of bed file, where we have
# CHR LOC DNASE_1 ... DNASE_N
# N is the size of the window around LOC, and DNASE_i are the DNASE values corresponding the window locations
# The output of this script shall be the input to the feature extraction part of the larger code
import gzip
import sys

# arbitrary!
WIN_SIZE = 25
WIN_EDGE = WIN_SIZE/2

def overlap(A1,A2,B1,B2):
    if A1 <= B1 and B1 <= A2:
        return True
    elif B1 <= A1 and A1 <= B2:
        return True
    else:
        return False

def in_range(x,A,B):
    if x < A or x > B:
        return False
    else:
        return True

def get_peak_range(pre_line):
    line = pre_line.strip()
    colon = line.find(':')
    hyphen = line.find('-')
    start = int(line[(colon+1):hyphen])
    end = int(line[(hyphen+1):])
    return (start,end)

def get_flanks(i,buff,WIN_EDGE):
# OUTSTANDING QUESTION: what to do when there is insufficient buffer to do this?
# Also WHY is that happening?
    flank_list = [element.split()[2] for element in buff[(i-WIN_EDGE):(i+WIN_EDGE+1)]]
#    if not len(flank_list)==(2*WIN_EDGE+1):
#        print buff[(i-WIN_EDGE):(i+WIN_EDGE+1)]
#        print WIN_EDGE
#        print i-WIN_EDGE
#        print buff[i+WIN_EDGE+1]
#        print buff[i-WIN_EDGE]
#        print len(flank_list)
#        sys.exit()
    flanks = ' '.join(flank_list)
    return flanks

def process_peak(peak_num,peak_start,peak_end,buff,outfile,new_peaks_file):
    # what we have: one line per location (good!), with CHR LOC VAL PEAK_NUM
    # peak_end is the last location in the peak, not the upper bound
    peak_length = peak_end-peak_start+1
    buff_length = len(buff)
    # what if there isn't enough buffer? NOTE: THIS MODIFIES THE PEAK CALLS, SO WE NEED TO USE THE OUTPUT OF THIS SCRIPT TO DEFINE OUR PEAKS FROM NOW ON... (the new_peaks_file)..
    buff_start = int(buff[0].split()[1])
    buff_end = int(buff[-1].split()[1])
    if (peak_start-buff_start)<WIN_EDGE:
        print peak_start
        print buff_start
        sys.exit()
        # move up the start of the peak
        peak_start = buff_start+WIN_EDGE
    if (buff_end-peak_end)<WIN_EDGE:
        # move in the end of the peak
        peak_end = buff_end-WIN_EDGE
    # this will be the total signal in the peak... can rewrite the dnase_peaks file to include this, i guess
    peak_total = 0
    for i in range(buff_length):
        element = buff[i]
        [chr,loc,val] = element.split()
        if in_range(int(loc),peak_start,peak_end):
            flanks = get_flanks(i,buff,WIN_EDGE)
            peak_total = peak_total + float(val)
            loc_line = chr+' '+str(loc)+' '+str(peak_num)+' '+flanks+'\n'
            outfile.write(loc_line)
    new_peaks_file.write(chr+'\t'+str(peak_start)+'\t'+str(peak_end)+'\t'+str(peak_total)+'\n') 

if len(sys.argv)<3:
    sys.exit('Requires .bedGraph and peaklist file!')

# each line of this file contains peak range
peaks_file = open(sys.argv[2],'r')
{'fraction_non_alphanumeric': 0.11496350364963503, 'fraction_numerical': 0.010617120106171201, 'mean_word_length': 3.0654079568442345, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2084342', 'n_tokens_mistral': 2198, 'n_tokens_neox': 2114, 'n_words': 628}