<commit_before>import numpy as np
import argparse
import glob

parser = argparse.ArgumentParser(description='Sort the spectra.')
parser.add_argument('folder', type=str, help='Folder with .lopa files to be processed')

args = parser.parse_args()
#set the necessary parameters for parsing

def sort_array(array, column, removeHeader):
    if removeHeader > 0:
        for i in range(0, removeHeader):
            array = np.delete(array, (0), axis=0)
    array = array[np.argsort(array[:,column])]
    return array
  

def deltaLambda(segment):
    """Calculates the deltaLambda for the given segment.

    Calculates the difference in wawelength in two adjancent data points in the segment.

    Parameters
    ----------
    segment : list
        list containing the current segment

    Returns
    -------
    list
        List with deltaLambdas as the third column.

    """
    for position in range(len(segment)-1):
        segment[position].append(segment[position][0] - segment[position+1][0])
    #segment[-1].append(segment[-2][-1])
    return segment

def deleteOverlapping(tempList, fileName):
    """Deletes the overlapping parts of a .lopa file.

    Each .lopa file contains parts which partially overlap with the next or the previous .lopa file, this function deletes those parts. If the file is first or last, it will only delete the last or first part respectively.

    Parameters
    ----------
    tempList : np.array()
        An array containing one segment of the .lopa file
    fileName : str
        Name of the current .lopa file.

    Returns
    -------
    np.array()
        Array without the overlapping parts.

    """
    # tempList = np.delete(tempList, (0), axis=0)
    # tempList = np.delete(tempList, (0), axis=0)
    # first_line = tempList[0]
    # tempList.pop(0)
    if fileName == 0: specialPosition = 0
    elif fileName == len(file_list)-1: specialPosition = -1
    else: specialPosition = 1
    fileName = int(file_list[fileName].split('/')[-1].split('.')[-2])
    # print fileName
    minimum = fileName - 5.
    maximum = fileName + 5.
    #print fileName, minimum, maximum

    if specialPosition == 1:
        tempTempList = [ line for line in tempList if (line[0] < maximum) and (line[0] > minimum)]
    elif specialPosition == -1:
        tempTempList = [ line for line in tempList if (line[0] < maximum)]      
    elif specialPosition == 0:
        tempTempList = [ line for line in tempList if (line[0] > minimum)]

    # tempTempList.insert(0, first_line)
    return tempTempList

file_list = []
#file_list = sorted(glob.glob('/scratch/cernetic/testRun/'+str(args.folder)+'/*.lopa'), reverse=True)
file_list = sorted(glob.glob(str(args.folder)+'/*.lopa'), reverse=True)
file_list=list(file_list[:3])
numberOfItems = len(file_list)
#create a list off all the .lopa files

firstFile = np.loadtxt(file_list[int(len(file_list)/2)])
depth=0
for array in firstFile:
    if ((array[0] % 1) == 0) and ((array[1] % 1) == 0) and (int(array[1]) > 0): #check whether we are in the header line
        depth += 1
print("Number of depthpoints: " + str(depth))
depthList = range(1,depth + 1)

depthLength = len(str(np.max(depthList)))
#get the list of all depth points from the first .lopa file
#depthLength is created as a global variable so all functions can access it


"""Load the files. Each item in data contains one lopa file"""
data = []
segments = []

for item in depthList: segments.insert(0, [])

print "Loading files"
for item in file_list:
    print "File: ", item
    # data.append(deleteOverlapping(np.loadtxt(item).tolist(), item))
    data.append(np.loadtxt(item).tolist())
print "Files loaded"

"""Walk over the already loaded lopa file"""
file_number = 0
for item in data:
    segment = -1
    segment_temp = []
    for line in item:
        if ((line[0] % 1) == 0) and ((line[1] % 1) == 0) and (int(line[1]) > 0):
            if segment == -1:
                pass
            else:
                segment_temp = deleteOverlapping(segment_temp, file_number)
                for itemm in segment_temp: segments[segment].append(itemm)
                segment_temp = []<commit_msg>remove testing lines<commit_after>import numpy as np
import argparse
import glob

parser = argparse.ArgumentParser(description='Sort the spectra.')
parser.add_argument('folder', type=str, help='Folder with .lopa files to be processed')

args = parser.parse_args()
#set the necessary parameters for parsing

def sort_array(array, column, removeHeader):
    if removeHeader > 0:
        for i in range(0, removeHeader):
            array = np.delete(array, (0), axis=0)
    array = array[np.argsort(array[:,column])]
    return array
  

def deltaLambda(segment):
    """Calculates the deltaLambda for the given segment.

    Calculates the difference in wawelength in two adjancent data points in the segment.

    Parameters
    ----------
    segment : list
        list containing the current segment

    Returns
    -------
    list
        List with deltaLambdas as the third column.

    """
    for position in range(len(segment)-1):
        segment[position].append(segment[position][0] - segment[position+1][0])
    #segment[-1].append(segment[-2][-1])
    return segment

def deleteOverlapping(tempList, fileName):
    """Deletes the overlapping parts of a .lopa file.

    Each .lopa file contains parts which partially overlap with the next or the previous .lopa file, this function deletes those parts. If the file is first or last, it will only delete the last or first part respectively.

    Parameters
    ----------
    tempList : np.array()
        An array containing one segment of the .lopa file
    fileName : str
        Name of the current .lopa file.

    Returns
    -------
    np.array()
        Array without the overlapping parts.

    """
    # tempList = np.delete(tempList, (0), axis=0)
    # tempList = np.delete(tempList, (0), axis=0)
    # first_line = tempList[0]
    # tempList.pop(0)
    if fileName == 0: specialPosition = 0
    elif fileName == len(file_list)-1: specialPosition = -1
    else: specialPosition = 1
    fileName = int(file_list[fileName].split('/')[-1].split('.')[-2])
    # print fileName
    minimum = fileName - 5.
    maximum = fileName + 5.
    #print fileName, minimum, maximum

    if specialPosition == 1:
        tempTempList = [ line for line in tempList if (line[0] < maximum) and (line[0] > minimum)]
    elif specialPosition == -1:
        tempTempList = [ line for line in tempList if (line[0] < maximum)]      
    elif specialPosition == 0:
        tempTempList = [ line for line in tempList if (line[0] > minimum)]

    # tempTempList.insert(0, first_line)
    return tempTempList

file_list = []
#file_list = sorted(glob.glob('/scratch/cernetic/testRun/'+str(args.folder)+'/*.lopa'), reverse=True)
file_list = sorted(glob.glob(str(args.folder)+'/*.lopa'), reverse=True)
#create a list off all the .lopa files

firstFile = np.loadtxt(file_list[int(len(file_list)/2)])
depth=0
for array in firstFile:
    if ((array[0] % 1) == 0) and ((array[1] % 1) == 0) and (int(array[1]) > 0): #check whether we are in the header line
        depth += 1
print("Number of depthpoints: " + str(depth))
depthList = range(1,depth + 1)

depthLength = len(str(np.max(depthList)))
#get the list of all depth points from the first .lopa file
#depthLength is created as a global variable so all functions can access it


"""Load the files. Each item in data contains one lopa file"""
data = []
segments = []

for item in depthList: segments.insert(0, [])

print "Loading files"
for item in file_list:
    print "File: ", item
    data.append(np.loadtxt(item).tolist())
print "Files loaded"

"""Walk over the already loaded lopa file"""
file_number = 0
for item in data:
    segment = -1
    segment_temp = []
    for line in item:
        if ((line[0] % 1) == 0) and ((line[1] % 1) == 0) and (int(line[1]) > 0):
            if segment == -1:
                pass
            else:
                segment_temp = deleteOverlapping(segment_temp, file_number)
                for itemm in segment_temp: segments[segment].append(itemm)
                segment_temp = []
{'fraction_non_alphanumeric': 0.1245708680725846, 'fraction_numerical': 0.01459048553212359, 'mean_word_length': 2.9926578560939796, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4839566', 'n_tokens_mistral': 2533, 'n_tokens_neox': 2450, 'n_words': 823}