<commit_before>"""
    Adds rejected steps in report

    To be run in simulation root folder

    Needs refactoring
"""

import numpy as np
import os, glob

"""
to be changed by user
"""
reportFilename = "report_"
reportOutputFilename = "allReport_"
stepsPerEpoch = 4
"""
"""

allFolders = os.listdir(".")
epochFolders = [epoch for epoch in allFolders if epoch.isdigit()]

for folder in epochFolders:
    print "Writing reports in folder", folder
    os.chdir(folder)

    reports = glob.glob(reportFilename + "*")
    for report in reports:
        f = open(report, "r")
        newReportContent = []
        reportContent = f.readlines()
        f.close()

        newReportContent.append(reportContent[0])
        count = 0
        for i in range(1, len(reportContent)-1): #the last one will be checked against steps per epoch
            splitContent = reportContent[i].split()
            thisAccStep = int(splitContent[1])
            nextAccStep = int(reportContent[i+1].split()[1])
            for j in range(nextAccStep-thisAccStep):
                splitContent[1] = str(count)
                newReportContent.append(" ".join(splitContent))
                count += 1

        #last one is always the steps per epoch (+1, since 0 is initial)
        splitContent = reportContent[-1].split()
        lastAccStep = int(splitContent[1])
        lastTotalStep = stepsPerEpoch + 1
        for i in range(lastTotalStep - lastAccStep):
            splitContent[1] = str(count)
            newReportContent.append(" ".join(splitContent))
            count += 1

        #write file
        reportNum = report.split("_")[-1]
        f = open(reportOutputFilename + reportNum, "w")
        for line in newReportContent:
            f.write(line + "\n")
        f.close()


    os.chdir("..")<commit_msg>Had problems when input file numbers where writen as floats (despite they were ints instead...)<commit_after>"""
    Adds rejected steps in report

    To be run in simulation root folder

    Needs refactoring
"""

import numpy as np
import os, glob

"""
to be changed by user
"""
reportFilename = "report_"
reportOutputFilename = "allReport_"
stepsPerEpoch = 4
"""
"""

allFolders = os.listdir(".")
epochFolders = [epoch for epoch in allFolders if epoch.isdigit()]

for folder in epochFolders:
    print "Writing reports in folder", folder
    os.chdir(folder)

    reports = glob.glob(reportFilename + "*")
    for report in reports:
        f = open(report, "r")
        newReportContent = []
        reportContent = f.readlines()
        f.close()

        newReportContent.append(reportContent[0])
        count = 0
        for i in range(1, len(reportContent)-1): #the last one will be checked against steps per epoch
            splitContent = reportContent[i].split()
            thisAccStep = int(float(splitContent[1]))
            nextAccStep = int(float(reportContent[i+1].split()[1]))
            for j in range(nextAccStep-thisAccStep):
                splitContent[1] = str(count)
                newReportContent.append(" ".join(splitContent))
                count += 1

        #last one is always the steps per epoch (+1, since 0 is initial)
        splitContent = reportContent[-1].split()
        lastAccStep = int(float(splitContent[1]))
        lastTotalStep = stepsPerEpoch + 1
        for i in range(lastTotalStep - lastAccStep):
            splitContent[1] = str(count)
            newReportContent.append(" ".join(splitContent))
            count += 1

        #write file
        reportNum = report.split("_")[-1]
        f = open(reportOutputFilename + reportNum, "w")
        for line in newReportContent:
            f.write(line + "\n")
        f.close()


    os.chdir("..")
{'fraction_non_alphanumeric': 0.1117694586587665, 'fraction_numerical': 0.009695663883652034, 'mean_word_length': 2.432532347504621, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6117008', 'n_tokens_mistral': 1086, 'n_tokens_neox': 1047, 'n_words': 309}