<commit_before>#! /usr/bin/env python
import argparse as ap
import glob
from astropy.table import Table
from astropy.table import Column
from astropy.io import ascii
import numpy as np
import os
import time


parser = ap.ArgumentParser()
parser.add_argument('-i','--filein', nargs=1, help = 'Directory of outputs of try_astride.py')
parser.add_argument('-a', '--alldir', nargs=1, help = 'All sub directories in given path')
parser.add_argument('-l', '--latest', nargs=1, help = 'looks at latest directory in given path')
args = vars(parser.parse_args())

all = False

table_set = []

if args['filein'] != None: 
    file_path = (args['filein'][0])
    table_set.append(time.gmtime(os.path.getctime(file_path)).tm_year)


if args['alldir'] != None:
    file_path = (args['alldir'][0])
    all = True

if args['latest'] != None:
    list = os.listdir(args['latest'][0])
    for i in range(len(list)):
        list[i] = args['latest'][0] + list[i]
    file_path = max(list, key=os.path.getctime)


try:
    t = ascii.read("table1.html")
except:
    arr = np.arange(3).reshape(1,3)
    t = Table(arr, names=('file', 'reg', 'diff'),dtype=('S32', 'S2', 'S2'))
    t.remove_row(0)


if all:
    list = os.listdir(file_path)
    list.sort()

    for file in list:
        table_set.append(time.gmtime(os.path.getctime(file_path)).tm_year)

    table_set = list(set(table_set))

    for file in list:
        row = [file]
        summary = open(file_path+file+"/summary.txt", 'r')
        for line in summary:
            line = line.strip()
            if line.startswith("Streaks detected:"):
                row.append(line[18:])
        while len(row) < 3:
            row.append('0')
        t.add_row(row)        
else:

    summary = open(file_path+"/summary.txt", 'r')
    row = [file_path]
    for line in summary:
            line = line.strip()
            if line.startswith("Streaks detected:"):
                row.append(line[18:])
    while len(row) < 3:
        row.append('0')
    t.add_row(row)


t.write("table1.html", format="html", overwrite = True)

print(t)

# if args['filein'] != None: 
#     file_path = (args['filein'][0])

# if args['alldir'] != None:
#     file_path = (args['alldir'][0])
#     all = True

# if args['latest'] != None:
#     list = os.listdir(args['latest'][0])
#     for i in range(len(list)):
#         list[i] = args['latest'][0] + list[i]
#     file_path = max(list, key=os.path.getctime)

# try:
#     t = ascii.read("table1.html")
# except:
#     arr = np.arange(3).reshape(1,3)
#     t = Table(arr, names=('file', 'reg', 'diff'),dtype=('S32', 'S2', 'S2'))
#     t.remove_row(0)


# if all:
#     list = os.listdir(file_path)
#     list.sort()

#     for file in list:
#         row = [file]
#         summary = open(file_path+file+"/summary.txt", 'r')
#         for line in summary:
#             line = line.strip()
#             if line.startswith("Streaks detected:"):
#                 row.append(line[18:])
#         while len(row) < 3:
#             row.append('0')
#         t.add_row(row)        
# else:

#     summary = open(file_path+"/summary.txt", 'r')
#     row = [file_path]
#     for line in summary:
#             line = line.strip()
#             if line.startswith("Streaks detected:"):
#                 row.append(line[18:])
#     while len(row) < 3:
#         row.append('0')
#     t.add_row(row)


# t.write("table1.html", format="html", overwrite = True)

# print(t)
<commit_msg>will now write data to different tables depending on the year of the files creation<commit_after>#! /usr/bin/env python
import argparse as ap
import glob
from astropy.table import Table
from astropy.table import Column
from astropy.io import ascii
import numpy as np
import os
import time

def listdir_fullpath(d):
    return [os.path.join(d, f) for f in os.listdir(d)]

def get_row(file_path):
    summary = open(file_path+"/summary.txt", 'r')
    row = [file_path]
    for line in summary:
            line = line.strip()
            if line.startswith("Streaks detected:"):
                row.append(line[18:])
    while len(row) < 3:
        row.append('0')

    return row

parser = ap.ArgumentParser()
parser.add_argument('-i','--filein', nargs=1, help = 'Directory of outputs of try_astride.py')
parser.add_argument('-a', '--alldir', nargs=1, help = 'All sub directories in given path')
parser.add_argument('-l', '--latest', nargs=1, help = 'looks at latest directory in given path')
args = vars(parser.parse_args())

time_set = []
table_set = []
file_paths = []

if args['filein'] != None: 
    file_paths = [(args['filein'][0])]
    time_set.append(time.gmtime(os.path.getctime(file_paths[0])).tm_year)

if args['alldir'] != None:
    file_paths = listdir_fullpath(args['alldir'][0])
    file_paths.sort()
    for file in file_paths:
        time_set.append(time.gmtime(os.path.getctime(file)).tm_year)
    time_set = list(set(time_set))
    time_set.sort()

if args['latest'] != None:
    file_paths = listdir_fullpath(args['latest'][0])
    file_paths = [max(file_paths, key=os.path.getctime)]
    time_set.append(time.gmtime(os.path.getctime(file_paths[0])).tm_year)



min_year = time_set[0]

for counter, year in enumerate(time_set):
    try:
        table_set.append(ascii.read("table"+year+".html"))
    except:
        arr = np.arange(3).reshape(1,3)
        table_set.append(Table(arr, names=('file', 'reg', 'diff'),dtype=('S32', 'S2', 'S2')))
        table_set[counter].remove_row(0)

for file in file_paths:
    year = time.gmtime(os.path.getctime(file)).tm_year
    table_set[year-min_year].add_row(get_row(file))

for counter, table in enumerate(table_set):
    table.write("table"+str(time_set[counter])+".html", format="html", overwrite = True)





{'fraction_non_alphanumeric': 0.16899685424676686, 'fraction_numerical': 0.01188395665851101, 'mean_word_length': 3.0588652482269505, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1846545', 'n_tokens_mistral': 2101, 'n_tokens_neox': 2059, 'n_words': 440}