<commit_before>#! /usr/bin/env python
import argparse as ap
import glob
from astropy.table import Table
from astropy.table import Column
from astropy.io import ascii
import numpy as np
import os


parser = ap.ArgumentParser()
parser.add_argument('-i','--filein', nargs=1, help = 'Directory of outputs of try_astride.py')
parser.add_argument('-a', '--alldir', nargs=1, help = 'All sub directories in given path')
parser.add_argument('-l', '--latest', nargs=1, help = 'looks at latest directory in given path')
args = vars(parser.parse_args())

all = False

table_set = {}

if args['filein'] != None: 
    file_path = (args['filein'][0])
    

if args['alldir'] != None:
    file_path = (args['alldir'][0])
    all = True

if args['latest'] != None:
    list = os.listdir(args['latest'][0])
    for i in range(len(list)):
        list[i] = args['latest'][0] + list[i]
    file_path = max(list, key=os.path.getctime)


try:
    t = ascii.read("table1.html")
except:
    arr = np.arange(3).reshape(1,3)
    t = Table(arr, names=('file', 'reg', 'diff'),dtype=('S32', 'S2', 'S2'))
    t.remove_row(0)


if all:
    list = os.listdir(file_path)
    list.sort()

    for file in list:
        row = [file]
        summary = open(file_path+file+"/summary.txt", 'r')
        for line in summary:
            line = line.strip()
            if line.startswith("Streaks detected:"):
                row.append(line[18:])
        while len(row) < 3:
            row.append('0')
        t.add_row(row)        
else:

    summary = open(file_path+"/summary.txt", 'r')
    row = [file_path]
    for line in summary:
            line = line.strip()
            if line.startswith("Streaks detected:"):
                row.append(line[18:])
    while len(row) < 3:
        row.append('0')
    t.add_row(row)

<commit_msg>started updates to building multiple tables for each year<commit_after>#! /usr/bin/env python
import argparse as ap
import glob
from astropy.table import Table
from astropy.table import Column
from astropy.io import ascii
import numpy as np
import os
import time


parser = ap.ArgumentParser()
parser.add_argument('-i','--filein', nargs=1, help = 'Directory of outputs of try_astride.py')
parser.add_argument('-a', '--alldir', nargs=1, help = 'All sub directories in given path')
parser.add_argument('-l', '--latest', nargs=1, help = 'looks at latest directory in given path')
args = vars(parser.parse_args())

all = False

table_set = []

if args['filein'] != None: 
    file_path = (args['filein'][0])
    table_set.append(time.gmtime(os.path.getctime(file_path)).tm_year)


if args['alldir'] != None:
    file_path = (args['alldir'][0])
    all = True

if args['latest'] != None:
    list = os.listdir(args['latest'][0])
    for i in range(len(list)):
        list[i] = args['latest'][0] + list[i]
    file_path = max(list, key=os.path.getctime)


try:
    t = ascii.read("table1.html")
except:
    arr = np.arange(3).reshape(1,3)
    t = Table(arr, names=('file', 'reg', 'diff'),dtype=('S32', 'S2', 'S2'))
    t.remove_row(0)


if all:
    list = os.listdir(file_path)
    list.sort()

    for file in list:
        table_set.append(time.gmtime(os.path.getctime(file_path)).tm_year)

    table_set = list(set(table_set))

    for file in list:
        row = [file]
        summary = open(file_path+file+"/summary.txt", 'r')
        for line in summary:
            line = line.strip()
            if line.startswith("Streaks detected:"):
                row.append(line[18:])
        while len(row) < 3:
            row.append('0')
        t.add_row(row)        
else:

    summary = open(file_path+"/summary.txt", 'r')
    row = [file_path]
    for line in summary:
            line = line.strip()
            if line.startswith("Streaks detected:"):
                row.append(line[18:])
    while len(row) < 3:
        row.append('0')
    t.add_row(row)


{'fraction_non_alphanumeric': 0.16093143596377749, 'fraction_numerical': 0.01241914618369987, 'mean_word_length': 2.8582834331337326, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3917535', 'n_tokens_mistral': 1387, 'n_tokens_neox': 1357, 'n_words': 326}