<commit_before>#!/usr/bin/env python

import sys
import re

def print_row(generation, quartile, value):
    print(str(generation) + "," + quartile + "," + str(value))

with open(sys.argv[1]) as f:
    print("Generation,Quartile,Homology")
    for line in f:
        # print("<", line, ">")
        is_gen_line = re.match("Processing generation: (\d+)", line.strip())
        # print(is_gen_line)
        if is_gen_line:
            generation = is_gen_line.groups()[0]
        # First quartile:  0.93203884
        is_first_quartile = re.match("First quartile:\s+(\d+\.\d+)", line.strip())
        # print(is_first_quartile)
        if is_first_quartile:
            first_quartile = is_first_quartile.groups()[0]
            print_row(generation, "'25%'", first_quartile)
        is_median = re.match("Median:\s+(\d+\.\d+)", line.strip())
        if is_median:
            median = is_median.groups()[0]
            print_row(generation, "'50%'", median)
        is_third_quartile = re.match("Third quartile:\s+(\d+\.\d+)", line.strip())
        if is_third_quartile:
            third_quartile = is_third_quartile.groups()[0]
            print_row(generation, "'75%'", third_quartile)<commit_msg>A working version of the homology scraper.<commit_after>#!/usr/bin/env python

# Takes a directory name as a single command line argument, and scrapes the
# homology data out of all the log*.txt files in that directory, putting the
# results in the name of the directory with "_homology.csv" appended.

import sys
import glob
import re

def print_row(run, generation, quartile, value):
    print(run + "," + str(generation) + "," + quartile + "," + str(value))

directory_to_scrape = sys.argv[1]

print("Run,Generation,Quartile,Homology")
for file in glob.glob(directory_to_scrape + "/log*.txt"):
  with open(file) as f:
      run = re.match(".*log(\d+).txt", file).groups()[0]
      for line in f:
          # print("<", line, ">")
          is_gen_line = re.match("Processing generation: (\d+)", line.strip())
          # print(is_gen_line)
          if is_gen_line:
              generation = is_gen_line.groups()[0]
          # First quartile:  0.93203884
          is_first_quartile = re.match("First quartile(\s+\(sample 1\))?:\s+(\d+\.\d+)", line.strip())
          # print(is_first_quartile)
          if is_first_quartile:
              first_quartile = is_first_quartile.groups()[1]
              print_row(run, generation, "0.25", first_quartile)
          is_median = re.match("Median(\s+\(sample 1\))?:\s+(\d+\.\d+)", line.strip())
          if is_median:
              median = is_median.groups()[1]
              print_row(run, generation, "0.5", median)
          is_third_quartile = re.match("Third quartile(\s+\(sample 1\))?:\s+(\d+\.\d+)", line.strip())
          if is_third_quartile:
              third_quartile = is_third_quartile.groups()[1]
              print_row(run, generation, "0.75", third_quartile)
{'fraction_non_alphanumeric': 0.15812841530054644, 'fraction_numerical': 0.015710382513661202, 'mean_word_length': 2.774484536082474, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4986574', 'n_tokens_mistral': 1008, 'n_tokens_neox': 926, 'n_words': 201}