<commit_before>#!/usr/bin/env python3
# Given two directories of synthesis outputs, compares them
# Assumes that docs have been generated for each run using `make html`
# Has options for quick checks, as well as formatted output for use in
#  release notes

import json
import argparse
import glob,os,re,csv
import requests
import peyotl.ott as ott
import subprocess
from collections import defaultdict


# From reference-taxonomy:org/opentreeoflife/taxa/Rank.java

_ranks = ["domain",
         "superkingdom",
         "kingdom",
         "subkingdom",
         "division",            # h2007
         "infrakingdom",        # worms
         "superphylum",
         "phylum",
         "subphylum",
         "infraphylum",         # worms
         "subdivision",         # worms
         "superclass",
         "class",
         "subclass",
         "infraclass",
         "subterclass",         # worms Colobognatha
         "cohort",              # NCBI Polyneoptera
         "superorder",
         "order",
         "suborder",
         "infraorder",
         "parvorder",
         "section",             # worms
         "subsection",          # worms
         "superfamily",
         "family",
         "subfamily",
         "supertribe",          # worms
         "tribe",
         "subtribe",
         "genus",
         "subgenus",
         "species group",
         "species subgroup",
         "species",
         "infraspecificname",
         "subspecies",
         "natio",               # worms
         "variety",
         "varietas",
         "subvariety",
         "form",                # 2016 GBIF
         "forma",
         "subform",
         "cluster",
          "no rank - terminal"
        ];

def rank_2_num_dict(ranks):
    i=0
    rank_dict = {}
    for rank in _ranks:
        rank_dict[rank] = i<commit_msg>Add rank "no rank" to avoid key errors.<commit_after>#!/usr/bin/env python3
# Given two directories of synthesis outputs, compares them
# Assumes that docs have been generated for each run using `make html`
# Has options for quick checks, as well as formatted output for use in
#  release notes

import json
import argparse
import glob,os,re,csv
import requests
import peyotl.ott as ott
import subprocess
from collections import defaultdict


# From reference-taxonomy:org/opentreeoflife/taxa/Rank.java

_ranks = ["domain",
         "superkingdom",
         "kingdom",
         "subkingdom",
         "division",            # h2007
         "infrakingdom",        # worms
         "superphylum",
         "phylum",
         "subphylum",
         "infraphylum",         # worms
         "subdivision",         # worms
         "superclass",
         "class",
         "subclass",
         "infraclass",
         "subterclass",         # worms Colobognatha
         "cohort",              # NCBI Polyneoptera
         "superorder",
         "order",
         "suborder",
         "infraorder",
         "parvorder",
         "section",             # worms
         "subsection",          # worms
         "superfamily",
         "family",
         "subfamily",
         "supertribe",          # worms
         "tribe",
         "subtribe",
         "genus",
         "subgenus",
         "species group",
         "species subgroup",
         "species",
         "infraspecificname",
         "subspecies",
         "natio",               # worms
         "variety",
         "varietas",
         "subvariety",
         "form",                # 2016 GBIF
         "forma",
         "subform",
         "cluster",
          "no rank - terminal",
          "no rank"
        ];

def rank_2_num_dict(ranks):
    i=0
    rank_dict = {}
    for rank in _ranks:
        rank_dict[rank] = i
{'fraction_non_alphanumeric': 0.10444145356662181, 'fraction_numerical': 0.0059219380888290716, 'mean_word_length': 1.5539518900343643, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5194243', 'n_tokens_mistral': 1118, 'n_tokens_neox': 1037, 'n_words': 272}