<commit_before>
import os
import re
import json

# ===================================================================

def createSpeciesJson(source_path, output_file):
    # create the species.json file using data from the specified path

    # here's a regex to test for species dirs:

    #                group(8) - Species --------------------------------------------.
    #                           (a literal underscore) --------------------------.  |
    #                group(7) - Genus ----------------------------------------.  |  |
    #                group(6) - Genus, again ---------------------------.     |  |  |
    #                group(5) - Family ---------------------------.     |     |  |  |
    #                group(4) - Order ----------------------.     |     |     |  |  |
    #                group(3) - Class ----------------.     |     |     |     |  |  |
    #                group(2) - Phylum ---------.     |     |     |     |     |  |  |
    #                group(1) - Kingdom --.     |     |     |     |     |     |  |  |
    #                                     V     V     V     V     V     V     V  V  V
    sppdir_regex = re.compile(r'species/(\w+)/(\w+)/(\w+)/(\w+)/(\w+)/(\w+)/(\w+)_(\w+)/summaries_temperature$')

    # Any dir that matches the regex (starting from source_path) is a dir
    # that contains species level data. For example:
    #
    #  Species -----------------------------------------------------------.
    #  Genus ----------------------------------------------------.        |
    #  Genus, again --------------------------------------.      |        |
    #  Family -------------------------------------.      |      |        |
    #  Order ------------------------------.       |      |      |        |
    #  Class ----------------------.       |       |      |      |        |
    #  Phylum ------------.        |       |       |      |      |        |
    #  Kingdom --.        |        |       |       |      |      |        |
    #            V        V        V       V       V      V      V        V
    # species/Animalia/Chordata/Amphibia/Anura/Alytidae/Alytes/Alytes_cisternasii/summaries_temperature/


    # ---- get common names -----------------------------------------

    # we'll get the species common name from here:
    common_names = {}

    cn_file = os.path.join(os.path.dirname(__file__), 'commonnames.json')
    try:
        # try reading in the list of sci-to-common species names<commit_msg>allow dashes in species names<commit_after>
import os
import re
import json

# ===================================================================

def createSpeciesJson(source_path, output_file):
    # create the species.json file using data from the specified path

    # here's a regex to test for species dirs:

    #                group(8) - Species ---------------------------------------------.
    #                           (a literal underscore) --------------------------.   |
    #                group(7) - Genus ----------------------------------------.  |   |
    #                group(6) - Genus, again ---------------------------.     |  |   |
    #                group(5) - Family ---------------------------.     |     |  |   |
    #                group(4) - Order ----------------------.     |     |     |  |   |
    #                group(3) - Class ----------------.     |     |     |     |  |   |
    #                group(2) - Phylum ---------.     |     |     |     |     |  |   |
    #                group(1) - Kingdom --.     |     |     |     |     |     |  |   |
    #                                     V     V     V     V     V     V     V  V   V
    sppdir_regex = re.compile(r'species/(\w+)/(\w+)/(\w+)/(\w+)/(\w+)/(\w+)/(\w+)_([-\w]+)/summaries_temperature$')

    # Any dir that matches the regex (starting from source_path) is a dir
    # that contains species level data. For example:
    #
    #  Species -----------------------------------------------------------.
    #  Genus ----------------------------------------------------.        |
    #  Genus, again --------------------------------------.      |        |
    #  Family -------------------------------------.      |      |        |
    #  Order ------------------------------.       |      |      |        |
    #  Class ----------------------.       |       |      |      |        |
    #  Phylum ------------.        |       |       |      |      |        |
    #  Kingdom --.        |        |       |       |      |      |        |
    #            V        V        V       V       V      V      V        V
    # species/Animalia/Chordata/Amphibia/Anura/Alytidae/Alytes/Alytes_cisternasii/summaries_temperature/


    # ---- get common names -----------------------------------------

    # we'll get the species common name from here:
    common_names = {}

    cn_file = os.path.join(os.path.dirname(__file__), 'commonnames.json')
    try:
        # try reading in the list of sci-to-common species names
{'fraction_non_alphanumeric': 0.3226192852864843, 'fraction_numerical': 0.003194250349371132, 'mean_word_length': 0.67, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6594355', 'n_tokens_mistral': 1317, 'n_tokens_neox': 1222, 'n_words': 245}