<jupyter_start><jupyter_text>Data preparation for AI training by NaturalisThis notebook executes all steps required to train a species recognition model based on various data sources, predominantly Artsobservasjoner.<jupyter_code>from process_AO import process_AO
import os

process_AO(os.path.join("./Input", "Artsobservasjoner.csv"), "./Output")
from process_GBIF import process_GBIF
import os

process_GBIF(os.path.join("./Input", "GBIF.zip"), "./Output")
from process_ML import process_ML
import os

process_ML(
    [
        os.path.join("./Input", "ML_snegl.csv"),
        os.path.join("./Input", "ML_fugl.csv"),
        os.path.join("./Input", "ML_lepi.csv"),
        os.path.join("./Input", "ML_meitemark.csv"),
        os.path.join("./Input", "ML_fremmed.csv"),

    ],
    "./Output")
from process_SUPP import process_supp
import os

process_supp(
    [
        os.path.join("./Input", "lichens_bold.csv"),
    ],
    "./Output",
    "Lichens"
    )
from process_SUPP import process_supp
import os

process_supp(
    [
        os.path.join("./Input", "Fiskebilder.csv"),
    ],
    "./Output",
    "Fish",
    checkfolder="/path/to/folder/with/img/files"
    )
from process_SUPP import process_supp
import os

process_supp(
    [
        os.path.join("./Input", "Slugs.csv"),
    ],
    "./Output",
    "Slugs"
    )
from combine import combine
import os

combine(
    [
        os.path.join("./Output", "AO_taxa.csv"),
        os.path.join("./Output", "GBIF_taxa.csv"),
        os.path.join("./Output", "ML_taxa.csv"),
        os.path.join("./Output", "Lichens_taxa.csv"),
        os.path.join("./Output", "Fish_taxa.csv"),
        os.path.join("./Output", "Slugs_taxa.csv"),
    ],
    [
        os.path.join("./Output", "AO_images.csv"),
        os.path.join("./Output", "GBIF_images.csv"),
        os.path.join("./Output", "ML_images.csv"),
        os.path.join("./Output", "Lichens_images.csv"),
        os.path.join("./Output", "Fish_images.csv"),
        os.path.join("./Output", "Slugs_images.csv"),
    ],
    outputfolder="./Output",
    previousImageList=".Input/previous_images.csv"
    )



from extensions import filter_extensions

filter_extensions(os.path.join("./Output", "images.csv"), ["png", "jpeg", "jpg"])
import pandas as pd
from tqdm import tqdm

tqdm.pandas()

# There are some taxa that have multiple "valid" entries. Replace those with the correct ones

df = pd.read_csv(os.path.join("./Output", "images.csv"))
df["accepted_taxon_id_at_source"] = df["accepted_taxon_id_at_source"].progress_apply(lambda x: "NBIC:100959" if x == "NBIC:217764" else ("NBIC:100392" if x == "NBIC:162803" else x))
df["taxon_id_at_source"] = df["accepted_taxon_id_at_source"]
print(f"{len(df)} images")
df.to_csv(os.path.join("./Output", "images.csv"), index=False)

df = pd.read_csv(os.path.join("./Output", "taxa.csv"))
df = df[(df["accepted_taxon_id_at_source"] != "NBIC:217764") & (df["accepted_taxon_id_at_source"] != "NBIC:162803")]
print(f"{len(df)} taxa")
df.to_csv(os.path.join("./Output", "taxa.csv"), index=False)<jupyter_output>/tmp/ipykernel_9296/145595103.py:8: DtypeWarning: Columns (8,10) have mixed types. Specify dtype option on import or set low_memory=False.
  df = pd.read_csv(os.path.join("./Output", "images.csv"))
100%|██████████| 1702829/1702829 [00:01<00:00, 1562923.91it/s]
{'fraction_non_alphanumeric': 0.1852187028657617, 'fraction_numerical': 0.026244343891402715, 'mean_word_length': 3.6770098730606486, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '363622', 'license': 'MIT', 'n_tokens_mistral': 1288, 'n_tokens_neox': 1210, 'n_words': 218, 'path': 'workflow.ipynb', 'repo_name': 'WouterKoch/Naturalis_data_preparation', 'size': '3315'}