<commit_before>#!/bin/bash

# Usage (example):
#  $ export PROPINQUITY_OUT_DIR=~/example2
#  $ export OTT_DIR=$(pwd)/examples/2/taxonomy/
#  $ make clean && ./build-from-newicks.sh examples/2/taxonomy/three-phylo-rankings.txt
#  $ make -j2 all extra
#
#  -- Continuing with this does not work: --
#  $ make clean
#  $ make -j2 all extra
#
#  -- It doesn't work because `make clean` removes the fake files.
#     If we make a fake phylesystem and collections, then it could work.
#
# * Right now we hardcode the cleaning flags to "major_rank_conflict,major_rank_conflict_inherited,environmental,unclassified_inherited,unclassified,viral,barren,not_otu,incertae_sedis,incertae_sedis_inherited,extinct_inherited,extinct,hidden,unplaced,unplaced_inherited,was_container,inconsistent,hybrid,merged"
#   + We could perhaps put a cleaning_flags file in the example directories...
#   + Should we then just have a single phylo-rankings.txt file per directory?

phyloranking="${1}"

if ! test -z "$PROPINQUITY_OUT_DIR" && [ "${PROPINQUITY_OUT_DIR:0:1}" != "/" ] ; then
    echo -e "build-from-newicks.sh: Error! PROPINQUITY_OUT_DIR=${PROPINQUITY_OUT_DIR} is a relative path!"
    exit 1
fi

PROPINQUITY_OUT_DIR=${PROPINQUITY_OUT_DIR:-.}
<commit_msg>mkdir for PROPINQUITY_OUT_DIR if it does not exist<commit_after>#!/bin/bash

# Usage (example):
#  $ export PROPINQUITY_OUT_DIR=~/example2
#  $ export OTT_DIR=$(pwd)/examples/2/taxonomy/
#  $ make clean && ./build-from-newicks.sh examples/2/taxonomy/three-phylo-rankings.txt
#  $ make -j2 all extra
#
#  -- Continuing with this does not work: --
#  $ make clean
#  $ make -j2 all extra
#
#  -- It doesn't work because `make clean` removes the fake files.
#     If we make a fake phylesystem and collections, then it could work.
#
# * Right now we hardcode the cleaning flags to "major_rank_conflict,major_rank_conflict_inherited,environmental,unclassified_inherited,unclassified,viral,barren,not_otu,incertae_sedis,incertae_sedis_inherited,extinct_inherited,extinct,hidden,unplaced,unplaced_inherited,was_container,inconsistent,hybrid,merged"
#   + We could perhaps put a cleaning_flags file in the example directories...
#   + Should we then just have a single phylo-rankings.txt file per directory?

phyloranking="${1}"

if ! test -z "$PROPINQUITY_OUT_DIR" && [ "${PROPINQUITY_OUT_DIR:0:1}" != "/" ] ; then
    echo -e "build-from-newicks.sh: Error! PROPINQUITY_OUT_DIR=${PROPINQUITY_OUT_DIR} is a relative path!"
    exit 1
fi

PROPINQUITY_OUT_DIR=${PROPINQUITY_OUT_DIR:-.}

if ! test -d "${PROPINQUITY_OUT_DIR}"
then
    mkdir "${PROPINQUITY_OUT_DIR}" || exit
fi

{'fraction_non_alphanumeric': 0.11891062523973916, 'fraction_numerical': 0.006904487917146145, 'mean_word_length': 4.995402298850575, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '195991', 'n_tokens_mistral': 945, 'n_tokens_neox': 904, 'n_words': 216}