<commit_before>#! /bin/bash
# Use the titles from various scrapers to compile a list of shows we might want info about.
# Create a dated shell script to obtain that info

while getopts ":ls" opt; do
    case $opt in
    l)
        LONG="yes"
        ;;
   s)
        SUMMARY="yes"
        ;;
    \?)
        echo "Ignoring invalid option: -$OPTARG" >&2
        ;;
    esac
done
shift $((OPTIND - 1))

DATE_ID="-$(date +%y%m%d)"
#
SCRIPT="getInfoFrom-IMDb$DATE_ID.sh"
#
COLUMNS="IMDb-columns"
mkdir -p $COLUMNS
#
ACORN_CSV=$(ls -1t Acorn_TV_Shows-*csv | head -1)
BBOX_CSV=$(ls -1t BBox_TV_Shows-*csv | head -1)
MHZ_CSV=$(ls -1t MHz_TV_Shows-*csv | head -1)
# echo ACORN=$ACORN
# echo BBOX=$BBOX
# echo MHZ=$MHZ
#
ACORN_TITLES="$COLUMNS/Acorn-titles$DATE_ID.txt"
BBOX_TITLES="$COLUMNS/BBox-titles$DATE_ID.txt"
MHZ_TITLES="$COLUMNS/MHz-titles$DATE_ID.txt"
#
ACORN_FIRST_SCRIPT="getIMDbFrom-Acorn_first$DATE_ID.sh"
BBOX_FIRST_SCRIPT="getIMDbFrom-BBox_first$DATE_ID.sh"
MHZ_FIRST_SCRIPT="getIMDbFrom-MHz_first$DATE_ID.sh"
#
ACORN_SEARCH_SCRIPT="getIMDbFrom-Acorn_search$DATE_ID.sh"
BBOX_SEARCH_SCRIPT="getIMDbFrom-BBox_search$DATE_ID.sh"
MHZ_SEARCH_SCRIPT="getIMDbFrom-MHz_search$DATE_ID.sh"
#
ACORN_FIRST_FILE="$COLUMNS/Acorn-first$DATE_ID.txt"
BBOX_FIRST_FILE="$COLUMNS/BBox-first$DATE_ID.txt"<commit_msg>move temp shell scripts to COLUMNS<commit_after>#! /bin/bash
# Use the titles from various scrapers to compile a list of shows we might want info about.
# Create a dated shell script to obtain that info

while getopts ":ls" opt; do
    case $opt in
    l)
        LONG="yes"
        ;;
   s)
        SUMMARY="yes"
        ;;
    \?)
        echo "Ignoring invalid option: -$OPTARG" >&2
        ;;
    esac
done
shift $((OPTIND - 1))

DATE_ID="-$(date +%y%m%d)"
#
SCRIPT="getInfoFrom-IMDb$DATE_ID.sh"
#
COLUMNS="IMDb-columns"
mkdir -p $COLUMNS
#
ACORN_CSV=$(ls -1t Acorn_TV_Shows-*csv | head -1)
BBOX_CSV=$(ls -1t BBox_TV_Shows-*csv | head -1)
MHZ_CSV=$(ls -1t MHz_TV_Shows-*csv | head -1)
# echo ACORN=$ACORN
# echo BBOX=$BBOX
# echo MHZ=$MHZ
#
ACORN_TITLES="$COLUMNS/Acorn-titles$DATE_ID.txt"
BBOX_TITLES="$COLUMNS/BBox-titles$DATE_ID.txt"
MHZ_TITLES="$COLUMNS/MHz-titles$DATE_ID.txt"
#
ACORN_FIRST_SCRIPT="$COLUMNS/getIMDbFrom-Acorn_first$DATE_ID.sh"
BBOX_FIRST_SCRIPT="$COLUMNS/getIMDbFrom-BBox_first$DATE_ID.sh"
MHZ_FIRST_SCRIPT="$COLUMNS/getIMDbFrom-MHz_first$DATE_ID.sh"
#
ACORN_SEARCH_SCRIPT="$COLUMNS/getIMDbFrom-Acorn_search$DATE_ID.sh"
BBOX_SEARCH_SCRIPT="$COLUMNS/getIMDbFrom-BBox_search$DATE_ID.sh"
MHZ_SEARCH_SCRIPT="$COLUMNS/getIMDbFrom-MHz_search$DATE_ID.sh"
#
ACORN_FIRST_FILE="$COLUMNS/Acorn-first$DATE_ID.txt"
BBOX_FIRST_FILE="$COLUMNS/BBox-first$DATE_ID.txt"
{'fraction_non_alphanumeric': 0.1440836132885405, 'fraction_numerical': 0.0059723777528928705, 'mean_word_length': 4.800865800865801, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5730417', 'n_tokens_mistral': 1234, 'n_tokens_neox': 1151, 'n_words': 184}