<commit_before># Verify that every show in BBoxPrograms is also in BBoxEpisodes

# INVOCATION:
#   awk -v EPISODES_SORTED_FILE=$EPISODES_SORTED_FILE -v SEASONS_SORTED_FILE=$SEASONS_SORTED_FILE \
#       -v TEMP_FILE=$TEMP_FILE -f verifyBBoxDownloadsFrom-webscraper.awk $PROGRAMS_SORTED_FILE \
#       >>$EPISODE_INFO_FILE

# Field numbers
#    1 web-scraper-start-url    2 URL    3 Program_Title    4 Sn_Years    5 Seasons    9  Description

BEGIN {
    FS="\t"
}

/\/us\// {
    programURL = $1
    nflds = split (programURL,fld,"/") 
    showType = fld[3]
    showType == "show" ? spacer = "  " : spacer = " "
    # Must have single quotes to handle special charaters in targets
    target = sprintf ("'%s'",fld[nflds])
    unquotedTarget = substr(target,2,length(target)-2)
    #
    cmd = "grep -c " target " " EPISODES_SORTED_FILE
    if ((cmd | getline NumEpisodes) > 0) {
        NumEpisodes == 1 ? epiStr = " episode" : epiStr = " episodes"
        print "==> " showType spacer target " has " NumEpisodes epiStr
    }
    close (cmd)
    if (NumEpisodes == 0) {
        badEpisodes += 1
        print "    " unquotedTarget >> TEMP_FILE
    }
    #
    cmd = "grep " target " " SEASONS_SORTED_FILE
    while ((cmd | getline seasonLine) > 0) {
        nfields = split (seasonLine, fields, "\t")
        seasonField = fields[nfields-4]
        episodeField = fields[nfields-2]
        if (seasonField == "") {
            print "==> " target " (blank Seasons field in " SEASONS_SORTED_FILE ")" \
                > "/dev/stderr"
            print "    " unquotedTarget " (blank Seasons field in " SEASONS_SORTED_FILE ")" \
                >> TEMP_FILE
            continue
        }
        print "          " target " " seasonField " has " episodeField

    }
    close (cmd)
}

END {
    if (badEpisodes > 0 ) {
        badEpisodes == 1 ? field = "URL" : field = "URLs"
        printf ("==> %2d extra /program/ %s in %s\n", badEpisodes, field, FILENAME) > "/dev/stderr"
    }
}<commit_msg>Add "in numSeasons seasons" to info about shows<commit_after># Verify that every show in BBoxPrograms is also in BBoxEpisodes

# INVOCATION:
#   awk -v EPISODES_SORTED_FILE=$EPISODES_SORTED_FILE -v SEASONS_SORTED_FILE=$SEASONS_SORTED_FILE \
#       -v TEMP_FILE=$TEMP_FILE -f verifyBBoxDownloadsFrom-webscraper.awk $PROGRAMS_SORTED_FILE \
#       >>$EPISODE_INFO_FILE

# Field numbers
#    1 web-scraper-start-url    2 URL    3 Program_Title    4 Sn_Years    5 Seasons    9  Description

BEGIN {
    FS="\t"
}

/\/us\// {
    programURL = $1
    numSeasons = $4

    seaStr = ""
    if (numSeasons != "null") {
        sub (/ Seasons?/,"",numSeasons)
        numSeasons == 1 ? seaStr = " in 1 season" : seaStr = " in " numSeasons " seasons"
    }

nflds = split (programURL,fld,"/") 
showType = fld[3]
showType == "show" ? spacer = "  " : spacer = " "
# Must have single quotes to handle special charaters in targets
target = sprintf ("'%s'",fld[nflds])
unquotedTarget = substr(target,2,length(target)-2)
#
cmd = "grep -c " target " " EPISODES_SORTED_FILE
if ((cmd | getline numEpisodes) > 0) {
    numEpisodes == 1 ? epiStr = " episode" : epiStr = " episodes"
    print "==> " showType spacer target " has " numEpisodes epiStr seaStr
}
close (cmd)
if (numEpisodes == 0) {
    badEpisodes += 1
    print "    " unquotedTarget >> TEMP_FILE
}
#
cmd = "grep " target " " SEASONS_SORTED_FILE
while ((cmd | getline seasonLine) > 0) {
    nfields = split (seasonLine, fields, "\t")
    seasonField = fields[nfields-4]
    episodeField = fields[nfields-2]
    if (seasonField == "") {
        print "==> " target " (blank Seasons field in " SEASONS_SORTED_FILE ")" \
            > "/dev/stderr"
        print "    " unquotedTarget " (blank Seasons field in " SEASONS_SORTED_FILE ")" \
            >> TEMP_FILE
        continue
    }
    print "          " target " " seasonField " has " episodeField

}
close (cmd)
}

END {
    if (badEpisodes > 0 ) {
        badEpisodes == 1 ? field = "URL" : field = "URLs"
        printf ("==> %2d extra /program/ %s in %s\n", badEpisodes, field, FILENAME) > "/dev/stderr"
    }
}
{'fraction_non_alphanumeric': 0.13008526187576128, 'fraction_numerical': 0.010475030450669914, 'mean_word_length': 2.536606373815676, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6187215', 'n_tokens_mistral': 1526, 'n_tokens_neox': 1483, 'n_words': 354}