<commit_before>USE  ${r"${hiveDB}"};
DROP TABLE IF EXISTS ${r"${downloadTableName}"}_interpreted;
DROP TABLE IF EXISTS ${r"${downloadTableName}"}_verbatim;
DROP TABLE IF EXISTS ${r"${downloadTableName}"}_citation;
DROP TABLE IF EXISTS ${r"${downloadTableName}"}_multimedia;
<#list extensions as extension>
-- ${extension.extension} extension
DROP TABLE IF EXISTS ${r"${downloadTableName}"}_ext_${extension.hiveTableName};
</#list>

dfs -rm -r ${r"${wfPath}"}/tmp/${r"${downloadTableName}"}-execute-extensions-query.q;
<commit_msg>usinf -f flag when removing extension temp qquery in case it doesn't exist<commit_after>USE  ${r"${hiveDB}"};
DROP TABLE IF EXISTS ${r"${downloadTableName}"}_interpreted;
DROP TABLE IF EXISTS ${r"${downloadTableName}"}_verbatim;
DROP TABLE IF EXISTS ${r"${downloadTableName}"}_citation;
DROP TABLE IF EXISTS ${r"${downloadTableName}"}_multimedia;
<#list extensions as extension>
-- ${extension.extension} extension
DROP TABLE IF EXISTS ${r"${downloadTableName}"}_ext_${extension.hiveTableName};
</#list>

dfs -rm -f ${r"${wfPath}"}/tmp/${r"${downloadTableName}"}-execute-extensions-query.q;

{'fraction_non_alphanumeric': 0.1784121320249777, 'fraction_numerical': 0.0, 'mean_word_length': 8.19672131147541, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '377503', 'n_tokens_mistral': 386, 'n_tokens_neox': 384, 'n_words': 88}