<jupyter_start><jupyter_text>Now that the data has loaded in, we can run the counter using the Counter class from collections. https://docs.python.org/3/library/collections.htmlcollections.Counter<jupyter_code>c = Counter()
def add_to_counter(df):
    temp = Counter()
    for col in df:
        series = df[col].str.lower().str.split("<|endoftext|>", expand=True)[0].apply(word_tokenize)
        for row in series:
            temp += Counter(row)
            
    return temp
c += add_to_counter(df1)
c += add_to_counter(df2)
c += add_to_counter(df3)
c += add_to_counter(df4)
c += add_to_counter(df5)
c += add_to_counter(df6)
c += add_to_counter(df7)
c += add_to_counter(df8)
c += add_to_counter(df9)
c += add_to_counter(df10)
c += add_to_counter(df11)
c += add_to_counter(df12)
c += add_to_counter(df13)
c += add_to_counter(df14)
c += add_to_counter(df15)
c += add_to_counter(df_1)
c += add_to_counter(df_2)
c += add_to_counter(df_3)
c += add_to_counter(df_4)
c += add_to_counter(df_5)
c += add_to_counter(df_6)
c += add_to_counter(df_7)
c += add_to_counter(df_8)
c += add_to_counter(df_9)
c += add_to_counter(df_10)
c += add_to_counter(df_11)
c += add_to_counter(df_12)
c += add_to_counter(df_13)
c += add_to_counter(df_14)
c += add_to_counter(df_15)
c += add_to_counter(df__1)
c += add_to_counter(df__2)
c += add_to_counter(df__3)
c += add_to_counter(df__4)
c += add_to_counter(df__5)
c += add_to_counter(df__6)
c += add_to_counter(df__7)
c += add_to_counter(df__8)
c += add_to_counter(df__9)
c += add_to_counter(df__10)
c += add_to_counter(df__11)
c += add_to_counter(df__12)
c += add_to_counter(df__13)
c += add_to_counter(df__14)
c += add_to_counter(df__15)
c += add_to_counter(df___1)
c += add_to_counter(df___2)
c += add_to_counter(df___3)
c += add_to_counter(df___4)
c += add_to_counter(df___5)
c += add_to_counter(df___6)
c += add_to_counter(df___7)
c += add_to_counter(df___8)
c += add_to_counter(df___9)
c += add_to_counter(df___10)
c += add_to_counter(df___11)
c += add_to_counter(df___12)
c += add_to_counter(df___13)
c += add_to_counter(df___14)
c += add_to_counter(df___15)
c += add_to_counter(df____1)
c += add_to_counter(df____2)
c += add_to_counter(df____3)
c += add_to_counter(df____4)
c += add_to_counter(df____5)
c += add_to_counter(df____6)
c += add_to_counter(df____7)
c += add_to_counter(df____8)
c += add_to_counter(df____9)
c += add_to_counter(df____10)
c += add_to_counter(df____11)
c += add_to_counter(df____12)
c += add_to_counter(df____13)
c += add_to_counter(df____14)
c += add_to_counter(df____15)
c += add_to_counter(df_____1)
c += add_to_counter(df_____2)
c += add_to_counter(df_____3)
c += add_to_counter(df_____4)
c += add_to_counter(df_____5)
c += add_to_counter(df_____6)
c += add_to_counter(df_____7)
c += add_to_counter(df_____8)
c += add_to_counter(df_____9)
c += add_to_counter(df_____10)
c += add_to_counter(df_____11)
c += add_to_counter(df_____12)
c += add_to_counter(df_____13)
c += add_to_counter(df_____14)
c += add_to_counter(df_____15)
c += add_to_counter(df______1)
c += add_to_counter(df______2)
c += add_to_counter(df______3)
c += add_to_counter(df______4)
c += add_to_counter(df______5)
c += add_to_counter(df______6)
c += add_to_counter(df______7)
c += add_to_counter(df______8)
c += add_to_counter(df______9)
c += add_to_counter(df______10)
c += add_to_counter(df______11)
c += add_to_counter(df______12)
c += add_to_counter(df______13)
c += add_to_counter(df______14)
c += add_to_counter(df______15)
c += add_to_counter(df_______1)
c += add_to_counter(df_______2)
c += add_to_counter(df_______3)
c += add_to_counter(df_______4)
c += add_to_counter(df_______5)
c += add_to_counter(df_______6)
c += add_to_counter(df_______7)
c += add_to_counter(df_______8)
c += add_to_counter(df_______9)
c += add_to_counter(df_______10)
c += add_to_counter(df_______11)
c += add_to_counter(df_______12)
c += add_to_counter(df_______13)
c += add_to_counter(df_______14)
c += add_to_counter(df_______15)
c += add_to_counter(df________1)
c += add_to_counter(df________2)
c += add_to_counter(df________3)
c += add_to_counter(df________4)
c += add_to_counter(df________5)
c += add_to_counter(df________6)
c += add_to_counter(df________7)
c += add_to_counter(df________8)
c += add_to_counter(df________9)
c += add_to_counter(df________10)
c += add_to_counter(df________11)
c += add_to_counter(df________12)
c += add_to_counter(df________13)
c += add_to_counter(df________14)
c += add_to_counter(df________15)
from nltk.corpus import stopwords
for word in stopwords.words('english'):
    try:
        c.pop(word)
    except:
        pass
import string
for punctuation in string.punctuation:
    try:
        c.pop(punctuation)
    except:
        pass
top = c.most_common(1000)
top
word, cnt = zip(*top)
plt.figure(figsize=(20,10))
plt.bar(word, cnt)
plt.xlabel("Word")
plt.ylabel("Count")
plt.title("Word vs. Count")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13431613431613432, 'fraction_numerical': 0.04074529074529074, 'mean_word_length': 6.956026058631922, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 30, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '169448', 'license': 'MIT', 'n_tokens_mistral': 2158, 'n_tokens_neox': 2087, 'n_words': 345, 'path': 'src/car_data/WordCount.ipynb', 'repo_name': 'albtang/Katona-AI-Research', 'size': '4884'}