<jupyter_start><jupyter_text>for df in dataframe_raw.values():    df["content"] = [BeautifulSoup(df.loc[i, "content"], "lxml").getText() for i, row in df.iterrows()]    df["title"] = df["title"].map(removePunctuation)    df["content"] = df["content"].map(removePunctuation)    df["title"] = df["title"].map(removeStopwords)    df["content"] = df["content"].map(removeStopwords)    df["tags"] = df["tags"].map(lambda x: x.split())dataframe["diy"].sample(3) for name, df in dataframe.items():     Saving to file    df.to_csv("../data/" + name + "_light.csv", index=False)<jupyter_code>dataframe = {
    "cooking": pd.read_csv("../data/cooking_light.csv"),
    "crypto": pd.read_csv("../data/crypto_light.csv"),
    "robotics": pd.read_csv("../data/robotics_light.csv"),
    "biology": pd.read_csv("../data/biology_light.csv"),
    "travel": pd.read_csv("../data/travel_light.csv"),
    "diy": pd.read_csv("../data/diy_light.csv"),
}
dataframe['diy'].sample(3)[["title", "content", "tags"]]
test.shape
dataframe['physics'] = test
dataframe.keys()
topics = list()
nExamples = list()
nTags = list()
tagsHist = list()
for topic in dataframe.keys():
    topics.append(topic)
    nExamples.append(dataframe[topic].shape[0])
    if topic == 'physics':
        nTags.append(0)
        print(topic + ": " + str(nExamples[-1]))
    else:
        tagCountList = [len(row['tags'].split('\', \'')) for i, row in dataframe[topic].iterrows()]
        nTags.append(np.sum(tagCountList))
        tagsHist.extend(tagCountList)
        print(topic + ": " + str(nExamples[-1]) + " tags: " + str(nTags[-1]))
x = np.arange(len(nExamples))
plt.figure(figsize=(14,6))
plt.bar(x, nExamples, align='center')
plt.xticks(x, topics)
plt.ylabel("Number of Examples")
bins=np.arange(1,7)
plt.figure(figsize=(14,6))
plt.hist(tagsHist, align='left', bins=bins)
plt.xticks(bins)
plt.xlabel("Tags per example")
dataframe['crypto']['tags'].apply(lambda x: len(x.split('\', \''))).sort_values(ascending=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2185, 'fraction_numerical': 0.0075, 'mean_word_length': 4.783236994219653, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '566743', 'license': 'MIT', 'n_tokens_mistral': 746, 'n_tokens_neox': 722, 'n_words': 114, 'path': 'src/Transfer Learning on Stack Exchange Tags.ipynb', 'repo_name': 'QEDan/transferLearningStackExchange', 'size': '2000'}