<jupyter_start><jupyter_text>Fitting TF-IDF on the subset of a German Wikipedia dumpDownload data from https://github.com/t-systems-on-site-services-gmbh/german-wikipedia-text-corpus<jupyter_code>import os
import numpy as np
import string, nltk
from sklearn.feature_extraction.text import TfidfVectorizer
from collections import defaultdict
folder = "" # specify location of dump
wiki_dump_path = os.path.join(folder, "wiki-all-shuf/wiki-all-shuf.txt")
wiki_file = open(wiki_dump_path, 'r')
lines = wiki_file.readlines()
print(len(lines))
def remove_punctuation(text):
    regular_punct = list(string.punctuation)
    for punc in regular_punct:
        if punc in text:
            text = text.replace(punc, " ")
    return text.strip()

texts = lines[:3000000] # specify size of subset
texts = [remove_punctuation(text) for text in texts]
tokenize = lambda text: nltk.tokenize.word_tokenize(text, language="german")
tfidf = TfidfVectorizer(analyzer='word', max_features=2000, tokenizer=tokenize,
                        lowercase=True)
tfidf.fit(texts)
max_idf = max(tfidf.idf_)
def dd():
    return defaultdict(max_idf)
word2weight = defaultdict(dd, [(w, tfidf.idf_[i]) for w, i in tfidf.vocabulary_.items()])
print(f"TFIDF features {len(tfidf.idf_)}")
dest_path = ""
np.save(dest_path, dict(word2weight), allow_pickle=True)
np.save(os.path.join(dest_path, "max_idf"), max(tfidf.idf_), allow_pickle=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12256267409470752, 'fraction_numerical': 0.009052924791086351, 'mean_word_length': 5.275109170305677, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '156517', 'license': 'MIT', 'n_tokens_mistral': 531, 'n_tokens_neox': 516, 'n_words': 108, 'path': 'notebooks/fit_tf_idf.ipynb', 'repo_name': 'krangelie/bias-in-german-nlg', 'size': '1436'}