<jupyter_start><jupyter_text>Stemming vs. Lemmatization:  stemming was found toreduce model fit, negligibly affect topic coherence, and negligibly or negatively affectmodel consistency across random initializations (Schofield and Mimno, 2016). In light of these results, authors recommended refraining from stemming the corpus as a pre-processing step and instead stemming the top-m word lists as a post-processing step, as needed. TO-DO: try a model with lemmatized and stemmed tokens as well as without it<jupyter_code>#Processing the text and returning tokens

def remove(tokens): 
    pattern = '[0-9]'
    new_tokens = [re.sub(pattern, '', i) for i in tokens]  #removing numbers from tokens
    removetable = str.maketrans('', '', '’“”–')  
    new_tokens = [s.translate(removetable) for s in new_tokens] #removing special characters
    return [x for x in new_tokens if len(x)>1]  #removing tokens with length 1 or empty


def text_processing(input_str):
    input_lower = input_str.lower()
    input_punctutation = input_lower.translate(str.maketrans(string.punctuation, ' ' * len(string.punctuation))).replace(' '*4, ' ').replace(' '*3, ' ').replace(' '*2, ' ').strip()
    input_tokens = nltk.word_tokenize(input_punctutation)
    input_stopwords = [i for i in input_tokens if not i in stop_words]
    lemmatizer=WordNetLemmatizer()  #lemmatization vs. stemming
    input_lemmatized = [lemmatizer.lemmatize(word) for word in input_stopwords]
    input_clean = remove(input_lemmatized)
    return (input_clean)
    
df['tokens'] = df['Text'].apply(lambda x: text_processing(x))
df['tokens'].sample(n=1).values
#count the frequency of words and return freq of corona-related terms

def wordListToFreqDict(wordlist, terms):
    wordfreq = [wordlist.count(p) for p in wordlist]
    res = dict(list(zip(wordlist,wordfreq)))
    return [(term, res[term]) for term in terms if term in res.keys()]

with open('corona_terms.txt', 'r') as corona_terms:
    terms = corona_terms.read().replace('\n', ' ').split(' ')

df['corona_terms'] = df['tokens'].apply(lambda x: wordListToFreqDict(x, terms))
df['corona_freq'] = df['corona_terms'].apply(lambda x: sum([item[1] for item in x]))
df['corona_freq'].describe()
df_corona = df[df['corona_freq'] >=3]
df_corona.shape
df['corona_terms']
df_corona.to_csv('preprocessed_results/mediacloud_parsed_corona_df_sep.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12353433835845896, 'fraction_numerical': 0.005862646566164154, 'mean_word_length': 5.3537234042553195, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '211114', 'license': 'MIT', 'n_tokens_mistral': 808, 'n_tokens_neox': 771, 'n_words': 222, 'path': 'Mediacloud_Analysis.ipynb', 'repo_name': 'gesiscss/media_frames', 'size': '2388'}