<jupyter_start><jupyter_text>In 2016, NYT became clickbait<jupyter_code>small_vals = NYT_all[['year', 'nouns_ratio', 'verbs_ratio', 'adjs_ratio', 'advs_ratio',
       'superlatives_ratio', 'w_words_ratio', 'uppercase_n',
       'question_marks_n', 'exclamation_marks_n', 'nouns_n_bin', 'verbs_n_bin',
       'adjs_n_bin', 'advs_n_bin', 'raps_n_bin', 'first_person_sg_bin',
       'first_person_pl_bin', 'future_bin', 'dems_bin', 'superlatives_n_bin',
       'w_words_n_bin', 'uppercase_n_bin', 'question_marks_n_bin',
       'exclamation_marks_n_bin']]

small_vals
#pickle the df with all features as ling_features

import pickle
NYT_all.to_pickle("./NYT_all_extended.pkl")<jupyter_output><empty_output><jupyter_text>THE END, lemmatization and zipf frequency (below) not used in present study<jupyter_code>import pickle
with open(file, 'rb') as pickle_file:
    NYT_all = pickle.load(pickle_file)
NYT_all
# lemmatize and preprocess 
import spacy
import en_core_web_sm
nlp = en_core_web_sm.load()

def lemmatize_text(text):
    doc = nlp(text)
    result = ' '.join([x.lemma_ for x in doc]) 
    return result
NYT_all['lemmatized_headline'] = NYT_all['headline'].apply(lemmatize_text)
NYT_all
NYT_all.rename(columns={'Unnamed: 0': 'number'}, inplace=True)
NYT_all = NYT_all[['number', 'tokenized_nltk', 'headline', 'lower_words_nltk', 'lemmatized_headline']]
NYT_all
from nltk.tokenize import TweetTokenizer

tweet_tokenizer = TweetTokenizer()

import string

def remove_punct_and_lowercase(text): 
    tokenized_tweet = tweet_tokenizer.tokenize(text)
    lower = [token.lower() for token in tokenized_tweet]
    no_punctuation = [token for token in lower if token not in string.punctuation] 
    return no_punctuation
NYT_all['preprocessed_headline'] = NYT_all['lemmatized_headline'].apply(remove_punct_and_lowercase)
NYT_all
NYT_all.to_pickle("./lemmatized_NYT.pkl")<jupyter_output><empty_output><jupyter_text>get freqs<jupyter_code>from wordfreq import zipf_frequency

def get_mean_freq(lemmatized):
    freqs = [zipf_frequency(word, 'en') for word in lemmatized]
    if sum(freqs):
        return sum(freqs) / len(freqs)
    else:
        return 0
NYT_all['avg_word_freq'] = NYT_all['preprocessed_headline'].apply(get_mean_freq)
NYT_all
import seaborn as sns

sns.set_theme(style = 'whitegrid')

sns.histplot(x="avg_word_freq", bins = 25, data=NYT_all)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1169811320754717, 'fraction_numerical': 0.0033542976939203353, 'mean_word_length': 6.409937888198757, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '500642', 'license': 'MIT', 'n_tokens_mistral': 952, 'n_tokens_neox': 931, 'n_words': 165, 'path': 'ANALYSIS/NYT_ling_features.ipynb', 'repo_name': 'pietronickl/clickbait', 'size': '2385'}