<jupyter_start><jupyter_text>Declare the dataset name (and topic, for Media Framing Corpus)<jupyter_code>dataset = 'peerread'
framing_topic = 'guncontrol'<jupyter_output><empty_output><jupyter_text>Processing data<jupyter_code>reload(ds)

label_is_bool=False

if dataset in TextResponseDataset.CLASSIFICATION_SETTINGS:
    label_is_bool=True

if dataset == 'amazon':
    datafile = '../dat/reviews_Office_Products_5.json'
elif dataset == 'amazon_binary':
    datafile = '../dat/reviews_Grocery_and_Gourmet_Food_5.json'
elif dataset == 'yelp':
    datafile = '../dat/yelp_review_polarity_csv/train.csv'
elif dataset == 'peerread':
    datafile = '../dat/peerread_abstracts.csv'
elif dataset == 'framing_corpus':
    datafile = '../dat/framing/'
else:
    datafile = '../dat/cs_papers.gz'

if dataset == 'framing_corpus':
    proc_file = '../dat/proc/' + dataset + '_' + framing_topic + '_proc.npz'
else:
    proc_file = '../dat/proc/' + dataset + '_proc.npz'

components = {'amazon':30, 
              'semantic_scholar':50, 
              'peerread':50, 'yelp':30, 
              'amazon_binary':20, 
              'framing_corpus':10
             }
text_dataset = ds.TextResponseDataset(dataset, 
                                      datafile, 
                                      proc_file, 
                                      use_bigrams=False,
                                      framing_topic=framing_topic)
text_dataset.process_dataset()
text_dataset.preprocessing()

counts = text_dataset.counts
labels= text_dataset.labels
vocab= text_dataset.vocab
docs = text_dataset.docs

n_components=components[dataset]
num_documents = counts.shape[0]
n_components, num_documents, counts.shape[1]<jupyter_output><empty_output><jupyter_text>Running PCA on cooccurence matrix of words to create embeddings of words for regression<jupyter_code>tf = csr_matrix(counts)
cooccurence = tf.T.dot(tf)
cooccurence = cooccurence.toarray()

pca = PCA(n_components=n_components)
embeddings = pca.fit_transform(cooccurence)

features = np.zeros((num_documents, n_components))
for i in range(num_documents):
    tf = counts[i,:]
    nonzero = (tf > 0)
    features[i] = embeddings[nonzero,:].sum(axis=0)

result_pca = run_cross_validation(features, labels, num_documents, label_is_bool=label_is_bool)
result_pca<jupyter_output><empty_output><jupyter_text>LDA features for regression<jupyter_code>if dataset == 'framing_corpus':
    pretraining_file = '../dat/proc/' + dataset + '_' + framing_topic + '_pretraining.npz'
else:
    pretraining_file = '../dat/proc/' + dataset + '_pretraining.npz'
    
if os.path.exists(pretraining_file):
    print("Loading saved results...")
    arr = np.load(pretraining_file)
    doc_rep = arr['theta'] 
    topics = arr['beta']
    print("Completed.")
else:
    lda_model = LDA(n_components=n_components)
    doc_rep = lda_model.fit_transform(counts)
    
    unnormalized_topics = lda_model.components_
    topics = lda_model.components_ / lda_model.components_.sum(axis=1)[:,np.newaxis]
    
    if dataset == 'framing_corpus':
        pretrained_out_file = '../dat/proc/' + dataset + '_' + framing_topic + '_pretraining'
    else:
        pretrained_out_file = '../dat/proc/' + dataset + '_pretraining'

    np.savez_compressed(pretrained_out_file, theta=doc_rep, beta=topics)
    
    print("Perplexity:", lda_model.perplexity(counts))
    
for k in range(n_components):
    beta = topics[k]
    top_words = (-beta).argsort()[:7]
    topic_words = [(vocab[t]) for t in top_words]
    print('Topic {}: {}'.format(k, topic_words))
results_lda = run_cross_validation(doc_rep, labels, num_documents, label_is_bool=label_is_bool)
results_lda<jupyter_output><empty_output><jupyter_text>BOW features for regression<jupyter_code>normalized = counts/counts.sum(axis=1)[:,np.newaxis]
result_bow = run_cross_validation(csr_matrix(normalized), labels, num_documents, label_is_bool=label_is_bool)
result_bow<jupyter_output><empty_output><jupyter_text>Regression adjusted for topic<jupyter_code>features = np.column_stack((normalized,doc_rep))
result_adjusted = run_cross_validation(csr_matrix(features), labels, num_documents, label_is_bool=label_is_bool)
result_adjusted<jupyter_output>[[0.81205147 0.43660107 0.78419711]
 [0.81887834 0.44377002 0.78589635]
 [0.77988073 0.43994415 0.7884452 ]
 [0.80625635 0.43761526 0.78759558]
 [0.82464898 0.42720401 0.79694138]]
{'fraction_non_alphanumeric': 0.12323140118667275, 'fraction_numerical': 0.03537197626654496, 'mean_word_length': 4.156470588235294, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '355927', 'license': 'MIT', 'n_tokens_mistral': 1639, 'n_tokens_neox': 1515, 'n_words': 244, 'path': 'src/run_baselines.ipynb', 'repo_name': 'dsridhar91/hstm', 'size': '4382'}