<jupyter_start><jupyter_text>Humicroedit dataset creation<jupyter_code>import warnings
warnings.filterwarnings('ignore')

from tqdm import tqdm
import numpy as np
# import torch
# from torchvision import datasets
# from torch import nn, optim, autograd
import matplotlib.pyplot as plt
import pandas as pd

# from sklearn.decomposition import LatentDirichletAllocation
# from sklearn.linear_model import LogisticRegression
# from sklearn.ensemble import RandomForestClassifier
# from sklearn.ensemble import RandomForestRegressor
# from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder, scale
# from sklearn.datasets import make_multilabel_classification

import random
# from sklearn import svm
# from sklearn import linear_model
# from scipy.sparse import csr_matrix
# from sklearn.metrics import classification_report
# from sklearn.cross_decomposition import CCA
from sklearn.utils import shuffle

import gensim, logging
import ast
# from sklearn.preprocessing import MinMaxScaler
# from sklearn.manifold import TSNE
# from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans
# from sklearn.metrics import silhouette_samples, silhouette_score
# from gensim.parsing.preprocessing import remove_stopwords
from cleantext import clean
# import matplotlib.cm as cm
# from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn import preprocessing
import re
from pathlib import Path
%matplotlib inline

np.random.seed(42)<jupyter_output><empty_output><jupyter_text>Read in data *Note*: to run this notebook you would need to download:1) Humicroedit data (from https://www.cs.rochester.edu/u/nhossain/humicroedit.html, Full Dataset Release)2) glove.6B (from https://nlp.stanford.edu/projects/glove/) and place them in the relevant paths below (in ../../data)<jupyter_code>train_df_task1 = pd.read_csv("../../data/semeval-2020-task-7-dataset/subtask-1/train.csv")
train_funline_df_task1 = pd.read_csv("../../data/semeval-2020-task-7-dataset/subtask-1/train_funlines.csv")
val_df_task1 = pd.read_csv("../../data/semeval-2020-task-7-dataset/subtask-1/dev.csv")
test_df_task1 = pd.read_csv("../../data/semeval-2020-task-7-dataset/subtask-1/test.csv")

train_df_task2 = pd.read_csv("../../data/semeval-2020-task-7-dataset/subtask-2/train.csv")
train_funline_df_task2 = pd.read_csv("../../data/semeval-2020-task-7-dataset/subtask-2/train_funlines.csv")
val_df_task2 = pd.read_csv("../../data/semeval-2020-task-7-dataset/subtask-2/dev.csv")
test_df_task2 = pd.read_csv("../../data/semeval-2020-task-7-dataset/subtask-2/test.csv")<jupyter_output><empty_output><jupyter_text>Helper functions<jupyter_code>def task2_convert_to_task1(df):
    dataset1 = df[['original1', 'edit1', 'grades1', 'meanGrade1']]
    dataset2 = df[['original2', 'edit2', 'grades2', 'meanGrade2']]
    dataset2.rename(columns={"original2": "original1", "edit2": "edit1", "grades2": "grades1", "meanGrade2": "meanGrade1"}, inplace=True)
    combined = dataset1.append(dataset2)
    combined.rename(columns={"original1": "original", "edit1": "edit", "grades1": "grades", "meanGrade1": "meanGrade"}, inplace=True)
    return combined
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
from collections import defaultdict
from nltk.stem.porter import PorterStemmer

## https://github.com/utkuozbulak/unsupervised-learning-document-clustering/blob/master/src/read_and_clean_documents.py
def clean_string(st):
    stemmer = PorterStemmer()
    items_to_clean = set(list(stopwords.words('english')) + ['\n','\n\n','\n\n\n','\n\n\n\n','ocroutput','',' '])
    # Items to clean
    regex_non_alphanumeric = re.compile('[^0-9a-zA-Z]')  # REGEX for non alphanumeric chars
    st = " ".join(regex_non_alphanumeric.sub(' ', st).split())  # Filter text, remove non alphanumeric chars
    st = st.lower()  # Lowercase the text
    st = stemmer.stem(st)  # Stem the text
    if len(st) < 3:  # If the length of item is lower than 3, remove item
        item = ''
    st = " ".join([elem for elem in st.split(" ") if elem not in items_to_clean])

    return st

def perform_edit(x):
    first_part = x['original'].split("<")[0]
    second_part = x['original'].split(">")[1]
    edit = x['edit']
    result = first_part + edit + second_part
#     return clean_string(result)
    return result

def replaced_word(x):
    return x[x.find("<")+len("<"):x.rfind("/>")]

def restructure_dataset(df):
    df['Z_raw'] = df['original'].apply(lambda x: x.replace("<","").replace("/>",""))
#     df['Z_raw'] = df['original'].apply(lambda x: clean_string(x.replace("<","").replace("/>","")))
    df['replaced'] = df['original'].apply(lambda x: replaced_word(x))
    df['W_raw'] = df['edit']
    df['X_raw'] = df.apply(lambda x: perform_edit(x), axis=1)
    df['Y'] = df['meanGrade']
    return df<jupyter_output>[nltk_data] Downloading package stopwords to
[nltk_data]     /Users/lgultchin/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!<jupyter_text>Structure dataset<jupyter_code>total_df_task1 = train_df_task1.append(train_funline_df_task1).append(val_df_task1).append(test_df_task1)
assert len(total_df_task1) == \
len(train_df_task1) + len(train_funline_df_task1) + \
len(val_df_task1) + len(test_df_task1)

total_df_task2 = train_df_task2.append(train_funline_df_task2).append(val_df_task2).append(test_df_task2)
assert len(total_df_task2) == \
len(train_df_task2) + len(train_funline_df_task2) + \
len(val_df_task2) + len(test_df_task2)
total_df_task2 = task2_convert_to_task1(total_df_task2)
total_df = total_df_task1.append(total_df_task2)
print(len(total_df))
total_df.drop_duplicates(inplace=True)
print(len(total_df))
total_df
total_df = restructure_dataset(total_df)
total_df<jupyter_output><empty_output><jupyter_text>Convert raw text columns to w2v<jupyter_code>model = gensim.models.KeyedVectors.load_word2vec_format('../../data/GoogleNews-vectors-negative300.bin.gz', binary=True)

from gensim.scripts.glove2word2vec import glove2word2vec
glove2word2vec(glove_input_file="../../data/glove.6B/glove.6B.300d.txt", 
               word2vec_output_file="../../data/glove.6B/gensim_glove_vectors.txt")

from gensim.models.keyedvectors import KeyedVectors
glove_model = KeyedVectors.load_word2vec_format("../../data/glove.6B/gensim_glove_vectors.txt", binary=False)
def word_vector(word2vec_model, word):
    try:
        return word2vec_model[word]
    except KeyError:
        return np.nan
    
def edits_diff(x,model=model):
    try:
        edit = model[x['edit']]
        init = model[x['replaced']]
        return edit-init
    except KeyError:
        return np.nan
    
def document_vector(word2vec_model, doc):
    # remove out-of-vocabulary words
    doc = [word for word in doc if word in word2vec_model.vocab]
    if len(doc)>0:
        return np.mean(word2vec_model[doc], axis=0)
    else:
        return np.nan
len(glove_model.vocab.keys()), len(model.vocab.keys())
%%time
total_df['W_vec'] = total_df['W_raw'].apply(lambda x: word_vector(model, x))
total_df['Z_vec'] = total_df['Z_raw'].apply(lambda x: document_vector(model, x))
total_df['X_vec'] = total_df['X_raw'].apply(lambda x: document_vector(model, x))
total_df['W_init_sub_edit_vec'] = total_df.apply(edits_diff, axis=1)
# glove vectors
total_df['W_glv_vec'] = total_df['W_raw'].apply(lambda x: word_vector(glove_model, clean_string(x)))
total_df['Z_glv_vec'] = total_df['Z_raw'].apply(lambda x: document_vector(glove_model, clean_string(x)))
total_df['X_glv_vec'] = total_df['X_raw'].apply(lambda x: document_vector(glove_model, clean_string(x)))

total_df['W_init_sub_edit_glv_vec'] = total_df.apply(lambda x: edits_diff(x, model=glove_model), axis=1)

total_df = total_df.dropna()
total_df<jupyter_output>CPU times: user 1min 19s, sys: 19.1 s, total: 1min 38s
Wall time: 3min 21s<jupyter_text>Cluster W<jupyter_code>W_glv_vec = total_df['W_glv_vec'].values
W_glv_vec = np.concatenate(W_glv_vec).ravel().reshape(len(W_glv_vec), -1)
clusterer = KMeans(n_clusters=20, random_state=10)
total_df['cluster_labels'] = clusterer.fit_predict(W_glv_vec)
enc = OneHotEncoder(handle_unknown='ignore')
enc.fit(np.array(total_df['cluster_labels']).reshape(-1,1))
def one_hot(x):
    return enc.transform(np.array([x]).reshape(-1,1)).toarray()[0]
total_df['cluster_labels_one_hot'] = total_df['cluster_labels'].apply(lambda x: one_hot(x))
clusters = {}
for i in range(20):
    clusters[i] = total_df[total_df['cluster_labels']==i]['W_raw']<jupyter_output><empty_output><jupyter_text>Prepare Phis: hypothsis testing based on Hossain et al.-------- Hossain et al, who compiled the Homicroedit dataset, analyzed and hypothesized what "humor features" exist in the humorous edits. Some of the candidate explainers were: Manual inspection1. Edit word forms a meaningful n-gram with adjacent words2. Connection b/w edit word and original word (e.g. semnatically distant, or similar pronounciation)3. Edit word makes a strong connection with an entity in the headline (Trump-hair, Obama-ears)4. Sarcasm5. Tension Supression/Relief Theory of Humor6. Edit word creates incongruity7. Setup and punchline: surprising edit at the end of the sentence----------------- Translation to our setup1. Length of resulting edited sentence (should not vary with w)2. Cosine distance b/w Glove of edit word and the rest of words in sentence3. Location index of replaced word (should not vary with w)4. Sentiment polarity of edit word5. Sentiment polarity of resulting sentence6. Cosine distance b/w Glove of edited word and Glove of original word7. Cosine distance from neigbouring words8. Distance of final sentence from cluster centroids 1. Length of resulting edited sentence (should not vary with w)<jupyter_code>total_df['phis0'] = total_df['X_raw'].apply(lambda x: len(x))<jupyter_output><empty_output><jupyter_text>2. Cosine distance b/w Glove of edit word and the rest of words in sentence<jupyter_code>def avg_dist_edit_rest(x):
    edit = x['edit']
    edit_vec = x['W_glv_vec']
    sent = x['X_raw']
    dists = []
    for word in sent.split(" "):
        if word!=edit:
            word_vec = word_vector(glove_model, clean_string(word))
            if not np.isnan(word_vec).any():
                dist = cosine_similarity(
                    word_vec.reshape(1,-1), 
                    edit_vec.reshape(1,-1))
                dists.append(dist)
    return np.mean(dists)

total_df['phis1'] = total_df.apply(avg_dist_edit_rest, axis=1)<jupyter_output><empty_output><jupyter_text>3. location index of replaced word (should not vary with w)<jupyter_code>def ind_of_edit(x):
    return x['X_raw'].split(" ").index(x['edit'])

total_df['phis2'] = total_df.apply(ind_of_edit, axis=1)<jupyter_output><empty_output><jupyter_text>4. Sentiment polarity of edit word<jupyter_code>import stanza
nlp = stanza.Pipeline(lang='en', processors='tokenize,sentiment')

def sentiment_of_word(x):
    sents = []
    doc = nlp(x)
    for sentence in doc.sentences:
        sent = sentence.sentiment
        sents.append(sent)
    return sents[0]

total_df['phis3'] = total_df['edit'].apply(sentiment_of_word)<jupyter_output>2021-06-04 13:08:59 INFO: Loading these models for language: en (English):
=======================
| Processor | Package |
-----------------------
| tokenize  | ewt     |
| sentiment | sstplus |
=======================

2021-06-04 13:08:59 INFO: Use device: cpu
2021-06-04 13:08:59 INFO: Loading: tokenize
2021-06-04 13:08:59 INFO: Loading: sentiment
2021-06-04 13:09:01 INFO: Done loading processors!<jupyter_text>5. Sentiment polarity of resulting sentence<jupyter_code>def sentiment_of_word(x):
    sents = []
    doc = nlp(x)
    for i, sentence in enumerate(doc.sentences):
        sent = sentence.sentiment
        sents.append(sent)
    return sents[0]

total_df['phis4'] = total_df['X_raw'].apply(sentiment_of_word)<jupyter_output><empty_output><jupyter_text>6. Cosine distance b/w Glove of edited word and Glove of original word<jupyter_code>total_df['replaced_glv_vec'] = total_df['replaced'].apply(lambda x: word_vector(glove_model, clean_string(x)))
total_df.dropna(inplace=True)
def cosine_distance_original_and_edit_word(x):
    edit_vec = x['W_glv_vec']
    original_vec = x['replaced_glv_vec']
    return cosine_similarity(
                    edit_vec.reshape(1,-1), 
                    original_vec.reshape(1,-1))

total_df['phis5'] = total_df.apply(cosine_distance_original_and_edit_word, axis=1)<jupyter_output><empty_output><jupyter_text>7. Cosine distance from neigbouring words<jupyter_code>def dist_from_neighb(x, neighb=1):
    edit_vec = x['W_glv_vec']
    sentence = x['X_raw']
    edit_index = x['phis2']
    neighb_word_vec = word_vector(glove_model, sentence[edit_index+neighb])
    if not np.isnan(neighb_word_vec).any():
        similar = cosine_similarity(
                        neighb_word_vec.reshape(1,-1), 
                        edit_vec.reshape(1,-1)).squeeze(axis=0)[0]
        return similar
    else:
        return -1

total_df['phis6'] = total_df.apply(dist_from_neighb, neighb=-2, axis=1)
total_df['phis7'] = total_df.apply(dist_from_neighb, neighb=-1, axis=1)
total_df['phis8'] = total_df.apply(dist_from_neighb, neighb=1, axis=1)
total_df['phis9'] = total_df.apply(dist_from_neighb, neighb=2, axis=1)<jupyter_output><empty_output><jupyter_text>8. Distance of final sentence from cluster centroids<jupyter_code>%%time 
cluster_centers = clusterer.cluster_centers_
avs_resulting_sent_vec = total_df['X_raw'].apply(lambda x: document_vector(glove_model, clean_string(x)))
distance_avg_sent_to_centroids = {10+i: [] for i in range(20)}
for sent in avs_resulting_sent_vec:
    for centroid_id, centroid in enumerate(cluster_centers):
        distance_avg_sent_to_centroids[centroid_id+10].append(
            cosine_similarity(sent.reshape(1,-1), 
                              centroid.reshape(1,-1)).item())
for key in distance_avg_sent_to_centroids.keys():
    total_df['phis'+str(key)] = distance_avg_sent_to_centroids[key]

len(list(total_df.columns[total_df.columns.str.contains('phis')]))
# Normalize phis and drop nans
for phi_col in list(total_df.columns[total_df.columns.str.contains('phis')]):
    total_df[phi_col] = scale(total_df[phi_col])
    
len(total_df)
unseen_clusters = 11
total_df_seen = total_df[total_df['cluster_labels'] != unseen_clusters]
total_df_unseen = total_df[total_df['cluster_labels'] == unseen_clusters]
print(total_df_seen.shape, total_df_unseen.shape)<jupyter_output>(6857, 949) (718, 949)<jupyter_text>Groundtruth Y<jupyter_code>np.random.seed(42)
weights_phis_groundtruth = np.random.uniform(-1, 1, 30)
zero_out_inds = np.random.choice(30, 9, replace=False)
weights_phis_groundtruth[zero_out_inds] = 0
# make sure at least one not partic. is varying w/ W
weights_phis_groundtruth[3] = 0
neg_inds = np.random.choice(30, 15, replace=False)
weights_phis_groundtruth[neg_inds] *= -1

def generate_groudtruth(x, weight_to_apply):
    phis = x[list(x.index[x.index.str.contains('phis')])].values
    try:
        result = (weight_to_apply@phis).squeeze(axis=0)
    except:
        result = (weight_to_apply@phis)
    # add noise to linear combin.
    result += np.random.normal(0, .5)
    return result
    
total_df['Y_groundtruth'] = total_df.apply(lambda x: generate_groudtruth(x, weights_phis_groundtruth), axis=1)
weights_phis_groundtruth<jupyter_output><empty_output><jupyter_text>Export dataset<jupyter_code># restructure dataframe: from single column multi-dimensional 
# to multi-column one-dimensional
total_df[['X_glv_vec'+str(i) for i in range(len(total_df['X_glv_vec'].iloc[0]))]] =\
    pd.DataFrame(total_df.X_glv_vec.tolist(), index = total_df.index)
total_df.drop('X_glv_vec', axis=1, inplace=True)

total_df[['Z_glv_vec'+str(i) for i in range(len(total_df['Z_glv_vec'].iloc[0]))]] =\
    pd.DataFrame(total_df.Z_glv_vec.tolist(), index = total_df.index)
total_df.drop('Z_glv_vec', axis=1, inplace=True)

total_df[['W_glv_vec'+str(i) for i in range(len(total_df['W_glv_vec'].iloc[0]))]] =\
    pd.DataFrame(total_df.W_glv_vec.tolist(), index = total_df.index)
total_df.drop('W_glv_vec', axis=1, inplace=True)

# total_df[['cluster_labels_one_hot'+str(i) for i in range(len(total_df['cluster_labels_one_hot'].iloc[0]))]] =\
#     pd.DataFrame(total_df.cluster_labels_one_hot.tolist(), index = total_df.index)
# total_df.drop('cluster_labels_one_hot', axis=1, inplace=True)
np.random.seed(42)
# # choice = int(np.random.choice(5, 1, replace=False))
# # dict_cluster_sizes = dict(total_df['cluster_labels'].value_counts())
# # keys=list(dict_cluster_sizes.keys())
# # values=list(dict_cluster_sizes.values())
# # value = sorted(values)[15+choice]
# # print("size of chosen split: ", value)
# # unseen_clusters = [keys[values.index(value)]]

# this is the cluster used for unseen W in paper
# chosen via process above
unseen_clusters = 11 

# total_df_seen = total_df[~total_df['cluster_labels'].isin([unseen_clusters])]
# total_df_unseen = total_df[total_df['cluster_labels'].isin([unseen_clusters])]

total_df_seen = total_df[total_df['cluster_labels'] != unseen_clusters]
total_df_unseen = total_df[total_df['cluster_labels'] == unseen_clusters]
print(total_df_seen.shape, total_df_unseen.shape)

total_df_seen = total_df_seen[
                    list(total_df_seen.columns[
                        total_df_seen.columns.str.contains('Z_glv_vec')])+
                    list(total_df_seen.columns[
                        total_df_seen.columns.str.contains('W_glv_vec')])+
                    list(total_df_seen.columns[
                        total_df_seen.columns.str.contains('X_glv_vec')])+['Y_groundtruth']
                    + list(total_df_seen.columns[
                        total_df_seen.columns.str.contains('phis')])
]


total_df_unseen = total_df_unseen[
                    list(total_df_unseen.columns[
                    total_df_unseen.columns.str.contains('Z_glv_vec')])+
                    list(total_df_unseen.columns[
                        total_df_unseen.columns.str.contains('W_glv_vec')])+
                    list(total_df_unseen.columns[
                        total_df_unseen.columns.str.contains('X_glv_vec')])+['Y_groundtruth']
                    + list(total_df_unseen.columns[
                        total_df_unseen.columns.str.contains('phis')])]

total_df_seen = shuffle(total_df_seen)
total_df_seen_train = total_df_seen.iloc[:int(len(total_df_seen)*0.9)]
total_df_seen_test = total_df_seen.iloc[int(len(total_df_seen)*0.9):]

total_df_unseen = shuffle(total_df_unseen)
df_test_unseen_train = total_df_unseen.iloc[:int(len(total_df_unseen)*0.8)]
df_test_unseen_test = total_df_unseen.iloc[int(len(total_df_unseen)*0.8):]

total_df_seen = total_df_seen.to_numpy()
total_df_seen_train = total_df_seen_train.to_numpy()
total_df_seen_test = total_df_seen_test.to_numpy()

testset_unseen = total_df_unseen.to_numpy()
test_unseen_train = df_test_unseen_train.to_numpy()
test_unseen_test = df_test_unseen_test.to_numpy()

print(total_df_seen_train.shape, total_df_seen_test.shape, 
      test_unseen_train.shape, test_unseen_test.shape)

Path('../../data/Humicroedit/').mkdir(parents=True, exist_ok=True)
np.savez_compressed('../../data/Humicroedit/Humicroedit_trainset_seen_train.npz', total_df_seen_train)
np.savez_compressed('../../data/Humicroedit/Humicroedit_trainset_seen_test.npz', total_df_seen_test)
np.savez_compressed('../../data/Humicroedit/Humicroedit_testset_unseen_train.npz', test_unseen_train)
np.savez_compressed('../../data/Humicroedit/Humicroedit_testset_unseen_test.npz', test_unseen_test)
np.savez_compressed('../../data/Humicroedit/Humicroedit_params.npz', weights_phis_groundtruth)<jupyter_output>(6857, 949) (718, 949)
(6171, 931) (686, 931) (574, 931) (144, 931)<jupyter_text>Additional experiments: create 100 versions of datasets with different coefficients for Y<jupyter_code>np.random.seed(42)
num_runs_diff_Y = 100
for run in range(num_runs_diff_Y):
    weights_phis_groundtruth = np.random.uniform(-1, 1, 30)
    zero_out_inds = np.random.choice(30, 9, replace=False)
    weights_phis_groundtruth[zero_out_inds] = 0
    # make sure at least one not partic. is varying w/ W
    weights_phis_groundtruth[3] = 0
    neg_inds = np.random.choice(30, 15, replace=False)
    weights_phis_groundtruth[neg_inds] *= -1

    def generate_groudtruth(x, weight_to_apply):
        phis = x[list(x.index[x.index.str.contains('phis')])].values
        try:
            result = (weight_to_apply@phis).squeeze(axis=0)
        except:
            result = (weight_to_apply@phis)
        # add noise to linear combin.
        result += np.random.normal(0, .5)
        return result

    total_df['Y_groundtruth'] = total_df.apply(lambda x: generate_groudtruth(x, weights_phis_groundtruth), axis=1)
    
    unseen_clusters = 11 

    total_df_seen = total_df[~total_df['cluster_labels'].isin([unseen_clusters])]
    total_df_unseen = total_df[total_df['cluster_labels'].isin([unseen_clusters])]
#     print(total_df_seen.shape, total_df_unseen.shape)

    total_df_seen = total_df_seen[
                        list(total_df_seen.columns[
                            total_df_seen.columns.str.contains('Z_glv_vec')])+
                        list(total_df_seen.columns[
                            total_df_seen.columns.str.contains('W_glv_vec')])+
                        list(total_df_seen.columns[
                            total_df_seen.columns.str.contains('X_glv_vec')])+['Y_groundtruth']
                        + list(total_df_seen.columns[
                            total_df_seen.columns.str.contains('phis')])
    ]


    total_df_unseen = total_df_unseen[
                        list(total_df_unseen.columns[
                        total_df_unseen.columns.str.contains('Z_glv_vec')])+
                        list(total_df_unseen.columns[
                            total_df_unseen.columns.str.contains('W_glv_vec')])+
                        list(total_df_unseen.columns[
                            total_df_unseen.columns.str.contains('X_glv_vec')])+['Y_groundtruth']
                        + list(total_df_unseen.columns[
                            total_df_unseen.columns.str.contains('phis')])]

    total_df_seen = shuffle(total_df_seen)
    total_df_seen_train = total_df_seen.iloc[:int(len(total_df_seen)*0.9)]
    total_df_seen_test = total_df_seen.iloc[int(len(total_df_seen)*0.9):]

    total_df_unseen = shuffle(total_df_unseen)
    df_test_unseen_train = total_df_unseen.iloc[:int(len(total_df_unseen)*0.8)]
    df_test_unseen_test = total_df_unseen.iloc[int(len(total_df_unseen)*0.8):]

    total_df_seen = total_df_seen.to_numpy()
    total_df_seen_train = total_df_seen_train.to_numpy()
    total_df_seen_test = total_df_seen_test.to_numpy()

    testset_unseen = total_df_unseen.to_numpy()
    test_unseen_train = df_test_unseen_train.to_numpy()
    test_unseen_test = df_test_unseen_test.to_numpy()

#     print(total_df_seen_train.shape, total_df_seen_test.shape, 
#           test_unseen_train.shape, test_unseen_test.shape)
    
    Path('../../data/Humicroedit/diffY/').mkdir(parents=True, exist_ok=True)    
    np.savez_compressed('../../data/Humicroedit/diffY/Humicroedit_unif{}_trainset_seen_train.npz'.format(run), total_df_seen_train)
    np.savez_compressed('../../data/Humicroedit/diffY/Humicroedit_unif{}_trainset_seen_test.npz'.format(run), total_df_seen_test)
    np.savez_compressed('../../data/Humicroedit/diffY/Humicroedit_unif{}_testset_unseen_train.npz'.format(run), test_unseen_train)
    np.savez_compressed('../../data/Humicroedit/diffY/Humicroedit_unif{}_testset_unseen_test.npz'.format(run), test_unseen_test)
    np.savez_compressed('../../data/Humicroedit/diffY/Humicroedit_unif{}_params.npz'.format(run), weights_phis_groundtruth)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.132279568990818, 'fraction_numerical': 0.018950987379984068, 'mean_word_length': 4.5599067599067595, 'pattern_counts': {'":': 8, '<': 70, '<?xml version=': 0, '>': 69, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '331916', 'license': 'CC0-1.0', 'n_tokens_mistral': 8885, 'n_tokens_neox': 8484, 'n_words': 1378, 'path': 'Python_Img_Humor/data_generation_notebooks/Humicroedit/Humicroedit.ipynb', 'repo_name': 'limorigu/ComplexCauses', 'size': '23851'}