<jupyter_start><jupyter_text>Analysis of Followers<jupyter_code>import glob
import pandas as pd
import seaborn as sns
import numpy as np
from matplotlib import pyplot as plt
sns.set_theme(style="darkgrid")<jupyter_output><empty_output><jupyter_text>First, read all the files belonging to the followers and group them by id to sum their counts<jupyter_code>df = pd.concat([pd.read_csv(f, index_col=0) for f in glob.glob('./data/*_followers.csv')], ignore_index=False)
# group them by id to sum the counts
df = df.groupby(["id"]).sum()
# store the total amount of interactions
df["total"] = df["quote_count"] + df["reply_count"] + df["retweet_count"] + df["like_count"]
# make data anonymous
df = df.reset_index().drop(columns=["id"])
df<jupyter_output><empty_output><jupyter_text>Inspect the distributions for each interaction<jupyter_code>fig, ax = plt.subplots(figsize=(15,7))
_ = sns.boxplot(data=df, ax=ax)
_ = sns.stripplot(data=df, color=".3", ax=ax)<jupyter_output><empty_output><jupyter_text>We are not interested in people without interactions (which are a lot) and there are outliers in the upper limit that need to be filtered.Let's get the distribution for each filtered quantity independently.<jupyter_code>def find_outliers_limit(df, col, z=1.5):
    q25, q75 = np.percentile(df[col], 25), np.percentile(df[col], 75)
    iqr = q75 - q25
    cut_off = iqr * z
    lower, upper = q25 - cut_off, q75 + cut_off
    return lower,upper

def get_outliers_filter(data, col):
    lower, upper = find_outliers_limit(data, col)
    return (data[col] < lower) | (data[col] > upper)

def remove_with_iqr(df, col):
    return df[ ~get_outliers_filter(df,col) ]

def remove_dummy(df, col):
    thresholds = {
        "reply_count" : 100,
        "retweet_count" : 100,
        "like_count" : 200,
        "total" : 300
    }
    return df[( df[col] < thresholds[col] )]

# remove zeros
df = df[ (df["total"]>0) ]
# remove outliers with 
cols=["reply_count", "like_count", "total"]
for col in cols:
    df = remove_dummy(df, col)
fig, ax = plt.subplots(ncols=len(df.columns))
plt.subplots_adjust(right=3, wspace=1)
for i, col in enumerate(df.columns):
    sns.boxplot(y=df[col], ax = ax[i])
    sns.stripplot(y=df[col], color=".3", ax=ax[i])
from matplotlib.ticker import MaxNLocator
fig, axs = plt.subplots(2, 3, figsize=(19,16))
axs[1,2].set_visible(False)

# for likes
ax = axs[0,0]
bins = np.arange(0,160,1)
#sns.histplot(data=df, x="like_count", bins=bins, alpha=0.4, ax=ax, kde=True)
sns.kdeplot(data=df, x="like_count", ax=ax, fill=True)
_ = ax.set_xlim(0,200)
ax.xaxis.set_major_locator(MaxNLocator(integer=True))

# for replies
ax = axs[0,1]
bins = np.arange(0,85,1)
##sns.histplot(data=df, x="reply_count", bins=bins, stat="probability", alpha=0.4, ax=ax)
sns.kdeplot(data=df, x="reply_count", ax=ax, fill=True)
_ = ax.set_xlim(0,85)

# for quotes
ax = axs[0,2]
bins = np.arange(0,2,0.5)
#sns.histplot(data=df, x="quote_count", bins=bins, stat="probability", alpha=0.4, ax=ax)
sns.kdeplot(data=df, x="quote_count", ax=ax, fill=True)
_ = ax.set_xlim(0,2)

# for retweets
ax = axs[1,0]
bins = np.arange(0,2,0.5)
#sns.histplot(data=df, x="retweet_count", bins=bins, stat="probability", alpha=0.4, ax=ax)
sns.kdeplot(data=df, x="retweet_count", ax=ax, fill=True)

# for total
ax = axs[1,1]
bins = np.arange(0,250,1)
#sns.histplot(data=df, x="total", bins=bins, stat="probability", alpha=0.4, ax=ax)
sns.kdeplot(data=df, x="total", ax=ax, fill=True)
_ = ax.set_xlim(0,250)
sns.pairplot(df, diag_kind="kde")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16421707179197287, 'fraction_numerical': 0.02882984737139627, 'mean_word_length': 5.627340823970037, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '666402', 'license': 'MIT', 'n_tokens_mistral': 1499, 'n_tokens_neox': 1425, 'n_words': 306, 'path': 'analysis_followers.ipynb', 'repo_name': 'peguerosdc/pegawards', 'size': '3538'}