<jupyter_start><jupyter_text>Filter and Merge Extracted Interactions===For <NAME>'s survival analysis, which requires different conditions.Builds on the three `Extraction` jupyter notebooks in this directory.<jupyter_code>%reload_ext autoreload
%autoreload 2
%matplotlib inline
import os
import re
import pandas as pd
import numpy as np

from collections import Counter, defaultdict
import sqlite3
from nltk import word_tokenize
from html.parser import HTMLParser
from tqdm import tqdm
import random
import pickle
import json

from datetime import datetime
from datetime import timezone
from pprint import pprint

import matplotlib.pyplot as plt
import matplotlib.dates as md
import matplotlib
import pylab as pl
from IPython.core.display import display, HTML
from pathlib import Path
git_root_dir = !git rev-parse --show-toplevel
git_root_dir = Path(git_root_dir[0].strip())
git_root_dir
import sys
caringbridge_core_path = "/home/srivbane/levon003/repos/caringbridge_core"
sys.path.append(caringbridge_core_path)
import cbcore.data.paths as paths
import cbcore.data.dates as dates
working_dir = "/home/lana/shared/caringbridge/data/projects/sna-social-support/survival_analysis_interactions"
assert os.path.exists(working_dir)<jupyter_output><empty_output><jupyter_text>Load the site metadata<jupyter_code>site_metadata_filepath = "/home/lana/shared/caringbridge/data/derived/site_metadata/site_metadata.feather"
site_df = pd.read_feather(site_metadata_filepath)
len(site_df)<jupyter_output><empty_output><jupyter_text>Filter by three conditions.<jupyter_code>len(site_df[site_df.visits > 2])
len(site_df[site_df.isSpam.isna()])
len(site_df[(site_df.isDeleted.isna())|(site_df.isDeleted == '0')])
subset_site_df = site_df[(site_df.visits > 2)&(site_df.isSpam.isna())&((site_df.isDeleted.isna())|(site_df.isDeleted == '0'))]
len(subset_site_df)
valid_site_ids = set(subset_site_df.site_id)
len(valid_site_ids)<jupyter_output><empty_output><jupyter_text>Define valid time range<jupyter_code>start_date = datetime.fromisoformat('2014-01-01').replace(tzinfo=timezone.utc)
start_timestamp = int(start_date.timestamp()) * 1000
end_date = datetime.fromisoformat('2019-02-19').replace(tzinfo=timezone.utc)
end_timestamp = int(end_date.timestamp()) * 1000
start_timestamp, end_timestamp<jupyter_output><empty_output><jupyter_text>Filter raw interactions filesFilters each interactions file to include only selected users.We filter first to greatly reduce the size of the data when we subsequently merge.<jupyter_code>source_dir = "/home/lana/shared/caringbridge/data/projects/sna-social-support/csv_data"
gb_filepath = os.path.join(source_dir, "old_gb_metadata_raw.csv")
comments_filepath = os.path.join(source_dir, "new_comments_metadata_raw.csv")
amps_filepath = os.path.join(source_dir, "new_amps_metadata_raw.csv")

gb_filtered_filepath = os.path.join(working_dir, "old_gb_metadata_filtered.csv")
comments_filtered_filepath = os.path.join(working_dir, "new_comments_metadata_filtered.csv")
amps_filtered_filepath = os.path.join(working_dir, "new_amps_metadata_filtered.csv")
for input_filepath, output_filepath in [(comments_filepath, comments_filtered_filepath), 
                                        (amps_filepath, amps_filtered_filepath)]:
    skipped_for_invalid_site_count = 0
    skipped_for_invalid_time_count = 0
    with open(input_filepath, 'r') as infile:
        with open(output_filepath, 'w') as outfile:
            for line in tqdm(infile, desc=os.path.basename(input_filepath)):
                if line.strip() == "":
                    continue
                site_id = int(line.split(",")[1])
                try:
                    created_at = int(line.split(",")[-2])
                except:
                    continue
                if site_id in valid_site_ids:
                    if created_at >= start_timestamp and created_at <= end_timestamp:
                        outfile.write(line)
                    else:
                        skipped_for_invalid_time_count += 1
                else:
                    skipped_for_invalid_site_count += 1
            print(output_filepath)
            print(f"Invalid site: {skipped_for_invalid_site_count}")
            print(f"Invalid time: {skipped_for_invalid_time_count}")

print("Finished filtering files.")<jupyter_output>new_comments_metadata_raw.csv: 31052715it [01:38, 314980.16it/s]
new_amps_metadata_raw.csv: 43469it [00:00, 434686.27it/s]<jupyter_text>Pre- and post-filtering word counts:```   63314738 new_amps_metadata_raw.csv   31052715 new_comments_metadata_raw.csv      36959218 new_amps_metadata_filtered.csv   29970237 new_comments_metadata_filtered.csv```<jupyter_code># proportion of remaining lines for amps and comments respectively
36959218 / 63314738, 29970237 / 31052715<jupyter_output><empty_output><jupyter_text>Read interactions data<jupyter_code>comments_cols = ['user_id','site_id','int_type','journal_oid','created_at','updated_at']
amps_cols = ['user_id','site_id','journal_oid','journal_user_id','created_at','updated_at']
s = datetime.now()
comments_df = pd.read_csv(comments_filtered_filepath, header=None, names=comments_cols)
print(datetime.now() - s)
len(comments_df)
s = datetime.now()
amps_df = pd.read_csv(amps_filtered_filepath, header=None, names=amps_cols)
print(datetime.now() - s)
len(amps_df)<jupyter_output>0:00:32.089755<jupyter_text>Merge with journal data<jupyter_code>amps_df['int_type'] = 'amps'
ints_df = pd.concat([comments_df, amps_df.drop(columns='journal_user_id')], sort=False)
ints_df.reset_index(drop=True, inplace=True)
len(ints_df)
assert len(ints_df) == len(amps_df) + len(comments_df)
ints_df.columns
ints_df.head()
# load the journal metadata
# load the dataframe with the index
s = datetime.now()
journal_metadata_dir = "/home/srivbane/shared/caringbridge/data/derived/journal_metadata"
journal_metadata_filepath = os.path.join(journal_metadata_dir, "journal_metadata.df")
journal_df = pd.read_feather(journal_metadata_filepath)
print(datetime.now() - s)
len(journal_df)
journal_df.head()
journal_df['int_type'] = 'journal'
journal_cols = ['user_id', 'site_id', 'int_type', 'created_at', 'updated_at', 'journal_oid', 'site_index', 'is_nontrivial']
merged_df = pd.concat([ints_df, journal_df[journal_cols]], sort=False)
assert len(merged_df) == len(ints_df) + len(journal_df)
len(merged_df)
merged_df.head()
# construct a map of (user_id) -> set(site_id)
# captures all sites on which a user_id has authored at least a single journal update
author_site_map = defaultdict(set)
for user_id, site_id in tqdm(zip(journal_df.user_id, journal_df.site_id), total=len(journal_df)):
    author_site_map[user_id].add(site_id)
s = datetime.now()
merged_df = merged_df.assign(is_self_interaction=merged_df.apply(lambda row: row.site_id in author_site_map[row.user_id], axis=1))
print(datetime.now() - s)
# quickly verify that journal OIDs are unique
assert len(journal_df) == len(set(journal_df.journal_oid))
journal_oid_index_map = {journal_oid: site_index
    for journal_oid, site_index in zip(journal_df.journal_oid, journal_df.site_index)
}
merged_df.head()
# add site_index info to all interactions with a journal_oid
s = datetime.now()
merged_df.loc[~pd.isnull(merged_df.journal_oid), 'site_index'] = merged_df.loc[~pd.isnull(merged_df.journal_oid), 'journal_oid'].map(lambda journal_oid: journal_oid_index_map[journal_oid] if journal_oid in journal_oid_index_map else -1)
print(datetime.now() - s)
# only 229 journals are reference by a journal oid that we don't have in the journal data
ints_journal_oid_set = set(ints_df.journal_oid)
journals_journal_oid_set = set(journal_df.journal_oid)
len(ints_journal_oid_set - journals_journal_oid_set)
len(merged_df[merged_df.site_index == -1])
merged_df.reset_index(drop=True, inplace=True)
merged_df.head()
user_metadata_dir = "/home/lana/shared/caringbridge/data/projects/sna-social-support/survival_analysis_interactions"
interaction_metadata_filepath = os.path.join(user_metadata_dir, "interaction_metadata.feather")
merged_df.to_feather(interaction_metadata_filepath)
print("Finished.")
s = datetime.now()
user_metadata_dir = "/home/lana/shared/caringbridge/data/projects/sna-social-support/survival_analysis_interactions"
interaction_metadata_filepath = os.path.join(user_metadata_dir, "interaction_metadata.csv")
merged_df.to_csv(interaction_metadata_filepath, index=False)
print(f"{datetime.now() - s}")
print("Finished.")
user_metadata_dir = "/home/lana/shared/caringbridge/data/projects/sna-social-support/survival_analysis_interactions"
interaction_metadata_filepath = os.path.join(user_metadata_dir, "interaction_metadata.feather")
merged_df =  pd.read_feather(interaction_metadata_filepath)
len(merged_df)<jupyter_output><empty_output><jupyter_text>Compute metrics and validate<jupyter_code># test site, that previously had omitted interactions
merged_df[merged_df.site_id == 1031356]
#compute number of amps, comments, and journals in the first 30 days on each site
days30_ms = 1000 * 60 * 60 * 24 * 30
df = merged_df
df.dtypes
s = datetime.now()
df.sort_values(by=['site_id', 'created_at'], inplace=True)
print(f"{datetime.now() - s}")
invalid_created_at = df.created_at <= 0
print(np.sum(invalid_created_at))
df.loc[invalid_created_at, 'created_at'] = df.loc[invalid_created_at, 'updated_at']
print(np.sum(df.created_at <= 0))
df.head()
first_journal_df = df[(df.int_type=='journal')&(df.is_nontrivial == True)]
first_journal_df = first_journal_df.drop_duplicates(subset='site_id')
assert len(first_journal_df) == len(set(first_journal_df.site_id))
site_start_dict = {row.site_id: row.created_at 
                   for row in tqdm(first_journal_df.itertuples(), total=len(first_journal_df))}
site_counts_list = []
curr_site_id = -1
site_start_timestamp = -1
skipped_site_count = 0
for row in tqdm(df.itertuples(), total=len(df)):
    site_id = row.site_id
    if site_id != curr_site_id:
        if curr_site_id != -1:
            if skip_site:
                skipped_site_count += 1
            else:
                assert max_observed_elapsed_time >= 0
                site_counts_list.append({
                    'site_id': curr_site_id,
                    'journal_count': journal_count,
                    'amps_count': amp_count,
                    'comment_count': comment_count,
                    'site_tenure': max_observed_elapsed_time
                })
        
        journal_count = 0
        amp_count = 0
        comment_count = 0
        max_observed_elapsed_time = -1
        skip_site = False
        
        curr_site_id = site_id
        if site_id not in site_start_dict:
            skip_site = True
            site_start_timestamp = 0
        else:
            site_start_timestamp = site_start_dict[site_id]
    elapsed_time = row.created_at - site_start_timestamp
    is_journal_int = row.int_type == 'journal'
    if is_journal_int and (row.created_at < start_timestamp or row.created_at > end_timestamp):
        # this site was created outside the analysis period! we only want to include sites created since 2014
        skip_site = True
    if skip_site:
        continue
    if is_journal_int and elapsed_time > max_observed_elapsed_time:
        max_observed_elapsed_time = elapsed_time
    if elapsed_time <= days30_ms:
        if is_journal_int:
            journal_count += 1
        elif not row.is_self_interaction:
            if row.int_type == 'amps':
                amp_count += 1
            elif row.int_type == 'comment':
                comment_count += 1
if not skip_site:
    site_counts_list.append({
        'site_id': curr_site_id,
        'journal_count': journal_count,
        'amps_count': amp_count,
        'comment_count': comment_count,
        'site_tenure': max_observed_elapsed_time
    })
len(site_counts_list), skipped_site_count
site_counts_df = pd.DataFrame(site_counts_list)
site_counts_df.head()
site_counts_df[site_counts_df.site_id == 1031356]
s = datetime.now()
user_metadata_dir = "/home/lana/shared/caringbridge/data/projects/sna-social-support/survival_analysis_interactions"
interaction_metadata_filepath = os.path.join(user_metadata_dir, "site_counts.csv")
site_counts_df.to_csv(interaction_metadata_filepath, index=False)
print(f"{datetime.now() - s}")
interaction_metadata_filepath
np.quantile(site_counts_df.journal_count, 0.25), np.quantile(site_counts_df.journal_count, 0.5), np.quantile(site_counts_df.journal_count, 0.75), np.quantile(site_counts_df.journal_count, 1)
np.quantile(site_counts_df.amps_count, 0.25), np.quantile(site_counts_df.amps_count, 0.5), np.quantile(site_counts_df.amps_count, 0.75), np.quantile(site_counts_df.amps_count, 1)<jupyter_output><empty_output><jupyter_text>Survival Analysis testing<jupyter_code># consider a site to be right-censored if there was a journal update within X days
right_censor_threshold_days = 365
right_censor_threshold_ms = 1000 * 60 * 60 * 24 * right_censor_threshold_days
right_censor_timestamp = end_timestamp - right_censor_threshold_ms
right_censor_timestamp
journal_df = df[(df.int_type == 'journal')&(df.site_id.isin(valid_site_ids))].copy()
len(journal_df)
journal_df['after_right_censor_threshold'] = journal_df.created_at >= right_censor_timestamp
right_censored_site_ids = set(journal_df[journal_df.after_right_censor_threshold].site_id)
len(right_censored_site_ids) / len(site_counts_df)
site_counts_df['is_right_censored'] = site_counts_df.site_id.map(lambda site_id: site_id in right_censored_site_ids)
site_counts_df['is_not_right_censored'] = ~site_counts_df.is_right_censored
np.sum(site_counts_df.is_right_censored), np.sum(site_counts_df.is_right_censored) / len(site_counts_df)
site_counts_df['interactions_count'] = site_counts_df.amps_count + site_counts_df.comment_count
site_counts_df['interactions_count_log'] = -1
site_counts_df.loc[site_counts_df.interactions_count > 0, 'interactions_count_log'] = np.log10(site_counts_df.loc[site_counts_df.interactions_count > 0, 'interactions_count'])
site_counts_df['journal_count_log'] = 0
site_counts_df.loc[site_counts_df.journal_count > 0, 'journal_count_log'] = np.log10(site_counts_df.loc[site_counts_df.journal_count > 0, 'journal_count'])

site_counts_df['has_received_interactions'] = site_counts_df.interactions_count > 0
site_counts_df['has_received_amps'] = site_counts_df.amps_count > 0
site_counts_df['has_received_comments'] = site_counts_df.comment_count > 0

site_counts_df['amps_count_log'] = 0  # these function as interactions ~ has_received_amps : amps_count_log
site_counts_df.loc[site_counts_df.amps_count > 0, 'amps_count_log'] = np.log10(site_counts_df.loc[site_counts_df.amps_count > 0, 'amps_count'])
site_counts_df['comment_count_log'] = 0
site_counts_df.loc[site_counts_df.comment_count > 0, 'comment_count_log'] = np.log10(site_counts_df.loc[site_counts_df.comment_count > 0, 'comment_count'])

site_counts_df['amps_frequency'] = site_counts_df.amps_count / site_counts_df.journal_count
site_counts_df['comment_frequency'] = site_counts_df.comment_count / site_counts_df.journal_count

site_counts_df['site_tenure_months'] = site_counts_df.site_tenure / (1000 * 60 * 60 * 24 * 30)
# Should Eliminate sites with apparent tenures that are greater than the dataset space
# But this is not a problem after correcting the journal creation times as above
len(site_counts_df[site_counts_df.site_tenure <= end_timestamp - start_timestamp]) / len(site_counts_df)
mdf = site_counts_df[(site_counts_df.site_tenure <= end_timestamp - start_timestamp)&(site_counts_df.journal_count<=100)]
len(mdf)
mdf['journal_comments_interaction'] = mdf.has_received_comments * mdf.journal_count
mdf['journal_amps_interaction'] = mdf.has_received_amps * mdf.journal_count
mdf['journal_frequency'] = pd.cut(mdf.journal_count, bins=[0, 2, np.max(mdf.journal_count)])
mdf = pd.concat([mdf, pd.get_dummies(mdf.journal_frequency, prefix='journal_frequency')], axis=1)
mdf.groupby('journal_frequency').size() / len(mdf)
mdf['journalfreq_comments_interaction'] = mdf.has_received_comments * mdf['journal_frequency_(2, 100]'] #* mdf['journal_frequency_(8, 100]']
mdf['journalfreq_amps_interaction'] = mdf.has_received_amps * mdf['journal_frequency_(2, 100]'] #* mdf['journal_frequency_(8, 100]']
mdf['journalfreq_commentcount_interaction'] = mdf.comment_count_log * mdf['journal_frequency_(2, 100]'] #* mdf['journal_frequency_(8, 100]']
mdf['journalfreq_ampscount_interaction'] = mdf.amps_count_log * mdf['journal_frequency_(2, 100]'] #* mdf['journal_frequency_(8, 100]']
#mdf = mdf[mdf.site_tenure > 0]
#len(mdf)
#mdf['site_tenure_months'] -= 1  # subtract 30 days, as we are predicting amount of time survived BEYOND the 30 day mark
#mdf = mdf[mdf.site_tenure_months >= 0]
#len(mdf)
np.max(mdf.site_tenure_months), len(mdf)
plt.hist(site_counts_df.site_tenure_months, bins=62, log=True)
plt.show()
np.sum(site_counts_df.journal_count >= 30) / len(site_counts_df), np.quantile(site_counts_df.journal_count, 0.999)
site_counts_df[site_counts_df.journal_count > 100]
mdf.head()
from lifelines import KaplanMeierFitter
T = mdf["site_tenure_months"]
E = ~mdf["is_right_censored"]

kmf = KaplanMeierFitter()
kmf.fit(T, event_observed=E)
kmf.plot()
print(kmf.median_survival_time_)
plt.show()
from lifelines import NelsonAalenFitter
naf = NelsonAalenFitter().fit(T, E)
naf.plot()
plt.show()
for right_censor_threshold_days in [0, 30, 90, 365, 365*2]:
    right_censor_threshold_ms = 1000 * 60 * 60 * 24 * right_censor_threshold_days
    right_censor_timestamp = end_timestamp - right_censor_threshold_ms
    print("Threshold (days):", right_censor_threshold_days)

    censored_journal_subset = journal_df[journal_df.created_at >= right_censor_timestamp]
    if len(censored_journal_subset) > 0:
        right_censored_site_ids = set(censored_journal_subset.site_id)
    else:
        right_censored_site_ids = set()

    is_right_censored = site_counts_df.site_id.map(lambda site_id: site_id in right_censored_site_ids)
    print("pct of sites considered right-censored:", np.sum(is_right_censored) / len(site_counts_df))

    kmf = KaplanMeierFitter()
    kmf.fit(site_counts_df.site_tenure_months, event_observed=~is_right_censored)
    median_survival_time = kmf.median_survival_time_
    print("KM median survival time:", median_survival_time)
# empirical observed tenure is well under a month
np.median(site_counts_df.site_tenure_months)
# a full 41% of the data have only a single journal update (so tenure = 0)
np.sum(site_counts_df.site_tenure_months == 0) / len(site_counts_df)
kmf = KaplanMeierFitter()
kmf.fit(site_counts_df.site_tenure_months, timeline=range(13), event_observed=np.ones(len(site_counts_df)))
kmf.plot()
kmf.median_survival_time_
from lifelines import CoxPHFitter
# assume no censoring
cph = CoxPHFitter()
cph.fit(mdf[['site_tenure_months', 'has_received_amps', 'has_received_comments', 'journal_count']], 
        duration_col='site_tenure_months',
        show_progress=True)
cph.print_summary()
cph.plot()
plt.show()
cph = CoxPHFitter()
cph.fit(mdf[['site_tenure_months', 'is_not_right_censored', 'has_received_amps', 'has_received_comments', 'journal_frequency_(2, 100]', 'amps_count_log', 'comment_count_log', 'journalfreq_comments_interaction', 'journalfreq_amps_interaction']], 
        duration_col='site_tenure_months', event_col='is_not_right_censored',
        show_progress=True)  # default step_size=0.95, can change to 0.5 or lower if needed
cph.print_summary()
cph.plot()
plt.show()
fig, ax = plt.subplots(1, 1, figsize=(12,12))
cph.plot_covariate_groups('journal_count', [1, 2, 5, 10, 30], cmap='coolwarm', ax=ax)
plt.show()
fig, ax = plt.subplots(1, 1, figsize=(12,12))
cph.plot_covariate_groups('has_received_comments', [0, 1], cmap='coolwarm', ax=ax)
plt.show()
fig, ax = plt.subplots(1, 1, figsize=(12,12))
cph.plot_covariate_groups(['journal_count', 'has_received_comments'], [(1, 0), (10, 0), (30, 0), (1, 1), (10, 1), (30, 1)], cmap='coolwarm', ax=ax)
plt.show()
fig, ax = plt.subplots(1, 1, figsize=(12,12))
cph.plot_covariate_groups(['journal_count', 'has_received_amps'], [(1, 0), (10, 0), (30, 0), (1, 1), (10, 1), (30, 1)], cmap='coolwarm', ax=ax)
plt.show()
fig, ax = plt.subplots(1, 1, figsize=(12,12))
cph.plot_covariate_groups(['has_received_comments', 'has_received_amps'], [(0, 0), (1, 0), (0, 1), (1, 1)], cmap='coolwarm', ax=ax)
plt.show()
fig, ax = plt.subplots(1, 1, figsize=(12,12))
cph.plot_covariate_groups('interactions_count_log', [0, 1, 5, 10], cmap='coolwarm', ax=ax)
plt.show()
mdf['amp_ratio'] = 0
mdf.loc[mdf.interactions_count > 0, 'amp_ratio'] = \
    (mdf.loc[mdf.interactions_count > 0, 'amps_count'] / mdf.loc[mdf.interactions_count > 0, 'interactions_count']) - 0.5
plt.hist(mdf.amp_ratio, log=True)
plt.show()
cph = CoxPHFitter()
cph.fit(mdf[['site_tenure_months', 'is_not_right_censored', 'journal_count', 'interactions_count', 'amp_ratio']], 
        duration_col='site_tenure_months', event_col='is_not_right_censored',
        show_progress=True)  # default step_size=0.95, can change to 0.5 or lower if needed
cph.print_summary()
cph.plot()
plt.show()
fig, ax = plt.subplots(1, 1, figsize=(12,12))
cph.plot_covariate_groups(['journal_count', 'amp_ratio'], [(1, -0.5), (10, -0.5), (30, -0.5), (1, 0.5), (10, 0.5), (30, 0.5)], cmap='coolwarm', ax=ax)
plt.show()
np.quantile(mdf.journal_count, 0.25), np.quantile(mdf.journal_count, 0.5), np.quantile(mdf.journal_count, 0.75)
mdf['journal_frequency'] = pd.cut(mdf.journal_count, bins=[0, 2, 8, np.max(mdf.journal_count)])
mdf.journal_frequency
# want to evaluate high vs low activity sites
cph = CoxPHFitter()
cph.fit(mdf[['site_tenure_months', 'is_not_right_censored', 'journal_frequency', 'interactions_count_log', 'has_received_comments', 'has_received_amps']], 
        duration_col='site_tenure_months', event_col='is_not_right_censored', strata='journal_frequency',
        show_progress=True)  # default step_size=0.95, can change to 0.5 or lower if needed
cph.print_summary()
cph.plot()
plt.show()
#fig, axes = plt.subplots(1, 3, figsize=(12,12))
cph.plot_covariate_groups('has_received_amps', [0, 1], cmap='coolwarm')
plt.show()
cph.predict_survival_function(pd.Series({
    'journal_frequency': mdf.groupby('journal_frequency').size().index[0],
    'has_received_amps': 0,
    'has_received_comments': 0,
    'is_not_right_censored': 1,
    'interactions_count_log': -1
}), [0, 1, 12]), \
cph.predict_survival_function(pd.Series({
    'journal_frequency': mdf.groupby('journal_frequency').size().index[0],
    'has_received_amps': 1,
    'has_received_comments': 1,
    'is_not_right_censored': 1,
    'interactions_count_log': np.log10(2)
}), [0, 1, 12])
0.325109 - 0.068964, 0.125701 - 0.007185
cph.predict_survival_function(pd.Series({
    'journal_frequency': mdf.groupby('journal_frequency').size().index[2],
    'has_received_amps': 0,
    'has_received_comments': 0,
    'is_not_right_censored': 1,
    'interactions_count_log': -1
}), [0, 1, 12]), \
cph.predict_survival_function(pd.Series({
    'journal_frequency': mdf.groupby('journal_frequency').size().index[2],
    'has_received_amps': 1,
    'has_received_comments': 1,
    'is_not_right_censored': 1,
    'interactions_count_log': np.log10(2)
}), [0, 1, 12])
0.644178 - 0.351100, 0.095447 - 0.003731
mdf.groupby('journal_frequency').size()
mdf = pd.concat([mdf, pd.get_dummies(mdf.journal_frequency, prefix='journal_frequency')], axis=1)
mdf.head(1)
# want to evaluate high vs low activity sites
cph = CoxPHFitter()
cph.fit(mdf[['site_tenure_months', 'is_not_right_censored', 'journal_frequency_(2, 8]', 'journal_frequency_(8, 100]', 'comment_count_log', 'amps_count_log', 'has_received_comments', 'has_received_amps']], 
        duration_col='site_tenure_months', event_col='is_not_right_censored',
        show_progress=True)  # default step_size=0.95, can change to 0.5 or lower if needed
cph.print_summary()
cph.plot()
plt.show()
cph.predict_survival_function(pd.Series({
    'journal_frequency_(2, 8]': 0, 
    'journal_frequency_(8, 100]': 0,
    'has_received_amps': 0,
    'has_received_comments': 0,
    'is_not_right_censored': 1,
    'interactions_count_log': -1
}), [0, 1, 6]), \
cph.predict_survival_function(pd.Series({
    'journal_frequency_(2, 8]': 0, 
    'journal_frequency_(8, 100]': 0,
    'has_received_amps': 1,
    'has_received_comments': 1,
    'is_not_right_censored': 1,
    'interactions_count_log': np.log10(2)
}), [0, 1, 6])
0.235869 - 0.018166, 0.011384 - 0.000004
cph.predict_survival_function(pd.Series({
    'journal_frequency_(2, 8]': 0, 
    'journal_frequency_(8, 100]': 1,
    'has_received_amps': 0,
    'has_received_comments': 0,
    'is_not_right_censored': 1,
    'interactions_count_log': -1
}), [0, 1, 6]), \
cph.predict_survival_function(pd.Series({
    'journal_frequency_(2, 8]': 0, 
    'journal_frequency_(8, 100]': 1,
    'has_received_amps': 1,
    'has_received_comments': 1,
    'is_not_right_censored': 1,
    'interactions_count_log': np.log10(2)
}), [0, 1, 6])
0.629923 - 0.277368, 0.367033 - 0.061963<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12433016076141726, 'fraction_numerical': 0.030072782532192272, 'mean_word_length': 5.677436582109479, 'pattern_counts': {'":': 1, '<': 48, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '259612', 'license': 'MIT', 'n_tokens_mistral': 9749, 'n_tokens_neox': 8937, 'n_words': 1563, 'path': 'data_pulling/FilterAndMergeExtractedInteractions-SurvivalAnalysis.ipynb', 'repo_name': 'levon003/cscw-caringbridge-interaction-network', 'size': '25010'}