<jupyter_start><jupyter_text>Cancer Site Analysis===<jupyter_code>%matplotlib inline
from IPython.core.display import display, HTML

import os
import numpy as np
import pandas as pd
import itertools

import matplotlib.pyplot as plt
import matplotlib.dates as md
import matplotlib
import pylab as pl

import datetime as dt
from datetime import datetime
import pytz
import time

from collections import Counter

import json
import os
import re
from html.parser import HTMLParser
import itertools
import multiprocessing as mp
from nltk import word_tokenize
from IPython.core.display import display, HTML
import datetime as dt
pd.read_feather
working_dir = "/home/srivbane/shared/caringbridge/data/projects/qual-health-journeys/extract_site_features"
feathered_site_df_filename = os.path.join(working_dir, "site_scrubbed.df")
df = pd.read_feather(feathered_site_df_filename)
min(df["createdAt"])
len(df[df["healthCondition_category"] == "Cancer"])
cancer_df = df[df["healthCondition_category"] == "Cancer"]
np.sum(cancer_df["numJournals"])
# load the actual counts
filename = "/home/srivbane/shared/caringbridge/data/raw/../projects/classify_health_condition/vw_experiments/site_survival_time.csv"
count_df = pd.read_csv(filename, header=None, names=["_id", "created_at_utc", "updated_at_utc", "first_journal_timestamp", "last_journal_timestamp", "numJournals", "trueNumJournals"])
count_df.head()
merged = pd.merge(cancer_df, count_df, how='left', on="_id")
merged[["numJournals_x", "numJournals_y", "trueNumJournals"]].describe()
np.sum(merged["trueNumJournals"])
utc1990 = 631152000
utc2000 = 946684800
of_interest = merged[(merged["trueNumJournals"] >= 3) & (merged["last_journal_timestamp"] >= utc1990) & (merged["first_journal_timestamp"] >= utc1990)]
len(of_interest)
# This should be the duration in years of the longest site, but it seems way too long...
np.max(of_interest["last_journal_timestamp"] - of_interest["first_journal_timestamp"]) / 60 / 60 / 24 / 365
of_interest["survival_time"] = of_interest["last_journal_timestamp"] - of_interest["first_journal_timestamp"]
longest_cancer_sites = of_interest[["_id", "created_at_utc", "updated_at_utc", "first_journal_timestamp", "last_journal_timestamp", "survival_time"]]
longest_cancer_sites = longest_cancer_sites.sort_values(by="survival_time", ascending=False)
longest_cancer_sites.head()
def convert_dates(created_at_utc):
    if created_at_utc < utc1990:
        return "%.2f years" % (created_at_utc / (60 * 60 * 24 * 365))
    
    unaware_datetime = datetime.utcfromtimestamp(created_at_utc)
    utc_datetime = unaware_datetime.replace(tzinfo=pytz.UTC)
    cst_datetime = utc_datetime.astimezone(tz=pytz.timezone("America/Chicago"))
    
    datetime_string = cst_datetime.strftime("%Y-%m-%d")
    return datetime_string

str_longest_df = longest_cancer_sites.copy()
for col in ['created_at_utc', 'updated_at_utc', 'first_journal_timestamp', 'last_journal_timestamp', 'survival_time']:
    str_longest_df[col] = str_longest_df[col].map(convert_dates)
str_longest_df.head(n=30)
len(longest_cancer_sites[longest_cancer_sites["created_at_utc"] > longest_cancer_sites["first_journal_timestamp"]])
# median survival time
convert_dates(np.median(of_interest["survival_time"]))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11638591117917305, 'fraction_numerical': 0.018376722817764167, 'mean_word_length': 7.709333333333333, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '435486', 'license': 'MIT', 'n_tokens_mistral': 1184, 'n_tokens_neox': 1100, 'n_words': 212, 'path': 'identify_candidate_sites/CancerSiteAnalysis.ipynb', 'repo_name': 'levon003/icwsm-cancer-journeys', 'size': '3265'}