<jupyter_start><jupyter_text>Examine time series for individual patients and quantify how much variance exists from which to make predictions. Upgrade libraries.<jupyter_code># !pip install dask --upgrade
# !pip install dask[complete]
# !pip install 'fsspec>=0.3.3'
# !pip install s3fs --upgrade
# !pip install msgpack==0.5.6<jupyter_output><empty_output><jupyter_text>Install to keep track of long running cell's progress<jupyter_code># !pip install tqdm<jupyter_output><empty_output><jupyter_text>Import packages.<jupyter_code>import boto3
import numpy as np
import dask.dataframe as dd
from dask.distributed import Client
from tqdm import tqdm
import pandas as pd
from sagemaker import get_execution_role
from d_types import dtypes
import s3fs

client = Client()
client
pd.set_option('display.max_columns', 250)
pd.set_option('display.max_rows', 250)<jupyter_output><empty_output><jupyter_text>Import data to dask dataframe.<jupyter_code>role = get_execution_role()

bucket='vbhdata'
data_key = '<KEY>'
data_location = 's3://{}/{}'.format(bucket, data_key)

df = dd.read_csv(data_location,
                 sep='|',
#                  parse_dates=['_Date'],
                 dtype=dtypes,
                 blocksize=64000000,
                 low_memory=False)
df.head()<jupyter_output><empty_output><jupyter_text>Get only what we need to isolate patients who have been hospitalized.<jupyter_code>df1 = df[['_Date', 'Clientid', 'Admissions']]
df1 = df1.dropna()
df1.head()
df1 = client.persist(df1)
df1 = df1.compute()<jupyter_output><empty_output><jupyter_text>Check for null values<jupyter_code>df2 = df1.isna().sum()
df2<jupyter_output><empty_output><jupyter_text>Filter for hopsitalized patients.<jupyter_code>hosp_df = df1[df1['Admissions'] == 1.0]
hosp_df = hosp_df.reset_index(drop=True)
hosp_df.head()<jupyter_output><empty_output><jupyter_text>Get list of clientid's of hospitalized pts and filter whole dataset for them.<jupyter_code>hosp_clients = hosp_df['Clientid'].unique()
hosp_clients_df = df[df['Clientid'].isin(hosp_clients)]
hosp_clients_df = client.persist(hosp_clients_df)
hosp_clients_df = hosp_clients_df.compute()
hosp_clients_df.head()
hosp_clients_df.shape<jupyter_output><empty_output><jupyter_text>Store dataframe for each client in a dict.<jupyter_code>hosp_clients_dict = {clientid: hosp_clients_df[hosp_clients_df['Clientid'] == clientid].set_index('_Date')
                     for clientid in tqdm(hosp_clients.tolist())}
def total_changes(df):
    """Total all changes in given client dataframe."""
    sums = []
    length = len(df)
    for column in df.columns:
        # Skip Admissions column, we need changes that may predict this.
        if column == 'Admissions':
            continue
        total = df[column].value_counts()
        if not total.empty:
            # Check that there are more than one value in the entire column.
            sums.append(total.values[0] != length)
    # Sum of all columns with multiple values.
    return sum(sums)<jupyter_output><empty_output><jupyter_text>Test it out on a few client's dataframes.<jupyter_code>total_changes(hosp_clients_dict[hosp_clients[1]])
total_changes(hosp_clients_dict[hosp_clients[2]])
total_changes(hosp_clients_dict[hosp_clients[3]])<jupyter_output><empty_output><jupyter_text>Get total number of columns containing more than one value for all clients.<jupyter_code>changes = [total_changes(hosp_clients_dict[client]) for client in tqdm(hosp_clients)]
sum(changes)<jupyter_output>100%|██████████| 1642/1642 [07:52<00:00,  3.47it/s]<jupyter_text>Total columns with multiple values per client.<jupyter_code>sum(changes) / len(changes)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11356209150326797, 'fraction_numerical': 0.01770152505446623, 'mean_word_length': 5.477954144620811, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '179621', 'license': 'MIT', 'n_tokens_mistral': 1330, 'n_tokens_neox': 1270, 'n_words': 269, 'path': 'Isolate_time_series.ipynb', 'repo_name': 'valley-behavioral-health/Data-Science', 'size': '3683'}