<jupyter_start><jupyter_text>Explore AppleHealthKit data<jupyter_code>import numpy as np
import pandas as pd
#import modin.pandas as pd
df = pd.read_csv('datasample.csv')
df.drop(columns=['#'], inplace=True)
df.head(2)<jupyter_output><empty_output><jupyter_text>Primary objective: To clean up the data with 23 million rows<jupyter_code>print(f'No of rows: {df.shape[0]}, No of columns: {df.shape[1]}')
# 2# Owner column name is little hard to read, let's label the owner with easy id
dicti = {}
x = 0
for i in df['owner'].unique():
    dicti[i] = x
    x += 1
df['easy_id'] = df['owner'].map(dicti)

# 3# Source needs to be cleaned: I am interested to know where the data is coming from
a = df['source'].nunique()

y = df['source'].str.split('healthkit:com.')
z = []
for i in y:
    try:
        string = i[1]
        str2 = string.split('.')
        str3 = str2[0]
        z.append(str3)
    except:
        continue

df['source'] = pd.Series(z)
b = df['source'].nunique()
print('No of unique source before cleaning: ', a)
print('No of unique source after cleaning: ', b)
# 4# sourceid very similar to _id # we can remove it
# 5# sourcename-user-friendly name for source of data eg-Fitbit, we get that from cleaned source column already
# 6# Type will be important for user-segmentation
df['type'].value_counts()
# 7#8# startdate, enddate would be used as a datetime column to aggregate activity /month/week etc
# 9# value column will be improtant to understand how active is the user
# 10# originaldata-remove no information
# 11#12# createdat,updatedat: will be remove- information will come from startdate and enddate anyways

# 13# Timezone is important to understand user demographics.
df['continent'] = df['timezone'].str.split('/').apply(lambda x: x[0])
df['city'] = df['timezone'].str.split('/').apply(lambda x: x[1])

# 14#15#16#18 # Only NaN values in the column
df[['mood', 'tags', '_fivetran_deleted', 'content']].nunique()
cols = ['sourceid', 'sourcename', 'createdat', 'updatedat', 'originaldata', 'timezone', 'mood', 'tags',
        '_fivetran_deleted', '_fivetran_synced', 'content']
df_clean = df.drop(columns=cols)
# arrange columns in a meaningful way
cols = ['_id', 'owner', 'easy_id', 'source', 'type',
        'startdate', 'enddate', 'value', 'continent', 'city']
df_clean = df_clean[cols]

# remove missing values in source column as 'Missing'
df_clean['source'].fillna('Missing', inplace=True)
df_clean.isna().sum().sum()
print(
    f'Before cleaning: No of rows: {df.shape[0]}, No of columns: {df.shape[1]}')
print(
    f'After cleaning: No of rows:  {df_clean.shape[0]}, No of columns: {df_clean.shape[1]}')
df_clean.to_csv("df_clean.csv", index=False)
df_clean_ = pd.read_csv("df_clean.csv", parse_dates=["startdate", "enddate"])
df_clean_.head(2)
######## Next: 2.Aggregation_AppleHealthKitData_G : aggregate AppleHealthKit Data ##########<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1570048309178744, 'fraction_numerical': 0.015527950310559006, 'mean_word_length': 4.977319587628866, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '149317', 'license': 'MIT', 'n_tokens_mistral': 1013, 'n_tokens_neox': 979, 'n_words': 304, 'path': 'Project/Part A. AppleHealthKitData Analysis/1.Cleaning_AppleHealthKitData_G.ipynb', 'repo_name': 'SayantanMitra87/OngoBoost_Insight_Project', 'size': '2898'}