<jupyter_start><jupyter_text>1. Read in data/drop columns to reduce size<jupyter_code>DATA_DIR = os.getenv("DATA_DIR")
REPORTS_DIR = os.getenv("REPORTS_DIR")
filename = "dlo_prelim_meta_standard_with_pageseq_from_29-10_to_04-11-2018.csv.gz"
# filename = "brexit_taxon_29_04.csv.gz"
# filename = "reduced_preprocessed_taxon_pageseq_20190114_20190116.csv.gz"
df_file = os.path.join(DATA_DIR, "processed_journey", filename)
df_reduced_file = os.path.join(DATA_DIR, "processed_journey", filename)
# df_rel_file = os.path.join(DATA_DIR, "processed_journey", "rel_"+filename)<jupyter_output><empty_output><jupyter_text>Load up unfiltered data: 14-16/01/19, one off run Original dataset shape (5048130, 17)<jupyter_code># df = pd.read_csv(df_file, sep="\t", compression = "gzip")
# print(df.shape)
# df.drop(['Dates', 'Page_List', 'Event_List', 'Taxon_List', 'Taxon_Page_List', 'Page_List_NL'],axis=1,inplace=True)
# df.to_csv(df_reduced_file, sep="\t", compression="gzip", index=False)<jupyter_output><empty_output><jupyter_text>2. User journey analysis<jupyter_code># sep="\t",
df = pd.read_csv(df_reduced_file, compression="gzip")
## Total volume of traffic
df.Occurrences.sum()
def column_eval(cols):
    for column in cols:
        if not isinstance(df[column].iloc[0],list):
            print(column)
            df[column] = df[column].map(literal_eval)
# 'Page_Event_List', 
columns = ['DeviceCategories', 'Event_cats_agg', 'Event_cat_act_agg']
column_eval(columns)<jupyter_output>DeviceCategories
Event_cats_agg
Event_cat_act_agg<jupyter_text>For older data cuts that do not include `PageSeq_Length`<jupyter_code>if "PageSeq_Length" not in df.columns:
    df['Page_List'] = df['Page_List'].map(literal_eval)
    df['PageSeq_Length'] = df['Page_List'].map(len)<jupyter_output><empty_output><jupyter_text>2.1 Out of all journeys, what is the % (volume, sum of Occurrences) of journeys including "related content" link clicks?<jupyter_code>n_rel = df[(df.Sequence.str.contains("Related content")) & (df.Sequence.str.contains("relatedLinkClicked"))].shape[0]
# 473975
"{}% of journeys that contain a \"related link\" click".format(round((n_rel*100)/df.Occurrences.sum(),2))<jupyter_output><empty_output><jupyter_text>Run this at some point to compare against "related content" link-specific dataset. There seems to be an indication that there are no major differences  2.2 Out of all journeys coming from mobiles vs desktops, what is the % (volume, sum of Occurrences) of journeys including "related content" link clicks?<jupyter_code>def more_device(x,device):
    return max(x,key=itemgetter(1))[0]==device 
def device_count(x,device):
    return sum([value for item, value in x if item==device])      
df["DesktopCount"] = df['DeviceCategories'].map(lambda x: device_count(x,"desktop"))
df["MobileCount"] = df['DeviceCategories'].map(lambda x: device_count(x,"mobile"))        
df["TabletCount"] = df['DeviceCategories'].map(lambda x: device_count(x,"tablet"))        
df["DesktopCount"].describe()<jupyter_output><empty_output><jupyter_text>2.3 Traffic volume Test implementation<jupyter_code>related = df[(df.Sequence.str.contains("Related content")) & (df.Sequence.str.contains("relatedLinkClicked"))].iloc[0]
related
explore = df[~(df.Sequence.str.contains("Related content")) & (df.Sequence.str.contains("relatedLinkClicked"))].iloc[0]
explore
explore.Event_cat_act_agg
#Compute whether a journey includes a related link click
def is_related(x):
    return all(cond in x for cond in ["relatedLinkClicked","Related content"])
print(is_related(related.Sequence))
print(is_related(df.Sequence.iloc[0]))
print(is_related(explore))
df["Has_Related"] = df["Sequence"].map(is_related)
df[df["Has_Related"]].Occurrences.sum()<jupyter_output><empty_output><jupyter_text>2.4 Journeys per deviceCounter({'desktop': 256791, 'tablet': 55546, 'mobile': 161638})<jupyter_code>index = ["All","All_related","Desktop","Desktop_rel","Mobile","Mobile_rel"]
columns = ["Volume", "Percentage", "Shape"]
df_stats = pd.DataFrame(index=index,columns=columns)<jupyter_output><empty_output><jupyter_text>Compute all volumes<jupyter_code>def compute_all_stats(df, desktop_journeys, filtering, df_stats):
    vol_all = df.Occurrences.sum()
    vol_all_related = df[df.Has_Related].Occurrences.sum()
    ## Number of journeys coming from desktops
    vol_desk = df["DesktopCount"].sum()
    ## Number of journeys coming from mobiles
    vol_mobile = df["MobileCount"].sum()
    ## Number of journeys coming from tablets
    vol_tablet = df["TabletCount"].sum()

    ## Seperate out desktop and mobile journeys
    desktop_journeys = df[df.DesktopCount>0]
    mobile_journeys = df[df.MobileCount>0]

    desk_rel_journeys = desktop_journeys[desktop_journeys.Has_Related]
    mobile_rel_journeys = mobile_journeys[mobile_journeys.Has_Related]

    ## Compute number of journeys from specific device that include related links
    ## Don't base counting on occurrences, will include excluded device
    vol_desk_rel = desktop_journeys[desktop_journeys.Has_Related].DesktopCount.sum()
    vol_mobile_rel = mobile_journeys[mobile_journeys.Has_Related].MobileCount.sum()
    
    vols = [vol_all, vol_all_related,
        vol_desk, vol_desk_rel,
        vol_mobile, vol_mobile_rel]
    
    percent_related = round((vol_all_related*100)/vol_all,2)
    percent_from_desk = round((vol_desk*100)/df.Occurrences.sum(),2)
    percent_from_mobile = round((vol_mobile*100)/df.Occurrences.sum(),2)
    percent_from_tablet = round((vol_tablet*100)/df.Occurrences.sum(),2)
    percent_from_desk_rel = round((vol_desk_rel*100)/vol_desk,2)
    percent_from_mobile_rel = round((vol_mobile_rel*100)/vol_mobile,2)
    
    percents = [100, percent_related,
            percent_from_desk, percent_from_desk_rel,
            percent_from_mobile, percent_from_mobile_rel]
    
    shape_all = df.shape[0]
    shape_all_rel = df[df.Has_Related].shape[0]
    shape_desk = desktop_journeys.shape[0]
    shape_desk_rel = desktop_journeys[desktop_journeys.Has_Related].shape[0]
    shape_mobile = mobile_journeys.shape[0]
    shape_mobile_rel = mobile_journeys[mobile_journeys.Has_Related].shape[0]
   
    shapes = [shape_all, shape_all_rel,
         shape_desk, shape_desk_rel,
         shape_mobile, shape_mobile_rel]
    
    df_stats["Volume"] = vols
    df_stats["Percentage"] = percents
    df_stats["Shape"] = shapes
    
    df_stats.to_csv(os.path.join(REPORTS_DIR,filename.replace(".csv.gz",filtering)+"_stats.csv"))
    
    print("{}% of journeys come from a desktop".format(percent_from_desk))
    print("{}% of journeys come from a mobile".format(percent_from_mobile))
    print("{}% of journeys come from a tablet".format(percent_from_tablet))
    print("{}% of desktop journeys include a \"Related content\" click".format(percent_from_desk_rel))
    print("{}% of mobile journeys include a \"Related content\" click".format(percent_from_mobile_rel))
compute_all_stats(df,"",df_stats)
df_stats
df_stats
compute_all_stats(df[df.PageSeq_Length>1], "_dlo",df_stats)
df_stats<jupyter_output><empty_output><jupyter_text>2.5 Chi-squared testCompute observations for a contingency table<jupyter_code>from scipy import stats
vol_desk = df_stats.loc['Desktop','Volume']
vol_mobile = df_stats.loc['Mobile','Volume']

vol_mobile_rel = df_stats.loc['Mobile_rel','Volume']
vol_mobile_no_rel = vol_mobile - vol_mobile_rel

vol_desk_rel = df_stats.loc['Desktop_rel','Volume']
vol_desk_no_rel = vol_desk - vol_desk_rel
def chi2_test(obs):
    chi2, p, dof, ex = stats.chi2_contingency(obs)
    print(chi2, p, dof, ex)
    
#     g, p, dof, expctd = stats.chi2_contingency(obs, lambda_="log-likelihood")

    with open(os.path.join(REPORTS_DIR, filename.replace(".csv.gz","")+"_chi2.csv"),"w") as f:
        f.write("chi2,p,dof\n")
        f.write("{},{},{}".format(chi2, p, dof))
obs = [[vol_mobile_rel, vol_mobile_no_rel], [vol_desk_rel, vol_desk_no_rel]]
obs
chi2_test(obs)
ch = pd.read_csv(os.path.join(REPORTS_DIR, filename.replace(".csv.gz","")+"_chi2.csv"))
ch<jupyter_output><empty_output><jupyter_text>2.6 User journey length<jupyter_code>def weight_seq_length(page_lengths, occurrences, name):
    length_occ = Counter()
    for length,occ in zip(page_lengths, occurrences):
           length_occ[length]+=occ
    data = []
    for key,value in length_occ.items():
        for i in range(value):
            data.append(key)
    return pd.Series(data,name=name)<jupyter_output><empty_output><jupyter_text>Everything together<jupyter_code>def describe_dfs(to_eval):
    descriptive = pd.DataFrame()
    for length,occ,name in to_eval:
        sr = weight_seq_length(length,occ,name).describe().apply(lambda x: format(x, '.3f'))
        descriptive[sr.name] = sr
    return descriptive
list_of_cols = [[df.PageSeq_Length, df.Occurrences, "All_Journeys"],
                [df[df.Has_Related].PageSeq_Length, df[df.Has_Related].Occurrences, "All_Journeys_Related"],
                [desktop_journeys.PageSeq_Length, desktop_journeys.DesktopCount,"All_Desktop"],
                [mobile_journeys.PageSeq_Length, mobile_journeys.MobileCount, "All_Mobile"],
                [desk_rel_journeys.PageSeq_Length, desk_rel_journeys.DesktopCount, "Desktop_Related"],
                [mobile_rel_journeys.PageSeq_Length, mobile_rel_journeys.MobileCount, "Mobile_Related"]]
describe = describe_dfs(list_of_cols)
describe.to_csv(os.path.join(REPORTS_DIR,filename.replace(".csv.gz","")+"_pageseq_length_all.csv"),index=False)
describe<jupyter_output><empty_output><jupyter_text>Exclude user journeys of length 1<jupyter_code>df_dlo = df[df.PageSeq_Length > 1]
## Seperate out desktop and mobile journeys
desktop_journeys_dlo = df_dlo[df_dlo.DesktopCount>0].copy(deep=True)
mobile_journeys_dlo =  df_dlo[df_dlo.MobileCount>0].copy(deep=True)
desk_rel_journeys_dlo = desktop_journeys_dlo[desktop_journeys_dlo.Has_Related]
mobile_rel_journeys_dlo = mobile_journeys_dlo[mobile_journeys_dlo.Has_Related]
list_of_columns = [[df_dlo.PageSeq_Length, df_dlo.Occurrences, "All_Journeys"],
                   [df_dlo.PageSeq_Length, df_dlo.Occurrences, "All_Journeys_Related"],
                   [desktop_journeys_dlo.PageSeq_Length, desktop_journeys_dlo.DesktopCount,"All_Desktop"],
                   [mobile_journeys_dlo.PageSeq_Length, mobile_journeys_dlo.MobileCount, "All_Mobile"],
                   [desk_rel_journeys_dlo.PageSeq_Length, desk_rel_journeys_dlo.DesktopCount, "Desktop_Related"],
                   [mobile_rel_journeys_dlo.PageSeq_Length, mobile_rel_journeys_dlo.MobileCount, "Mobile_Related"]]
describe_dlo = describe_dfs(list_of_cols)
describe_dlo.to_csv(os.path.join(REPORTS_DIR,filename.replace(".csv.gz","")+"_pageseq_length_dlo.csv"),index=False)<jupyter_output><empty_output><jupyter_text>2.7 Event frequencies overall<jupyter_code>df.columns
event_cats = Counter()
for event_agg in df.Event_cats_agg:
    for event,value in event_agg:
        event_cats[event] += value
len(event_cats.keys())
for key,value in event_cats.items():
    print(key,":",value)
# def mem_usage(pandas_obj):
#     if isinstance(pandas_obj,pd.DataFrame):
#         usage_b = pandas_obj.memory_usage(deep=True).sum()
#     else: # we assume if not a df it's a series
#         usage_b = pandas_obj.memory_usage(deep=True)
#     usage_mb = usage_b / 1024 ** 2 # convert bytes to megabytes
#     return "{:03.2f} MB".format(usage_mb)

# gl_int = df.select_dtypes(include=['int'])
# converted_int = gl_int.apply(pd.to_numeric,downcast='unsigned')

# print(mem_usage(gl_int))
# print(mem_usage(converted_int))

# compare_ints = pd.concat([gl_int.dtypes,converted_int.dtypes],axis=1)
# compare_ints.columns = ['before','after']
# compare_ints.apply(pd.Series.value_counts)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1305941522461853, 'fraction_numerical': 0.013639075952604211, 'mean_word_length': 5.78150289017341, 'pattern_counts': {'":': 1, '<': 56, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '231576', 'license': 'MIT', 'n_tokens_mistral': 4309, 'n_tokens_neox': 4068, 'n_words': 662, 'path': 'notebooks/eda/archived/related_links_eda_scriptify.ipynb', 'repo_name': 'alphagov-mirror/govuk-network-data', 'size': '11731'}