<jupyter_start><jupyter_text>Filtering Outlier POI from Patterns<jupyter_code>%load_ext autoreload
%autoreload 2
import sys
import os

module_path = os.path.abspath(os.path.join('..'))
if module_path not in sys.path:
    sys.path.append(module_path)
import pandas as pd
import numpy as np
from src.safegraph_eval.preprocessing import preprocessing
# ingest sample Monthly Patterns data from lowe's for 2021 Jan to Sep downloaded from shop
# https://shop.safegraph.com/?brands=SG_BRAND_4f8a39fb928268edb1ab256922e436d7&countries=US&poi=ALL&tab=datasets&cart=sg_p-202100-202108
patterns = pd.DataFrame()
for month in range(1,10,1):
    _ = pd.read_csv(f"data/Lowes-PATTERNS-2021_0{month}-2021-10-28/patterns.csv")
    patterns = patterns.append(_,ignore_index=True)
display(patterns.head()) # each row is Patterns data for a Lowe's POI for a specific month
# How many unique Lowe's have Patterns for this time period?
patterns["placekey"].drop_duplicates().size 
print(preprocessing.label_and_remove_outliers_iqr.__doc__)
patterns_filtered_default_params = preprocessing.label_and_remove_outliers_iqr(
    patterns, group_column = "brands", column_to_filter = "raw_visit_counts")
display(patterns_filtered_default_params.head())
# How many unique Lowe's remain?
patterns_filtered_default_params["placekey"].drop_duplicates().size 
# Which plackeys were removed?
removed_pks_default_params = set(patterns["placekey"]).difference(patterns_filtered_default_params["placekey"])
print(removed_pks_default_params)
# Let's look at one:
patterns.loc[patterns["placekey"]=="222-222@5p2-2tw-vs5",
             ["date_range_start","street_address", "city", "region", "raw_visit_counts"]]
# Raw visit counts are in the 1000s, so it was filtered out for being too high relative to the distribution of visits at other Lowe's.
# Let's say we think these are valid and want to increase the bound of what is considered an "outlier".

# We can increase the k parameter in the filtering function from 1.5 which is the default.
patterns_filtered_higher_k = preprocessing.label_and_remove_outliers_iqr(
    patterns, group_column = "brands", column_to_filter = "raw_visit_counts", k = 2)
# Which plackeys were removed this time?
removed_pks_higher_k = set(patterns["placekey"]).difference(patterns_filtered_higher_k["placekey"])
print(removed_pks_higher_k)
# The original placekey from Omaha NE is no longer removed.
"222-222@5p2-2tw-vs5" in removed_pks_higher_k<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09720534629404617, 'fraction_numerical': 0.03321182665046578, 'mean_word_length': 6.71875, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '388949', 'license': 'Apache-2.0', 'n_tokens_mistral': 900, 'n_tokens_neox': 815, 'n_words': 204, 'path': 'notebooks/Filtering Outlier POI from Patterns.ipynb', 'repo_name': 'echong-SG/safegraph_eval', 'size': '2469'}