<jupyter_start><jupyter_text>Imports<jupyter_code># note: must run in virtual environment
import os
import sys
venv_path = os.environ['VIRTUAL_ENV']
sys.path.append(os.path.join(os.environ['VIRTUAL_ENV'], "lib\site-packages"))
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from scipy import stats
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Consts<jupyter_code>PROVIDER_AND_ID = "provider_and_id"
CITY = "accident_yishuv_name"
LAT = "latitude"
LONG = "longitude"
X = "x"
Y = "y"<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>csv_path = "involved_markers_hebrew.csv"
data = pd.read_csv(csv_path)
# get only relevant data
filtered = data[[PROVIDER_AND_ID, CITY, LAT, LONG, X, Y]].drop_duplicates()<jupyter_output><empty_output><jupyter_text>Find outliers by zscore<jupyter_code>def calc_outliers_for_city(data, zscore_thresh, min_sample_size):
    coords = data[[X, Y]].dropna()
    
    # check if sample size is big enough for the statistic calculation
    if len(coords.drop_duplicates()) <= min_sample_size:
        return []
    
    # calculate zscores for each column
    zscores = stats.zscore(coords, axis = 0)
    zscores = np.nan_to_num(zscores)
    zscores = np.abs(zscores)
    # get indices to outliers
    outliers = np.argwhere(zscores > zscore_thresh)
    outlier_rows = np.unique(outliers[:,0])
    if len(outliers) > 0:
        outlier_rows = data.iloc[outlier_rows]
        return outlier_rows[[PROVIDER_AND_ID, CITY, LAT, LONG, X, Y]]
    return []

def get_outliers_by_z_scores(data, zscore_thresh, min_sample_size = 0):
    res = []
    for city, rows in data.groupby(CITY):
        outliers = calc_outliers_for_city(rows, zscore_thresh, min_sample_size)
        if len(outliers) > 0:
            res.append(outliers)
    return pd.concat(res)<jupyter_output><empty_output><jupyter_text>Visualize effect of different thresholds<jupyter_code>def plot_outliers_by_thresh(low, high):
    threshs = range(low, high+1)
    num_outliers = []
    for i in threshs:
        outliers = get_outliers_by_z_scores(filtered, i)
        num_outliers.append(len(outliers))
    plt.title("Number of outliers by threshold value")
    plt.plot(threshs, num_outliers)
    plt.show()
        
plot_outliers_by_thresh(3,10)
plot_outliers_by_thresh(7,15)<jupyter_output><empty_output><jupyter_text>Pick threshold and minimum sample size<jupyter_code>ZSCORE_THRESH = 10
MIN_SAMPLE_SIZE = 100<jupyter_output><empty_output><jupyter_text>Save output<jupyter_code>outliers = get_outliers_by_z_scores(filtered, ZSCORE_THRESH, MIN_SAMPLE_SIZE)
outliers.to_csv("outlier_points.csv", header=True, index=False, encoding='utf-8')<jupyter_output><empty_output><jupyter_text>Explore results<jupyter_code>def plot_city(data, city_name):
    city_data = data.loc[data[CITY] == city_name]
    outliers = calc_outliers_for_city(city_data, ZSCORE_THRESH, MIN_SAMPLE_SIZE)
    fig, (ax1, ax2) = plt.subplots(nrows = 2, ncols = 1, figsize=(15,15))
    ax1.plot(city_data[Y], city_data[X], "o")
    ax2.plot(city_data[LAT], city_data[LONG], "o")
    if len(outliers) > 0:
        ax1.plot(outliers[Y], outliers[X], "ro")
        ax2.plot(outliers[LAT], outliers[LONG], "ro")
    fig.show()
    
plot_city(filtered, "נצרת עילית")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11730827523491967, 'fraction_numerical': 0.009396786905122764, 'mean_word_length': 5.285714285714286, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '656916', 'license': 'MIT', 'n_tokens_mistral': 1341, 'n_tokens_neox': 1257, 'n_words': 221, 'path': 'learning_notebooks/filter_accident_outliers.ipynb', 'repo_name': 'dinasv/anyway-data-science', 'size': '3299'}