<jupyter_start><jupyter_text>Compare two datasetsLet's compare two pedestrian estimations with different methods: field count vs. cnn estimation.<jupyter_code>import pandas as pd
import statsmodels.api as sm
import altair as alt
import scipy
import numpy as np
import matplotlib.pyplot as plt
import statsmodels.graphics.api as smg
import itertools

alt.data_transformers.disable_max_rows()
df = pd.read_csv("../data/dataframe/df_count.csv")
df = df.set_index('loc')
df<jupyter_output><empty_output><jupyter_text>Correlation matrixlet's make correlation matrix between field counts * cnn counts<jupyter_code>counts_cols = list(df.filter(regex='field_(am|md|pm)_3y').columns)
cnn_cols = list(df.filter(regex='cnn_(count|sum)').columns)
corrs = []
for pair in itertools.product(counts_cols, cnn_cols):
    record = {}
    record['field'] = pair[0]
    record['cnn'] = pair[1]
    record['corr'] = scipy.stats.pearsonr(df[pair[0]], df[pair[1]])[0]
    corrs.append(record)
df_corr = pd.DataFrame(corrs)
df_corr
def draw_regplot(df, col1, col2):
    chart = alt.Chart(df).mark_circle().encode(
        alt.X(col1),
        alt.Y(col2),
        # alt.Color('borough', type='nominal')
    ).properties(width=300, height=300)

    line = chart.transform_regression(
        col1, 
        col2
    ).mark_line()
    
    corr = scipy.stats.pearsonr(df[col1], df[col2])[0]
    params = chart.mark_text(
        align='left',  
        text= f'Pearson\'s r = {corr:.3f}',
        # color='#000000'
    ).encode(
        x=alt.value(20),  # pixels from left
        y=alt.value(20),  # pixels from top
    )

    return chart  +  params + line
(draw_regplot(df, 'cnn_sum', 'field_pm_3y') | draw_regplot(df, 'cnn_count', 'field_pm_3y')) \
& \
(draw_regplot(df, 'cnn_sum', 'field_md_3y') | draw_regplot(df, 'cnn_count', 'field_md_3y')) \
& \
(draw_regplot(df, 'cnn_sum', 'field_am_3y') | draw_regplot(df, 'cnn_count', 'field_am_3y'))<jupyter_output><empty_output><jupyter_text>Comparison across aggregation methods<jupyter_code>alt.Chart(df_corr).mark_bar().encode(
    x='cnn:N',
    y='corr:Q',
    color='cnn:N',
    column='field:N'
).properties(width=100, height=300)<jupyter_output><empty_output><jupyter_text>Region<jupyter_code>def draw_regplot_c(df, col1, col2):
    chart = alt.Chart(df).mark_circle().encode(
        alt.X(col1),
        alt.Y(col2),
        alt.Color('borough', type='nominal')
    ).properties(width=300, height=300)

    line = chart.transform_regression(
        col1, 
        col2
    ).mark_line()
    
    corr = scipy.stats.pearsonr(df[col1], df[col2])[0]
    params = chart.mark_text(
        align='left',  
        text= f'Pearson\'s r = {corr:.3f}',
        # color='#000000'
    ).encode(
        x=alt.value(20),  # pixels from left
        y=alt.value(20),  # pixels from top
    )

    return chart  +  params + line
    chart = alt.Chart(df).mark_circle().encode(
        alt.X(col1),
        alt.Y(col2),
        color='borough:N'
    ).properties(width=300, height=300)

    line = chart.transform_regression(
        col1, 
        col2
    ).mark_line()
    
    corr = scipy.stats.pearsonr(df[col1], df[col2])[0]
    params = chart.mark_text(
        align='left',  
        text= f'Pearson\'s r = {corr:.3f}',
        # color='#000000'
    ).encode(
        x=alt.value(20),  # pixels from left
        y=alt.value(20),  # pixels from top
    )

    return chart  +  params + line
draw_regplot_c(df, 'cnn_sum', 'field_md_3y')<jupyter_output><empty_output><jupyter_text>Outliers are usually in Manhattan. Without Manhattan<jupyter_code>draw_regplot_c(df[df['borough']!='Manhattan'], 'cnn_sum', 'field_md_3y')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15365787326624966, 'fraction_numerical': 0.026108240413380474, 'mean_word_length': 3.5975, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '437287', 'license': 'MIT', 'n_tokens_mistral': 1436, 'n_tokens_neox': 1369, 'n_words': 233, 'path': 'notebooks/04-Compare-two-datasets.ipynb', 'repo_name': 'yonghah/nyc_ped_count_cnn', 'size': '3677'}