<jupyter_start><jupyter_text>Difference in Differences analysis for TCC Craigslist rent data<jupyter_code>import pandas as pd
import numpy as np
import geopandas as gpd

import statsmodels.api as sm
import statsmodels.formula.api as smf
fres_listings_20 = gpd.read_file('craigslist_data/2020/fresno_listings.geojson')
ont_listings_20 = gpd.read_file('craigslist_data/2020/ontario_listings.geojson')
la_listings_20 = gpd.read_file('craigslist_data/2020/la_listings.geojson')
fres_tracts = gpd.read_file('craigslist_data/tracts/fresno_tracts.geojson')
la_tracts = gpd.read_file('craigslist_data/tracts/la_tracts.geojson')
ont_tracts = gpd.read_file('craigslist_data/tracts/ontario_tracts.geojson')
def read_2014_data(city, tracts_gdf):
    
    def join_to_tracts(listing_gdf, tracts_gdf):
        sjoined_gdf = gpd.sjoin(listing_gdf, tracts_gdf, how='left', op='within')
        sjoined_gdf = (
                        #drop listings not in TCC nor control group
                        sjoined_gdf.dropna(subset=['group'])
                        #.rename(columns={'index_right': 'tract_id'})
                        .drop(columns='BASENAME')
                        )
        return sjoined_gdf
    
    assert city in ['la', 'ontario', 'fresno']
    subset = {'la':'losangeles', 'ontario':'inlandempire', 'fresno':'fresno'}
    rent_2014 = pd.read_csv(f'craigslist_data/2014/subset_{subset[city]}.csv')
    rent_2014 = rent_2014.drop_duplicates(subset=['rent', 'sqft', 'date'])
    rent_2014 = gpd.GeoDataFrame(
        rent_2014, geometry=gpd.points_from_xy(
            rent_2014['longitude'].astype('float64'), rent_2014['latitude'].astype('float64')))
    rent_2014 = rent_2014.dropna(subset=['rent'])
    rent_2014.crs = 'EPSG:4326'
    
    #add variable for DiD analysis
    rent_2014['time'] = 0
    joined_2014 = join_to_tracts(rent_2014, tracts_gdf)
    return joined_2014
fres_14 = read_2014_data('fresno', fres_tracts)
ont_14 = read_2014_data('ontario', ont_tracts)
la_14 = read_2014_data('la', la_tracts)
def join_prepare_data(listings_14, listings_20, tracts_gdf):

    listings_20 = listings_20.rename(columns={'index_right':'geoid'})
    all_listings = listings_14.append(listings_20)
    all_listings['post_treatment'] = all_listings['time'] * all_listings['group']
    all_listings['rent_sqft'] = all_listings['rent'] / all_listings['sqft']
    
    def remove_outliers(listing_gdf, quantiles):
        listing_gdf = listing_gdf.loc[listing_gdf['rent_sqft'] > listing_gdf['rent_sqft'].quantile(quantiles[0])]
        listing_gdf = listing_gdf.loc[listing_gdf['rent_sqft'] < listing_gdf['rent_sqft'].quantile(quantiles[1])]
        return listing_gdf

    all_listings = remove_outliers(all_listings, (.02, .98))
    #creates df with median of each statistic (rent_sqft) and a row for each tract at each time
    for_did = (all_listings.groupby(['geoid', 'time'])
               .median().reset_index()
               .drop(columns=['index_right', 'index']))
    
    return for_did
def estimate_did(df, model='single'):

    # choose a response and predictors
    response = 'rent_sqft'
    predictors = ['time', 'group', 'post_treatment']
    if model == 'pooled':
        predictors = predictors + ['is_fresno', 'is_ontario', 'is_la']


    # filter full dataset to retain only these columns and only rows without nulls in these columns
    data = df[[response] +  predictors].dropna()

    # create design matrix and response vector
    X = data[predictors]
    y = data[response]

    # estimate a simple linear regression model with OLS, using statsmodels
    model = sm.OLS(y, sm.add_constant(X))
    result = model.fit()
    print(result.summary())
##Example data for DiD regression
join_prepare_data(fres_14, fres_listings_20, fres_tracts).head(15)<jupyter_output><empty_output><jupyter_text>Fresno<jupyter_code>estimate_did(join_prepare_data(fres_14, fres_listings_20, fres_tracts))<jupyter_output>OLS Regression Results                            
==============================================================================
Dep. Variable:              rent_sqft   R-squared:                       0.699
Model:                            OLS   Adj. R-squared:                  0.668
Method:                 Least Squares   F-statistic:                     22.47
Date:                Tue, 11 Aug 2020   Prob (F-statistic):           1.02e-07
Time:                        10:59:19   Log-Likelihood:                 15.251
No. Observations:                  33   AIC:                            -22.50
Df Residuals:                      29   BIC:                            -16.52
Df Model:                           3                                         
Covariance Type:            nonrobust                                         
==================================================================================
                     coef    std err          t [...]<jupyter_text>Ontario<jupyter_code>estimate_did(join_prepare_data(ont_14, ont_listings_20, ont_tracts))<jupyter_output>OLS Regression Results                            
==============================================================================
Dep. Variable:              rent_sqft   R-squared:                       0.446
Model:                            OLS   Adj. R-squared:                  0.419
Method:                 Least Squares   F-statistic:                     16.64
Date:                Tue, 11 Aug 2020   Prob (F-statistic):           4.85e-08
Time:                        10:59:20   Log-Likelihood:                -14.252
No. Observations:                  66   AIC:                             36.50
Df Residuals:                      62   BIC:                             45.26
Df Model:                           3                                         
Covariance Type:            nonrobust                                         
==================================================================================
                     coef    std err          t [...]<jupyter_text>LA<jupyter_code>estimate_did(join_prepare_data(la_14, la_listings_20, la_tracts))<jupyter_output>OLS Regression Results                            
==============================================================================
Dep. Variable:              rent_sqft   R-squared:                       0.304
Model:                            OLS   Adj. R-squared:                  0.272
Method:                 Least Squares   F-statistic:                     9.601
Date:                Tue, 11 Aug 2020   Prob (F-statistic):           2.41e-05
Time:                        10:59:21   Log-Likelihood:                -74.856
No. Observations:                  70   AIC:                             157.7
Df Residuals:                      66   BIC:                             166.7
Df Model:                           3                                         
Covariance Type:            nonrobust                                         
==================================================================================
                     coef    std err          t [...]<jupyter_text>Pooled Model<jupyter_code>fres_for_did = join_prepare_data(fres_14, fres_listings_20, fres_tracts)
fres_for_did['is_fresno'] = 1
fres_for_did['is_la'] = 0
fres_for_did['is_ontario'] = 0
ont_for_did = join_prepare_data(ont_14, ont_listings_20, ont_tracts)
ont_for_did['is_fresno'] = 0
ont_for_did['is_la'] = 0
ont_for_did['is_ontario'] = 1
la_for_did = join_prepare_data(la_14, la_listings_20, ont_tracts)
la_for_did['is_fresno'] = 0
la_for_did['is_la'] = 1
la_for_did['is_ontario'] = 0

together_for_did = fres_for_did.append(ont_for_did).append(la_for_did)
estimate_did(together_for_did, model='pooled')<jupyter_output>OLS Regression Results                            
==============================================================================
Dep. Variable:              rent_sqft   R-squared:                       0.460
Model:                            OLS   Adj. R-squared:                  0.443
Method:                 Least Squares   F-statistic:                     27.77
Date:                Tue, 11 Aug 2020   Prob (F-statistic):           2.79e-20
Time:                        10:59:32   Log-Likelihood:                -125.91
No. Observations:                 169   AIC:                             263.8
Df Residuals:                     163   BIC:                             282.6
Df Model:                           5                                         
Covariance Type:            nonrobust                                         
==================================================================================
                     coef    std err          t [...]<jupyter_text>Some quick bar charts of the number of listings in each group in 2014 and 2020<jupyter_code>def listings_table(df_14, df_20):
    num_listings = df_14.append(df_20).groupby(['time', 'group']).count().iloc[:,1]
    df = pd.DataFrame(num_listings).reset_index().rename(columns={'rent':'num_listings'})
    times = {0:'2014', 1:'2020'}
    groups = {0:'control_', 1:'tcc_'}
    df['label'] = df.apply(lambda x: groups[x['group']] + times[x.time], axis=1)
    return df
listings_table(fres_14, fres_listings_20).plot.bar('label', 'num_listings');
listings_table(ont_14, ont_listings_20).plot.bar('label', 'num_listings');
listings_table(la_14, la_listings_20).plot.bar('label', 'num_listings');<jupyter_output><empty_output><jupyter_text>Rough histograms showing distribution of bedrooms by year<jupyter_code>la_14['bedrooms'].hist()
la_listings_20['bedrooms'].astype('float32').hist()
fres_14['bedrooms'].hist()
fres_listings_20['bedrooms'].astype('float32').hist()
ont_14['bedrooms'].hist()
ont_listings_20['bedrooms'].astype('float32').hist()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16210139002452983, 'fraction_numerical': 0.04251839738348324, 'mean_word_length': 2.193537859007833, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '533811', 'license': 'MIT', 'n_tokens_mistral': 3283, 'n_tokens_neox': 2880, 'n_words': 539, 'path': '.ipynb_checkpoints/Revised Craigslist DiD-checkpoint.ipynb', 'repo_name': 'jasonkarpman/TCC_Evaluation', 'size': '9784'}