<jupyter_start><jupyter_text>EDA  Initialization<jupyter_code>import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
import pandas as pd
import numpy as np
import sys
import os

# Visualization
import matplotlib.pyplot as plt
import seaborn as sns

import igraph

from geopandas import GeoDataFrame
from pyrosm import OSM
sns.set(rc={'figure.figsize':(11.7,8.27)})
# Self-made modules
import helpers as hp
pd.set_option('display.max_rows', 1000)
pd.set_option('display.max_columns', None)
pd.set_option('display.width', None)
pd.set_option('display.max_colwidth', None)<jupyter_output><empty_output><jupyter_text>Inputs<jupyter_code>buildings_int_path = '../data/02_intermediate/buildings_data/'
plz_ags_csv = '../data/01_raw/zuordnung_plz_ort_landkreis.csv'

# Demographics
ags_living_csv = '../data/01_raw/de_living_2019.csv'
ags_population_csv = '../data/01_raw/de_population_2019.csv'
ags_living = pd.read_csv(ags_living_csv, 
                         sep = ';', 
                         encoding = 'cp1250', 
                         dtype= {'1_Auspraegung_Code':str},
                         low_memory = False)
ags_population = pd.read_csv(ags_population_csv, 
                             sep = ';', 
                             dtype= {'1_Auspraegung_Code':str},
                             encoding = 'cp1250', 
                             low_memory = False)
# Contain local AGS codes (no regional)
plz_ags = pd.read_csv(plz_ags_csv, dtype= {'plz': str, 'ags': str})
test='1059187'
test.zfill(8)
plz_ags[plz_ags.ags == '01059187']<jupyter_output><empty_output><jupyter_text>PLZ AGS EDA<jupyter_code>plz_ags.head()
ags_bundesland = pd.DataFrame(plz_ags[['ags','bundesland']]
                              .drop_duplicates()
                              .groupby('bundesland')[['ags']]
                              .size()
                              .to_frame('ags_count')
                              .reset_index())
sns.set_style('white') 

g = sns.barplot(data = ags_bundesland,
                 y = 'bundesland',
                 x = 'ags_count',
                 palette = 'winter',
                 orient = 'h')

hp.show_values_on_bars(g, "h", 0.5)

plt.title('Distribution of municipality codes (AGS) in Germany', fontsize = 20)
plt.xlabel('Count', fontsize = 15)
plt.ylabel('State', fontsize = 15)
sns.despine()<jupyter_output><empty_output><jupyter_text>AGS Living<jupyter_code>ags_living.rename(columns = {'1_Auspraegung_Code':'ags',
                            '1_Auspraegung_Label':'place',
                            '2_Auspraegung_Label':'indication',
                            '2_Merkmal_Label':'measurement_type'}, inplace = True)
# Filter to get only local AGS codes
ags_living = ags_living[ags_living.ags.isin(plz_ags.ags)]
ags_living.ags.nunique()<jupyter_output><empty_output><jupyter_text>AGS Population<jupyter_code>ags_population.rename(columns = {'1_Auspraegung_Code':'ags',
                                 '1_Auspraegung_Label':'place',
                                 '2_Auspraegung_Label':'gender',
                                 '3_Auspraegung_Label':'age_group'}, inplace = True)

# Filter to get only local AGS codes
ags_population = ags_population[ags_population.ags.isin(plz_ags.ags)]
ags_population.ags.nunique()
ags_population[ags_population.ags == '01004000'].head()<jupyter_output><empty_output><jupyter_text>Plot examples<jupyter_code>id = 10
boundary_id = '09184123'
df = pd.read_csv(os.path.join(buildings_int_path, f'buildings_ags_{boundary_id}.csv'))
df.shape
plot_df = df.groupby('building_types').size().to_frame('count').reset_index()
plot_df
sns.set_style('white') 

g = sns.barplot(data = plot_df,
            x = 'count',
            y = 'building_types',
            palette = 'winter',
            orient = 'h')

hp.show_values_on_bars(g, "h", 0.5)

# plt.title('Distribution of municipality codes (AGS) in Germany', fontsize = 20)
plt.xlabel('Count', fontsize = 15)
plt.ylabel('Buliding type', fontsize = 15)
sns.despine()<jupyter_output><empty_output><jupyter_text>Get the number of residential buildings from officials<jupyter_code>ags_living['place'] = ags_living['place'].apply(lambda x: x.strip())
# Total buildings in the area that are classified as "residentials"
official_res_count = int(ags_living[(ags_living.ags == boundary_id) &
                                    (ags_living.measurement_type == 'Wohngebäude nach Anzahl der Wohnungen') &
                                    (ags_living.indication == 'Insgesamt')].BAUNW9__Wohngebaeude__Anzahl.iloc[0])
official_res_count
ags_place = ags_living[ags_living.ags == boundary_id].place.iloc[0]
ags_place
# OSM residential buildings count
osm_count = len(df[df.building_types == 'residential'])
osm_unidentified_count = len(df[df.building_types == 'to_be_classified'])
# differences in number
abs_diff = abs(osm_count-official_res_count)
# diff in percentage
pct_diff = round((abs_diff/official_res_count)*100,2)
f'{abs_diff} buildings & {pct_diff}% diff'<jupyter_output><empty_output><jupyter_text>Diff dist Get diff distribution of real residential buildings vs osm buildings<jupyter_code>res_diff_path = '../data/08_reporting/residential_diff.csv'
res_diff = pd.read_csv(res_diff_path,
                      dtype = {'ags':str})
res_diff = res_diff.drop_duplicates()
res_diff[res_diff.pct_diff == 100]
res_diff.pct_diff.max()
sns.distplot(res_diff[res_diff.pct_diff <=100].pct_diff,
            kde = False)
plt.xlim(0,100)
plt.title('Distribution of differences between official and OSM residential buildings count', fontsize = 18)
plt.xlabel('Percentage difference', fontsize = 15)
plt.ylabel('AGS count', fontsize = 15)
sns.despine()
from shapely import wkt
# Filter out NaN
df = df[df.geometry.isna() == False].reset_index(drop = True)

# Convert geometry to GeoSeries
df['geometry'] = df['geometry'].apply(wkt.loads)
# Convert to GeoPandas type
df_geo = GeoDataFrame(df, geometry='geometry')
df_geo = hp._swap_2_cols(df_geo, 'id','center.lon')
df_geo = hp._swap_2_cols(df_geo, 'type','center.lat')

df_geo.rename(columns = {'center.lat':'lat',
                        'center.lon':'lon'}, inplace = True)
# Plot
ax =   df_geo.plot(column='building_types', 
                   markersize=3, 
                   figsize=(12,20), 
                   legend=True, 
                   legend_kwds=dict(loc='upper left', ncol=1, bbox_to_anchor=(1, 1)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12600869025450032, 'fraction_numerical': 0.020018621973929236, 'mean_word_length': 3.3052772211088843, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '646065', 'license': 'MIT', 'n_tokens_mistral': 2339, 'n_tokens_neox': 2167, 'n_words': 366, 'path': 'notebooks/0_EDA.ipynb', 'repo_name': 'o0oBluePhoenixo0o/CheapAtlas', 'size': '6444'}