<jupyter_start><jupyter_text>Table of ContentsData LoadingRooftop ItemsRooftop StatisticsHistogram GraphsBuilding SamplingAppendix<jupyter_code>import sqlite3
import pandas as pd
import numpy as np
import seaborn as sns
from IPython.display import display
import matplotlib.pyplot as plt 
import matplotlib.ticker as mticker
from scipy.stats import gaussian_kde
from PIL import Image
sns.set()
sns.set_context("notebook", font_scale=1.5, rc={"lines.linewidth": 2.5})
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Data LoadingLoad the data and view a sample of the data frame. Surface details contains a list of named items on a rooftop. Their corresponsding quantity can be found in the quanity column.<jupyter_code>df_b = pd.read_csv("../assets/data/manhattan/buildings.csv", converters={'gps': lambda x: [float(coord) for coord in x.split(',')],
                                                     'quantity': lambda x: [int(quantity) for quantity in x.split(',')],
                                                     'surface details': lambda x: list(reversed(x.split(',')))})
df_b[['lat','lon']] = pd.DataFrame(df_b.gps.tolist(), index= df_b.index)
print("Buildings")
display(df_b.head())
print("Rooftop Items Count")
df_r = pd.read_csv("../assets/data/manhattan/rooftop-items.csv")
display(df_r.head())
df_b<jupyter_output><empty_output><jupyter_text>Rooftop ItemsLikelihood of having a rooftop item on a building.<jupyter_code>remove_list = ['pools', 'debris', 'solar-panel', 'human-occupation', 'empty', 'large-building', 'billboard', 'construction', 'multi-level']
items = np.unique([item for sublist in df_b['surface details'].values
                   for item in sublist if item not in remove_list]).tolist()
default_row = {k: 0 for k in items}
# This is a dataframe where each row is a building and each column is a different rooftip item
records_vector = []
# This is a dataframe where each row simply lists the amount footop item and the quantity of items on a roof
records_simple = []

for index, row in df_b.iterrows():
    new_row = default_row.copy()
    changes = {item: quantity for quantity, item in zip(row['quantity'], row['surface details']) if item not in remove_list}
    new_row.update(changes)
    records_vector.append(new_row)
    for k,v in new_row.items():
        records_simple.append({'Name': k, "Quantity": v})
          
df_vec = pd.DataFrame.from_records(records_vector)
df_simp = pd.DataFrame.from_records(records_simple)

print("Building View of Data")
display(df_vec.head())
print("Item View of Data")
display(df_simp.head())
with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also
    print(df_simp[df_simp['Name'] == 'water-tower'])<jupyter_output>Name  Quantity
11    water-tower         0
23    water-tower         0
35    water-tower         0
47    water-tower         0
59    water-tower         0
71    water-tower         0
83    water-tower         0
95    water-tower         1
107   water-tower         0
119   water-tower         0
131   water-tower         0
143   water-tower         0
155   water-tower         0
167   water-tower         0
179   water-tower         0
191   water-tower         0
203   water-tower         0
215   water-tower         0
227   water-tower         0
239   water-tower         0
251   water-tower         0
263   water-tower         0
275   water-tower         0
287   water-tower         0
299   water-tower         0
311   water-tower         0
323   water-tower         0
335   water-tower         0
347   water-tower         1
359   water-tower         0
371   water-tower         0
383   water-tower         0
395   water-tower         0
407   water-tower         0
419   water-tower   [...]<jupyter_text>Rooftop Statistics<jupyter_code>df_vec.describe()
asset_order = list(df_vec.mean().sort_values(ascending=False).index)
df_vec.mean().round(2).sort_values(ascending=False)
df_vec.std().round(2).sort_values(ascending=False)<jupyter_output><empty_output><jupyter_text>Histogram Graphs<jupyter_code>sns.set(style="ticks", font_scale=2.2, rc={"axes.facecolor": (0, 0, 0, 0)})
df = df_simp
# Initialize the FacetGrid object
g = sns.FacetGrid(df, col="Name",  aspect=2, col_wrap=3, col_order=asset_order)
# hist_kws = {'width': .4, "range": [0.0,12]}
kde_kws = {'clip': (-1.0, 12.0)}
# Draw the densities in a few steps
# g.map(sns.distplot, "Quantity", kde=True, bins=12, hist_kws=hist_kws, kde_kws=kde_kws)
g.map(sns.histplot, "Quantity", kde=False, bins=12, binwidth=1.0, binrange=[0.0,12], discrete=True)
g.set(yscale='log')
ticks = [1, 10, 100]
labels = [i for i in ticks]
xticks = range(13)
xlabels = [i for i in xticks]
g.set(yticks = ticks, yticklabels = labels, xticks=xticks, xticklabels=xlabels)
g.set(ylim=(0.8, 100))
g.savefig('../assets/imgs/hist.pdf')
# g.fig.get_axes()[0].yaxis.set_minor_locator(mticker.LogLocator(base=10, subs='all'))
# locmin = mticker.LogLocator(base=10, subs=np.arange(0.1,1,0.1),numticks=10)  
# g.fig.get_axes()[0].yaxis.set_minor_locator(locmin)
# g.fig.get_axes()[0].yaxis.set_minor_formatter(mticker.NullFormatter())
# # g.fig.get_axes()[0].yaxis.set_minor_formatter(mticker.NullFormatter())
# g.fig.get_axes()[0].grid(True, which="Minor")  
sns.set(style="ticks", font_scale=2.2, rc={"axes.facecolor": (0, 0, 0, 0)})
df = df_simp
# Initialize the FacetGrid object
g = sns.displot(df, x="Quantity", col="Name", kind='hist', aspect=2, col_wrap=3, 
                col_order=asset_order, bins=12, binwidth=1.0, binrange=[0.0,12], 
                discrete=True )

g.set(yscale='log')
ticks = [1, 10, 100]
labels = [i for i in ticks]
xticks = range(12)
xlabels = [i for i in xticks]
g.set(yticks = ticks, yticklabels = labels, xticks=xticks, xticklabels=xlabels)
g.set(ylim=(0.5, 100))
# g.savefig('../assets/imgs/hist.pdf')
SEED = 2
np.random.seed(SEED)
def histogram_sampling(x, size=None):
    if size is None:
        size=x.shape[0]
    quantities, counts = np.unique(x, return_counts=True)
    probs = counts / size
    return np.random.choice(quantities, size=size, p=probs)
def kde_sampling(x, size=None):
    if size is None:
        size=x.shape[0]
    kde = gaussian_kde(x)
    return np.maximum(kde.resample(x.shape[0]).squeeze() + 0.5, 0).astype(np.int)

records = []
for item in items:
    x = df_vec[item].values
    x_pred_kde = kde_sampling(x)
    x_pred_hist = histogram_sampling(x)
    records_x = [{'Name': item, 'Quantity': x_, 'Source': 'Manhattan Data'} for x_ in x]
    records_x_pred_kde = [{'Name': item, 'Quantity': x_, 'Source': 'KDE Sampling'} for x_ in x_pred_kde]
    records_x_pred_hist = [{'Name': item, 'Quantity': x_, 'Source': 'Histogram Sampling'} for x_ in x_pred_hist]
    records.extend(records_x)
    records.extend(records_x_pred_hist)
    records.extend(records_x_pred_kde)

df_kde = pd.DataFrame.from_records(records)
# sns.set(style="white", font_scale=1.5, rc={"axes.facecolor": (0, 0, 0, 0)})
# g = sns.FacetGrid(df_kde, col="Name",  aspect=2, hue="Source", col_wrap=3, legend_out =True)
# g.map(sns.distplot, "Quantity", kde=False, bins=range(0,11), hist_kws={'width': .4})
# g.add_legend()
allowed_columns = ['ac-unit', 'small-rooftop-entrance', 'air-vents', 'skylight']
# df_kde_filt = df_kde[df_kde['Name'].isin(allowed_columns)]
df_kde_filt = df_kde
df_kde_filt = df_kde_filt[df_kde_filt['Quantity'] < 10]

sns.set(style="white", font_scale=2.5, rc={"axes.facecolor": (0, 0, 0, 0)})
g = sns.catplot(x="Quantity", hue="Source", col="Name", col_wrap=3,
                data=df_kde_filt, kind="count", aspect=1.7, height=4);

g.fig.get_axes()[0].set_yscale('log')
# g.savefig('../assets/imgs/hist_sample_kde.pdf')
# for ax in g.axes:
#     ax.set(yscale='log')<jupyter_output><empty_output><jupyter_text>Building SamplingHow were the buildings samples? It was **not** uniform. * Start near 10th Ave and West 57th street.* Pick a block and start sampling buildings* Pick buildings that are mostly flat like  * No point sampling buildings we will never land on* Randomly pick another block that is nearby<jupyter_code>from folium.plugins import MousePosition
import io
import folium
m = folium.Map(location=[40.76483, -73.98962], zoom_start=16, tiles='Stamen Terrain')
for index, row in df_b.iterrows():
#     folium.Marker(row['gps']).add_to(m)
    folium.Circle(radius=10, location=row['gps'], fill=True).add_to(m)
MousePosition().add_to(m)
# png = m._to_png(delay=3, window_size=(700, 700))
# png = m._to_png(delay=3)
# image = Image.open(io.BytesIO(png))
# image.save('../assets/imgs/map_ny.png')
m
import plotly.express as px
fig = px.scatter_mapbox(df_b, lat="lat", lon="lon",zoom=15.8, height=300, )
fig.update_mapboxes(bearing=118)
fig.update_traces(marker_size=10)
fig.update_layout(mapbox_style="streets", mapbox=dict(center=dict(lat=40.76536, lon=-73.98898)), mapbox_accesstoken="<KEY>")
fig.update_layout(margin={"r":0,"t":0,"l":0,"b":0})
fig.write_image("../assets/imgs/map_ny_vector.pdf")
fig.show()<jupyter_output><empty_output><jupyter_text>Appendix<jupyter_code># SEED = 2
# np.random.seed(SEED)
# column_name = 'ac-unit'
# x = df_vec[column_name].as_matrix()
# print(x)
# kde = gaussian_kde(x)
# x_pred = np.maximum(kde.resample(x.shape[0]).squeeze() + 0.5, 0).astype(np.int)
# print(x_pred)
# print(np.unique(x))
# print(np.unique(x_pred))

class QuantitySampler(object):
    """Performs quantity sampling of roof top objects
    Uses a provided Pandas DataFrame where each row is a building observation and 
    a column is a rooftop object type. The cell contains the quantity observed.
    
    """

    def __init__(self, df, fit='histogram'):
        """Pass in a dataframe and specify if you want sampling to be from a histogram or 
        a kernel density estimator (kde).
        
        Arguments:
            df {DataFrame} -- DataFrame of Data
        
        Keyword Arguments:
            fit {str} -- How to sample from data. Histogram is just a weighted random choice (default: {'histogram'})
        """

        self.df = df
        self.fit = fit

        if fit == 'histogram':
            self.set_up_histogram()
        else:
            raise NotImplementedError("Have not implemented kde yet")
            pass

    def sample(self, key, size=1):
        if self.fit == 'histogram':
            quantity = self.histogram_sampling(**self.histogram_meta[key], size=size)
            return quantity if size > 1 else quantity[0]

    def set_up_histogram(self):
        items = self.df.columns
        histogram_meta = {}
        # Iterate through each column in dataframe
        for item in items:
            x = self.df[item].values
            quantities, counts = np.unique(x, return_counts=True)
            histogram_meta[item] = dict(quantities=quantities, probabilities=counts/x.shape[0])
        self.histogram_meta = histogram_meta
    
    def histogram_sampling(self, quantities, probabilities, size=1):
        return np.random.choice(quantities, size=size, p=probabilities)
    
QuantitySampler(df_vec, "histogram").sample('ac-unit')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13793103448275862, 'fraction_numerical': 0.0294276945296985, 'mean_word_length': 3.696481559983044, 'pattern_counts': {'":': 11, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '309670', 'license': 'MIT', 'n_tokens_mistral': 4166, 'n_tokens_neox': 3914, 'n_words': 920, 'path': 'notebooks/RooftopAnalysis-3.ipynb', 'repo_name': 'JeremyBYU/UnrealRooftopLanding', 'size': '11162'}