<jupyter_start><jupyter_text>Create Dublin Electoral Division basemap from CSO boundary files (already converted to geojson)<jupyter_code>with open("../data/raw/dublin_ed.geojson", "r") as f:
    dub = json.load(f)
# Creating new geojson for geometries and basic id info
# Including Ward names to incorporate 1961 data
dub2 = {'type':dub['type'], 'crs': dub['crs'], 'features': []}
for feat in dub['features']:
    prop = feat
    prop['properties'] = {"ed": feat['properties']['EDNAME'],
           "geogid": feat['properties']['GEOGID'],
           "osied": feat['properties']['OSIED'],
           "ward": re.sub("\s[A-Z]$", "", feat['properties']['EDNAME'])}
    dub2['features'].append(prop)

with open("../data/processed/dublin_ed_base.geojson", "w") as f:
    json.dump(dub2, f, indent=2, sort_keys=True)
#Create Geopandas data frame from dublin data
dub_gp = gp.GeoDataFrame.from_file("../data/processed/dublin_ed_base.geojson")
dub_gp.set_index("geogid")
dub_gp.geometry.total_bounds
dub_gp.plot(legend=True)
#Import census tenure data into dataframe
with open("../data/interim/ten_71_81_91_02_11.json", "r") as f:
    tenure = json.load(f)
ten_df = pd.DataFrame.from_dict(data)
ten_df[ten_df['year']==1971].describe()
for t in tenure:
    t['year'] = str(t['year'])

dub_df = pd.DataFrame.from_dict(sorted(({k:t[k] for k in t if k in ['area', 'geogid', "year"] or k.endswith("units")}
                                for t in tenure if t['geogid'] == "C02"), key=lambda x: x['year']))

dub_df.set_index("year")
dub_df.loc[dub_df['owner_occ_units'].isnull(), 'owner_occ_units'] = dub_df['owner_mortg_units']+dub_df['owner_no_mortg_units']
dub_df.loc[dub_df['private_rent_units'].isnull(), 'private_rent_units'] = dub_df['rent_furn_units']+dub_df['rent_unfurn_units']

ax = plt.figure(figsize=(7,4), dpi=300).add_subplot(111)
dub_df.plot(ax=ax, x="year", y=['rent_furn_units', 'rent_unfurn_units', "private_rent_units"])
#bx = plt.figure(figsize=(7,4), dpi=300).add_subplot(111)
dub_df.plot(x="year", y=['owner_occ_units', 'owner_mortg_units', "owner_no_mortg_units", "private_rent_units"])
dub71 = pd.DataFrame.from_dict({k:t[k] for k in t if k in ['area', 'geogid', "year"] or k.endswith("units")}
                                for t in tenure if t['geogid'] and t['geogid'].startswith("E") and t['year'] == "2002")
dub71.set_index("geogid")
dub71.describe()
bds = dub_gp.total_bounds
extra = 0.01
ll = (bds[0], bds[1])
ur = (bds[2], bds[3])
coords = list(chain(ll, ur))
w, h = coords[2] - coords[0], coords[3] - coords[1]
w, h = coords[2] - coords[0], coords[3] - coords[1]
extra = 0.01
?Basemap.readshapefile
m = Basemap(
    projection='tmerc',
    lon_0=-2.,
    lat_0=49.,
    ellps = 'WGS84',
    llcrnrlon=coords[0] - extra * w,
    llcrnrlat=coords[1] - extra + 0.01 * h,
    urcrnrlon=coords[2] + extra * w,
    urcrnrlat=coords[3] + extra + 0.01 * h,
    lat_ts=0,
    resolution='i',
    suppress_ticks=True)
m.readshapefile(
    '../data/external/census2011DED/Census2011_Electoral_Divisions_generalised20m',
    'Dublin City',
    color='none',
    zorder=2)
# Calculate Jenks natural breaks for density
breaks = nb(
    dub71[dub71['rent_furn_units'].notnull()].rent_unfurn_units.values,
    initial=300,
    k=5)
# the notnull method lets us match indices when joining
jb = pd.DataFrame({'jenks_bins': breaks.yb}, index=dub71[dub71['rent_furn_units'].notnull()].index)
dub71 = dub71.join(jb)
dub71.jenks_bins.fillna(-1, inplace=True)

jenks_labels = ["<= %0.f units (%s EDs)" % (b, c) for b, c in zip(
    breaks.bins, breaks.counts)]
#jenks_labels.insert(0, 'No plaques (%s wards)' % len(df_map[df_map['density_km'].isnull()]))
jenks_labels
#fig = plt.figure(figsize=(20,15), dpi=300)#.add_subplot(111)
plt.clf()
fig = plt.figure()

ax = fig.add_subplot(111, axisbg='w', frame_on=False)

# use a blue colour ramp - we'll be converting it to a map using cmap()
cmap = plt.get_cmap('Blues')
# draw wards with grey outlines
patches = []

d71 = pd.merge(dub_gp,dub71)
d71['patches'] = d71.geometry.map(lambda x: PolygonPatch(x, ec='#555555', lw=.2, alpha=1., zorder=4))
pc = PatchCollection(d71['patches'], match_original=True)
# impose our colour map onto the patch collection
norm = Normalize()
pc.set_facecolor(cmap(norm(d71['jenks_bins'].values)))
ax.add_collection(pc)

d71.plot(column="jenks_bins", colormap="Blues", 
                              categorical=True, 
                               alpha=1.0, 
                               
                               )
# Add a colour bar
cb = colorbar_index(ncolors=len(jenks_labels), cmap=cmap, shrink=0.5, labels=jenks_labels)
cb.ax.tick_params(labelsize=10)

# this will set the image width to 722px at 100dpi
plt.tight_layout()
fig.set_size_inches(12,8)
#plt.savefig('data/london_plaques.png', dpi=100, alpha=True)
plt.show()
# Convenience functions for working with colour ramps and bars
def colorbar_index(ncolors, cmap, labels=None, **kwargs):
    """
    This is a convenience function to stop you making off-by-one errors
    Takes a standard colour ramp, and discretizes it,
    then draws a colour bar with correctly aligned labels
    """
    cmap = cmap_discretize(cmap, ncolors)
    mappable = cm.ScalarMappable(cmap=cmap)
    mappable.set_array([])
    mappable.set_clim(-0.5, ncolors+0.5)
    colorbar = plt.colorbar(mappable, **kwargs)
    colorbar.set_ticks(np.linspace(0, ncolors, ncolors))
    colorbar.set_ticklabels(range(ncolors))
    if labels:
        colorbar.set_ticklabels(labels)
    return colorbar

def cmap_discretize(cmap, N):
    """
    Return a discrete colormap from the continuous colormap cmap.

        cmap: colormap instance, eg. cm.jet. 
        N: number of colors.

    Example
        x = resize(arange(100), (5,100))
        djet = cmap_discretize(cm.jet, 5)
        imshow(x, cmap=djet)

    """
    if type(cmap) == str:
        cmap = get_cmap(cmap)
    colors_i = np.concatenate((np.linspace(0, 1., N), (0., 0., 0., 0.)))
    colors_rgba = cmap(colors_i)
    indices = np.linspace(0, 1., N + 1)
    cdict = {}
    for ki, key in enumerate(('red', 'green', 'blue')):
        cdict[key] = [(indices[i], colors_rgba[i - 1, ki], colors_rgba[i, ki]) for i in range(N + 1)]
    return matplotlib.colors.LinearSegmentedColormap(cmap.name + "_%d" % N, cdict, 1024)
[(t['geogid'], t['area'], t['year']) for t in tenure if t['geogid'] == "E02146"]
dub_gp['geometry'].iloc[6:8].values
ax.collections<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1651688875116207, 'fraction_numerical': 0.0320731329408119, 'mean_word_length': 4.046911649726349, 'pattern_counts': {'":': 4, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '5554', 'license': 'MIT', 'n_tokens_mistral': 2635, 'n_tokens_neox': 2477, 'n_words': 518, 'path': 'notebooks/Dublin_ED_maps.ipynb', 'repo_name': 'stephenrigney/CIG_2016', 'size': '6454'}