<jupyter_start><jupyter_text>London Start-up Map This notebook contains the code performed for the analysis in this blog post.The end result is a map of London that displays relative start-up formation vs cessation:  Packages<jupyter_code>%%capture
!pip install pgeocode
!pip install plotly-express
!pip install numpy
!pip install matplotlib
!pip install pandas
!pip install imageio
!pip install visvis
%%capture
import pandas as pd
import datetime
from datetime import date
from dateutil.relativedelta import relativedelta
import pgeocode
import json
import numpy as np
import plotly.express as px
import copy
from matplotlib import cm
import matplotlib.pyplot as plt
import urllib.request
import requests
import string
import time
import pickle
import random
import io
import os
import sys
import collections
import imageio
import glob
import visvis as vv<jupyter_output><empty_output><jupyter_text>Downloading London Postcode GEO-code JSONWe will download the JSON file for the geographical boundaries of London postcodes from here.<jupyter_code>london_geojson_url = "https://raw.githubusercontent.com/sjwhitworth/london_geojson/master/london_postcodes.json"
urllib.request.urlretrieve(london_geojson_url,"london_postcodes.json")
with open("london_postcodes.json","r") as file:
    data = json.load(file)<jupyter_output><empty_output><jupyter_text>Scraping Companies House DataWe will now scrap the details of London-based companies from the Companies House website.You will need an API key, which you can set-up here.<jupyter_code>%%capture
api_key = "<KEY>"
url = "https://api.companieshouse.gov.uk/company/"
request_counter =  0
attempted_company_codes = []
scrape_start = 7500000
scrape_stop = 12480000
scraping_attempts = int(0) # Change to a larger number e.g 10000
current_company_id = 0
if (os.path.exists("companies_house_data.txt")):
    dataframe = pd.read_table("companies_house_data.txt",delimiter="\t")
else:
    dataframe = pd.DataFrame(columns=['number','formed','active_until','postcode'])
    dataframe.set_index('number')
for counter in range(scraping_attempts):
    while (current_company_id in attempted_company_codes):
        current_company_id = random.randint(scrape_start,scrape_stop)
    attempted_company_codes.append(current_company_id)
    str_company_id = str(current_company_id).zfill(8)
    request_result = requests.get(url = url + str_company_id, auth = (api_key,""))
    request_counter +=1
    json_fail = False
    json_result = request_result.json()
    try:
        sys.stdout = io.StringIO()
        print(json_result)
        sys.stdout = sys.__stdout__
    except:
        json_fail = True
    # Sleep when requests close to rate limit.
    if (request_counter > 550):
        request_counter = 0
        time.sleep(300)
        print("sleeping for 5 min")
    print("Counter: {}".format(counter))
    contains_registered_office = 'registered_office_address' in json_result
    is_situated_in_london = contains_registered_office and \
    'locality' in json_result['registered_office_address'] \
    and 'London' in json_result['registered_office_address']['locality']
    contains_company_status = 'company_status' in json_result
    contains_errors = 'errors' in json_result
    contains_postal_code = contains_registered_office and 'postal_code' in json_result['registered_office_address']
    is_error_free = json_fail == False and contains_errors == False
    if (is_error_free and contains_postal_code and is_situated_in_london and contains_company_status):
        cur_dict = {}
        cur_dict['number'] = json_result['company_number']
        cur_dict['formed'] = json_result['date_of_creation']
        cur_dict['active_until'] = json_result['date_of_cessation'] if 'date_of_cessation' in json_result else "2020-12-01"
        cur_dict['postcode'] = cessation,json_result['registered_office_address']['postal_code']
        dataframe.append(cur_dict,ignore_index=True)<jupyter_output><empty_output><jupyter_text>Aggregate the set of existing postcodes in the JSON and apply reformatting for further down the line:<jupyter_code>postcodes_extant = set()
for counter in range(len(data['features'])):
    data['features'][counter]['id'] = data['features'][counter]['properties']['Name']
    postcodes_extant.add(data['features'][counter]['properties']['Name'])<jupyter_output><empty_output><jupyter_text>Next, to compare the Companies House company postcode data vs. the London postcodes, we will use this utility function:<jupyter_code>def get_closest_postcode(postcode, postcodes_list):
    if (" " in postcode and postcode.split()[0] in postcodes_list):
        return postcode.split()[0]
    postcode = ''.join(postcode.split())
    longest = ""
    for pc in postcodes_list:
        result = postcode.find(pc)
        if (result == 0 and len(pc) > len(longest)):
            longest = pc
    return longest

postcodes_list = list(postcodes_extant)
dataframe['postcode'] = [get_closest_postcode(val.upper(), postcodes_list) for val in dataframe['postcode'].values.tolist()]<jupyter_output><empty_output><jupyter_text>Now, lets transform the company creation vs. cessation dates to python datetime format:<jupyter_code>dataframe['dt1'] = [datetime.date(int(val.split("-")[0]), int(val.split("-")[1]),1) for val in dataframe['formed'].values.tolist()]
dataframe['dt2'] = [datetime.date(int(val.split("-")[0]), int(val.split("-")[1]),1) for val in dataframe['active_until'].values.tolist()]

earliest = min(dataframe['dt1'].tolist())
latest = max(dataframe['dt1'].tolist())<jupyter_output><empty_output><jupyter_text>And another utility function for getting the time-difference between months:<jupyter_code>def get_months_dif(d1,d2):
    return d1.month - d2.month + 12*(d1.year-d2.year)<jupyter_output><empty_output><jupyter_text>Now, lets create a dictionary that maps postcodes to how many companies wereactive in a given period (represented as a numpy array):<jupyter_code>total_months = get_months_dif(latest,earliest)
postcode_dict = collections.defaultdict(lambda:np.zeros((total_months+1)))
total = np.zeros((total_months+1))
total2 = np.zeros((total_months+1))
for index,row in dataframe.iterrows():
    matrix = np.zeros((total_months+1))
    start_months = get_months_dif(row['dt1'],earliest)
    end_months = get_months_dif(min(row['dt2'],latest),earliest)
    matrix[start_months:end_months] = 1.0
    postcode_dict[row['postcode']] += matrix
    test = np.zeros((total_months+1))
    test2 = np.zeros((total_months+1))
    test[end_months] = 1
    test2[start_months] = 1
    total += matrix
    total2 += test2

postcode_vals = postcode_dict.values()

max_val = 0
#We will take the square root of the number of active companies for visual purposes
for i in postcode_vals:
    max_val = np.power(max(max_val,np.max(i,axis=0)),0.5)<jupyter_output><empty_output><jupyter_text>Now lets create a mapping of postcodes to transformed net-active company values:<jupyter_code>plotting_values = [pd.DataFrame.from_dict({'id': list(postcode_dict.keys()), 'val': [float(np.power(val[index],0.5)) for val in list(postcode_dict.values())]}) for index in range(total_months)]<jupyter_output><empty_output><jupyter_text>And finally, we will plot each month of the start-up map using the plotly express choropleth map,and save them as PNGs, which are then combined with imageio into an animated .gif image.<jupyter_code>viridis = cm.get_cmap('viridis', int(max_val)+1)
viridis.colors

def get_col(viridis_val):
    return 'rgb({},{},{})'.format(int(viridis_val[0]*255),int(viridis_val[1]*255),int(viridis_val[2]*255))

colorscale = {}
colorscale2 = ((0.0,get_col(viridis.colors[0])),)

for index in range(1,int(max_val)+1):
    colorscale[float(index)] = get_col(viridis.colors[index])
    colorscale2 += ((float(index),get_col(viridis.colors[index])),)

for index in range(total_months):
    day = earliest + relativedelta(months=index)
    fig = px.choropleth_mapbox(plotting_values[index], geojson=data, locations='id', color='val',
                               color_continuous_scale ="RdYlGn",
                               range_color=(0, float(max_val)),
                               mapbox_style='carto-positron',
                               zoom=10, center = {"lat": 51.5073, "lon": -0.1277},
                               opacity=0.1,
                               labels={'val':'(sqrt) Net Active'}
                              )
    fig.update_layout(title="   London: (sqrt of) Active Companies per Month, {}-{}".format(day.year,day.month), title_font_size=24)
    fig.layout.coloraxis.autocolorscale= False
    fig.layout.coloraxis.cauto= False
    fig.layout.coloraxis.cmin= 0.0
    fig.layout.coloraxis.cmax= float(max_val)
    fig.write_image("./images_output/image_{}_{}_{}.png".format(day.year,day.month//10,day.month%10))
    
images = []
imageio.plugins.freeimage.download()
for filename in sorted(glob.glob("./images_output/*.png")):
    images.append(imageio.imread(filename))
    images.append(imageio.imread(filename))
imageio.mimsave('animation.gif', images, 'GIF-FI')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11973148453835149, 'fraction_numerical': 0.014196104324859689, 'mean_word_length': 4.959344262295082, 'pattern_counts': {'":': 2, '<': 42, '<?xml version=': 0, '>': 44, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '72059', 'license': 'MIT', 'n_tokens_mistral': 3027, 'n_tokens_neox': 2899, 'n_words': 642, 'path': '.ipynb_checkpoints/London Start-up Map-checkpoint.ipynb', 'repo_name': 'nomadicmonad/london-startup-map', 'size': '9122'}