<jupyter_start><jupyter_text>Circular Night Time Lights PatchesCreate an HDF5 file with 10Km circular patchs of the nightime lights raster for each SEZ. This will create a patches file and publish it to the web, at: http://public.source.civicknowledge.com.s3.amazonaws.com/sez/sez_circ_patches.h5You can get these data packages via the ['sez' tag at the SDRDL Data Repository](https://data.sandiegodata.org/?s=sez)<jupyter_code>import seaborn as sns
import metapack as mp
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import rowgenerators as rg
import fiona
import rasterio
import rasterio.mask
from IPython.display import display 
from rasterio.plot import reshape_as_raster, reshape_as_image, show
from tqdm.notebook import tqdm
from joblib import Parallel, delayed
import requests   # to make GET request
import boto3
import h5py 

%matplotlib inline
sns.set_context('notebook')

%run ./lib.py

# Open the two main data packages
sez_p = mp.open_package('http://library.metatab.org/worldbank.org-sez-1.1.1.csv')
ntl_p = mp.open_package('http://library.metatab.org/figshare.com-harmonized_ntl-1.1.1.csv')

# Displaying the package shows metadata
sez_p

ntl_p

RADIUS = 5_000

# Create Dataframes
sez = sez_p.resource('sez').geoframe()

# The operational date col is a string, so make an int year column. 
sez['year'] = pd.to_numeric(sez.operational_date, errors='coerce')

sez.to_crs(3395).buffer(1000000).plot()

# Update the SEZ geometries to be a 20KM buffer around the point. We project to 
# EPSG:3395, World Mercator, to be able to make the buffer in meters. 
g =  sez.to_crs(3395).buffer(RADIUS)

sez['geo_area'] = g.area
sez['geometry'] = g.to_crs(4326)


# Generate all of the patches by masking each of the geometries in sez onto 
# all of the rasters in the ntl_p package. 

patches, exc, tasks = mp_ntl_mask(ntl_p, sez)

# Write all of the patches to an HDF5 file

source_fn = 'sez_circ_patches_{RADIUS}.h5'

def save_patches(patches, source_fn):
    with h5py.File(source_fn, 'w') as f:
        for year, sez_id,  e in patches:
            k = f'circle/{sez_id}/{year}'
            f[k] = e

            
save_patches(patches, source_fn)
# Upload the file to S3
# This won't work if you don't have access to the bucket, which 
# you probably don't

dest_fn = f'sez_circ_patches_{RADIUS}.h5'

def upload(source_fn, dest_fn):

    try:
        s3_client = boto3.client('s3')

        
        s3_client.upload_file(source_fn, 'public.source.civicknowledge.com', 'sez/'+dest_fn, ExtraArgs={'ACL': 'public-read'})
    except Exception as e:
        print(f"Nope: {e}")
        
upload(source_fn, dest_fn)<jupyter_output><empty_output><jupyter_text>Rings<jupyter_code>%run ./lib.py

sez_rings = make_rings(sez, RADIUS)
sez_rings.head()
sezr= sez.join(sez_rings.to_frame('ring'))

patches, exc, tasks = mp_ntl_mask(ntl_p, sezr, 'ring')
source_fn = 'sez_ring_patches_{RADIUS}.h5'
dest_fn = f'sez_ring_patches_{RADIUS}.h5'

save_patches(patches, source_fn)
upload(source_fn, dest_fn)<jupyter_output><empty_output><jupyter_text>Test the patches<jupyter_code># Test the file upload

# Download the file
dl_fn = 'patches_{RADIUS}.h5'
with open(dl_fn, "wb") as file:
    # get request
    response = requests.get(f'http://public.source.civicknowledge.com.s3.amazonaws.com/sez/sez_circ_patches_{RADIUS}.h5')
    # write to file
    file.write(response.content)
    
# Get a random selection of patches from the file
def get_hdf_keys(f):
    """Return all of the dataset keys in an HDF5 file"""
    keys = []
    f.visit(lambda key : keys.append(key) if isinstance(f[key], h5py.Dataset) else None)
    return keys

with h5py.File('patches.h5', 'r') as f:
    keys = get_hdf_keys(f)
    
# Plot a random sample
from random import shuffle
shuffle(keys)

fig, axes = plt.subplots(5,5, figsize=(10,10))
axes = np.ravel(axes)

with h5py.File('patches.h5', 'r') as f:
    for k, a in zip(keys[:len(axes)], axes):
        r = f[k][:]
        a.set_axis_off()
        show(r, ax=a)

fig.suptitle('Sample of 25 NTL Patches', fontsize=14)
plt.tight_layout()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11384690394929303, 'fraction_numerical': 0.017552413456850317, 'mean_word_length': 4.280566280566281, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '338742', 'license': 'CC0-1.0', 'n_tokens_mistral': 1615, 'n_tokens_neox': 1544, 'n_words': 405, 'path': 'notebooks/ericbusboom/01 Make Circular Patches.ipynb', 'repo_name': 'sandiegodata-projects/chartercitiesinstitute.org-sez', 'size': '4102'}