<jupyter_start><jupyter_text>Visualize US Data<jupyter_code>from geonotebook.wrappers import RasterData
rd = RasterData("geotrellis://http://localhost:8899/tiles/weld11")
M.add_layer(rd)<jupyter_output><empty_output><jupyter_text>Subset with a rectangle<jupyter_code>import findspark
findspark.init()

from pyspark import SparkContext, SparkConf
from pyproj import Proj, transform
from shapely.geometry import Polygon
from geopyspark.geotrellis.catalog import S3Catalog
from geopyspark.geopycontext import GeoPyContext
from geopyspark.avroregistry import AvroRegistry

def latLongToWebMercator(lat_long_coords):
    in_proj  = Proj("+init=EPSG:4326")
    out_proj = Proj("+init=EPSG:3857")
    return [transform(in_proj, out_proj, x, y) for (x, y) in lat_long_coords]

spark_config = SparkConf().setMaster('local')\
                          .setAppName('data_frame_test')
geopysc = GeoPyContext.construct(conf=spark_config)
avreg = AvroRegistry()
# import pudb; pu.db

catalog = S3Catalog(geopysc, avreg)

polygon = Polygon(
    latLongToWebMercator(list(M.layers.annotation.rectangles[0].exterior.coords)))
bucket = "kitware-catalog"
prefix = 'full-catalog'
layer_name = 'weld11'
layer_zoom = 13

(rdd, metadata) = catalog.query_spatial_singleband(
    bucket, prefix, layer_name, layer_zoom, polygon)
ta = rdd.collect()<jupyter_output><empty_output><jupyter_text>Investigate<jupyter_code>def stitch_tilearrays(ta):
    arr = None
    c_arr = None
    prev_col = None

    for key, _ta in sorted(ta, key=lambda x: (x[0].col, x[0].row)):
        # Build up the column
        if prev_col == None:
            c_arr = _ta
        elif prev_col == key.col:
            c_arr = np.vstack([c_arr, _ta])
        # We've finished with the column,  add it
        # to the array
        else:
            if arr is None:
                arr = c_arr
            else:
                arr = np.hstack([arr, c_arr])

            c_arr = _ta
        prev_col = key.col

    return np.hstack([arr, c_arr])

plt.imshow(stitch_tilearrays(ta))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10684798445847499, 'fraction_numerical': 0.011170471102476931, 'mean_word_length': 3.671201814058957, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '562508', 'license': 'Apache-2.0', 'n_tokens_mistral': 763, 'n_tokens_neox': 727, 'n_words': 150, 'path': 'notebooks/Example.ipynb', 'repo_name': 'OpenGeoscience/gnb_geotrellis', 'size': '2059'}