<jupyter_start><jupyter_text>vectorized Queen contiguity<jupyter_code>df = geopandas.read_file(geopandas.datasets.get_path('naturalearth_lowres'))
df = df.explode()
import itertools as it
import collections


def _get_verts(shape):
    return _get_boundary_points(shape)

def _get_boundary_points(shape):
    """
    Recursively handle polygons vs. multipolygons to
    extract the boundary point set from each. 
    """
    if shape.type.lower() == 'polygon':
        shape = shape.boundary
        return _get_boundary_points(shape)
    elif shape.type.lower() == 'linestring':
        return list(map(tuple, list(zip(*shape.coords.xy))))
    elif shape.type.lower() == 'multilinestring':
        return list(it.chain(*(list(zip(*shape.coords.xy))
                                 for shape in shape)))
    elif shape.type.lower() == 'multipolygon':
        return list(it.chain(*(_get_boundary_points(part.boundary) 
                               for part in shape)))
    else:
        raise TypeError('Input shape must be a Polygon, Multipolygon, LineString, '
                        ' or MultiLinestring and was '
                        ' instead: {}'.format(shape.type))
collection = df.geometry.values
numPoly = len(collection)

w = {}
for i in range(numPoly):
    w[i] = set()

geoms = []  # list of coordinate tuples
offsets = []  # identificators of origin geoms for tuples
c = 0  # PolyID Counter


for n in range(numPoly):
    verts = _get_verts(collection[n])
    offsets += [c] * len(verts)
    geoms += (verts)
    c += 1

items = collections.defaultdict(set)
for i, vertex in enumerate(geoms):
    items[vertex].add(offsets[i])

shared_vertices = []
for item, location in list(items.items()):
    if len(location) > 1:
        shared_vertices.append(location)

for vert_set in shared_vertices:
    for v in vert_set:
        w[v] = w[v] | vert_set
        try:
            w[v].remove(v)
        except:
            pass
%%timeit
numPoly = len(collection)

w = {}
for i in range(numPoly):
    w[i] = set()

geoms = []  # list of coordinate tuples
offsets = []  # identificators of origin geoms for tuples
c = 0  # PolyID Counter


for n in range(numPoly):
    verts = _get_verts(collection[n])
    offsets += [c] * len(verts)
    geoms += (verts)
    c += 1
w = {}
for i in range(numPoly):
    w[i] = set()
geoms = []
offsets = []
c = 0  # PolyID Counter
%%timeit
for n in range(numPoly):
    verts = _get_verts(collection[n])
    offsets += [c] * len(verts)
    geoms += (verts)
    c += 1
offsets
geoms
items = collections.defaultdict(set)
for i, vertex in enumerate(geoms):
    items[vertex].add(offsets[i])
items
# new implementation
%%timeit
geoms = pygeos.get_coordinates(collection.data)
indices = pygeos.get_num_coordinates(collection.data)
offsets = []
for i, c in enumerate(indices):
    offsets += [i] * c
geoms = np.array((map(tuple, geoms)))
geoms.shape
unique, counts = np.unique(geoms, axis=0, return_counts=True)
shared = unique[counts > 1]
len(shared)
geoms
geoms
np.where(np.all(geoms == shared[2], axis=1))[0]
shared_vertices
numPoly = len(collection)

w = {}
for i in range(numPoly):
    w[i] = set()

geoms = pygeos.get_coordinates(collection.data)
indices = pygeos.get_num_coordinates(collection.data)

offsets = []
for i, c in enumerate(indices):
    offsets += [i] * c
geoms = np.array((map(tuple, geoms)))

items = collections.defaultdict(set)
for i, vertex in enumerate(geoms):
    items[vertex].add(offsets[i])

shared_vertices = []
for item, location in list(items.items()):
    if len(location) > 1:
        shared_vertices.append(location)

for vert_set in shared_vertices:
    for v in vert_set:
        w[v] = w[v] | vert_set
        try:
            w[v].remove(v)
        except:
            pass
np.array([[1.1,1.1][1.1,1.1]])
ddf
def get_coordinates(series):
    return pygeos.get_coordinates(series.values.data)

geoms = ddf.geometry.map_partitions(get_coordinates, meta=np.array([[1.1,1.1], [1.1,1.1]]))

def get_indices(series):
    return pygeos.get_coordinates(series.values.data)

indices = ddf.geometry.map_partitions(get_indices, meta=np.array([1, 1]))
geoms.compute()
numPoly = len(ddf)

w = {}
for i in range(numPoly):
    w[i] = set()

def get_coordinates(series):
    return pygeos.get_coordinates(series.values.data)

geoms = ddf.buildings.map_partitions(get_coordinates).compute()

def get_indices(series):
    return pygeos.get_num_coordinates(series.values.data)

indices = ddf.buildings.map_partitions(get_indices, meta=np.array([1, 1])).compute()

offsets = []
for i, c in enumerate(indices):
    offsets += [i] * c
geoms = np.array((map(tuple, geoms)))

items = collections.defaultdict(set)
for i, vertex in enumerate(geoms):
    items[vertex].add(offsets[i])

shared_vertices = []
for item, location in list(items.items()):
    if len(location) > 1:
        shared_vertices.append(location)

for vert_set in shared_vertices:
    for v in vert_set:
        w[v] = w[v] | vert_set
        try:
            w[v].remove(v)
        except:
            pass
df = geopandas.read_file(geopandas.datasets.get_path('naturalearth_lowres'))
ddf = dg.from_geopandas(df, npartitions=14)
def get_coordinates(series):
    return pygeos.get_coordinates(series.values.data), pygeos.get_num_coordinates(series.values.data)

geoms = ddf.geometry.map_partitions(get_coordinates)
g = geoms.compute()
len(ddf)
%%time
numPoly = len(ddf)

w = {}
for i in range(numPoly):
    w[i] = set()

def get_coordinates(series):
    return pygeos.get_coordinates(series.values.data)

geoms = ddf.geometry.map_partitions(get_coordinates).compute()

def get_indices(series):
    return pygeos.get_num_coordinates(series.values.data)

indices = ddf.geometry.map_partitions(get_indices).compute()

offsets = []
for i, c in enumerate(indices):
    offsets += [i] * c
geoms = list(map(tuple, geoms))

items = collections.defaultdict(set)
for i, vertex in enumerate(geoms):
    items[vertex].add(offsets[i])

shared_vertices = []
for item, location in list(items.items()):
    if len(location) > 1:
        shared_vertices.append(location)

for vert_set in shared_vertices:
    for v in vert_set:
        w[v] = w[v] | vert_set
        try:
            w[v].remove(v)
        except:
            pass
%%time
collection = df.geometry.values
numPoly = len(df)

w = {}
for i in range(numPoly):
    w[i] = set()

geoms = []  # list of coordinate tuples
offsets = []  # identificators of origin geoms for tuples
c = 0  # PolyID Counter


for n in range(numPoly):
    verts = _get_verts(collection[n])
    offsets += [c] * len(verts)
    geoms += (verts)
    c += 1
%%time
items = collections.defaultdict(set)
for i, vertex in enumerate(geoms):
    items[vertex].add(offsets[i])

shared_vertices = []
for item, location in list(items.items()):
    if len(location) > 1:
        shared_vertices.append(location)

for vert_set in shared_vertices:
    for v in vert_set:
        w[v] = w[v] | vert_set
        try:
            w[v].remove(v)
        except:
            pass
client.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13348480545299632, 'fraction_numerical': 0.005680204487361545, 'mean_word_length': 3.4046278924327704, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '636396', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2519, 'n_tokens_neox': 2468, 'n_words': 566, 'path': 'docs/_sources/measuring/_dask_trials.ipynb', 'repo_name': 'alanurbangrammario/spatial_signatures', 'size': '7042'}