<jupyter_start><jupyter_text>Parking Violations - Street Name Clustering (Performance comparison)This notebook evaluates run times for clustering large datasets. It compares two options:- create clusters using the stream operator, and- extract distinct value stes using the stream operator and then cluster separately using multiple threads,<jupyter_code># Download the full 'NYC Parking Violations Issued - Fiscal Year 2014' dataset.
# Note that the downloaded full dataset file is about 380 MB in size! Use the
# alternative data file with 10,000 rows that is included in the repository if
# you do not want to download the full data file.

import gzip
import humanfriendly
import os

from openclean.data.source.socrata import Socrata

dataset = Socrata().dataset('jt7v-77mi')
datafile = './jt7v-77mi.tsv.gz'

# Download file only if it does not exist already.
if not os.path.isfile(datafile):
    with gzip.open(datafile, 'wb') as f:
        print('Downloading ...\n')
        dataset.write(f)


# As an alternative, you can also use the smaller dataset sample that is
# included in the repository.
#
#datafile = './data/jt7v-77mi.tsv.gz'

fsize = humanfriendly.format_size(os.stat(datafile).st_size)
print("Using '{}' in file {} of size {}".format(dataset.name, datafile, fsize))
# Create data stream for the downloaded file.

from openclean.pipeline import stream

ds = stream(datafile)<jupyter_output><empty_output><jupyter_text>Stream Clusters vs. Mutiple-ThreadsCompare run time for using the stream operator to perform the clustering or extracting the list of distinct street names first and then run clustering in parallel (using up to 4 threads).<jupyter_code>import time

from openclean.cluster.key import KeyCollision
from openclean.function.value.key.fingerprint import Fingerprint
from openclean_geo.address.usstreet import StandardizeUSStreetName
# Extract list of distinct street names.

start_parse = time.perf_counter()

streets = ds.distinct_values('Street')

end_parse = time.perf_counter()

print('Parse time {:0.4f} sec. ({} streets)'.format(end_parse - start_parse, len(streets)))
# Standardize street names in parallel.

f = StandardizeUSStreetName(characters='upper', alphanum=True, repeated=False)

for threads in range(1,5):
    start_std = time.perf_counter()
    streets_std = f.apply(streets, threads=threads)
    count = len(streets_std)
    end_std = time.perf_counter()
    exec_time = end_std - start_std
    print('Standardization time (using {} threads) {:0.4f} sec. ({} streets)'.format(threads, exec_time, count))
# Cluster standardized streen names.

for threads in range(1,5):
    f = KeyCollision(func=Fingerprint(), threads=threads)
    start_clstr = time.perf_counter()
    clusters = f.clusters(streets_std)
    count = len(clusters)
    end_clstr = time.perf_counter()
    exec_time = end_clstr - start_clstr
    print('Cluster time (using {} threads) {:0.4f} sec. ({} clusters)'.format(threads, exec_time, count))
# Run standardization and cluster generation as part of the
# stream processing.

start = time.perf_counter()

streets = ds\
    .select('Street')\
    .update('Street', StandardizeUSStreetName(characters='upper', alphanum=True, repeated=False))

clusters = streets.cluster(clusterer=KeyCollision(func=Fingerprint()))

end = time.perf_counter()

print('Runtime {:0.4f} sec. ({} clusters)'.format(end - start, len(clusters)))<jupyter_output>Runtime 325.9010 sec. (761 clusters)
{'fraction_non_alphanumeric': 0.10328775094559209, 'fraction_numerical': 0.012801862089031132, 'mean_word_length': 5.0, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '385068', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1094, 'n_tokens_neox': 1027, 'n_words': 331, 'path': 'examples/notebooks/Parking Violations - Street Name Clustering (Performance comparison).ipynb', 'repo_name': 'VIDA-NYU/openclean', 'size': '3437'}