<jupyter_start><jupyter_text>Identifying patterns in a date column<jupyter_code>import pandas as pd
dates = pd.read_csv('../resources/dev/dates.txt', header=None, squeeze=True)
dates.sample(5, random_state=42)
# Tokenize the column
from openclean_pattern.tokenize.factory import TokenizerFactory
tokenizer = TokenizerFactory.create_tokenizer('default')
tokenizer.encode(dates)
# Tokenize and convert column into internal token representations
enc = tokenizer.encode(dates)
enc
# Collect similar rows together

# from openclean_pattern.align.factory import CollectorFactory
# collector = CollectorFactory.create_collector('cluster')

from openclean_pattern.collect.cluster import Cluster
collector = Cluster(min_samples=1)

col = collector.collect(enc)
# encoded rows clustered together. -1 is noise but doesnt exist because we set min_samples=1 in dbscan
col
# Samples of encoded rows in each cluster
for j in col:
    print(j)
    for n, i in enumerate(col[j]):
        print(enc[i])
        if n > 5:
            break
    print()
# Compile patterns in each cluster
from openclean_pattern.regex.compiler import DefaultRegexCompiler

compiler = DefaultRegexCompiler()
compiled = compiler.compile(enc, col)
compiled
# Coverage of patterns in each cluster
for k, pattern in compiled.items():
    print(k, pattern.stats())<jupyter_output>0 defaultdict(<class 'float'>, {'NUMERIC PUNC NUMERIC': 1.0})
1 defaultdict(<class 'float'>, {'ALPHANUM': 1.0})
2 defaultdict(<class 'float'>, {'ALPHA \\S NUMERIC': 1.0})
{'fraction_non_alphanumeric': 0.10159362549800798, 'fraction_numerical': 0.010624169986719787, 'mean_word_length': 5.358649789029536, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '348644', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 472, 'n_tokens_neox': 441, 'n_words': 140, 'path': 'examples/Dates - Pattern Detection.ipynb', 'repo_name': 'VIDA-NYU/openclean-pattern', 'size': '1506'}