<jupyter_start><jupyter_text>🔗 Record Linkage- Record Linkage (groupings of various above algorithms)    - Reference https://en.wikipedia.org/wiki/Record_linkage    - Python lib https://github.com/J535D165/recordlinkage    - NLTK Distance references: http://www.nltk.org/_modules/nltk/metrics/distance.html    - Jaro-Winkler Distance    - Reference https://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance- Levenshtein Distance    - Reference https://en.wikipedia.org/wiki/Levenshtein_distance    - Python lib https://github.com/ztane/python-Levenshtein- Gestalt or Ratcliff/Obershelp Pattern Recognition    - Reference https://en.wikipedia.org/wiki/Gestalt_Pattern_Matching    - Python lib (SequenceMatcher) https://docs.python.org/3/library/difflib.html<jupyter_code># Modified from below
# reference: https://recordlinkage.readthedocs.io/en/latest/notebooks/link_two_dataframes.html
# note: The data used in this example is part of Febrl and is fictitious.
import pandas as pd
import recordlinkage
from recordlinkage.datasets import load_febrl4

dfA, dfB = load_febrl4()

dfA.head(2)
dfB.head(2)
indexer = recordlinkage.Index()
indexer.full()
pairs = indexer.index(dfA, dfB)
indexer = recordlinkage.Index()
indexer.block("given_name")
candidate_links = indexer.index(dfA, dfB)

print(len(candidate_links))
compare_cl = recordlinkage.Compare()
compare_cl.string("suburb", "suburb", method="jarowinkler", label="suburb")

features = compare_cl.compute(candidate_links, dfA, dfB)
features.sort_values(by="suburb", ascending=False).head(5)
features = features.rename(columns={"suburb": "suburb_match"})
features = features.reset_index(drop=False)

features_with_record = pd.merge(
    features,
    dfA.rename(columns={"suburb": "suburb_A"})["suburb_A"],
    how="left",
    left_on="rec_id_1",
    right_on=dfA.index,
)
features_with_record
features_with_record = pd.merge(
    features_with_record,
    dfB.rename(columns={"suburb": "suburb_B"})["suburb_B"],
    how="left",
    left_on="rec_id_2",
    right_on=dfB.index,
)
features_with_record
features_with_record[["suburb_match", "suburb_A", "suburb_B"]].sort_values(
    by="suburb_match", ascending=False
).head(5)
features_with_record[
    (features_with_record["suburb_match"] != 0)
    & (features_with_record["suburb_match"] != 1)
][["suburb_match", "suburb_A", "suburb_B"]].sort_values(
    by="suburb_match", ascending=False
).head(
    20
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13664341403364794, 'fraction_numerical': 0.009848173984407058, 'mean_word_length': 5.7162534435261705, 'pattern_counts': {'":': 3, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 8, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '390888', 'license': 'MIT', 'n_tokens_mistral': 887, 'n_tokens_neox': 860, 'n_words': 142, 'path': 'record_linkage.ipynb', 'repo_name': 'd33bs/notebook_letterpress', 'size': '2437'}