<jupyter_start><jupyter_text>Load the packages<jupyter_code># pip install jellyfish
# pip install fuzzywuzzy
# pip install xgboost
# conda install -c conda-forge python-levenshtein

import time
import re
import pandas as pd
import numpy as np
import jellyfish as jf
from fuzzywuzzy import fuzz
from fuzzywuzzy import process
from sklearn.neural_network import MLPClassifier
from xgboost import XGBClassifier
from sklearn.dummy import DummyClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.linear_model import Perceptron
from sklearn.svm import SVC
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.metrics import f1_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import roc_auc_score
from sklearn.metrics import jaccard_score<jupyter_output><empty_output><jupyter_text>Load the data<jupyter_code>df = pd.read_csv('./data/aux_municipality2.csv', sep = ",")
df
dfp = pd.read_csv('./data/lau.csv', sep = ",", encoding='latin1')
dfp
#df1 = pd.crosstab(df.Mun, dfp.lau_name)
from itertools import product

comb = list(product(df.Mun, dfp.lau_name))
df1 = pd.DataFrame(data=comb, columns=['internal_name','external_name'])
df1<jupyter_output><empty_output><jupyter_text>Feature engineering<jupyter_code>def jaccard_score(external_name, internal_name):

    external_chars = set(re.findall(r'[a-zA-Z0-9]+', external_name))
    internal_chars = set(re.findall(r'[a-zA-Z0-9]+', internal_name)) 
    union = external_chars.union(internal_chars)
    intersection = external_chars.intersection(internal_chars)

    if len(external_chars)==0 and len(internal_chars) == 0:
        return 1
    else:
        return (len(intersection)/ len(union))
def engineer_features(df):

    df['internal_name'] = df['internal_name'].str.lower()
    df['external_name'] = df['external_name'].str.lower()

    #df['levenshtein_distance'] = df.apply(
    #lambda x: jf.levenshtein_distance(x['external_name'], x['internal_name']), axis=1)

    #df['damerau_levenshtein_distance'] = df.apply(
    #lambda x: jf.damerau_levenshtein_distance(x['external_name'], x['internal_name']), axis=1)

    #df['hamming_distance'] = df.apply(
    #lambda x: jf.hamming_distance(x['external_name'], x['internal_name']), axis=1)

    #df['jaro_similarity'] = df.apply(
    #lambda x: jf.jaro_similarity(x['external_name'], x['internal_name']), axis=1)

    #df['jaro_winkler_similarity'] = df.apply(
    #lambda x: jf.jaro_winkler_similarity(x['external_name'], x['internal_name']), axis=1)

    #df['match_rating_comparison'] = df.apply(
    #lambda x: jf.match_rating_comparison(x['external_name'], x['internal_name']), axis=1).fillna(0).astype(int)

    df['jaccard_score'] = df.apply(
    lambda x: jaccard_score(x['external_name'], x['internal_name']), axis=1)

    df.replace([np.inf, -np.inf], np.nan, inplace=True)
    df.fillna(value=0, inplace=True)

    return df<jupyter_output><empty_output><jupyter_text>levenshtein_distance<jupyter_code>df1 = engineer_features(df1)
df1.sort_values('levenshtein_distance', ascending=True).drop_duplicates(['internal_name'])<jupyter_output><empty_output><jupyter_text>jaccard_score<jupyter_code>df1 = engineer_features(df1)
df1.sort_values('jaccard_score', ascending=False).drop_duplicates(['internal_name'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11941875825627477, 'fraction_numerical': 0.007397622192866579, 'mean_word_length': 6.8875, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '301520', 'license': 'MIT', 'n_tokens_mistral': 1345, 'n_tokens_neox': 1295, 'n_words': 242, 'path': '04_data_integration/3. data_profiling_notebooks_ipynb/03_matching_examples/simplified_matching_municipality_some_scores.ipynb', 'repo_name': 'rachelfanti/data_science', 'size': '3785'}