<jupyter_start><jupyter_text>Imports & setup<jupyter_code>import nltk
import os
import pandas as pd

# maximum number of columns
pd.set_option("display.max_rows", 101)
pd.set_option("display.max_columns", 101)<jupyter_output><empty_output><jupyter_text>Data ingestion<jupyter_code>file_name = "20210103_hundenamen.csv"
data_folder = os.path.join(os.getcwd(), "data")
file_path = os.path.join(data_folder, file_name)
try:
    df = pd.read_csv(file_path)
    print(df.head())
    
except FileNotFoundError:
    print(f"Error - cannot find {file_path}")<jupyter_output>HUNDENAME  GEBURTSJAHR_HUND GESCHLECHT_HUND
0                         Ituma              2011               w
1      "Bo" Bendy of Treegarden              2020               m
2         "Bobby" <NAME>clair              2009               m
3    "Buddy" <NAME>.              2011               m
4  "Fly" Showring i fly for you              2015               w<jupyter_text>EDA<jupyter_code>len(df.HUNDENAME), len(df.HUNDENAME.unique())
len(df[df.HUNDENAME =='unbekannt'])<jupyter_output><empty_output><jupyter_text>Feature Engineering & Data cleaning<jupyter_code># removing '"' in HUNDENAME
df.HUNDENAME =df.HUNDENAME.str.replace(r'\"', '')
df
def edit_distance(term, search_word):
    '''
    Calculate the Levenshtein distance (edit distance) between two strings after lowercasing them.
    
    Args:
        term (str): first string to compare
        searchword (str): the searchterm that we want to calculate the edit distance for
        
    Returns:
        int: the edit distance between the two terms
    '''
    
    try:
        term = nltk.edit_distance(term.lower(), search_word.lower())
    
    except:
        pass
    
    return term<jupyter_output><empty_output><jupyter_text>Searching for names similar to "Luca"<jupyter_code>search_word = 'Luca'
df['edit_distance']= df.HUNDENAME.apply(edit_distance, args=[search_word])
df
luca_df = df[df['edit_distance'] ==1]
luca_df
names = list(luca_df.HUNDENAME.unique())
names
# Try to find a not so common name
luca_df.HUNDENAME.value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10451306413301663, 'fraction_numerical': 0.019002375296912115, 'mean_word_length': 3.1868787276341948, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '657809', 'license': 'MIT', 'n_tokens_mistral': 760, 'n_tokens_neox': 721, 'n_words': 158, 'path': 'finding_dognames.ipynb', 'repo_name': 'subrockmann/finding_luca', 'size': '2118'}