<jupyter_start><jupyter_text>INCONSISTENT DATA ENTRY<jupyter_code># MODULES WE WILL USE
import pandas as pd 
import numpy as np 
# HELPFUL MODULES
import fuzzywuzzy
from fuzzywuzzy import process
import chardet
# we need to pip3 install fuzzywuzzy in anaconda cm !!
# read in all our data
ks = pd.read_csv('ks-projects-201801.csv')
ks.head()
# set seed for reproducibility
np.random.seed(0)
# clean up the category column to make sure there is not data entry inconsistencies
# get all the values in category coloumn 
category = ks['category'].unique()
# sort them alphabetically and then take a closer look
category.sort()
category
'''The first thing I'm going to do is make everything lower case (I can change it back at the end if I like) and remove any white spaces at the beginning and end of cells. Inconsistencies in capitalizations and trailing white spaces are very common in text data and you can fix a good 80% of your text data entry inconsistencies by doing this.'''
# convert to lower case
ks['category'] = ks['category'].str.lower()

# remove the trailing white spaces
ks['category'] = ks['category'].str.strip()
ks.head()
# use fuzzy matching to correct inconsistent data entry
# get all the unique values in the category column
categories = ks['category'].unique()

# sort them alphabetically and than take a closer look
categories.sort()
categories
# We're going to use the fuzzywuzzy package to help identify which strings are closest to each other<jupyter_output><empty_output><jupyter_text>Fuzzy matching: The process of automatically finding text strings that are very similar to the target string. In general, a string is considered "closer" to another one the fewer characters you'd need to change if you were transforming one string into another<jupyter_code>#Fuzzywuzzy returns a ratio given two strings. The closer the ratio is to 100
# get the top 10 closest matches to 'games'
matches = fuzzywuzzy.process.extract('games', categories, limit=10, scorer=fuzzywuzzy.fuzz.token_sort_ratio)
matches
# function to replace rows in the provided column of the provided data frame
# that match the provided string above the provided ratio with the provided string
def replace_matches_in_column(df, column, string_to_match, min_ratio = 67):
    # get a list of unique strings
    strings = df[column].unique()
    
    # get the top 10 closest matches to our input string
    matches = fuzzywuzzy.process.extract(string_to_match, strings, limit=10, scorer= fuzzywuzzy.fuzz.token_sort_ratio)
    
    # only get matches with a ratio > 90
    close_matches = [matches[0] for matches in matches if matches[1] >= min_ratio]
    
    # get the rows of all the close matches in our dataframe
    rows_with_matches = df[column].isin(close_matches)
    
    # replace all rows with close mathces with the input matches
    
    df.loc[rows_with_matches, column] = string_to_match

#let us know the function's done
print("all done!")
# use the function we just wrote to replace close matches to 'games' with 'games'
replace_matches_in_column(df=ks, column='category', string_to_match= 'games')<jupyter_output><empty_output><jupyter_text>and now let's check the unique values in our 'category' column  again and make sure we've tidied up 'games ' correctly<jupyter_code># get all the values in the 'category' column
categoriese = ks['category'].unique()

# sort them alphabtically and then take a closer look 
categories.sort()
categories
matches<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.07005455067470572, 'fraction_numerical': 0.007751937984496124, 'mean_word_length': 4.945392491467577, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '301174', 'license': 'Apache-2.0', 'n_tokens_mistral': 1031, 'n_tokens_neox': 958, 'n_words': 439, 'path': 'Inconsistent_DataEntry.ipynb', 'repo_name': 'pemtash/Inconsistent_DataEntry', 'size': '3483'}