<commit_before># $$$$$$$\                  $$\                                $$$$$$\   $$$$$$\  $$\    $$\    $$$$$$$\  $$$$$$$\  
# $$  __$$\                 $$ |                              $$  __$$\ $$  __$$\ $$ |   $$ |   $$  __$$\ $$  __$$\ 
# $$ |  $$ | $$$$$$\   $$$$$$$ |$$\   $$\  $$$$$$\   $$$$$$\  $$ /  \__|$$ /  \__|$$ |   $$ |   $$ |  $$ |$$ |  $$ |
# $$ |  $$ |$$  __$$\ $$  __$$ |$$ |  $$ |$$  __$$\ $$  __$$\ $$ |      \$$$$$$\  \$$\  $$  |   $$$$$$$  |$$$$$$$  |
# $$ |  $$ |$$$$$$$$ |$$ /  $$ |$$ |  $$ |$$ /  $$ |$$$$$$$$ |$$ |       \____$$\  \$$\$$  /    $$  ____/ $$  ____/ 
# $$ |  $$ |$$   ____|$$ |  $$ |$$ |  $$ |$$ |  $$ |$$   ____|$$ |  $$\ $$\   $$ |  \$$$  /     $$ |      $$ |      
# $$$$$$$  |\$$$$$$$\ \$$$$$$$ |\$$$$$$  |$$$$$$$  |\$$$$$$$\ \$$$$$$  |\$$$$$$  |   \$  /      $$ |      $$ |      
# \_______/  \_______| \_______| \______/ $$  ____/  \_______| \______/  \______/     \_/$$$$$$\\__|      \__|      
#                                         $$ |                                           \______|                   
#                                         $$ |                                                                      
#                                         \__|                                                                      
# ---------------------------------------------
#!/usr/bin/python3.4.3
# ---------------------------------------------
# Imports
# ---------------------------------------------
import csv
# ---------------------------------------------
# Global Variables
# ---------------------------------------------
CSVFilesHaveHeaderRow = True # True or False if input files include a header row
InputFile = "DedupeSampleData.csv" 
CleanOutput = "_CleanOutputPP.csv"
Dupes = "_DupesPP.csv" 
# ---------------------------------------------w
FName = 0
LName = 1
Address = 2
Zip = 5
Entries = set()
# ----------------------------------------------
# Objects
# ----------------------------------------------
Input = csv.reader(open(InputFile,'r'))
OutputClean = csv.writer(open(CleanOutput,'a'))<commit_msg>Updated ASCII Art Banner<commit_after># $$$$$$$\                $$\                            $$$$$$\  $$$$$$\ $$\    $$\  $$$$$$$\ $$$$$$$\  
# $$  __$$\               $$ |                          $$  __$$\$$  __$$\$$ |   $$ | $$  __$$\$$  __$$\ 
# $$ |  $$ |$$$$$$\  $$$$$$$ $$\   $$\ $$$$$$\  $$$$$$\ $$ /  \__$$ /  \__$$ |   $$ | $$ |  $$ $$ |  $$ |
# $$ |  $$ $$  __$$\$$  __$$ $$ |  $$ $$  __$$\$$  __$$\$$ |     \$$$$$$\ \$$\  $$  | $$$$$$$  $$$$$$$  |
# $$ |  $$ $$$$$$$$ $$ /  $$ $$ |  $$ $$ /  $$ $$$$$$$$ $$ |      \____$$\ \$$\$$  /  $$  ____/$$  ____/ 
# $$ |  $$ $$   ____$$ |  $$ $$ |  $$ $$ |  $$ $$   ____$$ |  $$\$$\   $$ | \$$$  /   $$ |     $$ |      
# $$$$$$$  \$$$$$$$\\$$$$$$$ \$$$$$$  $$$$$$$  \$$$$$$$\\$$$$$$  \$$$$$$  |  \$  /    $$ |     $$ |      
# \_______/ \_______|\_______|\______/$$  ____/ \_______|\______/ \______/    \_$$$$$$\__|     \__|      
#                                     $$ |                                      \______|                 
#                                     $$ |                                                               
#                                     \__|                                                                                                                                  
# ---------------------------------------------
#!/usr/bin/python3.4.3
# ---------------------------------------------
# Imports
# ---------------------------------------------
import csv
# ---------------------------------------------
# Global Variables
# ---------------------------------------------
CSVFilesHaveHeaderRow = True # True or False if input files include a header row
InputFile = "DedupeSampleData.csv" 
CleanOutput = "_CleanOutputPP.csv"
Dupes = "_DupesPP.csv" 
# ---------------------------------------------w
FName = 0
LName = 1
Address = 2
Zip = 5
Entries = set()
# ----------------------------------------------
# Objects
# ----------------------------------------------
Input = csv.reader(open(InputFile,'r'))
OutputClean = csv.writer(open(CleanOutput,'a'))
{'fraction_non_alphanumeric': 0.4182959208303162, 'fraction_numerical': 0.0033791938209027277, 'mean_word_length': 0.6907384740922072, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 35, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2796056', 'n_tokens_mistral': 1216, 'n_tokens_neox': 1135, 'n_words': 84}