<jupyter_start><jupyter_text>Get Appropriate Filepaths<jupyter_code>saved_files_path = "../../../saved_files"
task_name = os.path.basename(os.getcwd())
    
unlabeled_data_filepath = dm_file_checker.get_proper_unlabeled_data_filepath(task_name, saved_files_path)
settings_filepath =  dm_file_checker.get_filepath(task_name, "model_settings", saved_files_path)
labeled_data_filepath = dm_file_checker.get_filepath(task_name, "labeled_data", saved_files_path)
blocks_filepath = dm_file_checker.get_filepath(task_name, "blocks", saved_files_path)

numeric_fields = dm_file_checker.get_dataset_info(task_name, "numeric_fields", saved_files_path)
RECALL_TRAIN_VAL = dm_file_checker.get_task_info(task_name, "recall_train", saved_files_path)
print("Using recall value of {}".format(RECALL_TRAIN_VAL))
print("Numeric fields are {}".format(numeric_fields))<jupyter_output>Using recall value of 1.0
Numeric fields are []<jupyter_text>Define parameters for model<jupyter_code># default blocked_proportion is 50%
BLOCKED_PROPORTION = 0.5
# default sample size is 15000
#SAMPLE_SIZE = 30_000
SAMPLE_SIZE = 15_000<jupyter_output><empty_output><jupyter_text>Reading in Data<jupyter_code>unlabeled_data = dm_utils.read_unlabeled_data_json(unlabeled_data_filepath, numeric_fields = numeric_fields)<jupyter_output>converting 190 empty string values of column date_of_birth to None
converting 156 empty string values of column given_name to None
converting 79 empty string values of column surname to None
converting 245 empty string values of column street_number to None
converting 154 empty string values of column address_1 to None
converting 693 empty string values of column address_2 to None
converting 85 empty string values of column suburb to None
converting 0 empty string values of column postcode to None
converting 85 empty string values of column state to None
converting 0 empty string values of column soc_sec_id to None<jupyter_text>Training the Model- Link on syntax on how to define fields https://docs.dedupe.io/en/latest/Variable-definition.html  Load Pre-Trained Model if it Already Exists<jupyter_code># If a settings file already exists, we'll just load that and skip training
if os.path.exists(settings_filepath):
    print('reading from', settings_filepath)
    with open(settings_filepath, 'rb') as f:
        deduper = dedupe.StaticDedupe(f)
    skip_training = True
else:
    skip_training = False<jupyter_output><empty_output><jupyter_text>Set Data Fields for Model<jupyter_code>if not skip_training:
    fields = dm_file_checker.get_task_info(task_name, "fields", saved_files_path)
    print("Using the following fields for the model")
    print(fields)

    deduper = dedupe.Dedupe(fields, num_cores = 4)<jupyter_output>Using the following fields for the model
[{'field': 'street_number', 'type': 'ShortString', 'has missing': True}, {'field': 'postcode', 'type': 'ShortString'}, {'field': 'state', 'type': 'ShortString', 'has missing': True}, {'field': 'soc_sec_id', 'type': 'ShortString'}, {'field': 'given_name', 'type': 'String', 'has missing': True}, {'field': 'surname', 'type': 'String', 'has missing': True}, {'field': 'address_1', 'type': 'String', 'has missing': True}, {'field': 'address_2', 'type': 'String', 'has missing': True}, {'field': 'suburb', 'type': 'String', 'has missing': True}, {'field': 'date_of_birth', 'type': 'DateTime', 'has missing': True, 'fuzzy': False, 'yearfirst': True}]<jupyter_text>Load Data in Model (NOTE: this might take a while)<jupyter_code>%%time
if not skip_training:
    dm_utils.prepare_training_deduper(deduper, unlabeled_data, labeled_data_filepath, 
                                      blocked_proportion = BLOCKED_PROPORTION, 
                                      sample_size = SAMPLE_SIZE)<jupyter_output>INFO:dedupe.api:reading training from file<jupyter_text>Labeling of Data Proper Watch Out For the Following When Tagging1. Family Members    - Family members usually have many fields in common (last name, address, landline number, place of birth, mothers maiden name). Key distinguishing fields are first name, birthdate, social security number (SSS), and tax identification number (TIN)    - Most tricky are young siblings because only distinguishing fields would be first name and birthdate. No SSS and TIN because they're not yet of working age.2. Businesses Mistagged as Individuals    - If there's a pair of records with the same social security number (SSS) and tax identification number (TIN) but one of the records has no first name, middle name, and last name, then one of the records may be a business mistagged as an individual.    - Still treat these records as separate<jupyter_code>%%time
if not skip_training:
    # ## Active learning
    # Dedupe will find the next pair of records
    # it is least certain about and ask you to label them as duplicates
    # or not.
    # use 'y', 'n' and 'u' keys to flag duplicates
    # press 'f' when you are finished
    print('starting active labeling...')
    dedupe.console_label(deduper)

    # Using the examples we just labeled, train the deduper and learn blocking predicates
    deduper.train(recall = RECALL_TRAIN_VAL)
    
    dm_utils.save_trained_deduper(deduper, labeled_data_filepath, settings_filepath)<jupyter_output>starting active labeling...<jupyter_text>Retrain Model if Needed<jupyter_code>retrain_model = False
if retrain_model:
    for field in deduper.fingerprinter.index_fields:
        field_data = set(record[field] for key,record in unlabeled_data.items())
        deduper.fingerprinter.index(field_data, field)
   
    dedupe.console_label(deduper)
if retrain_model:

    # Using the examples we just labeled, train the deduper and learn blocking predicates
    deduper.train(recall = RECALL_TRAIN_VAL)
    
    dm_utils.save_trained_deduper(deduper, labeled_data_filepath, settings_filepath)<jupyter_output><empty_output><jupyter_text>Write Blocks to Disk<jupyter_code>dm_utils.write_deduper_blocks(deduper, unlabeled_data, blocks_filepath)<jupyter_output><empty_output><jupyter_text>Check performance of blocking method<jupyter_code>blocked_data = dm_utils.read_deduper_blocks(unlabeled_data, blocks_filepath)
num_candidate_pairs = dm_utils.count_blocked_pairs(deduper, blocked_data)
reduction_ratio = 1 - (num_candidate_pairs/(len(unlabeled_data)**2))

print("Number of candidate record pairs after blocking is {:,}".format(num_candidate_pairs))
print("Reduction ratio from blocking is {}%".format(reduction_ratio*100))
blocked_data = dm_utils.read_deduper_blocks(unlabeled_data, blocks_filepath)
dm_utils.check_block_sizes(blocked_data)<jupyter_output>Sizes of top 10 biggest blocks are: [2, 2, 2, 2, 2, 2, 2, 2, 2, 2]
Record pair contributions from top 10 biggest blocks are : [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
{'fraction_non_alphanumeric': 0.09140578854105139, 'fraction_numerical': 0.011813349084465446, 'mean_word_length': 5.759481037924152, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '32973', 'license': 'MIT', 'n_tokens_mistral': 2232, 'n_tokens_neox': 2060, 'n_words': 639, 'path': 'notebooks/dedup/dedup-febrl3/dedup-febrl3-1_train.ipynb', 'repo_name': 'joshuacortez/data-matching-workflow', 'size': '6772'}