<jupyter_start><jupyter_text>G2Engine Guide - Why EntitiesMore information:* [G2Engine Reference](senzing-G2Engine-reference.ipynb)  Prepare environment   Initialize Senzing configurationRun [senzing-G2ConfigMgr-reference.ipynb](senzing-G2ConfigMgr-reference.ipynb)to install a Senzing Engine configuration in the database.  Initialize python environment<jupyter_code>import argparse
import json
import operator
import os
import sys

# For RenderJSON

import uuid
from IPython.display import display_javascript, display_html, display<jupyter_output><empty_output><jupyter_text>Helper class for JSON renderingA class for pretty-printing JSON.Not required by Senzing, but helps visualize JSON.<jupyter_code>class RenderJSON(object):
    def __init__(self, json_data):
        if isinstance(json_data, dict):
            self.json_str = json.dumps(json_data)
        elif isinstance(json_data, bytearray):
            self.json_str = json_data.decode()
        else:
            self.json_str = json_data
        self.uuid = str(uuid.uuid4())

    def _ipython_display_(self):
        display_html('<div id="{}" style="height:100%; width:100%; background-color: LightCyan"></div>'.format(self.uuid), raw=True)
        display_javascript("""
        require(["https://rawgit.com/caldwell/renderjson/master/renderjson.js"], function() {
        document.getElementById('%s').appendChild(renderjson(%s))
        });
        """ % (self.uuid, self.json_str), raw=True)<jupyter_output><empty_output><jupyter_text>Utility functions<jupyter_code>def calculate_score_bucket_level(bucketString = None):
    score_bucket = {
        'SAME': 0,
        'CLOSE': 1,
        'LIKELY': 2,
        'PLAUSIBLE': 3, 
        'UNLIKELY': 4,
        'NO_CHANCE': 5,
    }
    return score_bucket.get(bucketString, 6)<jupyter_output><empty_output><jupyter_text>System pathUpdate system path.<jupyter_code>python_path = "{0}/python".format(
    os.environ.get("SENZING_G2_DIR", "/opt/senzing/g2"))
sys.path.append(python_path)<jupyter_output><empty_output><jupyter_text>Initialize variablesCreate variables used for G2Engine.<jupyter_code>%run senzing-init-config.ipynb
%store -r senzing_confifg_json
RenderJSON(senzing_config_json)<jupyter_output><empty_output><jupyter_text>G2Engine<jupyter_code>import G2Exception
from G2Engine import G2Engine<jupyter_output><empty_output><jupyter_text>G2Engine initializationDetails at [G2Engine initialization](senzing-G2Engine-reference.ipynbG2Engine-initialization).<jupyter_code>g2_engine = G2Engine()

try:
    g2_engine.initV2(
        module_name,
        senzing_config_json,
        verbose_logging)

except G2Exception.G2ModuleGenericException as err:
    print(g2_engine.getLastException())<jupyter_output><empty_output><jupyter_text>Purge prior repositoryDetails at [G2Engine.purgeRepository](senzing-G2Engine-reference.ipynbpurgeRepository).<jupyter_code>try:
    g2_engine.purgeRepository()

except G2Exception.G2ModuleGenericException as err:
    pring(g2_engine.getLastRepository())<jupyter_output><empty_output><jupyter_text>Insert recordsDetails at [G2Engine.addRecord](senzing-G2Engine-reference.ipynbaddRecord).  Insert record 1<jupyter_code>record_id = "1"
data = {
    "NAMES": [
        {
            "NAME_TYPE": "PRIMARY",
            "NAME_LAST": "Bob",
            "NAME_FIRST": "Max"
        }
    ],
    "PHONE_NUMBER": "123-456-7899"
}
data_as_json = json.dumps(data)
datasource_code = "TEST"
load_id = None
try:
    g2_engine.addRecord(
        datasource_code,
        record_id,
        data_as_json,
        load_id)

except G2Exception.G2ModuleGenericException as err:
    pring(g2_engine.getLastException())<jupyter_output><empty_output><jupyter_text>Insert record 2<jupyter_code>record_id = "2"
data = {
    "NAMES": [
        {
            "NAME_TYPE": "PRIMARY",
            "NAME_LAST": "Robart",
            "NAME_FIRST": "Max",
            "MIDDLE_NAME": "S"
        }
    ],
    "ADDR_FULL": "456 Funny ST"
}
data_as_json = json.dumps(data)
datasource_code = "TEST"
load_id = None

try:
    g2_engine.addRecord(
        datasource_code,
        record_id,
        data_as_json,
        load_id)

except G2Exception.G2ModuleGenericException as err:
    print(g2_engine.getLastException())<jupyter_output><empty_output><jupyter_text>Insert record 3<jupyter_code>record_id = "3"
data = {
    "NAMES": [
        {
            "NAME_TYPE": "PRIMARY",
            "NAME_LAST": "Robert",
            "NAME_FIRST": "Max",
            "MIDDLE_NAME":"Stewart"
        }
    ],
    "PHONE_NUMBER": "123-456-7899",
    "ADDR_FULL": "456 Funny st"
}
data_as_json = json.dumps(data)
datasource_code = "TEST"
load_id = None

try:
    g2_engine.addRecord(
        datasource_code,
        record_id,
        data_as_json,
        load_id)

except G2Exception.G2ModuleGenericException as err:
    print(g2_engine.getLastException())<jupyter_output><empty_output><jupyter_text>Ask WhyDetails at [G2Engine.whyEntityByRecordID](senzing-G2Engine-reference.ipynbwhyEntityByRecordID).<jupyter_code>record_id = "2"
response_bytearray = bytearray()

try:
    g2_engine.whyEntityByRecordID(
        datasource_code,
        record_id,
        response_bytearray)

    why_entity_dictionary = json.loads(response_bytearray.decode())

except G2Exception.G2ModuleGenericException as err:
    print(g2_engine.getLastException())
RenderJSON(response_bytearray)<jupyter_output><empty_output><jupyter_text>Start constructing final report<jupyter_code>final_result = {
    "Data_source": datasource_code,
    "Record_ID": record_id
}<jupyter_output><empty_output><jupyter_text>Get the main document data sections<jupyter_code>for why_result in why_entity_dictionary.get("WHY_RESULTS", []):
    for focus_record in why_result.get("FOCUS_RECORDS", []):
        if focus_record.get("DATA_SOURCE") == final_result.get("Data_source") \
        and focus_record.get("RECORD_ID") == final_result.get("Record_ID"):
            break
     
final_result["Entity_ID"] = why_result['ENTITY_ID']
RenderJSON(why_result)<jupyter_output><empty_output><jupyter_text>Get the basic why results<jupyter_code>match_info = why_result.get("MATCH_INFO", {})
final_result["Why_key"] = match_info.get("WHY_KEY")
final_result["Why_ER_rule_code"] = match_info.get("WHY_ERRULE_CODE")<jupyter_output><empty_output><jupyter_text>Get the matching candidate keys for the record<jupyter_code>name_feature_dictionary = {}

for candidate_key, candidate_values in match_info.get("CANDIDATE_KEYS", []).items():
    if candidate_key not in name_feature_dictionary:
        name_feature_dictionary[candidate_key] = []    
    for candidate_value in candidate_values:
        name_feature_dictionary[candidate_key].append(candidate_value.get("FEAT_DESC")) 
        
RenderJSON(name_feature_dictionary)<jupyter_output><empty_output><jupyter_text>Determine the best matches for each feature type<jupyter_code>best_behavior_level_by_types = {}
best_feature_match_by_types = {}

for feature_score_key, feature_score_values in match_info.get("FEATURE_SCORES", {}).items():
    
    if feature_score_key not in best_feature_match_by_types:
        best_behavior_level_by_types[feature_score_key] = calculate_score_bucket_level()
        
    for feature_score_value in feature_score_values:
        score_bucket = feature_score_value.get("SCORE_BUCKET")
        score_bucket_level = calculate_score_bucket_level(score_bucket)
        
        if best_behavior_level_by_types[feature_score_key] >= score_bucket_level:
            best_behavior_level_by_types[feature_score_key] = score_bucket_level
            best_feature_match_by_types[feature_score_key] = {
                "SCORE_BUCKET": score_bucket,
                "FEAT": feature_score_value.get("INBOUND_FEAT")
            }

RenderJSON(best_feature_match_by_types)<jupyter_output><empty_output><jupyter_text>Print the general "Why" results<jupyter_code>RenderJSON(final_result)<jupyter_output><empty_output><jupyter_text>Print the matching criteria<jupyter_code>print('Compared Features:')

for key, value in sorted(best_feature_match_by_types.items()):
    print("    {0}: '{1}' ({2})".format(key, value.get("FEAT"), value.get("SCORE_BUCKET")))<jupyter_output><empty_output><jupyter_text>Print the candidate keys<jupyter_code>print('Matching record candidate keys:')

for key, values in sorted(name_feature_dictionary.items()):
    print("    {0}:".format(key))
    values.sort()
    for value in values:
        print("        '{0}'".format(value))<jupyter_output><empty_output><jupyter_text>Shut down the engineDetails at [G2Engine.destroy](senzing-G2Engine-reference.ipynbdestroy).<jupyter_code>try:
    g2_engine.destroy()

except G2Exception.G2ModuleGenericException as err:
    print(g2_engine.getLastException)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1099601593625498, 'fraction_numerical': 0.012293682413204325, 'mean_word_length': 4.705194805194806, 'pattern_counts': {'":': 22, '<': 87, '<?xml version=': 0, '>': 88, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '279404', 'license': 'Apache-2.0', 'n_tokens_mistral': 3024, 'n_tokens_neox': 2952, 'n_words': 435, 'path': 'notebooks/senzing-examples/python/senzing-G2Engine-whyEntity.ipynb', 'repo_name': 'Senzing/docker-jupyter', 'size': '8785'}