<commit_before>from collections import namedtuple
import heapq as hq
import logging
import numpy
from operator import attrgetter
import pandas as pd

import py_entitymatching as em
import py_entitymatching.catalog.catalog_manager as cm

logger = logging.getLogger(__name__)


def debug_blocker(candset, ltable, rtable, output_size=200,
                  attr_corres=None, verbose=False):
    """
    This function debugs the blocker output and reports a list of potential
    matches that are discarded by a blocker (or a blocker sequence).

    Specifically,  this function takes in the two input tables for
    matching and the candidate set returned by a blocker (or a blocker
    sequence), and produces a list of tuple pairs which are rejected by the
    blocker but with high potential of being true matches.

    Args:
        candset (DataFrame): The candidate set generated by
            applying the blocker on the ltable and rtable.
        ltable,rtable (DataFrame): The input DataFrames that are used to
            generate the blocker output.
        output_size (int): The number of tuple pairs that will be
            returned (defaults to 200).
        attr_corres (list): A list of attribute correspondence tuples.
            When ltable and rtable have different schemas, or the same
            schema but different words describing the attributes, the
            user needs to manually specify the attribute correspondence.
            Each element in this list should be a tuple of strings
            which are the corresponding attributes in ltable and rtable.
            The default value is None, and if the user doesn't specify
            this list, a built-in function for finding the
            attribute correspondence list will be called. But we highly
            recommend the users manually specify the attribute
            correspondences, unless the schemas of ltable and rtable are
            identical (defaults to None).
        verbose (boolean):  A flag to indicate whether the debug information
         should be logged (defaults to False).

    Returns:
        A pandas DataFrame with 'output_size' number of rows. Each row in the
        DataFrame is a tuple pair which has potential of being a true
        match, but is rejected by the blocker (meaning that the tuple
        pair is in the Cartesian product of ltable and rtable subtracted
        by the candidate set). The fields in the returned DataFrame are
        from ltable and rtable, which are useful for determining similar
        tuple pairs.

    Raises:
        AssertionError: If `ltable`, `rtable` or `candset` is not of type
            pandas DataFrame.
        AssertionError: If `ltable` or `rtable` is empty (size of 0).
        AssertionError: If the output `size` parameter is less than or equal
            to 0.
        AssertionError: If the attribute correspondence (`attr_corres`) list is
            not in the correct format (a list of tuples).
        AssertionError: If the attribute correspondence (`attr_corres`)
            cannot be built correctly.
    """
    # Check input types.
    _validate_types(ltable, rtable, candset, output_size,
                    attr_corres, verbose)

    # Check table size.
    if len(ltable) == 0:
        raise AssertionError('Error: ltable is empty!')
    if len(rtable) == 0:
        raise AssertionError('Error: rtable is empty!')

    # Check the value of output size.
    if output_size <= 0:
        raise AssertionError('The input parameter: \'output_size\''
                             ' is less than or equal to 0. Nothing needs'
                             ' to be done!')

    # Get table metadata.
    l_key, r_key = cm.get_keys_for_ltable_rtable(ltable, rtable, logger, verbose)

    # Validate metadata
    cm._validate_metadata_for_table(ltable, l_key, 'ltable', logger, verbose)
    cm._validate_metadata_for_table(rtable, r_key, 'rtable', logger, verbose)

    # Check the user input field correst list (if exists) and get the raw
    # version of our internal correst list.
    _check_input_field_correspondence_list(ltable, rtable, attr_corres)
    corres_list = _get_field_correspondence_list(ltable, rtable,
                                                 l_key, r_key, attr_corres)

    # Build the (col_name: col_index) dict to speed up locating a field in
    # the schema.
    ltable_col_dict = _build_col_name_index_dict(ltable)
    rtable_col_dict = _build_col_name_index_dict(rtable)

    # Filter correspondence list to remove numeric types. We only consider
    # string types for document concatenation.
    _filter_corres_list(ltable, rtable, l_key, r_key,
                        ltable_col_dict, rtable_col_dict, corres_list)

    # Get field filtered new table.
    ltable_filtered, rtable_filtered = _get_filtered_table(
        ltable, rtable, l_key, r_key, corres_list)

    # Select a subset of fields with high scores.
    feature_list = _select_features(ltable_filtered, rtable_filtered, l_key)

    # Map the record key value to its index in the table.
    lrecord_id_to_index_map = _get_record_id_to_index_map(ltable_filtered, l_key)
    rrecord_id_to_index_map = _get_record_id_to_index_map(rtable_filtered, r_key)

    # Build the tokenized record list delimited by a white space on the
    # selected fields.
    lrecord_list = _get_tokenized_table(ltable_filtered, l_key, feature_list)
    rrecord_list = _get_tokenized_table(rtable_filtered, r_key, feature_list)

    # Reformat the candidate set from a dataframe to a list of record index
    # tuple pair.
    new_formatted_candidate_set = _index_candidate_set(
        candset, lrecord_id_to_index_map, rrecord_id_to_index_map, verbose)

    # Build the token order according to token's frequency. To run a
    # prefix filtering based similarity join algorithm, we first need
    # the global token order.
    order_dict = {}
    _build_global_token_order(lrecord_list, order_dict)
    _build_global_token_order(rrecord_list, order_dict)

    # Sort the tokens in each record by the global order.
    _sort_record_tokens_by_global_order(lrecord_list, order_dict)
    _sort_record_tokens_by_global_order(rrecord_list, order_dict)

    # Run the topk similarity join.
    topk_heap = _topk_sim_join(
        lrecord_list, rrecord_list, new_formatted_candidate_set, output_size)

    # Assemble the topk record list to a dataframe.
    ret_dataframe = _assemble_topk_table(topk_heap, ltable_filtered, rtable_filtered)
    return ret_dataframe


# Validate the types of input parameters.
def _validate_types(ltable, rtable, candidate_set, output_size,
                    attr_corres, verbose):
    if not isinstance(ltable, pd.DataFrame):
        logger.error('Input left table is not of type pandas data frame')
        raise AssertionError('Input left table is not of type pandas data frame')

    if not isinstance(rtable, pd.DataFrame):
        logger.error('Input right table is not of type pandas data frame')
        raise AssertionError('Input right table is not of type pandas data frame')

    if not isinstance(candidate_set, pd.DataFrame):
        logger.error('Input candidate set is not of type pandas dataframe')
        raise AssertionError('Input candidate set is not of type pandas dataframe')

    if not isinstance(output_size, int):
        logging.error('Output size is not of type int')
        raise AssertionError('Output size is not of type int')

    if attr_corres is not None:
        if not isinstance(attr_corres, list):
            logging.error('Input attribute correspondence is not of'
                          ' type list')
            raise AssertionError('Input attribute correspondence is'
                                 ' not of type list')

        for pair in attr_corres:
            if not isinstance(pair, tuple):
                logging.error('Pair in attribute correspondence list is not'
                              ' of type tuple')
                raise AssertionError('Pair in attribute correspondence list'
                                     ' is not of type tuple')<commit_msg>Refactored type validation code in debugblocker.py<commit_after>from collections import namedtuple
import heapq as hq
import logging
import numpy
from operator import attrgetter
import pandas as pd
from py_entitymatching.utils.generic_helper import validate_object_type

import py_entitymatching as em
import py_entitymatching.catalog.catalog_manager as cm

logger = logging.getLogger(__name__)


def debug_blocker(candset, ltable, rtable, output_size=200,
                  attr_corres=None, verbose=False):
    """
    This function debugs the blocker output and reports a list of potential
    matches that are discarded by a blocker (or a blocker sequence).

    Specifically,  this function takes in the two input tables for
    matching and the candidate set returned by a blocker (or a blocker
    sequence), and produces a list of tuple pairs which are rejected by the
    blocker but with high potential of being true matches.

    Args:
        candset (DataFrame): The candidate set generated by
            applying the blocker on the ltable and rtable.
        ltable,rtable (DataFrame): The input DataFrames that are used to
            generate the blocker output.
        output_size (int): The number of tuple pairs that will be
            returned (defaults to 200).
        attr_corres (list): A list of attribute correspondence tuples.
            When ltable and rtable have different schemas, or the same
            schema but different words describing the attributes, the
            user needs to manually specify the attribute correspondence.
            Each element in this list should be a tuple of strings
            which are the corresponding attributes in ltable and rtable.
            The default value is None, and if the user doesn't specify
            this list, a built-in function for finding the
            attribute correspondence list will be called. But we highly
            recommend the users manually specify the attribute
            correspondences, unless the schemas of ltable and rtable are
            identical (defaults to None).
        verbose (boolean):  A flag to indicate whether the debug information
         should be logged (defaults to False).

    Returns:
        A pandas DataFrame with 'output_size' number of rows. Each row in the
        DataFrame is a tuple pair which has potential of being a true
        match, but is rejected by the blocker (meaning that the tuple
        pair is in the Cartesian product of ltable and rtable subtracted
        by the candidate set). The fields in the returned DataFrame are
        from ltable and rtable, which are useful for determining similar
        tuple pairs.

    Raises:
        AssertionError: If `ltable`, `rtable` or `candset` is not of type
            pandas DataFrame.
        AssertionError: If `ltable` or `rtable` is empty (size of 0).
        AssertionError: If the output `size` parameter is less than or equal
            to 0.
        AssertionError: If the attribute correspondence (`attr_corres`) list is
            not in the correct format (a list of tuples).
        AssertionError: If the attribute correspondence (`attr_corres`)
            cannot be built correctly.
    """
    # Check input types.
    _validate_types(ltable, rtable, candset, output_size,
                    attr_corres, verbose)

    # Check table size.
    if len(ltable) == 0:
        raise AssertionError('Error: ltable is empty!')
    if len(rtable) == 0:
        raise AssertionError('Error: rtable is empty!')

    # Check the value of output size.
    if output_size <= 0:
        raise AssertionError('The input parameter: \'output_size\''
                             ' is less than or equal to 0. Nothing needs'
                             ' to be done!')

    # Get table metadata.
    l_key, r_key = cm.get_keys_for_ltable_rtable(ltable, rtable, logger, verbose)

    # Validate metadata
    cm._validate_metadata_for_table(ltable, l_key, 'ltable', logger, verbose)
    cm._validate_metadata_for_table(rtable, r_key, 'rtable', logger, verbose)

    # Check the user input field correst list (if exists) and get the raw
    # version of our internal correst list.
    _check_input_field_correspondence_list(ltable, rtable, attr_corres)
    corres_list = _get_field_correspondence_list(ltable, rtable,
                                                 l_key, r_key, attr_corres)

    # Build the (col_name: col_index) dict to speed up locating a field in
    # the schema.
    ltable_col_dict = _build_col_name_index_dict(ltable)
    rtable_col_dict = _build_col_name_index_dict(rtable)

    # Filter correspondence list to remove numeric types. We only consider
    # string types for document concatenation.
    _filter_corres_list(ltable, rtable, l_key, r_key,
                        ltable_col_dict, rtable_col_dict, corres_list)

    # Get field filtered new table.
    ltable_filtered, rtable_filtered = _get_filtered_table(
        ltable, rtable, l_key, r_key, corres_list)

    # Select a subset of fields with high scores.
    feature_list = _select_features(ltable_filtered, rtable_filtered, l_key)

    # Map the record key value to its index in the table.
    lrecord_id_to_index_map = _get_record_id_to_index_map(ltable_filtered, l_key)
    rrecord_id_to_index_map = _get_record_id_to_index_map(rtable_filtered, r_key)

    # Build the tokenized record list delimited by a white space on the
    # selected fields.
    lrecord_list = _get_tokenized_table(ltable_filtered, l_key, feature_list)
    rrecord_list = _get_tokenized_table(rtable_filtered, r_key, feature_list)

    # Reformat the candidate set from a dataframe to a list of record index
    # tuple pair.
    new_formatted_candidate_set = _index_candidate_set(
        candset, lrecord_id_to_index_map, rrecord_id_to_index_map, verbose)

    # Build the token order according to token's frequency. To run a
    # prefix filtering based similarity join algorithm, we first need
    # the global token order.
    order_dict = {}
    _build_global_token_order(lrecord_list, order_dict)
    _build_global_token_order(rrecord_list, order_dict)

    # Sort the tokens in each record by the global order.
    _sort_record_tokens_by_global_order(lrecord_list, order_dict)
    _sort_record_tokens_by_global_order(rrecord_list, order_dict)

    # Run the topk similarity join.
    topk_heap = _topk_sim_join(
        lrecord_list, rrecord_list, new_formatted_candidate_set, output_size)

    # Assemble the topk record list to a dataframe.
    ret_dataframe = _assemble_topk_table(topk_heap, ltable_filtered, rtable_filtered)
    return ret_dataframe


# Validate the types of input parameters.
def _validate_types(ltable, rtable, candidate_set, output_size,
                    attr_corres, verbose):
    validate_object_type(ltable, pd.DataFrame, 'Input left table')

    validate_object_type(rtable, pd.DataFrame, 'Input right table')

    validate_object_type(candidate_set, pd.DataFrame, 'Input candidate set')

    validate_object_type(output_size, int, 'Output size')

    if attr_corres is not None:
        if not isinstance(attr_corres, list):
            logging.error('Input attribute correspondence is not of'
                          ' type list')
            raise AssertionError('Input attribute correspondence is'
                                 ' not of type list')

        for pair in attr_corres:
            if not isinstance(pair, tuple):
                logging.error('Pair in attribute correspondence list is not'
                              ' of type tuple')
                raise AssertionError('Pair in attribute correspondence list'
                                     ' is not of type tuple')
{'fraction_non_alphanumeric': 0.052698011991164403, 'fraction_numerical': 0.001514673398548438, 'mean_word_length': 2.8256880733944953, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4815254', 'n_tokens_mistral': 4380, 'n_tokens_neox': 4206, 'n_words': 1645}