# Path: py_stringsimjoin/filter/filter_utils.py
# def get_prefix_length(num_tokens, sim_measure_type, threshold, tokenizer):
#     """Computes prefix length.                                     
# 
#     References:                                                                 
#         * String Similarity Joins: An Experimental Evaluation, VLDB 2014.       
#     """
# 
#     if num_tokens == 0:
#         return 0
# 
#     if sim_measure_type == 'COSINE':
#         return int(num_tokens -
#                    ceil(threshold * threshold * num_tokens) + 1)
#     elif sim_measure_type == 'DICE':
#         return int(num_tokens -
#                    ceil((threshold / (2 - threshold)) * num_tokens) + 1)
#     elif sim_measure_type == 'EDIT_DISTANCE':
#         return min(tokenizer.qval * threshold + 1, num_tokens)
#     elif sim_measure_type == 'JACCARD':
#         return int(num_tokens - ceil(threshold * num_tokens) + 1)
#     elif sim_measure_type == 'OVERLAP':
#         return max(num_tokens - threshold + 1, 0)
# 
# Path: py_stringsimjoin/index/index.py
# class Index(object):
#     pass
# 
# Path: py_stringsimjoin/utils/token_ordering.py
# def order_using_token_ordering(tokens, token_ordering):
#     ordered_tokens = []
# 
#     for token in tokens:
#         order = token_ordering.get(token)
#         if order is not None:
#             ordered_tokens.append(order)
#             
#     ordered_tokens.sort()
# 
#     return ordered_tokens


from py_stringsimjoin.filter.filter_utils import get_prefix_length
from py_stringsimjoin.index.index import Index
from py_stringsimjoin.utils.token_ordering import order_using_token_ordering
    """Builds an inverted index on the prefixes of the strings in the input 
    column in the input table.                                                                  
                                                                                
    Prefix index is used by prefix filter and edit distance join.      
    """                

    def __init__(self, table, index_attr, tokenizer, 
                 sim_measure_type, threshold, token_ordering):
        self.table = table
        self.index_attr = index_attr
        self.tokenizer = tokenizer
        self.sim_measure_type = sim_measure_type
        self.threshold = threshold
        self.token_ordering = token_ordering
        self.index = None
        super(self.__class__, self).__init__()

    def build(self, cache_empty_records=True):
        """Build prefix index."""                                             
        self.index = {}
        empty_records = []
        row_id = 0
        for row in self.table:
            # tokenize string and order the tokens using the token ordering
            index_string = row[self.index_attr]
            index_attr_tokens = order_using_token_ordering(
                self.tokenizer.tokenize(index_string), self.token_ordering)

            # compute prefix length
            num_tokens = len(index_attr_tokens)

# Path: py_stringsimjoin/filter/filter_utils.py
# def get_prefix_length(num_tokens, sim_measure_type, threshold, tokenizer):
#     """Computes prefix length.                                     
# 
#     References:                                                                 
#         * String Similarity Joins: An Experimental Evaluation, VLDB 2014.       
#     """
# 
#     if num_tokens == 0:
#         return 0
# 
#     if sim_measure_type == 'COSINE':
#         return int(num_tokens -
#                    ceil(threshold * threshold * num_tokens) + 1)
#     elif sim_measure_type == 'DICE':
#         return int(num_tokens -
#                    ceil((threshold / (2 - threshold)) * num_tokens) + 1)
#     elif sim_measure_type == 'EDIT_DISTANCE':
#         return min(tokenizer.qval * threshold + 1, num_tokens)
#     elif sim_measure_type == 'JACCARD':
#         return int(num_tokens - ceil(threshold * num_tokens) + 1)
#     elif sim_measure_type == 'OVERLAP':
#         return max(num_tokens - threshold + 1, 0)
# 
# Path: py_stringsimjoin/index/index.py
# class Index(object):
#     pass
# 
# Path: py_stringsimjoin/utils/token_ordering.py
# def order_using_token_ordering(tokens, token_ordering):
#     ordered_tokens = []
# 
#     for token in tokens:
#         order = token_ordering.get(token)
#         if order is not None:
#             ordered_tokens.append(order)
#             
#     ordered_tokens.sort()
# 
#     return ordered_tokens


# Path: py_stringsimjoin/index/prefix_index.py
from py_stringsimjoin.filter.filter_utils import get_prefix_length
from py_stringsimjoin.index.index import Index
from py_stringsimjoin.utils.token_ordering import order_using_token_ordering

    """Builds an inverted index on the prefixes of the strings in the input 
    column in the input table.                                                                  
                                                                                
    Prefix index is used by prefix filter and edit distance join.      
    """                

    def __init__(self, table, index_attr, tokenizer, 
                 sim_measure_type, threshold, token_ordering):
        self.table = table
        self.index_attr = index_attr
        self.tokenizer = tokenizer
        self.sim_measure_type = sim_measure_type
        self.threshold = threshold
        self.token_ordering = token_ordering
        self.index = None
        super(self.__class__, self).__init__()

    def build(self, cache_empty_records=True):
        """Build prefix index."""                                             
        self.index = {}
        empty_records = []
        row_id = 0
        for row in self.table:
            # tokenize string and order the tokens using the token ordering
            index_string = row[self.index_attr]
            index_attr_tokens = order_using_token_ordering(
                self.tokenizer.tokenize(index_string), self.token_ordering)

            # compute prefix length
            num_tokens = len(index_attr_tokens)

            prefix_length = get_prefix_length(