# Path: py_stringsimjoin/filter/filter_utils.py
# def get_prefix_length(num_tokens, sim_measure_type, threshold, tokenizer):
#     """Computes prefix length.                                     
# 
#     References:                                                                 
#         * String Similarity Joins: An Experimental Evaluation, VLDB 2014.       
#     """
# 
#     if num_tokens == 0:
#         return 0
# 
#     if sim_measure_type == 'COSINE':
#         return int(num_tokens -
#                    ceil(threshold * threshold * num_tokens) + 1)
#     elif sim_measure_type == 'DICE':
#         return int(num_tokens -
#                    ceil((threshold / (2 - threshold)) * num_tokens) + 1)
#     elif sim_measure_type == 'EDIT_DISTANCE':
#         return min(tokenizer.qval * threshold + 1, num_tokens)
#     elif sim_measure_type == 'JACCARD':
#         return int(num_tokens - ceil(threshold * num_tokens) + 1)
#     elif sim_measure_type == 'OVERLAP':
#         return max(num_tokens - threshold + 1, 0)
# 
# Path: py_stringsimjoin/index/index.py
# class Index(object):
#     pass
# 
# Path: py_stringsimjoin/utils/token_ordering.py
# def order_using_token_ordering(tokens, token_ordering):
#     ordered_tokens = []
# 
#     for token in tokens:
#         order = token_ordering.get(token)
#         if order is not None:
#             ordered_tokens.append(order)
#             
#     ordered_tokens.sort()
# 
#     return ordered_tokens


from sys import maxsize
from py_stringsimjoin.filter.filter_utils import get_prefix_length
from py_stringsimjoin.index.index import Index
from py_stringsimjoin.utils.token_ordering import order_using_token_ordering
    """Builds a position index on the input column in the input table.                                                                  
                                                                                
    Position index is used by position filter, dice join, cosine join and 
    jaccard join.               
    """   

    def __init__(self, table, index_attr, tokenizer, 
                 sim_measure_type, threshold, token_ordering):
        self.table = table
        self.index_attr = index_attr
        self.tokenizer = tokenizer
        self.sim_measure_type = sim_measure_type
        self.threshold = threshold
        self.token_ordering = token_ordering
        self.index = None
        self.size_cache = None
        self.min_length = maxsize
        self.max_length = 0
        super(self.__class__, self).__init__()

    def build(self, cache_empty_records=True, cache_tokens=False):
        """Build position index."""
        self.index = {}
        self.size_cache = []
        cached_tokens = []
        empty_records = []
        row_id = 0
        for row in self.table:
            # tokenize string and order the tokens using the token ordering     
            index_string = row[self.index_attr]

# Path: py_stringsimjoin/filter/filter_utils.py
# def get_prefix_length(num_tokens, sim_measure_type, threshold, tokenizer):
#     """Computes prefix length.                                     
# 
#     References:                                                                 
#         * String Similarity Joins: An Experimental Evaluation, VLDB 2014.       
#     """
# 
#     if num_tokens == 0:
#         return 0
# 
#     if sim_measure_type == 'COSINE':
#         return int(num_tokens -
#                    ceil(threshold * threshold * num_tokens) + 1)
#     elif sim_measure_type == 'DICE':
#         return int(num_tokens -
#                    ceil((threshold / (2 - threshold)) * num_tokens) + 1)
#     elif sim_measure_type == 'EDIT_DISTANCE':
#         return min(tokenizer.qval * threshold + 1, num_tokens)
#     elif sim_measure_type == 'JACCARD':
#         return int(num_tokens - ceil(threshold * num_tokens) + 1)
#     elif sim_measure_type == 'OVERLAP':
#         return max(num_tokens - threshold + 1, 0)
# 
# Path: py_stringsimjoin/index/index.py
# class Index(object):
#     pass
# 
# Path: py_stringsimjoin/utils/token_ordering.py
# def order_using_token_ordering(tokens, token_ordering):
#     ordered_tokens = []
# 
#     for token in tokens:
#         order = token_ordering.get(token)
#         if order is not None:
#             ordered_tokens.append(order)
#             
#     ordered_tokens.sort()
# 
#     return ordered_tokens


# Path: py_stringsimjoin/index/position_index.py
from sys import maxsize
from py_stringsimjoin.filter.filter_utils import get_prefix_length
from py_stringsimjoin.index.index import Index
from py_stringsimjoin.utils.token_ordering import order_using_token_ordering

    """Builds a position index on the input column in the input table.                                                                  
                                                                                
    Position index is used by position filter, dice join, cosine join and 
    jaccard join.               
    """   

    def __init__(self, table, index_attr, tokenizer, 
                 sim_measure_type, threshold, token_ordering):
        self.table = table
        self.index_attr = index_attr
        self.tokenizer = tokenizer
        self.sim_measure_type = sim_measure_type
        self.threshold = threshold
        self.token_ordering = token_ordering
        self.index = None
        self.size_cache = None
        self.min_length = maxsize
        self.max_length = 0
        super(self.__class__, self).__init__()

    def build(self, cache_empty_records=True, cache_tokens=False):
        """Build position index."""
        self.index = {}
        self.size_cache = []
        cached_tokens = []
        empty_records = []
        row_id = 0
        for row in self.table:
            # tokenize string and order the tokens using the token ordering     
            index_string = row[self.index_attr]

            index_attr_tokens = order_using_token_ordering(