def main():
    long_tokens = ['this', 'is', 'a', 'sentence', 'that', 'is', 'over', 'ten', 'embeddings', 'long', 'and', 'that', 'there', 'are', 'punctuations', '.', 'this', 'is', 'a', 'sentence', 'that', 'is', 'over', 'ten', 'embeddings', 'long', 'and', 'that', 'there', 'are', 'punctuations', '.']
    long_tokens2 = ['.', '.', 'gonna', 'be', 'a', 'long', 'in', 'order', 'for', 'the', 'testing', 'of', 'the', 'code', '.', 'there', 'will', 'be', 'some', 'weird', 'tokens', 'hello', 'this', 'spellings', 'to', 'see', 'how', "that's", 'this', 'will', 'be', 'the']
    long_embeddings = [[1.2, 3.34], [2.3, 3.5], [5.6, 6.6], [5.1, 2.3], [2.3, 4.4], [3.3, 5.8], [8.8, 7.7], [1.1, 2.3], [9.9, 1.2], [2.1, 2.1], [1.0, 1.0], [1.1, 3.4], [1.2, 3.2], [3.4, 4.0], [1.1, 2.3], [1.1, 1.1], [1.2, 3.34], [2.3, 3.5], [5.6, 6.6], [5.1, 2.3], [2.3, 4.4], [3.3, 5.8], [8.8, 7.7], [1.1, 2.3], [9.9, 1.2], [2.1, 2.1], [1.0, 1.0], [1.1, 3.4], [1.2, 3.2], [3.4, 4.0], [1.1, 2.3], [1.1, 1.1]]
    print('test standardize_by_averaging')
    print('before; tokens:\n', long_tokens)
    print('before; embeddings:\n', long_embeddings, '\n\n')
    (tokens, embeddings) = standardize_by_averaging(long_tokens, long_embeddings)
    print('after; tokens:\n', tokens)
    print('after; embeddings:\n', embeddings, '\n\n')
    print('test standardize_by_averaging#2')
    print('before; tokens:\n', long_tokens2)
    print('before; embeddings:\n', long_embeddings, '\n\n')
    (tokens, embeddings) = standardize_by_averaging(long_tokens2, long_embeddings)
    print('after; tokens:\n', tokens)
    print('after; embeddings:\n', embeddings, '\n\n')
    short_tokens = ['This', 'is', 'looking', 'Bullish']
    short_embeddings = [[1.2, 3.34], [2.3, 3.5], [5.6, 6.6], [5.1, 2.3]]
    print('test standardize_by_duplicating')
    print('before; tokens:\n', short_tokens)
    print('before embeddings:\n', short_embeddings, '\n\n')
    (tokens, embeddings) = standardize_by_duplicating(short_tokens, short_embeddings)
    print('after; tokens:\n', tokens)
    print('after; embeddings:\n', embeddings, '\n\n')
    return
def main():
    long_tokens = ['this', 'is', 'a', 'sentence', 'that', 'is', 'over', 'ten', 'embeddings', 'long', 'and', 'that', 'there', 'are', 'punctuations', '.', 'this', 'is', 'a', 'sentence', 'that', 'is', 'over', 'ten', 'embeddings', 'long', 'and', 'that', 'there', 'are', 'punctuations', '.']
    long_tokens2 = ['.', '.', 'gonna', 'be', 'a', 'long', 'in', 'order', 'for', 'the', 'testing', 'of', 'the', 'code', '.', 'there', 'will', 'be', 'some', 'weird', 'tokens', 'hello', 'this', 'spellings', 'to', 'see', 'how', "that's", 'this', 'will', 'be', 'the']
    long_embeddings = [[1.2, 3.34], [2.3, 3.5], [5.6, 6.6], [5.1, 2.3], [2.3, 4.4], [3.3, 5.8], [8.8, 7.7], [1.1, 2.3], [9.9, 1.2], [2.1, 2.1], [1.0, 1.0], [1.1, 3.4], [1.2, 3.2], [3.4, 4.0], [1.1, 2.3], [1.1, 1.1], [1.2, 3.34], [2.3, 3.5], [5.6, 6.6], [5.1, 2.3], [2.3, 4.4], [3.3, 5.8], [8.8, 7.7], [1.1, 2.3], [9.9, 1.2], [2.1, 2.1], [1.0, 1.0], [1.1, 3.4], [1.2, 3.2], [3.4, 4.0], [1.1, 2.3], [1.1, 1.1]]
    print('test standardize_by_averaging')
    print('before; tokens:\n', long_tokens)
    print('before; embeddings:\n', long_embeddings, '\n\n')
    (tokens, embeddings) = standardize_by_averaging(long_tokens, long_embeddings)
    print('after; tokens:\n', tokens)
    print('after; embeddings:\n', embeddings, '\n\n')
    print('test standardize_by_averaging#2')
    print('before; tokens:\n', long_tokens2)
    print('before; embeddings:\n', long_embeddings, '\n\n')
    (tokens, embeddings) = standardize_by_averaging(long_tokens2, long_embeddings)
    print('after; tokens:\n', tokens)
    print('after; embeddings:\n', embeddings, '\n\n')
    short_tokens = ['This', 'is', 'looking', 'Bullish']
    short_embeddings = [[1.2, 3.34], [2.3, 3.5], [5.6, 6.6], [5.1, 2.3]]
    print('test standardize_by_duplicating')
    print('before; tokens:\n', short_tokens)
    print('before embeddings:\n', short_embeddings, '\n\n')
    (tokens, embeddings) = standardize_by_duplicating(short_tokens, short_embeddings)
    print('after; tokens:\n', tokens)
    print('after; embeddings:\n', embeddings, '\n\n')
    return