<commit_before>"""
This linter checks for ...
"""
from collections import OrderedDict

from indexdigest.utils import LinterEntry

# skip small tables
ROWS_COUNT_THRESHOLD = 1000

# cardinality threshold
INDEX_CARDINALITY_THRESHOLD = 5

# the least frequent value should be used at most by x% rows
INDEX_VALUE_PERCENTAGE_THRESHOLD = 20


def get_low_cardinality_indices(database):
    """
    :type database  indexdigest.database.Database
    :rtype: list
    """
    for table_name in database.get_tables():
        rows_count = database.get_table_rows_estimate(table_name)
        if rows_count < ROWS_COUNT_THRESHOLD:
            continue

        # get table indices statistics
        # @see https://dev.mysql.com/doc/refman/5.7/en/show-index.html
        # @see https://www.percona.com/blog/2007/08/28/do-you-always-need-index-on-where-column/
        indices = database.query_dict_rows(
            "select TABLE_NAME, INDEX_NAME, COLUMN_NAME, CARDINALITY from"
            " INFORMATION_SCHEMA.STATISTICS where"
            " TABLE_NAME = '{table_name}' AND TABLE_SCHEMA = '{database_name}'".format(
                table_name=table_name, database_name=database.db_name)
        )

        for index in indices:
            # the cardinality is too high
            if index['CARDINALITY'] > INDEX_CARDINALITY_THRESHOLD:
                continue

            yield (table_name, rows_count, index)

<commit_msg>get_low_cardinality_indices: add debug print()<commit_after>"""
This linter checks for ...
"""
from collections import OrderedDict

from indexdigest.utils import LinterEntry

# skip small tables
ROWS_COUNT_THRESHOLD = 1000

# cardinality threshold
INDEX_CARDINALITY_THRESHOLD = 5

# the least frequent value should be used at most by x% rows
INDEX_VALUE_PERCENTAGE_THRESHOLD = 20


def get_low_cardinality_indices(database):
    """
    :type database  indexdigest.database.Database
    :rtype: list
    """
    for table_name in database.get_tables():
        rows_count = database.get_table_rows_estimate(table_name)
        if rows_count < ROWS_COUNT_THRESHOLD:
            continue

        # get table indices statistics
        # @see https://dev.mysql.com/doc/refman/5.7/en/show-index.html
        # @see https://www.percona.com/blog/2007/08/28/do-you-always-need-index-on-where-column/
        indices = database.query_dict_rows(
            "select TABLE_NAME, INDEX_NAME, COLUMN_NAME, CARDINALITY from"
            " INFORMATION_SCHEMA.STATISTICS where"
            " TABLE_NAME = '{table_name}' AND TABLE_SCHEMA = '{database_name}'".format(
                table_name=table_name, database_name=database.db_name)
        )

        print('get_low_cardinality_indices', list(indices))

        for index in indices:
            # the cardinality is too high
            if index['CARDINALITY'] > INDEX_CARDINALITY_THRESHOLD:
                continue

            yield (table_name, rows_count, index)


{'fraction_non_alphanumeric': 0.08743169398907104, 'fraction_numerical': 0.011612021857923498, 'mean_word_length': 2.8794701986754965, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1368325', 'n_tokens_mistral': 957, 'n_tokens_neox': 906, 'n_words': 195}