<commit_before>import re
import scoring
import settings as namebot_settings

"""Recommended usage: filtered results using a list comprehension
of all filters 'anded' together.
e.g:
filtered = [word for word in words if
            filter_length(word, max_length=7) and
            filter_startswith(word, beginning='c') and
            filter_startswith(word, beginning='c') and
            filter_endswith(word, ending='e') ...]
"""


def filter_vowel_cons_ratio(word, ratio=0.5):
    """Return True if the ratio of vowels to consonants is > `ratio`.
    This can be used as an ad-hoc pronunciation filter."""
    vowels = re.compile(r'[aeiouy]')
    consonants = re.compile(r'[^aeyiuo]')
    vmatch = re.findall(vowels, word.lower())
    cmatch = re.findall(consonants, word.lower())
    _ratio = float(len(vmatch)) / float(len(cmatch))
    return _ratio > ratio


def filter_length(word,
                  min_length=namebot_settings.MIN_LENGTH,
                  max_length=namebot_settings.MAX_LENGTH):
    return len(word) >= min_length and len(word) <= max_length


def filter_startswith(word, beginning=None):
    return word.lower().startswith(beginning)


def filter_endswith(word, ending=None):
    return word.lower().endswith(ending)


def filter_tld(word, tld='.com'):
    if tld.startswith('.'):
        tld = tld.replace('.', '')
    return filter_endswith(word, ending=tld)


def filter_metaphone(word):
    """Get the metaphone value, and if the second key is None
    pronunciation is unlikely."""
    score = scoring.score_dmetaphone(word)
    return score[1] is not None


def filter_soundex(word):
    # TODO
    # res = nameScoring.score_soundex(word)
    pass


def filter_nysiis(word):
    # TODO
    # res = nameScoring.score_nysiis(word)
    pass<commit_msg>Add missing docstrings and fill out remaining filter methods.<commit_after>"""Provides filtering functions for reducing large sets of generated words.

Recommended usage: filtered results using a list comprehension
of all filters 'anded' together.
e.g:
filtered = [word for word in words if
            filter_length(word, max_length=7) and
            filter_startswith(word, beginning='c') and
            filter_startswith(word, beginning='c') and
            filter_endswith(word, ending='e') ...]
"""

import re
import scoring
import settings as namebot_settings


def filter_vowel_cons_ratio(word, ratio=0.5):
    """Return True if the ratio of vowels to consonants is > `ratio`.

    This can be used as an ad-hoc pronunciation filter.
    """
    vowels = re.compile(r'[aeiouy]')
    consonants = re.compile(r'[^aeyiuo]')
    vmatch = re.findall(vowels, word.lower())
    cmatch = re.findall(consonants, word.lower())
    _ratio = float(len(vmatch)) / float(len(cmatch))
    return _ratio > ratio


def filter_length(word,
                  min_length=namebot_settings.MIN_LENGTH,
                  max_length=namebot_settings.MAX_LENGTH):
    """Filter based on min and max lengths.

    Args:
        word (str): The word.
        min_length (int, optional): The minimum allowed length.
        max_length (int, optional): The maximum allowed length.

    Returns:
        bool: The resulting check.
    """
    return len(word) >= min_length and len(word) <= max_length


def filter_startswith(word, beginning=None):
    """Filter a word starting with specified string.

    Args:
        word (str): The word.
        beginning (str, optional): The beginning string to check.

    Returns:
        bool: The resulting check.
    """
    return word.lower().startswith(beginning)


def filter_endswith(word, ending=None):
    """Filter words ending with a specified suffix.

    Args:
        word (str): A word.
        ending (str, optional): The optional ending to check.

    Returns:
        bool: The resulting check.
    """
    return word.lower().endswith(ending)


def filter_tld(word, tld='.com'):
    """Check if a word ends with a TLD suffix (can be used to make a valid TLD).

    Args:
        word (str): The wor.d
        tld (str, optional): The TLD to check against, with or without a dot.

    Returns:
        bool: The resulting check.
    """
    if tld.startswith('.'):
        tld = tld.replace('.', '')
    return filter_endswith(word, ending=tld)


def filter_metaphone(word):
    """Get metaphone value, checking score.

    Where second key being None, an easy pronunciation is unlikely.

    Args:
        word (str): The word.

    Returns:
        bool: The resulting check.
    """
    score = scoring.score_dmetaphone(word)
    return score[1] is not None


def filter_soundex(word, code=None):
    """Get soundex value, checking result against a given code.

    Args:
        word (str): The word.
        code (str, optional): The soundex pronunciation code.

    Returns:
        bool: The resulting check.
    """
    if code is None:
        return True
    return scoring.score_soundex(word).split(':')[1].strip() == code


def filter_nysiis(word, code=None):
    """Get nysiis value, checking score.

    Args:
        word (str): The word.
        code (str, optional): The nysiis pronunciation code.

    Returns:
        bool: The resulting check.
    """
    if code is None:
        return True
    return scoring.score_nysiis(word).split(':')[1].strip() == code
{'fraction_non_alphanumeric': 0.11008482563619228, 'fraction_numerical': 0.001885014137606032, 'mean_word_length': 2.93620178041543, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7515352', 'n_tokens_mistral': 1669, 'n_tokens_neox': 1607, 'n_words': 497}