<commit_before>from __future__ import absolute_import
from __future__ import division

from random import choice
import re
import nltk

from . import settings as namebot_settings
from . import normalization


___prefixes___ = namebot_settings.PREFIXES
___suffixes___ = namebot_settings.SUFFIXES
___alphabet___ = namebot_settings.ALPHABET
___vowels___ = namebot_settings.VOWELS
___regexes___ = namebot_settings.regexes


def spoonerism(words):
    "First: [f]oo [b]ar => boo far"
    new_words = []
    for k, word in enumerate(words):
        try:
            new_words.append('{}{} {}{}'.format(
                words[k + 1][0],  # 2nd word, 1st letter
                word[1:],  # 1st word, 2nd letter to end
                word[0],  # 1st word, 1st letter
                words[k + 1][1:]))  # 2nd word, 2nd letter to end
        except IndexError:
            continue
    return new_words


def kniferism(words):
    "Mid: f[o]o b[a]r => fao bor"
    new_words = []
    for k, word in enumerate(words):
        try:
            middle_second = int(len(words[k + 1]) / 2)
            middle_first = int(len(word) / 2)
            new_words.append('{}{}{} {}{}{}'.format(
                word[:middle_first],
                words[k + 1][middle_second],
                word[middle_first + 1:],
                words[k + 1][:middle_second],
                word[middle_first],
                words[k + 1][middle_second + 1:]))
        except IndexError:
            continue
    return new_words


def forkerism(words):
    "Last: fo[o] ba[r] => for bao"
    new_words = []
    for k, word in enumerate(words):
        try:
            s_word = words[k + 1]
            s_word_len = len(s_word)
            f_word_len = len(word)
            f_w_last_letter = word[f_word_len - 1]
            s_w_last_letter = words[k + 1][s_word_len - 1]
            new_words.append('{}{} {}{}'.format(
                word[:f_word_len - 1],  # 1st word, 1st letter to last - 1
                s_w_last_letter,  # 2nd word, last letter
                s_word[:s_word_len - 1],  # 2nd word, 1st letter to last - 1
                f_w_last_letter))  # 1st word, last letter
        except IndexError:
            continue
    return new_words


def reduplication_ablaut(words, count=1):
    """
    http://phrases.org.uk/meanings/reduplication.html
    A technique to combine words and altering the vowels
    e.g ch[i]t-ch[a]t, d[i]lly, d[a]lly
    """

    new_words = []
    for word in words:
        new_words.append(
            '{} {}'.format(
                word,
                re.sub(r'a|e|i|o|u',
                       choice(___vowels___),
                       word,
                       count=count)))
    return new_words


def affix_words(the_arr, affix_type):
    """
    Do some type of affixing technique,
    such as prefixing or suffixing.

    MASTER LIST:

    TODO FINISH *-fixes from article

    """
    new_arr = []
    if len(the_arr):

        if affix_type is 'prefix':
            for v in the_arr:
                for v1 in ___prefixes___:
                    if v1 is not None:
                        if re.search(
                            ___regexes___['no_vowels'], v[0]) or re.search(
                                ___regexes___['no_vowels'], v1[0]):
                                    # if there's a vowel at the end of
                                    # prefix but not at the beginning
                                    # of the word (or vice versa)
                                    if re.search(
                                        r'a|e|i|o|u', v1[-1:]) or re.search(
                                            r'^a|e|i|o|u', v[:1]):
                                        new_arr.append('{}{}'.format(v1, v))

        elif affix_type is 'suffix':
            for v in the_arr:
                for suffix in ___suffixes___:
                    if suffix is not None:
                        if re.search(___regexes___['all_vowels'], v[0]) or re.search(
                                ___regexes___['all_vowels'], suffix[0]):
                                if suffix is "ify":
                                    if v[-1] is "e":
                                        if v[-2] is not "i":
                                            new_arr.append(v[:-2] + suffix)
                                        else:
                                            new_arr.append(v[:-1] + suffix)
                                    new_arr.append(v + suffix)
                                else:
                                    new_arr.append(v + suffix)

        elif affix_type is 'duplifix':
            """
            makes duplification
            (e.g: teeny weeny, etc...)
            """
            for v in the_arr:
                for letter in ___alphabet___:
                    # check if the first letter is
                    # NOT the same as the second letter in reduplication
                    if v[0] is not letter:
                        # check if the first word is
                        # NOT the same as the second word. (or letter)
                        if v is not letter + v[1:]:
                            if re.match('[aeiou]', v[1]):
                                if re.match('[^aeiou]', letter):
                                    new_arr.append(
                                        '{} {} {}'.format(
                                            v, letter, v[1:]))
                            elif re.match('[^aeiou]', v[1]):
                                if re.match('[aeiou]', letter):
                                    new_arr.append(
                                        '{} {} {}'.format(
                                            v, letter, v[1:]))
        elif affix_type is "infix":
            pass<commit_msg>Use pre-compiled regex, fix space<commit_after>from __future__ import absolute_import
from __future__ import division

from random import choice
import re
import nltk

from . import settings as namebot_settings
from . import normalization


___prefixes___ = namebot_settings.PREFIXES
___suffixes___ = namebot_settings.SUFFIXES
___alphabet___ = namebot_settings.ALPHABET
___vowels___ = namebot_settings.VOWELS
___regexes___ = namebot_settings.regexes


def spoonerism(words):
    "First: [f]oo [b]ar => boo far"
    new_words = []
    for k, word in enumerate(words):
        try:
            new_words.append('{}{} {}{}'.format(
                words[k + 1][0],  # 2nd word, 1st letter
                word[1:],  # 1st word, 2nd letter to end
                word[0],  # 1st word, 1st letter
                words[k + 1][1:]))  # 2nd word, 2nd letter to end
        except IndexError:
            continue
    return new_words


def kniferism(words):
    "Mid: f[o]o b[a]r => fao bor"
    new_words = []
    for k, word in enumerate(words):
        try:
            middle_second = int(len(words[k + 1]) / 2)
            middle_first = int(len(word) / 2)
            new_words.append('{}{}{} {}{}{}'.format(
                word[:middle_first],
                words[k + 1][middle_second],
                word[middle_first + 1:],
                words[k + 1][:middle_second],
                word[middle_first],
                words[k + 1][middle_second + 1:]))
        except IndexError:
            continue
    return new_words


def forkerism(words):
    "Last: fo[o] ba[r] => for bao"
    new_words = []
    for k, word in enumerate(words):
        try:
            s_word = words[k + 1]
            s_word_len = len(s_word)
            f_word_len = len(word)
            f_w_last_letter = word[f_word_len - 1]
            s_w_last_letter = words[k + 1][s_word_len - 1]
            new_words.append('{}{} {}{}'.format(
                word[:f_word_len - 1],  # 1st word, 1st letter to last - 1
                s_w_last_letter,  # 2nd word, last letter
                s_word[:s_word_len - 1],  # 2nd word, 1st letter to last - 1
                f_w_last_letter))  # 1st word, last letter
        except IndexError:
            continue
    return new_words


def reduplication_ablaut(words, count=1):
    """
    http://phrases.org.uk/meanings/reduplication.html
    A technique to combine words and altering the vowels
    e.g ch[i]t-ch[a]t, d[i]lly, d[a]lly
    """

    new_words = []
    for word in words:
        new_words.append(
            '{} {}'.format(
                word,
                re.sub(r'a|e|i|o|u',
                       choice(___vowels___),
                       word,
                       count=count)))
    return new_words


def affix_words(the_arr, affix_type):
    """
    Do some type of affixing technique,
    such as prefixing or suffixing.

    MASTER LIST:

    TODO FINISH *-fixes from article

    """
    new_arr = []
    if len(the_arr):

        if affix_type is 'prefix':
            for v in the_arr:
                for v1 in ___prefixes___:
                    if v1 is not None:
                        if re.search(
                            ___regexes___['no_vowels'], v[0]) or re.search(
                                ___regexes___['no_vowels'], v1[0]):
                                    # if there's a vowel at the end of
                                    # prefix but not at the beginning
                                    # of the word (or vice versa)
                                    if re.search(
                                        r'a|e|i|o|u', v1[-1:]) or re.search(
                                            r'^a|e|i|o|u', v[:1]):
                                        new_arr.append('{}{}'.format(v1, v))

        elif affix_type is 'suffix':
            for v in the_arr:
                for suffix in ___suffixes___:
                    if suffix is not None:
                        if re.search(___regexes___['all_vowels'], v[0]) or re.search(
                                ___regexes___['all_vowels'], suffix[0]):
                                if suffix is "ify":
                                    if v[-1] is "e":
                                        if v[-2] is not "i":
                                            new_arr.append(v[:-2] + suffix)
                                        else:
                                            new_arr.append(v[:-1] + suffix)
                                    new_arr.append(v + suffix)
                                else:
                                    new_arr.append(v + suffix)

        elif affix_type is 'duplifix':
            """
            makes duplification
            (e.g: teeny weeny, etc...)
            """
            for v in the_arr:
                for letter in ___alphabet___:
                    # check if the first letter is
                    # NOT the same as the second letter in reduplication
                    if v[0] is not letter:
                        # check if the first word is
                        # NOT the same as the second word. (or letter)
                        if v is not letter + v[1:]:
                            if re.match(___regexes___['all_vowels'], v[1]):
                                if re.match(___regexes___['no_vowels'], letter):
                                    new_arr.append(
                                        '{} {}{}'.format(
                                            v, letter, v[1:]))
                            elif re.match(___regexes___['no_vowels'], v[1]):
                                if re.match(___regexes___['all_vowels'], letter):
                                    new_arr.append(
                                        '{} {}{}'.format(
                                            v, letter, v[1:]))
        elif affix_type is "infix":
            pass
{'fraction_non_alphanumeric': 0.10042553191489362, 'fraction_numerical': 0.009872340425531916, 'mean_word_length': 1.164885777450258, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 68, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2852635', 'n_tokens_mistral': 3435, 'n_tokens_neox': 3211, 'n_words': 855}