<commit_before>"""
Future plans

1. Ability to randomize a generators return values.
2. Enforce Uniqueness across results.
"""
import random

from pyrite.base import (BaseGenerator, WordGenerator, MultiGenerator,
        RepeatValueGenerator, IntegerGenerator)
from pyrite.content import (MALE_NAMES, FEMALE_NAMES, LAST_NAMES, FIRST_NAMES,
        WORD_LIST, USERNAMES)


class MaleNameGenerator(WordGenerator):
    """
    Generates male names from a wordlist of common male names.
    """
    words = MALE_NAMES


class FemaleNameGenerator(WordGenerator):
    """
    Generates female names from a wordlist of common female names.
    """
    words = FEMALE_NAMES


class FirstNameGenerator(WordGenerator):
    """
    Generates first names from a wordlist of common male and female names.
    """
    words = FIRST_NAMES


class LastNameGenerator(WordGenerator):
    """
    Generates last names from a wordlist of common last names.
    """
    words = LAST_NAMES


class UsernameGenerator(WordGenerator):
    """
    Generates usernames from a wordlist of common usernames.
    """
    words = USERNAMES


class FullNameGenerator(BaseGenerator):
    """
    Generates 2-tuples of first_name, last_name pairs from a wordlist of common
    names.
    """
    first_names = FIRST_NAMES
    last_names = LAST_NAMES

    def __init__(self, size=None, **kwargs):
        max_size = len(self.first_names) * len(self.last_names)
        kwargs['size'] = size or max_size
        super(FullNameGenerator, self).__init__(**kwargs)
        if self.unique and not self.size == max_size and self.size > max_size / 4:
            raise RuntimeWarning("FullNameGenerator's performance will degrade at higher iteration counts when set to return unique results")

    def inner_generator(self):
        while True:
            if len(self.hashes) == self.size:
                raise StopIteration
            first_name = random.choice(self.first_names)
            last_name = random.choice(self.last_names)
            yield first_name, last_name


class SingleLineTextGenerator(BaseGenerator):
    """
    Returns single lines of text between ``self.min_length`` and
    ``self.max_length`` in length.
    """
    size = 20
    words = WORD_LIST
    max_length = 80
    min_length = 50

    def inner_generator(self):
        while True:
            title = ''
            length = random.randint(self.min_length, self.max_length)

            words = []
            while True:
                if not words:
                    words = random.sample(self.words, len(self.words) / 2 + 1)
                word = words.pop()
                if len(title) + len(word) + 1 > self.max_length:
                    break
                title += ' ' + word
                title = title.strip()
                if len(title) > length:
                    break
            yield title


class TitleGenerator(SingleLineTextGenerator):
    """
    Returns the same valus as ``SingleLineTextGenerator`` but with their first
    letter capitalized.
    """
    def inner_generator(self):
        for title in super(TitleGenerator, self).inner_generator():
            yield title[0].upper() + title[1:]


class SimplePasswordGenerator(RepeatValueGenerator):
    value = '<PASSWORD>'


class GmailGenerator(RepeatValueGenerator):
    value = 'gmail'


class EmailAddressGenerator(MultiGenerator):
    """
    Returns gmail addresses using ``UsernameGenerator`` for the username
    portion of the email address.
    """
    format_string = '{username}@{domain}.com'
    generator_classes = {
            'username': UsernameGenerator,
            'domain': GmailGenerator,
            }


class OrderedIntegerGenerator(IntegerGenerator):
    """
    Returns ordered integers over the range ``self.lower_bound``,
    ``self.upper_bound``, repeating until termination
    """
    shuffle = False


class RandomIntegerGenerator(IntegerGenerator):
    """
    Returns random integers over the range ``self.lower_bound``,
    ``self.upper_bound``, repeating until termination
    """
    pass
<commit_msg>removed outdated docstring from pyrite.generators<commit_after>import random

from pyrite.base import (BaseGenerator, WordGenerator, MultiGenerator,
        RepeatValueGenerator, IntegerGenerator)
from pyrite.content import (MALE_NAMES, FEMALE_NAMES, LAST_NAMES, FIRST_NAMES,
        WORD_LIST, USERNAMES)


class MaleNameGenerator(WordGenerator):
    """
    Generates male names from a wordlist of common male names.
    """
    words = MALE_NAMES


class FemaleNameGenerator(WordGenerator):
    """
    Generates female names from a wordlist of common female names.
    """
    words = FEMALE_NAMES


class FirstNameGenerator(WordGenerator):
    """
    Generates first names from a wordlist of common male and female names.
    """
    words = FIRST_NAMES


class LastNameGenerator(WordGenerator):
    """
    Generates last names from a wordlist of common last names.
    """
    words = LAST_NAMES


class UsernameGenerator(WordGenerator):
    """
    Generates usernames from a wordlist of common usernames.
    """
    words = USERNAMES


class FullNameGenerator(BaseGenerator):
    """
    Generates 2-tuples of first_name, last_name pairs from a wordlist of common
    names.
    """
    first_names = FIRST_NAMES
    last_names = LAST_NAMES

    def __init__(self, size=None, **kwargs):
        max_size = len(self.first_names) * len(self.last_names)
        kwargs['size'] = size or max_size
        super(FullNameGenerator, self).__init__(**kwargs)
        if self.unique and not self.size == max_size and self.size > max_size / 4:
            raise RuntimeWarning("FullNameGenerator's performance will degrade at higher iteration counts when set to return unique results")

    def inner_generator(self):
        while True:
            if len(self.hashes) == self.size:
                raise StopIteration
            first_name = random.choice(self.first_names)
            last_name = random.choice(self.last_names)
            yield first_name, last_name


class SingleLineTextGenerator(BaseGenerator):
    """
    Returns single lines of text between ``self.min_length`` and
    ``self.max_length`` in length.
    """
    size = 20
    words = WORD_LIST
    max_length = 80
    min_length = 50

    def inner_generator(self):
        while True:
            title = ''
            length = random.randint(self.min_length, self.max_length)

            words = []
            while True:
                if not words:
                    words = random.sample(self.words, len(self.words) / 2 + 1)
                word = words.pop()
                if len(title) + len(word) + 1 > self.max_length:
                    break
                title += ' ' + word
                title = title.strip()
                if len(title) > length:
                    break
            yield title


class TitleGenerator(SingleLineTextGenerator):
    """
    Returns the same valus as ``SingleLineTextGenerator`` but with their first
    letter capitalized.
    """
    def inner_generator(self):
        for title in super(TitleGenerator, self).inner_generator():
            yield title[0].upper() + title[1:]


class SimplePasswordGenerator(RepeatValueGenerator):
    value = '<PASSWORD>'


class GmailGenerator(RepeatValueGenerator):
    value = 'gmail'


class EmailAddressGenerator(MultiGenerator):
    """
    Returns gmail addresses using ``UsernameGenerator`` for the username
    portion of the email address.
    """
    format_string = '{username}@{domain}.com'
    generator_classes = {
            'username': UsernameGenerator,
            'domain': GmailGenerator,
            }


class OrderedIntegerGenerator(IntegerGenerator):
    """
    Returns ordered integers over the range ``self.lower_bound``,
    ``self.upper_bound``, repeating until termination
    """
    shuffle = False


class RandomIntegerGenerator(IntegerGenerator):
    """
    Returns random integers over the range ``self.lower_bound``,
    ``self.upper_bound``, repeating until termination
    """
    pass

{'fraction_non_alphanumeric': 0.08921059143776293, 'fraction_numerical': 0.0034644889878742884, 'mean_word_length': 2.637713771377138, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5791011', 'n_tokens_mistral': 2264, 'n_tokens_neox': 2158, 'n_words': 624}