<commit_before>from hypothesis import settings
from hypothesis.strategies import (
    fixed_dictionaries, lists,
    booleans, integers, text, none,
    composite, sampled_from, one_of,
    just)

settings.register_profile("unit", settings(
    database=None,
    max_examples=1,
))
settings.load_profile("unit")


@composite
def requirements_list(draw, length, answers=False):
    if answers:
        elements = booleans() if length is not None else one_of(booleans(), none())
    else:
        elements = text(min_size=1, average_size=10, max_size=300, alphabet='abcdefgh')
    return draw(lists(
        elements=elements,
        min_size=length,
        max_size=length if length is not None else 10
    ))


def brief_response_data(essential_count=5, nice_to_have_count=5):
    return fixed_dictionaries({
        "essentialRequirements": requirements_list(essential_count, answers=True),
        "niceToHaveRequirements": requirements_list(nice_to_have_count, answers=True),
        "availability": text(min_size=1, average_size=10, max_size=100, alphabet='abcdefghijkl'),
        "respondToEmailAddress": just("<EMAIL>"),
    })


def specialists_brief_response_data(min_day_rate=1, max_day_rate=1000):
    return fixed_dictionaries({
        "essentialRequirements": requirements_list(5, answers=True),
        "niceToHaveRequirements": requirements_list(5, answers=True),
        "respondToEmailAddress": just("<EMAIL>"),
        "availability": text(min_size=1, average_size=10, max_size=100, alphabet='abcdefghijkl'),
        "dayRate": integers(min_value=min_day_rate, max_value=max_day_rate).map(lambda x: str(x)),
    })


def brief_data(essential_count=5, nice_to_have_count=5):
    return fixed_dictionaries({
        'specialistRole': just('developer'),
        'location': text(min_size=1, average_size=10, alphabet='abcdefghijkl'),
        'essentialRequirements': requirements_list(essential_count),
        'niceToHaveRequirements': requirements_list(nice_to_have_count),
    })
<commit_msg>hypothesis example_listings: improve the diversity of generated descriptive strings with unicode characters and spaces<commit_after># -*- coding: utf-8 -*-
from hypothesis import settings
from hypothesis.strategies import (
    fixed_dictionaries, lists,
    booleans, integers, text, none,
    composite, sampled_from, one_of,
    just)

from functools import partial
import re

settings.register_profile("unit", settings(
    database=None,
    max_examples=1,
))
settings.load_profile("unit")


_descriptive_alphabet = (
    # start with some normal alphanumerics
    u"abcdefghijklmnopABCDEFGHIJKLMNOP123"
    # some characters likely to fudge up poor escaping
    + u"\"\'<&%"
    # some printable unicode oddities
    + u"£Ⰶⶼ"
    # various weird types of spaces
    + u" \u200d\u2029\u202f"
)
_nonspace_cluster_re = re.compile(r"\S+", flags=re.UNICODE)


# we use this filter against hypothesis-generated strings so we don't unnecessarily trigger
# min/max word limits
# hint: use partial for tinkering with the kwargs
def _word_count_filter(s, min_words=1, max_words=None):
    count = len(_nonspace_cluster_re.findall(s))
    return (
        min_words is None or min_words <= count
    ) and (
        max_words is None or count <= max_words
    )


@composite
def requirements_list(draw, length, answers=False):
    if answers:
        elements = booleans() if length is not None else one_of(booleans(), none())
    else:
        elements = text(min_size=1, average_size=10, max_size=300, alphabet=_descriptive_alphabet).filter(
            partial(_word_count_filter, max_words=30)
        )
    return draw(lists(
        elements=elements,
        min_size=length,
        max_size=length if length is not None else 10
    ))


_brief_response_availability = text(min_size=1, average_size=10, max_size=100, alphabet=_descriptive_alphabet).filter(
    _word_count_filter
)


def brief_response_data(essential_count=5, nice_to_have_count=5):
    return fixed_dictionaries({
        "essentialRequirements": requirements_list(essential_count, answers=True),
        "niceToHaveRequirements": requirements_list(nice_to_have_count, answers=True),
        "availability": _brief_response_availability,
        "respondToEmailAddress": just("<EMAIL>"),
    })


def specialists_brief_response_data(min_day_rate=1, max_day_rate=1000):
    return fixed_dictionaries({
        "essentialRequirements": requirements_list(5, answers=True),
        "niceToHaveRequirements": requirements_list(5, answers=True),
        "respondToEmailAddress": just("<EMAIL>"),
        "availability": _brief_response_availability,
        "dayRate": integers(min_value=min_day_rate, max_value=max_day_rate).map(lambda x: str(x)),
    })


def brief_data(essential_count=5, nice_to_have_count=5):
    return fixed_dictionaries({
        'specialistRole': just('developer'),
        'location': text(min_size=1, average_size=10, alphabet='abcdefghijkl'),
        'essentialRequirements': requirements_list(essential_count),
        'niceToHaveRequirements': requirements_list(nice_to_have_count),
    })

{'fraction_non_alphanumeric': 0.10546951578121937, 'fraction_numerical': 0.015879239364830426, 'mean_word_length': 4.533622559652929, 'pattern_counts': {'":': 18, '<': 10, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1521101', 'n_tokens_mistral': 1645, 'n_tokens_neox': 1553, 'n_words': 324}