<commit_before># from hypothesis import composite
from hypothesis import given, settings
from hypothesis.strategies import (
    text, integers, lists, composite, uuids, sampled_from, none, one_of,
    permutations)

RESOURCES = [
    'unit',
    'organization',
    'department',
    'ontologyword',
    'ontologytree',
]


def int_keys(draw):
    return draw(lists(
        integers(min_value=1, max_value=15),
        min_size=1, max_size=3, unique=True))


def uuid_keys(draw):
    return draw(lists(uuids(), min_size=2, max_size=3))


def translated_field(draw, name, allow_none=True, languages=['fi', 'sv', 'en']):
    result = {}
    for lang in languages:
        if allow_none:
            val = draw(one_of(text(), none()))
        else:
            val = draw(text())
        if val is not None:
            result['{}_{}'.format(name, lang)] = val
    return result


VIEWPOINTS = ['00', '11', '12', '13', '21', '22', '23', '31', '32', '33', '41',
              '51', '52', '61']

VIEWPOINT_STATES = ['green', 'red', 'unknown']

MUNICIPALITIES = [  # TODO: read from independent source
    {
        'fi': 'Helsinki',
        'sv': 'Helsingfors',
        'en': 'Helsinki'
    },
    {
        'fi': 'Espoo',
        'sv': 'Esbo',
        'en': 'Espoo'
    },
    {
        'fi': 'Kauniainen',
        'sv': 'Grankulla',
        'en': 'Kauniainen'
    },
    {
        'fi': 'Vantaa',
        'sv': 'Vanda',
        'en': 'Vantaa'
    }
]


def accessibility_viewpoints(draw):
    return ','.join([
        '{}:{}'.format(key, draw(sampled_from(VIEWPOINT_STATES))) for key in VIEWPOINTS])


PROVIDER_TYPES = [
    "CONTRACT_SCHOOL",
    "OTHER_PRODUCTION_METHOD",
    "PAYMENT_COMMITMENT",
    "PURCHASED_SERVICE",
    "SELF_PRODUCED",
    "SUPPORTED_OPERATIONS",
    "UNKNOWN_PRODUCTION_METHOD"]

ORGANIZER_TYPES = [
    "ASSOCIATION",
    "FOUNDATION",
    "GOVERNMENT",
    "GOVERNMENTAL_COMPANY",
    "JOINT_MUNICIPAL_AUTHORITY",
    "MUNICIPAL_ENTERPRISE_GROUP",
    "MUNICIPALITY",
    "MUNICIPALLY_OWNED_COMPANY",
    "ORGANIZATION",
    "OTHER_REGIONAL_COOPERATION_ORGANIZATION",
    "PRIVATE_ENTERPRISE",
    "UNKNOWN"
]


@composite
def make_source(draw):
    return {'id': draw(text()), 'source': draw(text())}


def unit_maker(draw, resource_ids):
    def make_unit(uid):
        result = {
            'id': uid,
            'org_id': str(draw(sampled_from(resource_ids['organization']))),
            'dept_id': str(draw(sampled_from(resource_ids['department']))),
            'ontologyword_ids': draw(permutations(resource_ids['ontologyword'])),
            'ontologytree_ids': draw(permutations(resource_ids['ontologytree'])),
            'accessibility_viewpoints': accessibility_viewpoints(draw),
            'sources': draw(lists(make_source(), min_size=0, max_size=2)),
            'provider_type': draw(sampled_from(PROVIDER_TYPES)),
            'address_city': draw(sampled_from(MUNICIPALITIES)),
            'accessibility_email': draw(one_of(text(), none()))  # TODO: map to another field
        }
        result.update(translated_field(draw, 'name', allow_none=False))
        result.update(translated_field(draw, 'address_postal_full', allow_none=False))

        result.update(translated_field(draw, 'call_charge_info', allow_none=True))
        result.update(translated_field(draw, 'desc', allow_none=True))
        result.update(translated_field(draw, 'picture_caption', allow_none=True))
        return result
    return make_unit


def organization_maker(*args):
    return lambda x: {'id': str(x)}


def department_maker(draw, resource_ids):
    def make_department(did):
        return {
            'id': str(did),
            'hierarchy_level': 0,
            'org_id': str(draw(sampled_from(resource_ids['organization'])))
        }
    return make_department


def ontologyword_maker(*args):
    return lambda x: {'id': x}

<commit_msg>Rename option for clarity<commit_after># from hypothesis import composite
from hypothesis import given, settings
from hypothesis.strategies import (
    text, integers, lists, composite, uuids, sampled_from, none, one_of,
    permutations)

RESOURCES = [
    'unit',
    'organization',
    'department',
    'ontologyword',
    'ontologytree',
]


def int_keys(draw):
    return draw(lists(
        integers(min_value=1, max_value=15),
        min_size=1, max_size=3, unique=True))


def uuid_keys(draw):
    return draw(lists(uuids(), min_size=2, max_size=3))


def translated_field(draw, name, allow_missing=True, languages=['fi', 'sv', 'en']):
    result = {}
    for lang in languages:
        if allow_missing:
            val = draw(one_of(text(), none()))
        else:
            val = draw(text())
        if val is not None:
            result['{}_{}'.format(name, lang)] = val
    return result


VIEWPOINTS = ['00', '11', '12', '13', '21', '22', '23', '31', '32', '33', '41',
              '51', '52', '61']

VIEWPOINT_STATES = ['green', 'red', 'unknown']

MUNICIPALITIES = [  # TODO: read from independent source
    {
        'fi': 'Helsinki',
        'sv': 'Helsingfors',
        'en': 'Helsinki'
    },
    {
        'fi': 'Espoo',
        'sv': 'Esbo',
        'en': 'Espoo'
    },
    {
        'fi': 'Kauniainen',
        'sv': 'Grankulla',
        'en': 'Kauniainen'
    },
    {
        'fi': 'Vantaa',
        'sv': 'Vanda',
        'en': 'Vantaa'
    }
]


def accessibility_viewpoints(draw):
    return ','.join([
        '{}:{}'.format(key, draw(sampled_from(VIEWPOINT_STATES))) for key in VIEWPOINTS])


PROVIDER_TYPES = [
    "CONTRACT_SCHOOL",
    "OTHER_PRODUCTION_METHOD",
    "PAYMENT_COMMITMENT",
    "PURCHASED_SERVICE",
    "SELF_PRODUCED",
    "SUPPORTED_OPERATIONS",
    "UNKNOWN_PRODUCTION_METHOD"]

ORGANIZER_TYPES = [
    "ASSOCIATION",
    "FOUNDATION",
    "GOVERNMENT",
    "GOVERNMENTAL_COMPANY",
    "JOINT_MUNICIPAL_AUTHORITY",
    "MUNICIPAL_ENTERPRISE_GROUP",
    "MUNICIPALITY",
    "MUNICIPALLY_OWNED_COMPANY",
    "ORGANIZATION",
    "OTHER_REGIONAL_COOPERATION_ORGANIZATION",
    "PRIVATE_ENTERPRISE",
    "UNKNOWN"
]


@composite
def make_source(draw):
    return {'id': draw(text()), 'source': draw(text())}


def unit_maker(draw, resource_ids):
    def make_unit(uid):
        result = {
            'id': uid,
            'org_id': str(draw(sampled_from(resource_ids['organization']))),
            'dept_id': str(draw(sampled_from(resource_ids['department']))),
            'ontologyword_ids': draw(permutations(resource_ids['ontologyword'])),
            'ontologytree_ids': draw(permutations(resource_ids['ontologytree'])),
            'accessibility_viewpoints': accessibility_viewpoints(draw),
            'sources': draw(lists(make_source(), min_size=0, max_size=2)),
            'provider_type': draw(sampled_from(PROVIDER_TYPES)),
            'address_city': draw(sampled_from(MUNICIPALITIES)),
            'accessibility_email': draw(one_of(text(), none()))  # TODO: map to another field
        }
        result.update(translated_field(draw, 'name', allow_missing=False))

        result.update(translated_field(draw, 'address_postal_full', allow_missing=True))
        result.update(translated_field(draw, 'call_charge_info', allow_missing=True))
        result.update(translated_field(draw, 'desc', allow_missing=True))
        result.update(translated_field(draw, 'picture_caption', allow_missing=True))
        return result
    return make_unit


def organization_maker(*args):
    return lambda x: {'id': str(x)}


def department_maker(draw, resource_ids):
    def make_department(did):
        return {
            'id': str(did),
            'hierarchy_level': 0,
            'org_id': str(draw(sampled_from(resource_ids['organization'])))
        }
    return make_department


def ontologyword_maker(*args):
    return lambda x: {'id': x}


{'fraction_non_alphanumeric': 0.14996154832094336, 'fraction_numerical': 0.009741092027685208, 'mean_word_length': 3.172727272727273, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1205579', 'n_tokens_mistral': 2763, 'n_tokens_neox': 2647, 'n_words': 487}