<commit_before>from uuid import uuid4

__test_missile = """\
POST /example/search HTTP/1.1\r
Connection: close\r
Host: example.org\r
Content-length: 32\r
\r
param1=50&param2=0&param3=hello
"""


def __mark_by_uri(missile):
    return '_'.join(missile.split('\n', 1)[0].split(' ', 2)[1].split('/'))


__markers = {
    'uniq': lambda m: uuid4().hex,
    'uri': __mark_by_uri,
    '1': __mark_by_uri,
}


def get_marker(marker_type):
    '''
    Returns a marker function of the requested marker_type

    >>> marker = get_marker('uniq')(__test_missile)
    >>> type(marker)<commit_msg>throw out params in uri marker<commit_after>from uuid import uuid4

__test_missile = """\
POST /example/search HTTP/1.1\r
Connection: close\r
Host: example.org\r
Content-length: 32\r
\r
param1=50&param2=0&param3=hello
"""

__test_missile = """\
GET /example/search?param1=50&param2=0&param3=hello HTTP/1.1\r
Connection: close\r
Host: example.org\r
Content-length: 32\r
\r
"""


def __mark_by_uri(missile):
    return '_'.join(missile.split('\n', 1)[0].split(' ', 2)[1].split('?')[0].split('/'))


__markers = {
    'uniq': lambda m: uuid4().hex,
    'uri': __mark_by_uri,
    '1': __mark_by_uri,
}


def get_marker(marker_type):
    '''
    Returns a marker function of the requested marker_type

    >>> marker = get_marker('uniq')(__test_missile)
    >>> type(marker)
{'fraction_non_alphanumeric': 0.17590539541759054, 'fraction_numerical': 0.03325942350332594, 'mean_word_length': 4.207692307692308, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '469570', 'n_tokens_mistral': 533, 'n_tokens_neox': 520, 'n_words': 103}