<commit_before># test
import hypothesis
from hypothesis import strategies as st
# uncompyle6
from uncompyle6 import PYTHON_VERSION, deparse_code


@st.composite
def expressions(draw):
    # todo : would be nice to generate expressions using hypothesis however
    # this is pretty involved so for now just use a corpus of expressions
    # from which to select.
    return draw(st.sampled_from((
        'abc',
        'len(items)',
        'x + 1',
        'lineno',
        'container',
        'self.attribute',
        'self.method()',
        'sorted(items, key=lambda x: x.name)',
        'func(*args, **kwargs)',
        'text or default',
    )))


@st.composite
def format_specifiers(draw):
    """
    Generate a valid format specifier using the rules:

    format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
    fill        ::=  <any character>
    align       ::=  "<" | ">" | "=" | "^"
    sign        ::=  "+" | "-" | " "
    width       ::=  integer
    precision   ::=  integer
    type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"

    See https://docs.python.org/2/library/string.html

    :param draw: Let hypothesis draw from other strategies.

    :return: An example format_specifier.
    """
    alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
    fill = draw(st.one_of(alphabet_strategy, st.none()))
    align = draw(st.sampled_from(list('<>=^')))
    fill_align = (fill + align or '') if fill else ''

    type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
    can_have_sign = type_ in 'deEfFgGnoxX%'
    can_have_comma = type_ in 'deEfFgG%'
    can_have_precision = type_ in 'fFgG'
    can_have_pound = type_ in 'boxX%'
    can_have_zero = type_ in 'oxX'

    sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
    pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
    zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''

    int_strategy = st.integers(min_value=1, max_value=1000)

    width = draw(st.one_of(int_strategy, st.none()))
    width = str(width) if width is not None else ''

    comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
    if can_have_precision:
        precision = draw(st.one_of(int_strategy, st.none()))
        precision = '.' + str(precision) if precision else ''
    else:
        precision = ''

    return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))


@st.composite
def fstrings(draw):
    """
    Generate a valid f-string.
    See https://www.python.org/dev/peps/pep-0498/#specification

    :param draw: Let hypothsis draw from other strategies.

    :return: A valid f-string.
    """
    prefix = draw(st.sampled_from('fF'))
    raw = draw(st.sampled_from(list('rR') + ['']))
    integer_strategy = st.integers(min_value=0, max_value=3)
    expression_count = draw(integer_strategy)
    content = []
    for _ in range(expression_count):
        expression = draw(expressions())
        #conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
        #specifier = draw(st.sampled_from(format_specifiers(), st.just('')))
        content.append(f'{{{expression}}}')
    content = ''.join(content)

    return f"{prefix}{raw}'{content}'"


@hypothesis.given(format_specifiers())
def test_format_specifiers(format_specifier):
    """Verify that format_specifiers generates valid specifiers"""
    try:
        exec('"{:' + format_specifier + '}".format(0)')
    except ValueError as e:
        if 'Unknown format code' not in str(e):
            raise


@hypothesis.given(fstrings())
def test_uncompyle_fstring(fstring):
    """Verify uncompyling fstring bytecode"""
    hypothesis.assume('{' in fstring)  # ignore fstring with no expressions
    expr = f'{fstring}\n'
    code = compile(expr, '<string>', 'single')
    assert deparse_code(PYTHON_VERSION, code, compile_mode='single').text == expr<commit_msg>added examples for known failures<commit_after># test
import pytest
import hypothesis
from hypothesis import strategies as st
# uncompyle6
from uncompyle6 import PYTHON_VERSION, deparse_code


@st.composite
def expressions(draw):
    # todo : would be nice to generate expressions using hypothesis however
    # this is pretty involved so for now just use a corpus of expressions
    # from which to select.
    return draw(st.sampled_from((
        'abc',
        'len(items)',
        'x + 1',
        'lineno',
        'container',
        'self.attribute',
        'self.method()',
        'sorted(items, key=lambda x: x.name)',
        'func(*args, **kwargs)',
        'text or default',
    )))


@st.composite
def format_specifiers(draw):
    """
    Generate a valid format specifier using the rules:

    format_spec ::=  [[fill]align][sign][#][0][width][,][.precision][type]
    fill        ::=  <any character>
    align       ::=  "<" | ">" | "=" | "^"
    sign        ::=  "+" | "-" | " "
    width       ::=  integer
    precision   ::=  integer
    type        ::=  "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"

    See https://docs.python.org/2/library/string.html

    :param draw: Let hypothesis draw from other strategies.

    :return: An example format_specifier.
    """
    alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
    fill = draw(st.one_of(alphabet_strategy, st.none()))
    align = draw(st.sampled_from(list('<>=^')))
    fill_align = (fill + align or '') if fill else ''

    type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
    can_have_sign = type_ in 'deEfFgGnoxX%'
    can_have_comma = type_ in 'deEfFgG%'
    can_have_precision = type_ in 'fFgG'
    can_have_pound = type_ in 'boxX%'
    can_have_zero = type_ in 'oxX'

    sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
    pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
    zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''

    int_strategy = st.integers(min_value=1, max_value=1000)

    width = draw(st.one_of(int_strategy, st.none()))
    width = str(width) if width is not None else ''

    comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
    if can_have_precision:
        precision = draw(st.one_of(int_strategy, st.none()))
        precision = '.' + str(precision) if precision else ''
    else:
        precision = ''

    return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))


@st.composite
def fstrings(draw):
    """
    Generate a valid f-string.
    See https://www.python.org/dev/peps/pep-0498/#specification

    :param draw: Let hypothsis draw from other strategies.

    :return: A valid f-string.
    """
    prefix = draw(st.sampled_from('f'))
    raw = draw(st.sampled_from(list('r') + ['']))
    integer_strategy = st.integers(min_value=0, max_value=3)
    expression_count = draw(integer_strategy)
    content = []
    for _ in range(expression_count):
        expression = draw(expressions())
        #conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
        #specifier = draw(st.sampled_from(format_specifiers(), st.just('')))
        content.append('{{{}}}'.format(expression))
    content = ''.join(content)

    return "{}{}'{}'".format(prefix, raw, content)


@hypothesis.given(format_specifiers())
def test_format_specifiers(format_specifier):
    """Verify that format_specifiers generates valid specifiers"""
    try:
        exec('"{:' + format_specifier + '}".format(0)')
    except ValueError as e:
        if 'Unknown format code' not in str(e):
            raise


@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason="requires python3.6")
@hypothesis.given(fstrings())
@hypothesis.example("f'{abc}'")                # BUG: strings with a single expression do not uncompyle correctly.
@hypothesis.example("fr'{abc}{xyz}'")          # BUG: no support for raw f strings.
@hypothesis.example("f'{len(items)}{abc}'")    # BUG: more complicated expressions than LOAD_NAME don't work
def test_uncompyle_fstring(fstring):
    """Verify uncompyling fstring bytecode"""

    # ignore fstring with no expressions an fsring with
    # no expressions just gets compiled to a normal string.
    hypothesis.assume('{' in fstring)

    expr = fstring + '\n'
    code = compile(expr, '<string>', 'single')
    deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')

    assert deparsed.text == expr
{'fraction_non_alphanumeric': 0.15960950364620088, 'fraction_numerical': 0.004704775346977182, 'mean_word_length': 3.2387836490528414, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1840277', 'n_tokens_mistral': 2929, 'n_tokens_neox': 2848, 'n_words': 670}