<commit_before>from data.graph import bloom_node, regex
from spec.mamba import *

_GOAL = "BloomNode('', '#', 1)"


with description('parse'):
  with it('rejects unsupported input'):
    expect(calling(regex.parse, '(matching group)')).to(
        raise_error(NotImplementedError))
    expect(calling(regex.parse, 'a*')).to(
        raise_error(NotImplementedError, 'Unable to repeat MAXREPEAT'))
    expect(calling(regex.parse, 'a+')).to(
        raise_error(NotImplementedError, 'Unable to repeat MAXREPEAT'))

  with it('accepts simple input'):
    expect(calling(regex.parse, 'simple')).not_to(raise_error)
    expect(calling(regex.parse, 's.mple')).not_to(raise_error)

  with it('produces simple graphs'):
    parsed = regex.parse('a')
    expect(parsed).to(be_a(bloom_node.BloomNode))
    expect(repr(parsed)).to(equal("BloomNode('A', ' #', 0)"))
    expect(repr(parsed['a'])).to(equal(_GOAL))

  with it('produces longer graphs'):
    parsed = regex.parse('abc')
    expect(parsed).to(be_a(bloom_node.BloomNode))
    expect(repr(parsed)).to(equal("BloomNode('ABC', '   #', 0)"))
    expect(repr(parsed['a']['b']['c'])).to(equal(_GOAL))

  with description('with capitals'):
    with it('produces simple graphs'):
      parsed = regex.parse('aA')
      expect(repr(parsed)).to(equal("BloomNode('A;A', '  #', 0)"))

    with it('with "."'):
      parsed = regex.parse('a.Z')
      expect(repr(parsed)).to(equal(
          "BloomNode('Abcdefghijklmnopqrstuvwxyz;ABCDEFGHIJKLMNOPQRSTUVWXYZ',"
          " '   #', 0)"))

  with description('with "." character'):
    with it('produces simple graphs'):
      parsed = regex.parse('.')
      expect(parsed).to(be_a(bloom_node.BloomNode))
      expect(repr(parsed)).to(equal(
          "BloomNode('abcdefghijklmnopqrstuvwxyz', ' #', 0)"))
      expect(repr(parsed['z'])).to(equal(_GOAL))

    with it('produces wildcard prefix'):
      parsed = regex.parse('.b')
      expect(parsed).to(be_a(bloom_node.BloomNode))
      expect(repr(parsed)).to(equal(
          "BloomNode('aBcdefghijklmnopqrstuvwxyz', '  #', 0)"))
      expect(repr(parsed['z'])).to(equal("BloomNode('B', ' #', 0)"))
      expect(repr(parsed['z']['b'])).to(equal(_GOAL))

    with it('produces wildcard suffix'):
      parsed = regex.parse('a.')
      expect(parsed).to(be_a(bloom_node.BloomNode))
      expect(repr(parsed)).to(equal(
          "BloomNode('Abcdefghijklmnopqrstuvwxyz', '  #', 0)"))
      expect(repr(parsed['a']['z'])).to(equal(_GOAL))

    with it('produces wildcard middle'):
      parsed = regex.parse('a.c')
      expect(parsed).to(be_a(bloom_node.BloomNode))
      expect(repr(parsed)).to(equal(
          "BloomNode('AbCdefghijklmnopqrstuvwxyz', '   #', 0)"))
      expect(repr(parsed['a']['z'])).to(equal("BloomNode('C', ' #', 0)"))
      expect(repr(parsed['a']['z']['c'])).to(equal(_GOAL))

    with it('does not accept invalid paths through graph'):
      parsed = regex.parse('a.c')
      expect(repr(parsed)).to(equal(
          "BloomNode('AbCdefghijklmnopqrstuvwxyz', '   #', 0)"))
      expect(lambda: parsed['a']['z']['z']).to(raise_error(KeyError))

  with description('whitespace'):
    with it('accepts simple example'):
      expect(calling(regex.parse, 'multiple words')).not_to(raise_error)

    with it('produces simple graphs'):
      parsed = regex.parse('a b')
      expect(repr(parsed)).to(equal("BloomNode('A; ', ' #', 0)"))
      expect(repr(parsed['a'])).to(equal("BloomNode(' ', '#', 0)"))
      expect(repr(parsed['a'][' '])).to(equal("BloomNode('B', ' #', 0)"))
      expect(repr(parsed['a'][' ']['b'])).to(equal(_GOAL))

  with it('supports small character groups'):
    node = regex.parse('[abc][def]')
    expect(repr(node)).to(equal("BloomNode('abcdef', '  #', 0)"))
    expect(repr(node['a'])).to(equal("BloomNode('def', ' #', 0)"))

  with it('supports character groups with spaces'):
    node = regex.parse('[abc][abc ][abc]')
    expect(repr(node)).to(equal("BloomNode('abc; ', ' # #', 0)"))
    expect(repr(node['a'])).to(equal("BloomNode('abc; ', '# #', 0)"))
    expect(repr(node['a'][' '])).to(equal("BloomNode('abc', ' #', 0)"))
    expect(repr(node['a'][' ']['b'])).to(equal("BloomNode('', '#', 1)"))

  with it('supports small A|B expressions'):
    expect(repr(regex.parse('a|b'))).to(equal("BloomNode('ab', ' #', 0)"))

  with it('supports optional (x?) characters'):
    node = regex.parse('abc?')
    expect(repr(node)).to(equal("BloomNode('ABc', '  ##', 0)"))
    expect(repr(node['a'])).to(equal("BloomNode('Bc', ' ##', 0)"))
    expect(repr(node['a']['b'])).to(equal("BloomNode('C', '##', 1)"))
    expect(repr(node['a']['b']['c'])).to(equal("BloomNode('', '#', 1)"))

  with it('supports repeat ranges'):
    node = regex.parse('ab{1,3}')
    expect(repr(node)).to(equal("BloomNode('AB', '  ###', 0)"))
    expect(repr(node['a'])).to(equal("BloomNode('B', ' ###', 0)"))
    expect(repr(node['a']['b'])).to(equal("BloomNode('B', '###', 1)"))
    expect(repr(node['a']['b']['b'])).to(equal("BloomNode('B', '##', 1)"))
    expect(repr(node['a']['b']['b']['b'])).to(equal("BloomNode('', '#', 1)"))

with description('normalize'):
  with it('leaves normal input alone'):
    expect(regex.normalize('asdf')).to(equal('asdf'))

  with it('converts ALL CAPS'):
    expect(regex.normalize('ALL CAPS')).to(equal('all caps'))<commit_msg>Add regex test to verify graphs are DFAs.<commit_after>from data.graph import bloom_node, regex
from spec.mamba import *

_GOAL = "BloomNode('', '#', 1)"


with description('parse'):
  with it('rejects unsupported input'):
    expect(calling(regex.parse, '(matching group)')).to(
        raise_error(NotImplementedError))
    expect(calling(regex.parse, 'a*')).to(
        raise_error(NotImplementedError, 'Unable to repeat MAXREPEAT'))
    expect(calling(regex.parse, 'a+')).to(
        raise_error(NotImplementedError, 'Unable to repeat MAXREPEAT'))

  with it('accepts simple input'):
    expect(calling(regex.parse, 'simple')).not_to(raise_error)
    expect(calling(regex.parse, 's.mple')).not_to(raise_error)

  with it('produces simple graphs'):
    parsed = regex.parse('a')
    expect(parsed).to(be_a(bloom_node.BloomNode))
    expect(repr(parsed)).to(equal("BloomNode('A', ' #', 0)"))
    expect(repr(parsed['a'])).to(equal(_GOAL))

  with it('produces longer graphs'):
    parsed = regex.parse('abc')
    expect(parsed).to(be_a(bloom_node.BloomNode))
    expect(repr(parsed)).to(equal("BloomNode('ABC', '   #', 0)"))
    expect(repr(parsed['a']['b']['c'])).to(equal(_GOAL))

  with description('with capitals'):
    with it('produces simple graphs'):
      parsed = regex.parse('aA')
      expect(repr(parsed)).to(equal("BloomNode('A;A', '  #', 0)"))

    with it('with "."'):
      parsed = regex.parse('a.Z')
      expect(repr(parsed)).to(equal(
          "BloomNode('Abcdefghijklmnopqrstuvwxyz;ABCDEFGHIJKLMNOPQRSTUVWXYZ',"
          " '   #', 0)"))

  with description('with "." character'):
    with it('produces simple graphs'):
      parsed = regex.parse('.')
      expect(parsed).to(be_a(bloom_node.BloomNode))
      expect(repr(parsed)).to(equal(
          "BloomNode('abcdefghijklmnopqrstuvwxyz', ' #', 0)"))
      expect(repr(parsed['z'])).to(equal(_GOAL))

    with it('produces wildcard prefix'):
      parsed = regex.parse('.b')
      expect(parsed).to(be_a(bloom_node.BloomNode))
      expect(repr(parsed)).to(equal(
          "BloomNode('aBcdefghijklmnopqrstuvwxyz', '  #', 0)"))
      expect(repr(parsed['z'])).to(equal("BloomNode('B', ' #', 0)"))
      expect(repr(parsed['z']['b'])).to(equal(_GOAL))

    with it('produces wildcard suffix'):
      parsed = regex.parse('a.')
      expect(parsed).to(be_a(bloom_node.BloomNode))
      expect(repr(parsed)).to(equal(
          "BloomNode('Abcdefghijklmnopqrstuvwxyz', '  #', 0)"))
      expect(repr(parsed['a']['z'])).to(equal(_GOAL))

    with it('produces wildcard middle'):
      parsed = regex.parse('a.c')
      expect(parsed).to(be_a(bloom_node.BloomNode))
      expect(repr(parsed)).to(equal(
          "BloomNode('AbCdefghijklmnopqrstuvwxyz', '   #', 0)"))
      expect(repr(parsed['a']['z'])).to(equal("BloomNode('C', ' #', 0)"))
      expect(repr(parsed['a']['z']['c'])).to(equal(_GOAL))

    with it('does not accept invalid paths through graph'):
      parsed = regex.parse('a.c')
      expect(repr(parsed)).to(equal(
          "BloomNode('AbCdefghijklmnopqrstuvwxyz', '   #', 0)"))
      expect(lambda: parsed['a']['z']['z']).to(raise_error(KeyError))

  with description('whitespace'):
    with it('accepts simple example'):
      expect(calling(regex.parse, 'multiple words')).not_to(raise_error)

    with it('produces simple graphs'):
      parsed = regex.parse('a b')
      expect(repr(parsed)).to(equal("BloomNode('A; ', ' #', 0)"))
      expect(repr(parsed['a'])).to(equal("BloomNode(' ', '#', 0)"))
      expect(repr(parsed['a'][' '])).to(equal("BloomNode('B', ' #', 0)"))
      expect(repr(parsed['a'][' ']['b'])).to(equal(_GOAL))

  with it('supports small character groups'):
    node = regex.parse('[abc][def]')
    expect(repr(node)).to(equal("BloomNode('abcdef', '  #', 0)"))
    expect(repr(node['a'])).to(equal("BloomNode('def', ' #', 0)"))

  with it('supports character groups with spaces'):
    node = regex.parse('[abc][abc ][abc]')
    expect(repr(node)).to(equal("BloomNode('abc; ', ' # #', 0)"))
    expect(repr(node['a'])).to(equal("BloomNode('abc; ', '# #', 0)"))
    expect(repr(node['a'][' '])).to(equal("BloomNode('abc', ' #', 0)"))
    expect(repr(node['a'][' ']['b'])).to(equal(_GOAL))

  with it('supports small A|B expressions'):
    expect(repr(regex.parse('a|b'))).to(equal("BloomNode('ab', ' #', 0)"))

  with it('supports optional (x?) characters'):
    node = regex.parse('abc?')
    expect(repr(node)).to(equal("BloomNode('ABc', '  ##', 0)"))
    expect(repr(node['a'])).to(equal("BloomNode('Bc', ' ##', 0)"))
    expect(repr(node['a']['b'])).to(equal("BloomNode('C', '##', 1)"))
    expect(repr(node['a']['b']['c'])).to(equal(_GOAL))

  with it('supports repeat ranges'):
    node = regex.parse('ab{1,3}')
    expect(repr(node)).to(equal("BloomNode('AB', '  ###', 0)"))
    expect(repr(node['a'])).to(equal("BloomNode('B', ' ###', 0)"))
    expect(repr(node['a']['b'])).to(equal("BloomNode('B', '###', 1)"))
    expect(repr(node['a']['b']['b'])).to(equal("BloomNode('B', '##', 1)"))
    expect(repr(node['a']['b']['b']['b'])).to(equal(_GOAL))

  with it('supports repeat ranges + suffix'):
    node = regex.parse('b{1,3}c')
    expect(repr(node)).to(equal("BloomNode('BC', '  ###', 0)"))
    expect(repr(node['b'])).to(equal("BloomNode('bC', ' ###', 0)"))
    expect(repr(node['b']['c'])).to(equal(_GOAL))
    expect(repr(node['b']['b'])).to(equal("BloomNode('bC', ' ##', 0)"))
    expect(repr(node['b']['b']['c'])).to(equal(_GOAL))
    expect(repr(node['b']['b']['b'])).to(equal("BloomNode('C', ' #', 0)"))
    expect(repr(node['b']['b']['b']['c'])).to(equal(_GOAL))

with description('normalize'):
  with it('leaves normal input alone'):
    expect(regex.normalize('asdf')).to(equal('asdf'))

  with it('converts ALL CAPS'):
    expect(regex.normalize('ALL CAPS')).to(equal('all caps'))
{'fraction_non_alphanumeric': 0.24764214497440043, 'fraction_numerical': 0.006197790353004581, 'mean_word_length': 4.671930718288334, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5298837', 'n_tokens_mistral': 4200, 'n_tokens_neox': 4144, 'n_words': 531}