<commit_before>describe "Perl 6 grammar", ->
  grammar = null

  beforeEach ->
    waitsForPromise ->
      atom.packages.activatePackage("language-perl")

    runs ->
      grammar = atom.grammars.grammarForScopeName("source.perl6")

  it "parses the grammar", ->
    expect(grammar).toBeDefined()
    expect(grammar.scopeName).toBe "source.perl6"

  describe "identifiers", ->
    it "should match simple scalar identifiers", ->
      {tokens} = grammar.tokenizeLine('$a')
      expect(tokens[0]).toEqual value: '$a', scopes: [
        'source.perl6'
        'variable.other.identifier.perl6'
      ]

    it "should match simple array identifiers", ->
      {tokens} = grammar.tokenizeLine('@a')
      expect(tokens[0]).toEqual value: '@a', scopes: [
        'source.perl6'
        'variable.other.identifier.perl6'
      ]

    it "should match simple hash identifiers", ->
      {tokens} = grammar.tokenizeLine('%a')
      expect(tokens[0]).toEqual value: '%a', scopes: [
        'source.perl6'
        'variable.other.identifier.perl6'
      ]

    it "should match simple hash identifiers", ->
      {tokens} = grammar.tokenizeLine('&a')
      expect(tokens[0]).toEqual value: '&a', scopes: [
        'source.perl6'
        'variable.other.identifier.perl6'
      ]
    it "should match unicode identifiers", ->
      {tokens} = grammar.tokenizeLine('$cööl-páttérn')
      expect(tokens[0]).toEqual value: '$cööl-páttérn', scopes: [
        'source.perl6'
        'variable.other.identifier.perl6'
      ]
<commit_msg>More tests for Perl 6 identifiers with a dash<commit_after>describe "Perl 6 grammar", ->
  grammar = null

  beforeEach ->
    waitsForPromise ->
      atom.packages.activatePackage("language-perl")

    runs ->
      grammar = atom.grammars.grammarForScopeName("source.perl6")

  it "parses the grammar", ->
    expect(grammar).toBeDefined()
    expect(grammar.scopeName).toBe "source.perl6"

  describe "identifiers", ->
    it "should match simple scalar identifiers", ->
      {tokens} = grammar.tokenizeLine('$a')
      expect(tokens[0]).toEqual value: '$a', scopes: [
        'source.perl6'
        'variable.other.identifier.perl6'
      ]

    it "should match simple array identifiers", ->
      {tokens} = grammar.tokenizeLine('@a')
      expect(tokens[0]).toEqual value: '@a', scopes: [
        'source.perl6'
        'variable.other.identifier.perl6'
      ]

    it "should match simple hash identifiers", ->
      {tokens} = grammar.tokenizeLine('%a')
      expect(tokens[0]).toEqual value: '%a', scopes: [
        'source.perl6'
        'variable.other.identifier.perl6'
      ]

    it "should match simple hash identifiers", ->
      {tokens} = grammar.tokenizeLine('&a')
      expect(tokens[0]).toEqual value: '&a', scopes: [
        'source.perl6'
        'variable.other.identifier.perl6'
      ]

    it "should match unicode identifiers", ->
      {tokens} = grammar.tokenizeLine('$cööl-páttérn')
      expect(tokens[0]).toEqual value: '$cööl-páttérn', scopes: [
        'source.perl6'
        'variable.other.identifier.perl6'
      ]

    it "identifiers with dash can contain other keywords", ->
      {tokens} = grammar.tokenizeLine('start-from-here')
      expect(tokens.length).toEqual 1
      expect(tokens[0]).toEqual value: 'start-from-here', scopes: [
        'source.perl6'
      ]

    it "identifiers with dash can contain other keywords", ->
      {tokens} = grammar.tokenizeLine('start-here')
      expect(tokens.length).toEqual 1
      expect(tokens[0]).toEqual value: 'start-here', scopes: [
        'source.perl6'
      ]

    it "identifiers with dash can contain other keywords", ->
      {tokens} = grammar.tokenizeLine('is-required')
      expect(tokens.length).toEqual 1
      expect(tokens[0]).toEqual value: 'is-required', scopes: [
        'source.perl6'
      ]

    it "identifiers with dash can contain other keywords", ->
      {tokens} = grammar.tokenizeLine('is-utf8')
      expect(tokens.length).toEqual 1
      expect(tokens[0]).toEqual value: 'is-utf8', scopes: [
        'source.perl6'
      ]

    it "identifiers with dash can contain other keywords", ->
      {tokens} = grammar.tokenizeLine('is-')
      expect(tokens.length).toEqual 1
      expect(tokens[0]).toEqual value: 'is-', scopes: [
        'source.perl6'
      ]

{'fraction_non_alphanumeric': 0.14617169373549885, 'fraction_numerical': 0.012529002320185615, 'mean_word_length': 2.748695652173913, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2622035', 'n_tokens_mistral': 1404, 'n_tokens_neox': 1331, 'n_words': 273}