<commit_before>%YAML 1.2
---
name: SQL (YAML Macros example)
scope: source.sql
hidden: true

variables:
  identifier: \b[A-Za-z_]\w*\b
  reserved_words: (?:\b(?i:select|distinct|from|as)\b)

  var_name: (?:(?!{{reserved_words}}){{identifier}})

contexts:
  main:
  - match: ;
    scope: punctuation.terminator.statement.sql

  - match: (?=\S)
    push: query

  prototype:
  - include: comments

  comments:
  - match: (--).*
    scope: comment.line.sql
    captures:
      1: punctuation.definition.comment.sql

  else-pop:
  - match: (?=\S)
    pop: true

  query:
  - meta_scope: meta.query.sql
  - pop: true
    match: (?=(?:;))

  - match: (?:\b(?i:select)\b)
    scope: keyword.other.select.sql
    push:
    - - meta_scope: meta.select.sql
      - pop: true
        match: ''
    - select-list
    - - pop: true
        match: (?:\b(?i:distinct)\b)
        scope: keyword.other.sql
      - pop: true
        match: (?=\S)

  - match: (?:\b(?i:from)\b)
    scope: keyword.other.from.sql
    push:
    - - meta_scope: meta.from.sql
      - pop: true
        match: ''
    -                        

      - pop: true
        match: '{{var_name}}'
        scope: variable.other.table.sql
      - set:
        - meta_scope: meta.string.sql
        - meta_content_scope: variable.other.table.sql
        - pop: true
          match: '"'
          scope: punctuation.definition.string.end.sql
        match: '"'
        scope: punctuation.definition.string.begin.sql
      - pop: true
        match: (?=\S)
  select-list:
  - set:
    - select-list-rest
    - select-list-item
    match: (?=\S)

  select-list-rest:
  - match: ','
    scope: punctuation.separator.comma.sql
    push: select-list-item
  - include: else-pop

  select-list-item:
  - set:
    - - meta_scope: meta.select-item.sql
      - pop: true
        match: ''
    - - pop: true
        match: '{{var_name}}'
        scope: entity.name.alias.sql
      - set:
        - meta_scope: meta.string.sql
        - meta_content_scope: entity.name.alias.sql
        - pop: true
          match: '"'
          scope: punctuation.definition.string.end.sql
        match: '"'
        scope: punctuation.definition.string.begin.sql
      - pop: true
        match: (?=\S)
    - - pop: true
        match: (?:\b(?i:as)\b)
        scope: keyword.other.sql
      - pop: true
        match: (?=\S)
    - expression
    match: (?=\S)
  expression:
  - set:
    - expression-tail
    - expression-head
    match: (?=\S)

  expression-tail:
  - include: infix-operators
  - include: else-pop

  expression-head:
  - match: \*
    scope: keyword.other.star.sql
    pop: true

  - include: variable
  - include: literal-number

  - include: else-pop

  variable:             

    - pop: true
      match: '{{var_name}}'
      scope: variable.other.table.sql
    - set:
      - meta_scope: meta.string.sql
      - meta_content_scope: variable.other.table.sql
      - pop: true
        match: '"'
        scope: punctuation.definition.string.end.sql
      match: '"'
      scope: punctuation.definition.string.begin.sql
  literal-number:
  - match: \b\d+\b
    scope: constant.numeric.sql
    pop: true

  infix-operators:
  - match: '[-+*/]'
    scope: keyword.operator.numeric.sql
    push: expression-head
<commit_msg>Rebuilt SQL example<commit_after>%YAML 1.2
---
name: SQL (YAML Macros example)
scope: source.sql
hidden: true

variables:
  identifier: \b[A-Za-z_]\w*\b
  reserved_words: (?:\b(?i:select|distinct|from|as)\b)

  var_name: (?:(?!{{reserved_words}}){{identifier}})

contexts:
  main:
  - match: ;
    scope: punctuation.terminator.statement.sql

  - match: (?=\S)
    push: query

  prototype:
  - include: comments

  comments:
  - match: (--).*
    scope: comment.line.sql
    captures:
      1: punctuation.definition.comment.sql

  else-pop:
  - match: (?=\S)
    pop: true

  query:
  - meta_scope: meta.query.sql
  - pop: true
    match: (?=(?:;))

  - match: (?:\b(?i:select)\b)
    scope: keyword.other.select.sql
    push:
    - - meta_scope: meta.select.sql
      - pop: true
        match: ''
    - select-list
    - - scope: keyword.other.sql
        pop: true
        match: (?:\b(?i:distinct)\b)
      - pop: true
        match: (?=\S)

  - match: (?:\b(?i:from)\b)
    scope: keyword.other.from.sql
    push:
    - - meta_scope: meta.from.sql
      - pop: true
        match: ''
    -                        

      - scope: variable.other.table.sql
        pop: true
        match: '{{var_name}}'
      - scope: punctuation.definition.string.begin.sql
        match: '"'
        set:
        - meta_scope: meta.string.sql
        - meta_content_scope: variable.other.table.sql
        - scope: punctuation.definition.string.end.sql
          pop: true
          match: '"'
      - pop: true
        match: (?=\S)
  select-list:
  - match: (?=\S)
    set:
    - select-list-rest
    - select-list-item

  select-list-rest:
  - match: ','
    scope: punctuation.separator.comma.sql
    push: select-list-item
  - include: else-pop

  select-list-item:
  - match: (?=\S)
    set:
    - - meta_scope: meta.select-item.sql
      - pop: true
        match: ''
    - - scope: entity.name.alias.sql
        pop: true
        match: '{{var_name}}'
      - scope: punctuation.definition.string.begin.sql
        match: '"'
        set:
        - meta_scope: meta.string.sql
        - meta_content_scope: entity.name.alias.sql
        - scope: punctuation.definition.string.end.sql
          pop: true
          match: '"'
      - pop: true
        match: (?=\S)
    - - scope: keyword.other.sql
        pop: true
        match: (?:\b(?i:as)\b)
      - pop: true
        match: (?=\S)
    - expression
  expression:
  - match: (?=\S)
    set:
    - expression-tail
    - expression-head

  expression-tail:
  - include: infix-operators
  - include: else-pop

  expression-head:
  - match: \*
    scope: keyword.other.star.sql
    pop: true

  - include: variable
  - include: literal-number

  - include: else-pop

  variable:             

    - scope: variable.other.table.sql
      pop: true
      match: '{{var_name}}'
    - scope: punctuation.definition.string.begin.sql
      match: '"'
      set:
      - meta_scope: meta.string.sql
      - meta_content_scope: variable.other.table.sql
      - scope: punctuation.definition.string.end.sql
        pop: true
        match: '"'
  literal-number:
  - match: \b\d+\b
    scope: constant.numeric.sql
    pop: true

  infix-operators:
  - match: '[-+*/]'
    scope: keyword.operator.numeric.sql
    push: expression-head

{'fraction_non_alphanumeric': 0.15377560379088964, 'fraction_numerical': 0.0009171507184347294, 'mean_word_length': 1.848498040922943, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5885221', 'n_tokens_mistral': 2309, 'n_tokens_neox': 2239, 'n_words': 432}