<commit_before>- token_auto = token.auto ? "token-auto" : nil
- value = token.token_type
- token_class = nil

- if token.token_type == :identifier
  - token_class = "token-identifier"
  - value = token.raw
- elsif token.token_type == :number
  - token_class = "token-number"
  - value = token.raw
- elsif token.token_type == :string
  - token_class = "token-string"
  - value = "\"#{token.raw}\""
- elsif token.token_type == :left_paren
  - token_class = "token-paren"
  - value = "("
- elsif token.token_type == :right_paren
  - token_class = "token-paren"
  - value = ")"
- elsif token.token_type == :greater
  - token_class = "token-operator"
  - value = ">"
- elsif token.token_type == :less
  - token_class = "token-operator"
  - value = "<"
- elsif token.token_type == :greater_equal
  - token_class = "token-operator"
  - value = ">="
- elsif token.token_type == :less_equal
  - token_class = "token-operator"
  - value = "<="
- elsif token.token_type == :bang_equal
  - token_class = "token-operator"
  - value = "!="
- elsif token.token_type == :equal
  - token_class = "token-operator"
  - value = "="
- elsif token.token_type == :minus
  - token_class = "token-operator"
  - value = "-"
- elsif token.token_type == :plus
  - token_class = "token-operator"
  - value = "+"
- elsif token.token_type == :star
  - token_class = "token-operator"
  - value = "*"
- elsif token.token_type == :slash
  - token_class = "token-operator"
  - value = "/"
- elsif token.token_type == :bang
  - token_class = "token-operator"
  - value = "!"
- elsif token.token_type.in?([:between, :and, :or, :at])
  - token_class = "token-operator"
- elsif token.token_type.in?([:entered, :any, :missing, :unentered])
  - token_class = "token-value"
- elsif token.token_type.in?([:true, :false, :nil])
  - token_class = "token-boolean"
<commit_msg>Add styling for power operator, addresses #691.<commit_after>- token_auto = token.auto ? "token-auto" : nil
- value = token.token_type
- token_class = nil

- if token.token_type == :identifier
  - token_class = "token-identifier"
  - value = token.raw
- elsif token.token_type == :number
  - token_class = "token-number"
  - value = token.raw
- elsif token.token_type == :string
  - token_class = "token-string"
  - value = "\"#{token.raw}\""
- elsif token.token_type == :left_paren
  - token_class = "token-paren"
  - value = "("
- elsif token.token_type == :right_paren
  - token_class = "token-paren"
  - value = ")"
- elsif token.token_type == :greater
  - token_class = "token-operator"
  - value = ">"
- elsif token.token_type == :less
  - token_class = "token-operator"
  - value = "<"
- elsif token.token_type == :greater_equal
  - token_class = "token-operator"
  - value = ">="
- elsif token.token_type == :less_equal
  - token_class = "token-operator"
  - value = "<="
- elsif token.token_type == :bang_equal
  - token_class = "token-operator"
  - value = "!="
- elsif token.token_type == :equal
  - token_class = "token-operator"
  - value = "="
- elsif token.token_type == :minus
  - token_class = "token-operator"
  - value = "-"
- elsif token.token_type == :plus
  - token_class = "token-operator"
  - value = "+"
- elsif token.token_type == :power
  - token_class = "token-operator"
  - value = "^"
- elsif token.token_type == :star
  - token_class = "token-operator"
  - value = "*"
- elsif token.token_type == :slash
  - token_class = "token-operator"
  - value = "/"
- elsif token.token_type == :bang
  - token_class = "token-operator"
  - value = "!"
- elsif token.token_type.in?([:between, :and, :or, :at])
  - token_class = "token-operator"
- elsif token.token_type.in?([:entered, :any, :missing, :unentered])
  - token_class = "token-value"
- elsif token.token_type.in?([:true, :false, :nil])
  - token_class = "token-boolean"

{'fraction_non_alphanumeric': 0.17407407407407408, 'fraction_numerical': 0.0007936507936507937, 'mean_word_length': 3.2293064876957494, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5195424', 'n_tokens_mistral': 1388, 'n_tokens_neox': 1416, 'n_words': 268}