<commit_before>/** @babel */

import { Point, Range } from 'atom'

const juliaGrammar = atom.grammars.grammarForScopeName('source.julia')
const juliaScopes = ['source.julia', 'source.embedded.julia']
const openers = [
  'if', 'while', 'for', 'begin', 'function', 'macro', 'module', 'baremodule', 'type', 'immutable',
  'struct', 'mutable struct', 'try', 'let', 'do', 'quote', 'abstract type', 'primitive type'
]
const reopeners = [ 'else', 'elseif', 'catch', 'finally' ]

function isKeywordScope (scopes) {
  // Skip 'source.julia'
  return scopes.slice(1).some(scope => {
    return scope.indexOf('keyword') > -1
  })
}

function isStringScope (scopes) {
  // Skip 'source.julia'
  return scopes.slice(1).some(scope => {
    return scope.indexOf('string') > -1
  })
}

function forRange (editor, range) {
  const scopes = []
  let n_parens = 0
  let n_brackets = 0
  const text = editor.getTextInBufferRange(range)
  juliaGrammar.tokenizeLines(text).forEach(lineTokens => {
    lineTokens.forEach(token => {
      const { value } = token
      if (!isStringScope(token.scopes)) {
        if (n_parens > 0 && value === ')') {
          n_parens -= 1
          scopes.splice(scopes.lastIndexOf('paren'), 1)
          return
        } else if (n_brackets > 0 && value === ']') {
          n_brackets -= 1
          scopes.splice(scopes.lastIndexOf('bracket'), 1)
          return
        } else if (value === '(') {
          n_parens += 1
          scopes.push('paren')
          return
        } else if (value === '[') {
          n_brackets += 1<commit_msg>fix grammar undefined bug in scopes.js<commit_after>/** @babel */

import { Point, Range } from 'atom'

const juliaScopes = ['source.julia', 'source.embedded.julia']
const openers = [
  'if', 'while', 'for', 'begin', 'function', 'macro', 'module', 'baremodule', 'type', 'immutable',
  'struct', 'mutable struct', 'try', 'let', 'do', 'quote', 'abstract type', 'primitive type'
]
const reopeners = [ 'else', 'elseif', 'catch', 'finally' ]

function isKeywordScope (scopes) {
  // Skip 'source.julia'
  return scopes.slice(1).some(scope => {
    return scope.indexOf('keyword') > -1
  })
}

function isStringScope (scopes) {
  // Skip 'source.julia'
  return scopes.slice(1).some(scope => {
    return scope.indexOf('string') > -1
  })
}

function forRange (editor, range) {
  // this should happen here and not a top-level so that we aren't relying on
  // Atom to load packages in a specific order:
  const juliaGrammar = atom.grammars.grammarForScopeName('source.julia')

  if (juliaGrammar === undefined) return []

  const scopes = []
  let n_parens = 0
  let n_brackets = 0
  const text = editor.getTextInBufferRange(range)
  juliaGrammar.tokenizeLines(text).forEach(lineTokens => {
    lineTokens.forEach(token => {
      const { value } = token
      if (!isStringScope(token.scopes)) {
        if (n_parens > 0 && value === ')') {
          n_parens -= 1
          scopes.splice(scopes.lastIndexOf('paren'), 1)
          return
        } else if (n_brackets > 0 && value === ']') {
          n_brackets -= 1
          scopes.splice(scopes.lastIndexOf('bracket'), 1)
          return
        } else if (value === '(') {
          n_parens += 1
          scopes.push('paren')
          return
        } else if (value === '[') {
          n_brackets += 1
{'fraction_non_alphanumeric': 0.1655110039192041, 'fraction_numerical': 0.008441362677117878, 'mean_word_length': 2.9127358490566038, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6156277', 'n_tokens_mistral': 1149, 'n_tokens_neox': 1121, 'n_words': 262}