<commit_before>module.exports =
class ASStest
  id: null
  lines: null
  input: null
  tokens: null
  currentScope: null
  isValid: false

  constructor: (@id, @lines) ->
    @parseData()

  parseData: ->
    # set @id if specified as first-line
    if (@lines[0][0] is '@')
      @id = @lines.shift().substring(1)

    @input = ""
    while (@lines[0][0] is '"') or (@lines[0][0] is "'")
      input = @lines.shift()
      # remove last character (either { or +) and trim whitespace
      input = input.substring(0, input.length - 1).trim()
      # remove encasing quotes
      input = input.substring(1, input.length - 1)
      # transform \n into proper new lines
      input = input.replace(/\\n/g, '\n')
      # insert a new line when concatenating multiple input lines
      if @input then @input += '\n'
      @input += input

      # NOTE
      # If @input.length is 0 then test is empty, and thus can be skipped
      @isValid = true if @input.length

    # gather tokens and scopes
    @tokens = []
    @currentScope = []
    for line in @lines
      if line isnt '}'
        line = line.trim()

        # add scope
        if line[line.length - 1] is '{'
          line = line.substring(0, line.length - 1).trim()
          @addScope(line)

        # remove scope
        else if line is '}'
          @removeScope()

        # create token if (value)
        else if line[0] is line[line.length - 1]
          token =
            value: line.substring(1, line.length - 1)
            scopes: @getScope()
          @tokens.push(token)

        # create token if (value: scope)
        else
          position = null
          for i in [line.length - 1 .. 0]
            if line[i] is ':'
              position = i
              break
          if position isnt null
            @addScope(line.substring(position + 1).trim())
            token =
              value: line.substring(1, position - 1)
              scopes: @getScope()
            @removeScope()
            @tokens.push(token)

  addScope: (scope) ->
    @currentScope.push(scope)

  removeScope: ->
    removed = @currentScope.pop()

  getScope: -><commit_msg>Skip tests when input or tokens is null<commit_after>module.exports =
class ASStest
  id: null
  lines: null
  input: null
  tokens: null
  currentScope: null
  isValid: false

  constructor: (@id, @lines) ->
    @parseData()

  parseData: ->
    # set @id if specified as first-line
    if (@lines[0][0] is '@')
      @id = @lines.shift().substring(1)

    @input = ""
    while (@lines[0][0] is '"') or (@lines[0][0] is "'")
      input = @lines.shift()
      # remove last character (either { or +) and trim whitespace
      input = input.substring(0, input.length - 1).trim()
      # remove encasing quotes
      input = input.substring(1, input.length - 1)
      # transform \n into proper new lines
      input = input.replace(/\\n/g, '\n')
      # insert a new line when concatenating multiple input lines
      if @input then @input += '\n'
      @input += input

    # gather tokens and scopes
    @tokens = []
    @currentScope = []
    for line in @lines
      if line isnt '}'
        line = line.trim()

        # add scope
        if line[line.length - 1] is '{'
          line = line.substring(0, line.length - 1).trim()
          @addScope(line)

        # remove scope
        else if line is '}'
          @removeScope()

        # create token if (value)
        else if line[0] is line[line.length - 1]
          token =
            value: line.substring(1, line.length - 1)
            scopes: @getScope()
          @tokens.push(token)

        # create token if (value: scope)
        else
          position = null
          for i in [line.length - 1 .. 0]
            if line[i] is ':'
              position = i
              break
          if position isnt null
            @addScope(line.substring(position + 1).trim())
            token =
              value: line.substring(1, position - 1)
              scopes: @getScope()
            @removeScope()
            @tokens.push(token)

    # NOTE
    # If @input.length or @tokens.length is 0, then the test is invalid and can be skipped
    @isValid = true if @input.length and @tokens.length

  addScope: (scope) ->
    @currentScope.push(scope)

  removeScope: ->
    removed = @currentScope.pop()

  getScope: ->
{'fraction_non_alphanumeric': 0.12465373961218837, 'fraction_numerical': 0.0110803324099723, 'mean_word_length': 1.8246414602346805, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2805344', 'n_tokens_mistral': 1326, 'n_tokens_neox': 1277, 'n_words': 401}