<commit_before>fs = require 'fs'
{EventEmitter} = require 'events'
{pointToErrorLocation} = require './helpers'
StringScanner = require 'StringScanner'

inspect = (o) -> (require 'util').inspect o, no, 9e9, yes


# TODO: better comments
# TODO: support win32-style line endings

@Preprocessor = class Preprocessor extends EventEmitter

  ws = '\\t\\x0B\\f \\xA0\\u1680\\u180E\\u2000-\\u200A\\u202F\\u205F\\u3000\\uFEFF'
  INDENT = '\uEFEF'
  DEDENT = '\uEFFE'
  TERM   = '\uEFFF'

  constructor: ->
    # `base` is either `null` or a regexp that matches the base indentation
    @base = null
    # `indents` is an array of successive indentation characters.
    @indents = []
    @context = []
    @context.peek = -> if @length then this[@length - 1] else null
    @context.err = (c) ->
      throw new Error "Unexpected " + inspect c
    @context.observe = (c) ->
      top = @peek()
      switch c
        # opening token is closing token
        when '"""', '\'\'\'', '"', '\'', '###', '`', '///', '/'
          if top is c then do @pop
          else @push c
        # strictly opening tokens
        when INDENT, '#', '#{', '[', '(', '{', '\\', 'regexp-[', 'regexp-(', 'regexp-{', 'heregexp-#', 'heregexp-[', 'heregexp-(', 'heregexp-{'
          @push c
        # strictly closing tokens
        when DEDENT
          (@err c) unless top is INDENT
          do @pop
        when '\n'
          (@err c) unless top in ['#', 'heregexp-#']
          do @pop
        when ']'
          (@err c) unless top in ['[', 'regexp-[', 'heregexp-[']
          do @pop
        when ')'
          (@err c) unless top in ['(', 'regexp-(', 'heregexp-(']
          do @pop
        when '}'
          (@err c) unless top in ['#{', '{', 'regexp-{', 'heregexp-{']
          do @pop
        when 'end-\\'
          (@err c) unless top is '\\'
          do @pop
        else throw new Error "undefined token observed: " + c
      this
    @ss = new StringScanner ''

  p: (s) ->
    if s? then @emit 'data', s
    s

  scan: (r) -> @p @ss.scan r

  processInput = (isEnd) -> (data) ->
    @ss.concat data unless isEnd

    until @ss.eos()
      switch @context.peek()
        when null, INDENT, '#{', '[', '(', '{'
          if @ss.bol() or @scan /// (?:[#{ws}]* \n)+ ///

            @scan /// (?: [#{ws}]* (\#\#?(?!\#)[^\n]*)? \n )+ ///

            # we might require more input to determine indentation
            return if not isEnd and (@ss.check /// [#{ws}\n]* $ ///)?

            if @base?
              unless (@scan @base)?
                throw new Error "inconsistent base indentation"
            else
              @base = /// #{@scan /// [#{ws}]* ///} ///

            i = 0
            lines = @ss.str.substr(0, @ss.pos).split(/\n/) || ['']
            while i < @indents.length
              indent = @indents[i]
              if @ss.check /// #{indent} ///
                # an existing indent
                @scan /// #{indent} ///
              else if @ss.check /// [^#{ws}] ///
                # we lost an indent
                @indents.splice i--, 1
                @context.observe DEDENT
                @p "#{DEDENT}#{TERM}"
              else
                # Some ambiguous dedent
                lines = @ss.str.substr(0, @ss.pos).split(/\n/) || ['']
                message = "Syntax error on line #{lines.length}: indention is ambiguous"
                lineLen = @indents.reduce ((l, r) -> l + r.length), 0
                context = pointToErrorLocation @ss.str, lines.length, lineLen
                throw new Error "#{message}\n#{context}"
              i++
            if @ss.check /// [#{ws}]+ [^#{ws}#] ///
              # an indent
              @indents.push @scan /// [#{ws}]+ ///
              @context.observe INDENT
              @p INDENT

          tok = switch @context.peek()
            when '['
              # safe things, but not closing bracket
              @scan /[^\n'"\\\/#`[({\]]+/
              @scan /\]/<commit_msg>Removed unnedded code.<commit_after>fs = require 'fs'
{EventEmitter} = require 'events'
{pointToErrorLocation} = require './helpers'
StringScanner = require 'StringScanner'

inspect = (o) -> (require 'util').inspect o, no, 9e9, yes


# TODO: better comments
# TODO: support win32-style line endings

@Preprocessor = class Preprocessor extends EventEmitter

  ws = '\\t\\x0B\\f \\xA0\\u1680\\u180E\\u2000-\\u200A\\u202F\\u205F\\u3000\\uFEFF'
  INDENT = '\uEFEF'
  DEDENT = '\uEFFE'
  TERM   = '\uEFFF'

  constructor: ->
    # `base` is either `null` or a regexp that matches the base indentation
    @base = null
    # `indents` is an array of successive indentation characters.
    @indents = []
    @context = []
    @context.peek = -> if @length then this[@length - 1] else null
    @context.err = (c) ->
      throw new Error "Unexpected " + inspect c
    @context.observe = (c) ->
      top = @peek()
      switch c
        # opening token is closing token
        when '"""', '\'\'\'', '"', '\'', '###', '`', '///', '/'
          if top is c then do @pop
          else @push c
        # strictly opening tokens
        when INDENT, '#', '#{', '[', '(', '{', '\\', 'regexp-[', 'regexp-(', 'regexp-{', 'heregexp-#', 'heregexp-[', 'heregexp-(', 'heregexp-{'
          @push c
        # strictly closing tokens
        when DEDENT
          (@err c) unless top is INDENT
          do @pop
        when '\n'
          (@err c) unless top in ['#', 'heregexp-#']
          do @pop
        when ']'
          (@err c) unless top in ['[', 'regexp-[', 'heregexp-[']
          do @pop
        when ')'
          (@err c) unless top in ['(', 'regexp-(', 'heregexp-(']
          do @pop
        when '}'
          (@err c) unless top in ['#{', '{', 'regexp-{', 'heregexp-{']
          do @pop
        when 'end-\\'
          (@err c) unless top is '\\'
          do @pop
        else throw new Error "undefined token observed: " + c
      this
    @ss = new StringScanner ''

  p: (s) ->
    if s? then @emit 'data', s
    s

  scan: (r) -> @p @ss.scan r

  processInput = (isEnd) -> (data) ->
    @ss.concat data unless isEnd

    until @ss.eos()
      switch @context.peek()
        when null, INDENT, '#{', '[', '(', '{'
          if @ss.bol() or @scan /// (?:[#{ws}]* \n)+ ///

            @scan /// (?: [#{ws}]* (\#\#?(?!\#)[^\n]*)? \n )+ ///

            # we might require more input to determine indentation
            return if not isEnd and (@ss.check /// [#{ws}\n]* $ ///)?

            if @base?
              unless (@scan @base)?
                throw new Error "inconsistent base indentation"
            else
              @base = /// #{@scan /// [#{ws}]* ///} ///

            i = 0
            while i < @indents.length
              indent = @indents[i]
              if @ss.check /// #{indent} ///
                # an existing indent
                @scan /// #{indent} ///
              else if @ss.check /// [^#{ws}] ///
                # we lost an indent
                @indents.splice i--, 1
                @context.observe DEDENT
                @p "#{DEDENT}#{TERM}"
              else
                # Some ambiguous dedent
                lines = @ss.str.substr(0, @ss.pos).split(/\n/) || ['']
                message = "Syntax error on line #{lines.length}: indention is ambiguous"
                lineLen = @indents.reduce ((l, r) -> l + r.length), 0
                context = pointToErrorLocation @ss.str, lines.length, lineLen
                throw new Error "#{message}\n#{context}"
              i++
            if @ss.check /// [#{ws}]+ [^#{ws}#] ///
              # an indent
              @indents.push @scan /// [#{ws}]+ ///
              @context.observe INDENT
              @p INDENT

          tok = switch @context.peek()
            when '['
              # safe things, but not closing bracket
              @scan /[^\n'"\\\/#`[({\]]+/
              @scan /\]/
{'fraction_non_alphanumeric': 0.186578546009151, 'fraction_numerical': 0.009023894255210982, 'mean_word_length': 1.5164694595458905, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2724389', 'n_tokens_mistral': 2590, 'n_tokens_neox': 2463, 'n_words': 705}