<commit_before>class AsciiIo.VT

  constructor: (@cols, @lines, @view) ->
    @sgrInterpreter = new AsciiIo.SgrInterpreter()
    @reset()

  reset: ->
    @data = ''
    @resetTerminal()

  handleData: (data) ->
    if data.match(/^\x1b[\x00-\x1f]/)
      @handleControlCharacter(data[1])
      return 2

    else if match = data.match(/^(\x1b\x5d|\x9d).*?(\x1b\\|\x9c|\x07)/)
      # OSC seq
      return match[0].length

    else if match = data.match(/^(\x1b[PX_^]|[\x90\x98\x9e\x9f]).*?(\x1b\\|\x9c)/)
      # DCS/SOS/PM/APC seq
      return match[0].length

    else if match = data.match(/^(?:\x1b\x5b|\x9b)([\x30-\x3f]*?)[\x20-\x2f]*?[\x40-\x7e]/)
      # Control sequences
      @handleControlSequence(match[0], match[1], match)
      return match[0].length

    else if match = data.match(/^\x1b[\x20-\x2f]*?[\x30-\x3f]/)
      @handlePrivateEscSeq(match[0])
      return match[0].length

    else if match = data.match(/^\x1b[\x20-\x2f]*?[\x40-\x5a\x5c-\x7e]/) # excluding \x5b "["
      @handleStandardEscSeq(match[0])
      return match[0].length

    else if data.match(/^\x1b\x7f/) # DELETE
      return 2

    else if data.match(/^[\x00-\x1a\x1c-\x1f]/) # excluding \x1b "ESC"
      @handleControlCharacter(data[0])
      return 1

    else if match = data.match(/^([\x20-\x7e]|[\xe2\xe3]..|[\xa1-\xfe])+/)
      @handlePrintableCharacters(match[0])
      return match[0].length

    else if data[0] is "\x7f"
      # DELETE, always and everywhere ignored
      return 1

    else if data.match(/^[\x80-\x9f]/)
      @handleControlCharacter(data[0])
      return 1
<commit_msg>Again, better handling of UTF8 printing<commit_after>class AsciiIo.VT

  constructor: (@cols, @lines, @view) ->
    @sgrInterpreter = new AsciiIo.SgrInterpreter()
    @reset()

  reset: ->
    @data = ''
    @resetTerminal()

  handleData: (data) ->
    if data.match(/^\x1b[\x00-\x1f]/)
      @handleControlCharacter(data[1])
      return 2

    else if match = data.match(/^(\x1b\x5d|\x9d).*?(\x1b\\|\x9c|\x07)/)
      # OSC seq
      return match[0].length

    else if match = data.match(/^(\x1b[PX_^]|[\x90\x98\x9e\x9f]).*?(\x1b\\|\x9c)/)
      # DCS/SOS/PM/APC seq
      return match[0].length

    else if match = data.match(/^(?:\x1b\x5b|\x9b)([\x30-\x3f]*?)[\x20-\x2f]*?[\x40-\x7e]/)
      # Control sequences
      @handleControlSequence(match[0], match[1], match)
      return match[0].length

    else if match = data.match(/^\x1b[\x20-\x2f]*?[\x30-\x3f]/)
      @handlePrivateEscSeq(match[0])
      return match[0].length

    else if match = data.match(/^\x1b[\x20-\x2f]*?[\x40-\x5a\x5c-\x7e]/) # excluding \x5b "["
      @handleStandardEscSeq(match[0])
      return match[0].length

    else if data.match(/^\x1b\x7f/) # DELETE
      return 2

    else if data.match(/^[\x00-\x1a\x1c-\x1f]/) # excluding \x1b "ESC"
      @handleControlCharacter(data[0])
      return 1

    else if match = data.match(/^([\x20-\x7e]|[\xe2-\xe8]..|[\xa1-\xfe])+/)
      @handlePrintableCharacters(match[0])
      return match[0].length

    else if data[0] is "\x7f"
      # DELETE, always and everywhere ignored
      return 1

    else if data.match(/^[\x80-\x9f]/)
      @handleControlCharacter(data[0])
      return 1

{'fraction_non_alphanumeric': 0.20242914979757085, 'fraction_numerical': 0.054500155714730616, 'mean_word_length': 2.5103825136612024, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2481953', 'n_tokens_mistral': 1356, 'n_tokens_neox': 1285, 'n_words': 211}