<commit_before>_ = require 'underscore-plus'
{Model} = require 'theorist'
EmitterMixin = require('emissary').Emitter
{Emitter} = require 'event-kit'
{Point, Range} = require 'text-buffer'
Serializable = require 'serializable'
TokenizedLine = require './tokenized-line'
Token = require './token'
Grim = require 'grim'

module.exports =
class TokenizedBuffer extends Model
  Serializable.includeInto(this)

  @property 'tabLength'

  grammar: null
  currentGrammarScore: null
  buffer: null
  tokenizedLines: null
  chunkSize: 50
  invalidRows: null
  visible: false

  constructor: ({@buffer, @tabLength, @invisibles}) ->
    @emitter = new Emitter

    @tabLength ?= atom.config.get('editor.tabLength')

    @subscribe atom.syntax.onDidAddGrammar(@grammarAddedOrUpdated)
    @subscribe atom.syntax.onDidUpdateGrammar(@grammarAddedOrUpdated)

    @subscribe @buffer.onDidChange (e) => @handleBufferChange(e)
    @subscribe @buffer.onDidChangePath (@bufferPath) => @reloadGrammar()

    @subscribe @$tabLength.changes, (tabLength) => @retokenizeLines()

    @subscribe atom.config.onDidChange 'editor.tabLength', (value) => @setTabLength(value)

    @reloadGrammar()

  serializeParams: ->
    bufferPath: @buffer.getPath()
    tabLength: @tabLength
    invisibles: _.clone(@invisibles)

  deserializeParams: (params) ->
    params.buffer = atom.project.bufferForPathSync(params.bufferPath)<commit_msg>Fix onDidChange usage<commit_after>_ = require 'underscore-plus'
{Model} = require 'theorist'
EmitterMixin = require('emissary').Emitter
{Emitter} = require 'event-kit'
{Point, Range} = require 'text-buffer'
Serializable = require 'serializable'
TokenizedLine = require './tokenized-line'
Token = require './token'
Grim = require 'grim'

module.exports =
class TokenizedBuffer extends Model
  Serializable.includeInto(this)

  @property 'tabLength'

  grammar: null
  currentGrammarScore: null
  buffer: null
  tokenizedLines: null
  chunkSize: 50
  invalidRows: null
  visible: false

  constructor: ({@buffer, @tabLength, @invisibles}) ->
    @emitter = new Emitter

    @tabLength ?= atom.config.get('editor.tabLength')

    @subscribe atom.syntax.onDidAddGrammar(@grammarAddedOrUpdated)
    @subscribe atom.syntax.onDidUpdateGrammar(@grammarAddedOrUpdated)

    @subscribe @buffer.onDidChange (e) => @handleBufferChange(e)
    @subscribe @buffer.onDidChangePath (@bufferPath) => @reloadGrammar()

    @subscribe @$tabLength.changes, (tabLength) => @retokenizeLines()

    @subscribe atom.config.onDidChange 'editor.tabLength', ({newValue}) => @setTabLength(newValue)

    @reloadGrammar()

  serializeParams: ->
    bufferPath: @buffer.getPath()
    tabLength: @tabLength
    invisibles: _.clone(@invisibles)

  deserializeParams: (params) ->
    params.buffer = atom.project.bufferForPathSync(params.bufferPath)
{'fraction_non_alphanumeric': 0.12571022727272727, 'fraction_numerical': 0.0014204545454545455, 'mean_word_length': 5.058064516129032, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2332740', 'n_tokens_mistral': 907, 'n_tokens_neox': 873, 'n_words': 183}