<commit_before>{Emitter} = require 'emissary'

module.exports =
class NullGrammar
  Emitter.includeInto(this)

  name: 'Null Grammar'
  scopeName: 'text.plain.null-grammar'

  constructor: ->
    @scopes = ['null-grammar.text.plain']

  getScore: -> 0

  tokenizeLine: (line) ->
    tokens: [{value: line, @scopes}]

  tokenizeLines: (text) -><commit_msg>Return new scopes array for each line tokenized<commit_after>{Emitter} = require 'emissary'

module.exports =
class NullGrammar
  Emitter.includeInto(this)

  name: 'Null Grammar'
  scopeName: 'text.plain.null-grammar'

  getScore: -> 0

  tokenizeLine: (line) ->
    tokens: [{value: line, scopes: ['null-grammar.text.plain']}]

  tokenizeLines: (text) ->
{'fraction_non_alphanumeric': 0.14767932489451477, 'fraction_numerical': 0.0028129395218002813, 'mean_word_length': 4.159420289855072, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7114689', 'n_tokens_mistral': 260, 'n_tokens_neox': 248, 'n_words': 55}