<commit_before>JS.require('JS.Class');

var __ = require('./import')('types');

exports.Token = new JS.Class('Token', {
  initialize: function (startMark, endMark) {
    this.startMark = startMark;
    this.endMark = endMark;
  },
  hash: function () {
    var values = [], self = this;
    
    Object.getOwnPropertyNames(this).forEach(function (key) {
      values.push(key + ':' + self[key]);
    });

    return this.klass + '(' + values.join(', ') + ')';
  }
});


exports.Token.include(__.Hashable);


exports.DirectiveToken = new JS.Class('DirectiveToken', exports.Token, {
  extend: { id: '<directive>' },
  initialize: function (name, value, startMark, endMark) {
    this.name = name;
    this.value = value;
    this.startMark = startMark;
    this.endMark = endMark;
  }
});


exports.DocumentStartToken = new JS.Class('DocumentStartToken', exports.Token, {
  extend: { id: '<document start>' }
});


exports.DocumentEndToken = new JS.Class('DocumentEndToken', exports.Token, {
  extend: { id: '<document end>' }
});

<commit_msg>Cleaned hashing of tokens (do not show marks)<commit_after>JS.require('JS.Class');

var __ = require('./import')('types');

exports.Token = new JS.Class('Token', {
  initialize: function (startMark, endMark) {
    this.startMark = startMark;
    this.endMark = endMark;
  },
  hash: function () {
    var values = [], self = this;
    
    Object.getOwnPropertyNames(this).forEach(function (key) {
      if (!/startMark|endMark|__meta__/.test(key)) {
        values.push(key + ':' + self[key]);
      }
    });

    return this.klass + '(' + values.join(', ') + ')';
  }
});


exports.Token.include(__.Hashable);


exports.DirectiveToken = new JS.Class('DirectiveToken', exports.Token, {
  extend: { id: '<directive>' },
  initialize: function (name, value, startMark, endMark) {
    this.name = name;
    this.value = value;
    this.startMark = startMark;
    this.endMark = endMark;
  }
});


exports.DocumentStartToken = new JS.Class('DocumentStartToken', exports.Token, {
  extend: { id: '<document start>' }
});


exports.DocumentEndToken = new JS.Class('DocumentEndToken', exports.Token, {
  extend: { id: '<document end>' }
});


{'fraction_non_alphanumeric': 0.17714547957778798, 'fraction_numerical': 0.0, 'mean_word_length': 3.6088794926004226, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '620564', 'n_tokens_mistral': 713, 'n_tokens_neox': 704, 'n_words': 139}