<commit_before>import Ember from "ember";

var get = Ember.get;
var set = Ember.set;

var TokenList = function () {
  this.tokens = [];
  this.length = 0;
};

TokenList.prototype.push = function (token) {
  var previous = this.tokens.slice(-1)[0];
  if (previous) {
    set(token, 'previous', previous);
    set(previous, 'next', token);

    if (get(token, 'separator') == null) {
      set(token, 'separator', get(previous, 'separator'));
    }

    if (get(token, 'prepend') == null) {
      set(token, 'prepend', get(previous, 'prepend'));
    }
  }

  if (get(token, 'replace')) {
    var cursor = get(token, 'previous');
    while (cursor) {
      set(cursor, 'hidden', true);
      if (get(cursor, 'replace')) {
        break;
      }
      cursor = get(cursor, 'previous');
    }
  }

  this.tokens.push(token);
  this.length++;
};

TokenList.prototype.remove = function (token) {
  var next = get(token, 'next');
  var previous = get(token, 'previous');
  if (next) {
    set(next, 'previous', previous);
  }

  if (previous) {
    set(previous, 'next', next);
  }

  if (get(token, 'replace')) {
    var cursor = get(token, 'previous');
    while (cursor) {
      set(cursor, 'hidden', false);
      if (get(cursor, 'replace')) {
        break;
      }
      cursor = get(cursor, 'previous');
    }
  }

  set(token, 'previous', null);
  set(token, 'next', null);

  this.tokens.removeObject(token);
  this.length--;
};

export default TokenList;
<commit_msg>refactor backtracking to be an iterator method<commit_after>import Ember from "ember";

var get = Ember.get;
var set = Ember.set;

var backtrackFrom = function (start, func) {
  var cursor = get(start, 'previous');
  while (cursor) {
    var result = func(cursor);
    if (result === false) {
      break;
    }
    cursor = get(cursor, 'previous');
  }
};

var TokenList = function () {
  this.tokens = [];
  this.length = 0;
};

TokenList.prototype.push = function (token) {
  var previous = this.tokens.slice(-1)[0];
  if (previous) {
    set(token, 'previous', previous);
    set(previous, 'next', token);

    if (get(token, 'separator') == null) {
      set(token, 'separator', get(previous, 'separator'));
    }

    if (get(token, 'prepend') == null) {
      set(token, 'prepend', get(previous, 'prepend'));
    }
  }

  if (get(token, 'replace')) {
    backtrackFrom(token, function (token) {
      set(token, 'hidden', true);
      return !get(token, 'replace');
    });
  }

  this.tokens.push(token);
  this.length++;
};

TokenList.prototype.remove = function (token) {
  var next = get(token, 'next');
  var previous = get(token, 'previous');
  if (next) {
    set(next, 'previous', previous);
  }

  if (previous) {
    set(previous, 'next', next);
  }

  if (get(token, 'replace')) {
    backtrackFrom(token, function (token) {
      set(token, 'hidden', false);
      return !get(token, 'replace');
    });
  }

  set(token, 'previous', null);
  set(token, 'next', null);

  this.tokens.removeObject(token);
  this.length--;
};

export default TokenList;

{'fraction_non_alphanumeric': 0.1825657894736842, 'fraction_numerical': 0.001973684210526316, 'mean_word_length': 3.1205962059620598, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4301563', 'n_tokens_mistral': 1048, 'n_tokens_neox': 1022, 'n_words': 249}