<commit_before>var PropertyOptimizer = require('../../properties/optimizer');
var CleanUp = require('./clean-up');

var extractProperties = require('../../properties/extractor');
var canReorder = require('../../properties/reorderable').canReorder;
var canReorderSingle = require('../../properties/reorderable').canReorderSingle;
var stringifyBody = require('../../utils/stringify-tokens').body;
var stringifySelector = require('../../utils/stringify-tokens').selector;

function AdvancedOptimizer(options, context) {
  this.options = options;
  this.propertyOptimizer = new PropertyOptimizer(this.options, context);
}

function unsafeSelector(value) {
  return /\.|\*| :/.test(value);
}

function naturalSorter(a, b) {
  return a > b;
}

AdvancedOptimizer.prototype.isSpecial = function (selector) {
  return this.options.compatibility.selectors.special.test(selector);
};

AdvancedOptimizer.prototype.removeDuplicates = function (tokens) {
  var matched = {};
  var forRemoval = [];

  for (var i = 0, l = tokens.length; i < l; i++) {
    var token = tokens[i];
    if (token[0] != 'selector')
      continue;

    var id = stringifyBody(token[2]) + '@' + stringifySelector(token[1]);
    var alreadyMatched = matched[id];

    if (alreadyMatched) {
      forRemoval.push(alreadyMatched[0]);
      alreadyMatched.unshift(i);
    } else {
      matched[id] = [i];
    }
  }

  forRemoval = forRemoval.sort(function(a, b) {
    return a > b ? 1 : -1;
  });

  for (var j = 0, n = forRemoval.length; j < n; j++) {
    tokens.splice(forRemoval[j] - j, 1);
  }
};

AdvancedOptimizer.prototype.mergeAdjacent = function (tokens) {
  var forRemoval = [];
  var lastToken = [null, [], []];
  var adjacentSpace = this.options.compatibility.selectors.adjacentSpace;

  for (var i = 0, l = tokens.length; i < l; i++) {
    var token = tokens[i];

    if (token[0] != 'selector') {
      lastToken = [null, [], []];
      continue;
    }

    if (lastToken[0] == 'selector' && stringifySelector(token[1]) == stringifySelector(lastToken[1])) {
      var joinAt = [lastToken[2].length];<commit_msg>Speeds up duplication removal in advanced optimizer.<commit_after>var PropertyOptimizer = require('../../properties/optimizer');
var CleanUp = require('./clean-up');

var extractProperties = require('../../properties/extractor');
var canReorder = require('../../properties/reorderable').canReorder;
var canReorderSingle = require('../../properties/reorderable').canReorderSingle;
var stringifyBody = require('../../utils/stringify-tokens').body;
var stringifySelector = require('../../utils/stringify-tokens').selector;

function AdvancedOptimizer(options, context) {
  this.options = options;
  this.propertyOptimizer = new PropertyOptimizer(this.options, context);
}

function unsafeSelector(value) {
  return /\.|\*| :/.test(value);
}

function naturalSorter(a, b) {
  return a > b;
}

AdvancedOptimizer.prototype.isSpecial = function (selector) {
  return this.options.compatibility.selectors.special.test(selector);
};

AdvancedOptimizer.prototype.removeDuplicates = function (tokens) {
  var matched = {};
  var moreThanOnce = [];
  var id, token;
  var body, bodies;

  for (var i = 0, l = tokens.length; i < l; i++) {
    token = tokens[i];
    if (token[0] != 'selector')
      continue;

    id = stringifySelector(token[1]);

    if (matched[id] && matched[id].length == 1)
      moreThanOnce.push(id);
    else
      matched[id] = matched[id] || [];

    matched[id].push(i);
  }

  for (i = 0, l = moreThanOnce.length; i < l; i++) {
    id = moreThanOnce[i];
    bodies = [];

    for (var j = matched[id].length - 1; j >= 0; j--) {
      token = tokens[matched[id][j]];
      body = stringifyBody(token[2]);

      if (bodies.indexOf(body) > -1)
        token[2] = [];
      else
        bodies.push(body);
    }
  }
};

AdvancedOptimizer.prototype.mergeAdjacent = function (tokens) {
  var forRemoval = [];
  var lastToken = [null, [], []];
  var adjacentSpace = this.options.compatibility.selectors.adjacentSpace;

  for (var i = 0, l = tokens.length; i < l; i++) {
    var token = tokens[i];

    if (token[0] != 'selector') {
      lastToken = [null, [], []];
      continue;
    }

    if (lastToken[0] == 'selector' && stringifySelector(token[1]) == stringifySelector(lastToken[1])) {
      var joinAt = [lastToken[2].length];
{'fraction_non_alphanumeric': 0.1741339491916859, 'fraction_numerical': 0.007159353348729792, 'mean_word_length': 3.5256008359456636, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5307857', 'n_tokens_mistral': 1424, 'n_tokens_neox': 1384, 'n_words': 294}