<commit_before>(function () {
  'use strict';

  var assert = require('assert');
  var _ = require('underscore');
  var summarizer = require('../src/summarizer');
  var sha1 = require('../src/sha1');
  var utils = require('./utils_typedarrays');

  function assertThatSetOfArrayEquals(arr1, arr2) {
    assert.equal(arr1.lenght, arr2.lenght);
    assert.ok(_(arr1).every(function (item1) {
      return _(arr2).some(function (item2) {
        return utils.isEqual(item1, item2);
      });
    }));
  }

  function testSummarizer(builder) {
    it('generate summary with input items', function(done) {
      function serialize(value) {
        return new Int8Array(value).buffer;
      }

      builder([[1, 2], [2, 2], [3, 2]], serialize)(5).then(function (summary) {
        var diff = summary._asDifference();
        assertThatSetOfArrayEquals(diff.added, [[1, 2], [2, 2], [3, 2]]);
        assert.equal(0, diff.removed.length);
        done();
      }, function (err) {
        done(err);
      });
    });
  }

  describe('Summarizer', function() {<commit_msg>Use sha1 module from core in generator package<commit_after>(function () {
  'use strict';

  var assert = require('assert');
  var _ = require('underscore');
  var summarizer = require('../src/summarizer');
  var sha1 = require('mathsync/src/sha1');
  var utils = require('./utils_typedarrays');

  function assertThatSetOfArrayEquals(arr1, arr2) {
    assert.equal(arr1.lenght, arr2.lenght);
    assert.ok(_(arr1).every(function (item1) {
      return _(arr2).some(function (item2) {
        return utils.isEqual(item1, item2);
      });
    }));
  }

  function testSummarizer(builder) {
    it('generate summary with input items', function(done) {
      function serialize(value) {
        return new Int8Array(value).buffer;
      }

      builder([[1, 2], [2, 2], [3, 2]], serialize)(5).then(function (summary) {
        var diff = summary._asDifference();
        assertThatSetOfArrayEquals(diff.added, [[1, 2], [2, 2], [3, 2]]);
        assert.equal(0, diff.removed.length);
        done();
      }, function (err) {
        done(err);
      });
    });
  }

  describe('Summarizer', function() {
{'fraction_non_alphanumeric': 0.17058552328261872, 'fraction_numerical': 0.025357307514983864, 'mean_word_length': 3.0789473684210527, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3002358', 'n_tokens_mistral': 721, 'n_tokens_neox': 706, 'n_words': 145}