<commit_before>var Variable  = require('../../index').Variable;
var utils  = require('../../base/utils');
var expect = require('chai').expect;

describe('order', function() {
   var v = new Variable([2, 2.1, 4.3, 1.5, -2, 1, 3]);
   it('works with numerical inputs', function() {
      expect(v).to.respondTo('order');
      expect(v.order().toArray()).to.deep.equal([5, 6, 4, 1, 2, 7, 3]);
   });
   it('can make descending order', function() {
      expect(v.order(true).toArray()).to.deep.equal([3, 7, 2, 1, 4, 6, 5]);
   });
   it('can use a provided comparator', function() {
      f = function(a, b) { 
         return a.length < b.length ? -1 : a.length === b.length ? 0 : 1; 
      }
      var w = Variable.string(["zach","c","antelope","zoo","foods"]);
      expect(w.order(f).toArray()).to.deep.equal([2,4,1,5,3]);
   });
   it('sends NaN values to the end', function() {
      var u = new Variable([2, NaN, 4.3, 1.5, 100, 1, -3]);
      expect(u.order().toArray()).to.deep.equal([7, 6, 4, 1, 3, 5, 2]);
      expect(u.order(true).toArray()).to.deep.equal([5, 3, 1, 4, 6, 7, 2]);
      u = new Variable([2, NaN, 4.3, 1.5, 100, 1, NaN, -3]);
      expect(u.order().toArray()).to.deep.equal([8, 6, 4, 1, 3, 5, 2, 7]);
      expect(u.order(true).toArray()).to.deep.equal([5, 3, 1, 4, 6, 8, 2, 7]);
   });
   it('works for factors', function() {
      var w = Variable.factor(["A", "A", "B", "A", "C", "B"]);
      expect(w.order().toArray()).to.deep.equal([1, 2, 4, 3, 6, 5]);
      expect(w.order(true).toArray()).to.deep.equal([5, 3, 6, 1, 2, 4]);
   });
});
describe('sort', function() {
   var v = new Variable([2, 2.1, 4.3, 1.5, -2, 1, 3]);
   it('works with numerical inputs', function() {
      expect(v).to.respondTo('order');
      expect(v.sort().toArray()).to.deep.equal([-2, 1, 1.5, 2, 2.1, 3, 4.3]);
   });
   it('can make descending order', function() {
      expect(v.sort(true).toArray()).to.deep.equal([4.3, 3, 2.1, 2, 1.5, 1, -2]);
   });
   it('can use a provided comparator', function() {
      f = function(a, b) { 
         return a.length < b.length ? -1 : a.length === b.length ? 0 : 1; 
      }
      var w = new Variable.string(["zach","c","antelope","zoo","foods"]);
      expect(w.sort(f).toArray()).to.deep.equal(["c", "zoo", "zach", "foods", "antelope"]);
   });
   it('sends NaN values to the end', function() {
      var u = new Variable([2, NaN, 4.3, 1.5, 100, 1, -3]);
      expect(u.sort().toArray().slice(0,6)).to.deep.equal([-3, 1, 1.5, 2, 4.3, 100]);
      expect(u.sort(true).toArray().slice(0,6)).to.deep.equal([100, 4.3, 2, 1.5, 1, -3]);
      u = new Variable([2, NaN, 4.3, 1.5, 100, 1, NaN, -3]);
      expect(u.sort().toArray().slice(0,6)).to.deep.equal([-3, 1, 1.5, 2, 4.3, 100]);
      expect(u.sort(true).toArray().slice(0,6)).to.deep.equal([100, 4.3, 2, 1.5, 1, -3]);
   });
   it('with random numbers', function() {
      var v = new Variable(new Variable.Vector(Math.random, 100));
      var w = v.sort();
      for (var i = 1; i < w.length; i+= 1) {
         expect(w.get(i)).to.not.be.greaterThan(w.get(i + 1));
      }
      w = v.sort(true);
      for (var i = 1; i < w.length; i+= 1) {
         expect(w.get(i)).to.not.be.lessThan(w.get(i + 1));
      }
   });
   it('works for factors', function() {
      var w = Variable.factor(["A", "A", "B", "A", "C", "B"]);
      expect(w.sort().asScalar().toArray()).to.deep.equal([1, 1, 1, 2, 2, 3]);
      expect(w.sort(true).asScalar().toArray()).to.deep.equal([3, 2, 2, 1, 1, 1]);
   });
});
describe('quantile', function() {
   var v1 = new Variable([3, 4, 5, 6, 7]);
   var v2 = new Variable([3, 4, NaN, 5, 6, 7, NaN]);
   it('works when no missing values', function() {
      expect(v1.quantile(.2).toArray()).to.deep.equal([3.8]);
      expect(v1.quantile(.25).toArray()).to.deep.equal([4]);
      expect(v1.quantile(0).toArray()).to.deep.equal([3]);
      expect(v1.quantile(1).toArray()).to.deep.equal([7]);
   });
   it('skips values missing in the variable if skipMissing', function() {
      expect(v2.quantile(.2, true).toArray()).to.deep.equal([3.8]);
      expect(v2.quantile(.25, true).toArray()).to.deep.equal([4]);
      expect(v2.quantile(0, true).toArray()).to.deep.equal([3]);
      expect(v2.quantile(1, true).toArray()).to.deep.equal([7]);
   });
   it('throws an error if missing and not skipMissing', function() {
      expect(function() { v2.quantile(.2)}).to.throw(Error);
   });
   it('preserves missing values in the `probs` argument', function() {
      expect(utils.areEqualArrays(v1.quantile([.2, NaN, null, .5, undefined]).toArray(),
         [3.8, utils.missing, utils.missing, 5, utils.missing])).to.be.true;
   });
   it('works for different `probs` formats: 1 value, array, vector, variable', function() {
      expect(v1.quantile(new Variable.Vector([.2, .25, 1])).toArray())
         .to.deep.equal([3.8, 4, 7]);
      expect(v1.quantile(new Variable([.2, .25, 1])).toArray())
         .to.deep.equal([3.8, 4, 7]);
   });
   it('generates appropriate name labels', function() {
            expect(v1.quantile(new Variable([.2, .25, 1]))
               .names().toArray()).to.deep.equal(['20%', '25%', '100%']);
   });
   it('errors when prob < 0 or prob > 1', function() {
      expect(function() { v1.quantile(-2)}).to.throw(Error);
      expect(function() { v1.quantile(-.02)}).to.throw(Error);
      expect(function() { v1.quantile(1.1)}).to.throw(Error);
   });
});
describe('median', function() {
   var v1 = new Variable([3, 4, 5, 6, 7]);
   var v2 = new Variable([3, 4, NaN, 5, 6, 7, NaN]);
   it('returns a (the correct) number', function() {
      expect(v1.median()).to.equal(5);
      expect(new Variable([3,4,5,6]).median()).to.equal(4.5);
      expect(v2.median(true)).to.equal(5);
      expect(utils.isMissing(v2.median())).to.be.true;
   });
});
describe('fiveNum', function() {
   var v1 = new Variable([3, 4, 5, 6, 7]);
   var v2 = new Variable([3, 4, NaN, 5, 6, 7, NaN]);
   it('returns the correct numbers', function() {
      expect(v1.fiveNum().toArray()).to.deep.equal([3,4,5,6,7]);
      expect(new Variable([3,4,5,6]).fiveNum().toArray())
         .to.deep.equal([3, 3.75, 4.5, 5.25, 6]);
      expect(v2.median(true)).to.equal(5);
      expect(utils.isMissing(v2.median())).to.be.true;
   });
});
describe('table', function() {
   var v1 = new Variable([4.5, 2.3, 2.3, 4.5, 1.2, 4.5]);
   it('returns a variable of frequencies (with names)', function() {
      expect(v1.table().toArray()).to.deep.equal([1, 2, 3]);
      expect(v1.table().names().asScalar().toArray()).to.deep.equal([1.2, 2.3, 4.5]);
   });
   it('tabulates missing values at the end', function() {
      var v2 = new Variable([3, 4, NaN, 3, 6, 4, NaN]);
      expect(v2.table().toArray()).to.deep.equal([2, 2, 1, 2]);
      expect(utils.areEqualArrays(v2.table().names().asScalar().toArray(),
                                  [3, 4, 6, utils.missing]));
   });
});<commit_msg>Add test for freq table of factor variable.<commit_after>var Variable  = require('../../index').Variable;
var utils  = require('../../base/utils');
var expect = require('chai').expect;

describe('order', function() {
   var v = new Variable([2, 2.1, 4.3, 1.5, -2, 1, 3]);
   it('works with numerical inputs', function() {
      expect(v).to.respondTo('order');
      expect(v.order().toArray()).to.deep.equal([5, 6, 4, 1, 2, 7, 3]);
   });
   it('can make descending order', function() {
      expect(v.order(true).toArray()).to.deep.equal([3, 7, 2, 1, 4, 6, 5]);
   });
   it('can use a provided comparator', function() {
      f = function(a, b) { 
         return a.length < b.length ? -1 : a.length === b.length ? 0 : 1; 
      }
      var w = Variable.string(["zach","c","antelope","zoo","foods"]);
      expect(w.order(f).toArray()).to.deep.equal([2,4,1,5,3]);
   });
   it('sends NaN values to the end', function() {
      var u = new Variable([2, NaN, 4.3, 1.5, 100, 1, -3]);
      expect(u.order().toArray()).to.deep.equal([7, 6, 4, 1, 3, 5, 2]);
      expect(u.order(true).toArray()).to.deep.equal([5, 3, 1, 4, 6, 7, 2]);
      u = new Variable([2, NaN, 4.3, 1.5, 100, 1, NaN, -3]);
      expect(u.order().toArray()).to.deep.equal([8, 6, 4, 1, 3, 5, 2, 7]);
      expect(u.order(true).toArray()).to.deep.equal([5, 3, 1, 4, 6, 8, 2, 7]);
   });
   it('works for factors', function() {
      var w = Variable.factor(["A", "A", "B", "A", "C", "B"]);
      expect(w.order().toArray()).to.deep.equal([1, 2, 4, 3, 6, 5]);
      expect(w.order(true).toArray()).to.deep.equal([5, 3, 6, 1, 2, 4]);
   });
});
describe('sort', function() {
   var v = new Variable([2, 2.1, 4.3, 1.5, -2, 1, 3]);
   it('works with numerical inputs', function() {
      expect(v).to.respondTo('order');
      expect(v.sort().toArray()).to.deep.equal([-2, 1, 1.5, 2, 2.1, 3, 4.3]);
   });
   it('can make descending order', function() {
      expect(v.sort(true).toArray()).to.deep.equal([4.3, 3, 2.1, 2, 1.5, 1, -2]);
   });
   it('can use a provided comparator', function() {
      f = function(a, b) { 
         return a.length < b.length ? -1 : a.length === b.length ? 0 : 1; 
      }
      var w = new Variable.string(["zach","c","antelope","zoo","foods"]);
      expect(w.sort(f).toArray()).to.deep.equal(["c", "zoo", "zach", "foods", "antelope"]);
   });
   it('sends NaN values to the end', function() {
      var u = new Variable([2, NaN, 4.3, 1.5, 100, 1, -3]);
      expect(u.sort().toArray().slice(0,6)).to.deep.equal([-3, 1, 1.5, 2, 4.3, 100]);
      expect(u.sort(true).toArray().slice(0,6)).to.deep.equal([100, 4.3, 2, 1.5, 1, -3]);
      u = new Variable([2, NaN, 4.3, 1.5, 100, 1, NaN, -3]);
      expect(u.sort().toArray().slice(0,6)).to.deep.equal([-3, 1, 1.5, 2, 4.3, 100]);
      expect(u.sort(true).toArray().slice(0,6)).to.deep.equal([100, 4.3, 2, 1.5, 1, -3]);
   });
   it('with random numbers', function() {
      var v = new Variable(new Variable.Vector(Math.random, 100));
      var w = v.sort();
      for (var i = 1; i < w.length; i+= 1) {
         expect(w.get(i)).to.not.be.greaterThan(w.get(i + 1));
      }
      w = v.sort(true);
      for (var i = 1; i < w.length; i+= 1) {
         expect(w.get(i)).to.not.be.lessThan(w.get(i + 1));
      }
   });
   it('works for factors', function() {
      var w = Variable.factor(["A", "A", "B", "A", "C", "B"]);
      expect(w.sort().asScalar().toArray()).to.deep.equal([1, 1, 1, 2, 2, 3]);
      expect(w.sort(true).asScalar().toArray()).to.deep.equal([3, 2, 2, 1, 1, 1]);
   });
});
describe('quantile', function() {
   var v1 = new Variable([3, 4, 5, 6, 7]);
   var v2 = new Variable([3, 4, NaN, 5, 6, 7, NaN]);
   it('works when no missing values', function() {
      expect(v1.quantile(.2).toArray()).to.deep.equal([3.8]);
      expect(v1.quantile(.25).toArray()).to.deep.equal([4]);
      expect(v1.quantile(0).toArray()).to.deep.equal([3]);
      expect(v1.quantile(1).toArray()).to.deep.equal([7]);
   });
   it('skips values missing in the variable if skipMissing', function() {
      expect(v2.quantile(.2, true).toArray()).to.deep.equal([3.8]);
      expect(v2.quantile(.25, true).toArray()).to.deep.equal([4]);
      expect(v2.quantile(0, true).toArray()).to.deep.equal([3]);
      expect(v2.quantile(1, true).toArray()).to.deep.equal([7]);
   });
   it('throws an error if missing and not skipMissing', function() {
      expect(function() { v2.quantile(.2)}).to.throw(Error);
   });
   it('preserves missing values in the `probs` argument', function() {
      expect(utils.areEqualArrays(v1.quantile([.2, NaN, null, .5, undefined]).toArray(),
         [3.8, utils.missing, utils.missing, 5, utils.missing])).to.be.true;
   });
   it('works for different `probs` formats: 1 value, array, vector, variable', function() {
      expect(v1.quantile(new Variable.Vector([.2, .25, 1])).toArray())
         .to.deep.equal([3.8, 4, 7]);
      expect(v1.quantile(new Variable([.2, .25, 1])).toArray())
         .to.deep.equal([3.8, 4, 7]);
   });
   it('generates appropriate name labels', function() {
            expect(v1.quantile(new Variable([.2, .25, 1]))
               .names().toArray()).to.deep.equal(['20%', '25%', '100%']);
   });
   it('errors when prob < 0 or prob > 1', function() {
      expect(function() { v1.quantile(-2)}).to.throw(Error);
      expect(function() { v1.quantile(-.02)}).to.throw(Error);
      expect(function() { v1.quantile(1.1)}).to.throw(Error);
   });
});
describe('median', function() {
   var v1 = new Variable([3, 4, 5, 6, 7]);
   var v2 = new Variable([3, 4, NaN, 5, 6, 7, NaN]);
   it('returns a (the correct) number', function() {
      expect(v1.median()).to.equal(5);
      expect(new Variable([3,4,5,6]).median()).to.equal(4.5);
      expect(v2.median(true)).to.equal(5);
      expect(utils.isMissing(v2.median())).to.be.true;
   });
});
describe('fiveNum', function() {
   var v1 = new Variable([3, 4, 5, 6, 7]);
   var v2 = new Variable([3, 4, NaN, 5, 6, 7, NaN]);
   it('returns the correct numbers', function() {
      expect(v1.fiveNum().toArray()).to.deep.equal([3,4,5,6,7]);
      expect(new Variable([3,4,5,6]).fiveNum().toArray())
         .to.deep.equal([3, 3.75, 4.5, 5.25, 6]);
      expect(v2.median(true)).to.equal(5);
      expect(utils.isMissing(v2.median())).to.be.true;
   });
});
describe('table', function() {
   var v1 = new Variable([4.5, 2.3, 2.3, 4.5, 1.2, 4.5]);
   it('returns a variable of frequencies (with names)', function() {
      expect(v1.table().toArray()).to.deep.equal([1, 2, 3]);
      expect(v1.table().names().asScalar().toArray()).to.deep.equal([1.2, 2.3, 4.5]);
   });
   it('tabulates missing values at the end', function() {
      var v2 = new Variable([3, 4, NaN, 3, 6, 4, NaN]);
      expect(v2.table().toArray()).to.deep.equal([2, 2, 1, 2]);
      expect(utils.areEqualArrays(v2.table().names().asScalar().toArray(),
                                  [3, 4, 6, utils.missing]));
   });
   it('works for factor variables', function() {
      var v3 = new Variable(["d","c","d","a"]);
      expect(v3.table().toArray()).to.deep.equal([1, 1, 2]);
      expect(v3.table().names().toArray()).to.deep.equal(["a","c","d"]);
   });
});
{'fraction_non_alphanumeric': 0.24851021566401815, 'fraction_numerical': 0.05746311010215664, 'mean_word_length': 2.995748299319728, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '904920', 'n_tokens_mistral': 5892, 'n_tokens_neox': 5326, 'n_words': 1150}