<commit_before>{expect} = require "chai"
utils = require "../utils"

{Collections} = utils.require "common/base"

describe "categorical mapper", ->
  # FIXME Would be nice to randomize the numbers
  factors = ["foo", "bar", "baz"]
  start = 20
  end = 80

  generateMapper = ->
    Collections("CategoricalMapper").create
      source_range: Collections("FactorRange").create factors: factors
      target_range: Collections("Range1d").create start: start, end: end

  testMapping = (key, expected) ->
    mapper = generateMapper()
    expect(mapper.map_to_target key).to.equal expected

  it "should map first category to bottom third", ->
    testMapping "foo", 30

  it "should map second category to middle", ->
    testMapping "bar", 50

  it "should map third category to upper third", ->
    testMapping "baz", 70

  describe "vector mapping", ->
    vectors = generateMapper().v_map_to_target factors
    it "should map to a Float64Array", ->
      expect(vectors).to.be.an.instanceof Float64Array

    it "should be evenly distributed", ->
      expect(vectors).to.deep.equal new Float64Array [30, 50, 70]<commit_msg>Change spec wording<commit_after>{expect} = require "chai"
utils = require "../utils"

{Collections} = utils.require "common/base"

describe "categorical mapper", ->
  # FIXME Would be nice to randomize the numbers
  factors = ["foo", "bar", "baz"]
  start = 20
  end = 80

  generateMapper = ->
    Collections("CategoricalMapper").create
      source_range: Collections("FactorRange").create factors: factors
      target_range: Collections("Range1d").create start: start, end: end

  testMapping = (key, expected) ->
    mapper = generateMapper()
    expect(mapper.map_to_target key).to.equal expected

  it "should map first category to bottom third", ->
    testMapping "foo", 30

  it "should map second category to middle", ->
    testMapping "bar", 50

  it "should map third category to upper third", ->
    testMapping "baz", 70

  describe "vector mapping", ->
    vectors = generateMapper().v_map_to_target factors
    it "should be an instance of Float64Array", ->
      expect(vectors).to.be.an.instanceof Float64Array

    it "should be evenly distributed", ->
      expect(vectors).to.deep.equal new Float64Array [30, 50, 70]
{'fraction_non_alphanumeric': 0.1199823555359506, 'fraction_numerical': 0.020291133656815175, 'mean_word_length': 3.6666666666666665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6716454', 'n_tokens_mistral': 702, 'n_tokens_neox': 642, 'n_words': 212}