<commit_before>
var fs = require('fs')

  , test = require('tape')

  , uncompress = require('../lib/uncompress')

  , small   = fs.readFileSync(__dirname + '/fixtures/small-compressed.bin')
  , small2  = fs.readFileSync(__dirname + '/fixtures/small2-compressed.bin')
  , random  = fs.readFileSync(__dirname + '/fixtures/random-compressed.bin')
  , medium  = fs.readFileSync(__dirname + '/fixtures/medium-compressed.bin')
  , large   = fs.readFileSync(__dirname + '/fixtures/large-compressed.bin')
  , urls    = fs.readFileSync(__dirname + '/fixtures/urls.10K-compressed.bin')

  , randomRaw = fs.readFileSync(__dirname + '/fixtures/random-raw.bin')
  , urlsRaw   = fs.readFileSync(__dirname + '/fixtures/urls.10K-raw.txt')

test('uncompress small', function(t) {
  var m = uncompress(small)
  var data = 'foo'

  t.equal(m.length, data.length, 'should have correct length')
  t.equal(m.toString(), 'foo', 'should have correct data')
  t.end()
})

test('uncompress small2', function(t) {
  var m = uncompress(small2)
  var data = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ123456789!@3$%^&*()_'

  t.equal(m.length, data.length, 'should have correct length')
  t.equal(m.toString(), data, 'should have correct data')
  t.end()
})

test('uncompress random', function(t) {
  var m = uncompress(random)

  t.equal(m.length, 1024, 'should have correct length')
  t.deepEqual(m.toString('hex'), randomRaw.toString('hex'), 'should have correct data')
  t.end()
})

test('uncompress urls', function(t) {
  var m = uncompress(urls)

  t.equal(m.length, urlsRaw.length, 'should have correct length')

  t.deepEqual(m.toString().slice(0, 600), urlsRaw.toString().slice(0, 600), 'should have correct data')
  t.end()
})

test('uncompress medium', function(t) {
  var m = uncompress(medium)
  var data = Array(21).join('abcdefghijklmnopqrstuvwxyz')

  t.equal(m.length, data.length, 'should have correct length')
  t.equal(m.toString(), data, 'should have correct data')
  t.end()
})

test('uncompress large', function(t) {
  var m = uncompress(large)
  var data = Array(10001).join('abcdefghijklmnopqrstuvwxyz')

  t.equal(m.length, 260000, 'should have correct length')
  t.equal(m.toString(), data, 'should have correct data')
  t.end()
})<commit_msg>Test full url-page<commit_after>
var fs = require('fs')

  , test = require('tape')

  , uncompress = require('../lib/uncompress')

  , small   = fs.readFileSync(__dirname + '/fixtures/small-compressed.bin')
  , small2  = fs.readFileSync(__dirname + '/fixtures/small2-compressed.bin')
  , random  = fs.readFileSync(__dirname + '/fixtures/random-compressed.bin')
  , medium  = fs.readFileSync(__dirname + '/fixtures/medium-compressed.bin')
  , large   = fs.readFileSync(__dirname + '/fixtures/large-compressed.bin')
  , urls    = fs.readFileSync(__dirname + '/fixtures/urls.10K-compressed.bin')

  , randomRaw = fs.readFileSync(__dirname + '/fixtures/random-raw.bin')
  , urlsRaw   = fs.readFileSync(__dirname + '/fixtures/urls.10K-raw.txt')

test('uncompress small', function(t) {
  var m = uncompress(small)
  var data = 'foo'

  t.equal(m.length, data.length, 'should have correct length')
  t.equal(m.toString(), 'foo', 'should have correct data')
  t.end()
})

test('uncompress small2', function(t) {
  var m = uncompress(small2)
  var data = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ123456789!@3$%^&*()_'

  t.equal(m.length, data.length, 'should have correct length')
  t.equal(m.toString(), data, 'should have correct data')
  t.end()
})

test('uncompress random', function(t) {
  var m = uncompress(random)

  t.equal(m.length, 1024, 'should have correct length')
  t.deepEqual(m.toString('hex'), randomRaw.toString('hex'), 'should have correct data')
  t.end()
})

test('uncompress urls', function(t) {
  var m = uncompress(urls)

  t.equal(m.length, urlsRaw.length, 'should have correct length')

  t.deepEqual(m.toString(), urlsRaw.toString(), 'should have correct data')
  t.end()
})

test('uncompress medium', function(t) {
  var m = uncompress(medium)
  var data = Array(21).join('abcdefghijklmnopqrstuvwxyz')

  t.equal(m.length, data.length, 'should have correct length')
  t.equal(m.toString(), data, 'should have correct data')
  t.end()
})

test('uncompress large', function(t) {
  var m = uncompress(large)
  var data = Array(10001).join('abcdefghijklmnopqrstuvwxyz')

  t.equal(m.length, 260000, 'should have correct length')
  t.equal(m.toString(), data, 'should have correct data')
  t.end()
})
{'fraction_non_alphanumeric': 0.16581120285650525, 'fraction_numerical': 0.01740682883284981, 'mean_word_length': 4.58852867830424, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3708274', 'n_tokens_mistral': 1665, 'n_tokens_neox': 1584, 'n_words': 320}