<commit_before>'use strict';

var zstd = require('../');
var expect = require('chai').expect;
var Writable = require('stream').Writable;
var util = require('util');
var fs = require('fs');
var path = require('path');

function BufferWriter() {
  Writable.call(this);
  this.data = new Buffer(0);
}
util.inherits(BufferWriter, Writable);

BufferWriter.prototype._write = function(chunk, encoding, next) {
  this.data = Buffer.concat([this.data, chunk], this.data.length + chunk.length);
  next();
};

function testStream(method, bufferFile, resultFile, done, params) {
  var writeStream = new BufferWriter();

  fs.createReadStream(path.join(__dirname, '/fixtures/', bufferFile))
    .pipe(method(params))
    .pipe(writeStream);

  writeStream.on('finish', function() {
    var result = fs.readFileSync(path.join(__dirname, '/fixtures/', resultFile));
    expect(writeStream.data).to.deep.equal(result);
    done();
  });
}

describe('ZSTD Stream', function() {
  describe('compress', function() {
    it('should compress binary data', function(done) {
      testStream(zstd.compressStream, 'data10k.bin', 'data10k.bin.zst', done);
    });

    it('should compress text data', function(done) {
      testStream(zstd.compressStream, 'data.txt', 'data.txt.zst', done);
    });

    it('should compress text data with level=3', function(done) {
      testStream(zstd.compressStream, 'data.txt', 'data.txt.zst.03', done, {level: 3});
    });

    it('should compress text data with level=9', function(done) {
      testStream(zstd.compressStream, 'data.txt', 'data.txt.zst.09', done, {level: 9});
    });

    it('should compress an empty buffer', function(done) {
      testStream(zstd.compressStream, 'empty', 'empty.zst', done);
    });

    it('should compress a large buffer', function(done) {
      this.timeout(30000);
      testStream(zstd.compressStream, 'large.txt', 'large.txt.zst', done);
    });
  });

  describe('decompress', function() {
    it('should decompress binary data', function(done) {
      testStream(zstd.decompressStream, 'data10k.bin.zst', 'data10k.bin', done);
    });

    it('should decompress text data', function(done) {
      testStream(zstd.decompressStream, 'data.txt.zst', 'data.txt', done);
    });

    it('should decompress to an empty buffer', function(done) {
      testStream(zstd.decompressStream, 'empty.zst', 'empty', done);
    });

    it('should decompress to a large buffer', function(done) {
      testStream(zstd.decompressStream, 'large.zst', 'large', done);
    });

    it('should decompress to another large buffer', function(done) {
      testStream(zstd.decompressStream, 'large.txt.zst', 'large.txt', done);<commit_msg>test<zstdStream>: add dict testcase for stream<commit_after>'use strict';

var zstd = require('../');
var expect = require('chai').expect;
var Writable = require('stream').Writable;
var util = require('util');
var fs = require('fs');
var path = require('path');

function BufferWriter() {
  Writable.call(this);
  this.data = new Buffer(0);
}
util.inherits(BufferWriter, Writable);

BufferWriter.prototype._write = function(chunk, encoding, next) {
  this.data = Buffer.concat([this.data, chunk], this.data.length + chunk.length);
  next();
};

function testStream(method, bufferFile, resultFile, done, params) {
  var writeStream = new BufferWriter();

  fs.createReadStream(path.join(__dirname, '/fixtures/', bufferFile))
    .pipe(method(params))
    .pipe(writeStream);

  writeStream.on('finish', function() {
    var result = fs.readFileSync(path.join(__dirname, '/fixtures/', resultFile));
    expect(writeStream.data).to.deep.equal(result);
    done();
  });
}

describe('ZSTD Stream', function() {
  describe('compress', function() {
    it('should compress binary data', function(done) {
      testStream(zstd.compressStream, 'data10k.bin', 'data10k.bin.zst', done);
    });

    it('should compress text data', function(done) {
      testStream(zstd.compressStream, 'data.txt', 'data.txt.zst', done);
    });

    it('should compress text data with level=3', function(done) {
      testStream(zstd.compressStream, 'data.txt', 'data.txt.zst.03', done, {level: 3});
    });

    it('should compress text data with level=9', function(done) {
      testStream(zstd.compressStream, 'data.txt', 'data.txt.zst.09', done, {level: 9});
    });

    it('should compress text data with dict=(new Buffer("hello zstd"))', function(done) {
      testStream(zstd.compressStream, 'data.txt', 'data.txt.zst.09.dict', done, {level: 9, dict: new Buffer("hello zstd")});
    });

    it('should compress an empty buffer', function(done) {
      testStream(zstd.compressStream, 'empty', 'empty.zst', done);
    });

    it('should compress a large buffer', function(done) {
      this.timeout(30000);
      testStream(zstd.compressStream, 'large.txt', 'large.txt.zst', done);
    });
  });

  describe('decompress', function() {
    it('should decompress binary data', function(done) {
      testStream(zstd.decompressStream, 'data10k.bin.zst', 'data10k.bin', done);
    });

    it('should decompress text data', function(done) {
      testStream(zstd.decompressStream, 'data.txt.zst', 'data.txt', done);
    });

    it('should decompress text data with dict=(new Buffer("hello zstd"))', function() {
      testStream(zstd.decompressStream, 'data.txt.zst.09.dict', 'data.txt', {level: 9, dict: new Buffer("hello zstd")});
    });

    it('should decompress to an empty buffer', function(done) {
      testStream(zstd.decompressStream, 'empty.zst', 'empty', done);
    });

    it('should decompress to a large buffer', function(done) {
      testStream(zstd.decompressStream, 'large.zst', 'large', done);
    });

    it('should decompress to another large buffer', function(done) {
      testStream(zstd.decompressStream, 'large.txt.zst', 'large.txt', done);
{'fraction_non_alphanumeric': 0.17230135575767977, 'fraction_numerical': 0.00858074480864939, 'mean_word_length': 4.19892952720785, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3555957', 'n_tokens_mistral': 1886, 'n_tokens_neox': 1829, 'n_words': 411}