<commit_before>/*
 * pygments.js: A node.js wrapper for pygments
 *
 * (C) 2011, <NAME>
 *
 */

var spawn = require('child_process').spawn,
    exists = require('path').existsSync,
    fs = require('fs');

var _ = require('underscore');

var pygments = exports;

//
// Default timeout threshold for pygments
//
pygments.timeout = 10

//
// Default binary path for pygmentize
//
pygments.bin = 'pygmentize'

//
// ### function colorize
// #### @target {String} Target to be highlighted
// #### @lexer {String} Lexer to use for highlighting
// #### @format {String} Format for the output
// #### @options {Object} Other options
//
pygments.colorize = function(target, lexer, format, callback, options) {
  var options = options || {};
  if(lexer) options['l'] = lexer;
  if(format) options['f'] = format;

  options = this.merge_options(options);
  target = this.stringize(target, (options['force']));
  delete options['force'];

  this.execute(target, options, callback);
}

//
// ### function execute
// #### @target {String} Target to be highlighted
// #### @options {Object} Options
//
pygments.execute = function(target, options, callback) {
  var pyg = spawn(this.bin, this.convert_options(options));
  pyg.stdout.on('data', function(data) {
    callback(data.toString());
  });
  pyg.stderr.on('data', function (data) {
    console.log(data.toString());
  });
  pyg.stdin.write(target);
  pyg.stdin.end();
}
<commit_msg>fix: incomplete result for large data<commit_after>/*
 * pygments.js: A node.js wrapper for pygments
 *
 * (C) 2011, <NAME>
 *
 */

var spawn = require('child_process').spawn,
    exists = require('path').existsSync,
    fs = require('fs');

var _ = require('underscore');

var pygments = exports;

//
// Default timeout threshold for pygments
//
pygments.timeout = 10

//
// Default binary path for pygmentize
//
pygments.bin = 'pygmentize'

//
// ### function colorize
// #### @target {String} Target to be highlighted
// #### @lexer {String} Lexer to use for highlighting
// #### @format {String} Format for the output
// #### @options {Object} Other options
//
pygments.colorize = function(target, lexer, format, callback, options) {
  var options = options || {};
  if(lexer) options['l'] = lexer;
  if(format) options['f'] = format;

  options = this.merge_options(options);
  target = this.stringize(target, (options['force']));
  delete options['force'];

  this.execute(target, options, callback);
}

//
// ### function execute
// #### @target {String} Target to be highlighted
// #### @options {Object} Options
//
pygments.execute = function(target, options, callback) {
  var pyg = spawn(this.bin, this.convert_options(options));
  var chunks = [];
  pyg.stdout.on('data', function(chunk) {
    chunks.push(chunk);
  });
  pyg.stderr.on('data', function (data) {
    console.log(data.toString());
  });
  pyg.on('exit', function() {
    var length = 0;
    chunks.forEach(function(chunk) {
      length += chunk.length;
    });
    var content = new Buffer(length);
    var index = 0;
    chunks.forEach(function(chunk) {
      chunk.copy(content, index, 0, chunk.length);
      index += chunk.length;
    });
    callback(content.toString());
  });
  pyg.stdin.write(target);
  pyg.stdin.end();
}

{'fraction_non_alphanumeric': 0.17996289424860853, 'fraction_numerical': 0.00463821892393321, 'mean_word_length': 3.6884057971014492, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3974300', 'n_tokens_mistral': 1068, 'n_tokens_neox': 1038, 'n_words': 258}