<commit_before>var fs = require('fs');
var path = require('path');

var libpath = process.env.TEST_COV ? '../lib-cov/' : '../lib/';
var detector = require(libpath + 'detector');

var handlers = {};
var types = ['png', 'gif', 'bmp', 'psd', 'jpg'];

// load all available handlers
types.forEach(function (type) {
  handlers[type] = require(libpath + 'types/' + type);
});

/// Maximum buffer size, with a default of 4096 bytes.
var bufferSize = 4096;

function lookup (buffer) {
  // detect the file type.. don't rely on the extension
  var type = detector(buffer);
  // find an appropriate handler for this file type
  if (type in handlers) {
    return handlers[type](buffer);
  }
  // throw up, if we can't understand the file
  else {
    throw new TypeError('unsupported file type');
  }
}

function asyncFileToBuffer (filepath, buffer, callback) {
  // open the file in read only mode
  fs.open(filepath, 'r', function (err, descriptor) {
    if (err) { throw err; }
    // read first buffer block from the file, asynchronously
    fs.read(descriptor, buffer, 0, bufferSize, 0, function (err) {
      if (err) { throw err; }
      // no errors, return the buffer
      callback();
    });
  });
}

function syncFileToBuffer (filepath, buffer) {
  // read from the file, synchronously
  var descriptor = fs.openSync(filepath, 'r');
  fs.readSync(descriptor, buffer, 0, bufferSize, 0);
}

/**
 * @params input - buffer or relative/absolute path of the image file
 * @params callback - optional function for async detection
 */
module.exports = function (input, callback) {

  // Handle buffer input
  if (input instanceof Buffer) {
    return lookup(input);
  }

  // input should be a string at this point
  if (typeof input !== 'string') {
    throw new TypeError('invalid invocation');
  }

  // resolve the file path
  var filepath = path.resolve(input);

  // create a buffer to read data in
  var buffer = new Buffer(bufferSize);

  if (typeof callback === 'function') {
    asyncFileToBuffer(filepath, buffer, function () {
      // return the dimensions
      callback(null, lookup(buffer));
    });
  } else {
    syncFileToBuffer(filepath, buffer);
    // return the dimensions
    return lookup(buffer);
  }
};

// Export the lookup method for benchmarking
module.exports.lookup = lookup;
<commit_msg>bump up the buffer size from 4KB to 32KB, handle larger files.<commit_after>var fs = require('fs');
var path = require('path');

var libpath = process.env.TEST_COV ? '../lib-cov/' : '../lib/';
var detector = require(libpath + 'detector');

var handlers = {};
var types = ['png', 'gif', 'bmp', 'psd', 'jpg'];

// load all available handlers
types.forEach(function (type) {
  handlers[type] = require(libpath + 'types/' + type);
});

/// Maximum buffer size, with a default of 32 kilobytes.
var bufferSize = 32*1024;

function lookup (buffer) {
  // detect the file type.. don't rely on the extension
  var type = detector(buffer);
  // find an appropriate handler for this file type
  if (type in handlers) {
    return handlers[type](buffer);
  }
  // throw up, if we can't understand the file
  else {
    throw new TypeError('unsupported file type');
  }
}

function asyncFileToBuffer (filepath, buffer, callback) {
  // open the file in read only mode
  fs.open(filepath, 'r', function (err, descriptor) {
    if (err) { throw err; }
    // read first buffer block from the file, asynchronously
    fs.read(descriptor, buffer, 0, bufferSize, 0, function (err) {
      if (err) { throw err; }
      // no errors, return the buffer
      callback();
    });
  });
}

function syncFileToBuffer (filepath, buffer) {
  // read from the file, synchronously
  var descriptor = fs.openSync(filepath, 'r');
  fs.readSync(descriptor, buffer, 0, bufferSize, 0);
}

/**
 * @params input - buffer or relative/absolute path of the image file
 * @params callback - optional function for async detection
 */
module.exports = function (input, callback) {

  // Handle buffer input
  if (input instanceof Buffer) {
    return lookup(input);
  }

  // input should be a string at this point
  if (typeof input !== 'string') {
    throw new TypeError('invalid invocation');
  }

  // resolve the file path
  var filepath = path.resolve(input);

  // create a buffer to read data in
  var buffer = new Buffer(bufferSize);

  if (typeof callback === 'function') {
    asyncFileToBuffer(filepath, buffer, function () {
      // return the dimensions
      callback(null, lookup(buffer));
    });
  } else {
    syncFileToBuffer(filepath, buffer);
    // return the dimensions
    return lookup(buffer);
  }
};

// Export the lookup method for benchmarking
module.exports.lookup = lookup;

{'fraction_non_alphanumeric': 0.13049978641606153, 'fraction_numerical': 0.005766766339171294, 'mean_word_length': 3.2923923006416134, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3770536', 'n_tokens_mistral': 1422, 'n_tokens_neox': 1363, 'n_words': 501}