<commit_before>/*
 * GET users listing.
 */

var _ = require('underscore')
  , mongodb = require('mongodb')
  , GridStore = mongodb.GridStore
  , fs = require('fs')
  , async = require('async')
  , spawn = require('child_process').spawn
  , csv = require('csv')
  ;

exports.batch = function (db, src, dst, condition, callback) {
  var collection = db.collection(src + '.files');
  var cursor = collection.find(condition);

  var array = [];
  var count = 0;
  cursor.each(function (err, item) {
    if (item) {
      count++;
      var option = {root: src};
      var gs = new GridStore(db, item._id, 'r', option);
      gs.open(function (err, gs) {
        err && end(err);
        exports.parse(db, gs, function (err, result) {
          array.push(result);
          end(err);
        })
      });
    }
    ;
  });

  function end(err) {
    if (--count == 0) {
      callback(err, array, array.length);
    }
  }
}

exports.parse = function (db, gs, callback) {

  var mecab = spawn('mecab');

  var result = {};
  var total = 0;

  var process = csv().from.stream(mecab.stdout, {delimiter: '\t', quote: null});

  gs.seek(0, function (err, store) {
    var stream = store.stream(true);
    stream.pipe(mecab.stdin);
  });

  process.on('record', function (record, index) {
    if (1 < record.length) {
      var term = record[1].split(',');
      if (term[0] == '名詞') {
        var count = result[record[0]];
        result[record[0]] = count ? count + 1 : 1;
        total++;
      }
    }
  });

  process.on('error', function (error) {
    console.log(error);
    callback(error);
  });

  process.on('end', function (count) {
    callback(null, result, total);
  });

};

/**
 * DocCat
 *
 * @param req
 * @param res
 */
exports.doccat = function (req, res) {

  // アップロードファイル
  var file = req.files.file.path;
<commit_msg>すこし、うまくいっている<commit_after>/*
 * GET users listing.
 */

var _ = require('underscore')
  , mongodb = require('mongodb')
  , GridStore = mongodb.GridStore
  , fs = require('fs')
  , async = require('async')
  , spawn = require('child_process').spawn
  , csv = require('csv')
  ;

exports.batch = function (db, src, dst, condition, callback) {
  var collection = db.collection(src + '.files');
  var cursor = collection.find(condition);

  var array = [];
  var count = 0;
  cursor.each(function (err, item) {
    if (item) {
      count++;
      var option = {root: src};
      var gs = new GridStore(db, item._id, 'r', option);
      gs.open(function (err, gs) {
        err && end(err);
        exports.parse(db, gs, function (err, result) {
          array.push(result);
          end(err);
        })
      });
    }
    ;
  });

  function end(err) {
    if (--count == 0) {
      callback(err, array, array.length);
    }
  }
}

exports.parse = function (db, gs, callback) {

  var mecab = spawn('mecab');

  var result = {};
  var total = 0;

  var process = csv().from.stream(mecab.stdout, {delimiter: '\t', quote: null});

  gs.seek(0, function (err, store) {
    var stream = store.stream(true);
    stream.pipe(mecab.stdin);
  });

  process.on('record', function (record, index) {
    if (1 < record.length) {
      var term = record[1].split(',');
      if (term[0] == '名詞') {
        var count = result[record[0]];
        result[record[0]] = count ? count + 1 : 1;
        total++;
      }
    }
  });

  process.on('error', function (error) {
    callback(error);
  });

  process.on('end', function (count) {
    callback(null, result, total);
  });

};

/**
 * DocCat
 *
 * @param req
 * @param res
 */
exports.doccat = function (req, res) {

  // アップロードファイル
  var file = req.files.file.path;

{'fraction_non_alphanumeric': 0.1819928630249794, 'fraction_numerical': 0.0060389788635739775, 'mean_word_length': 2.5207729468599034, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4847093', 'n_tokens_mistral': 1332, 'n_tokens_neox': 1276, 'n_words': 295}