<commit_before>#!/usr/bin/env node

/**
 * Module dependencies.
 */

var program = require('commander')
  , fs = require('fs')
  , mongoose = require('mongoose')

program
  .version('0.0.1')
  .option('-u, --mongourl <url>', 'URL to connect to mongo')
  .option('-j, --jsonfile [file]', 'File to export json to')

program.on('--help', function() {
  console.log('You can get url with `meteor mongo --url [site domain]`.');
  console.log('When process of saving stops, you can write file with Ctrl-C.');
});

program.parse(process.argv);

var mongoose = require('mongoose');
mongoose.connect(program.mongourl);
var model = require('./mongoose-model.js');
var Type = model.Type;
var TypeArg = model.TypeArg;
var Verb = model.Verb;
var Arg = model.Arg;

var data = {}

var dataFromArg = function(arg) {
  var dataArg = {}
  if (arg.prep) {
    dataArg.prep = arg.prep;
  }
  if (arg.noun_case) {
    dataArg.noun_case = arg.noun_case;
  }
  if (arg.example) {
    dataArg.example = arg.example;
  }
  return dataArg;
};

Arg.find({}, function(err, args) {
  //console.log(args);
  for (var i = 0; i < args.length; ++i) {
    (function(arg) {
      Verb.findOne({_id: arg.verb_id}, function(err, verb) {
        Type.findOne({_id: verb.type_id}, function(err, type) {
          if (!data[type.name]) {
            data[type.name] = {};
          }
          if (!data[type.name][verb.inf]) {
            data[type.name][verb.inf] = [];
          }
          if (arg.type_arg_id) {
            TypeArg.findOne({_id: arg.type_arg_id}, function(err, typeArg) {
              var dataArg = dataFromArg(arg);
              dataArg["type_arg"] = typeArg.name;
              data[type.name][verb.inf].push(dataArg);
              console.log('Saved: ' + arg);
            });
          } else {
            //data[type.name][verb.inf].push(dataFromArg(arg));
            //console.log('Saved: ' + arg);
          }
        });
      });
    })(args[i]);
  }
});

process.stdin.resume();

process.on('SIGINT', function() {
  console.log('Writing file...');
  fs.writeFile(program.jsonfile, JSON.stringify(data, null, 4), function (err) {
    if (err) throw err;
    console.log('It\'s saved!');
    process.exit();
  });
});<commit_msg>Exit automatically on exporting<commit_after>#!/usr/bin/env node

/**
 * Module dependencies.
 */

var program = require('commander')
  , fs = require('fs')
  , mongoose = require('mongoose')

program
  .version('0.0.1')
  .option('-u, --mongourl <url>', 'URL to connect to mongo')
  .option('-j, --jsonfile [file]', 'File to export json to')

program.on('--help', function() {
  console.log('You can get url with `meteor mongo --url [site domain]`.');
  console.log('When process of saving stops, you can write file with Ctrl-C.');
});

program.parse(process.argv);

var mongoose = require('mongoose');
mongoose.connect(program.mongourl);
var model = require('./mongoose-model.js');
var Type = model.Type;
var TypeArg = model.TypeArg;
var Verb = model.Verb;
var Arg = model.Arg;

var data = {}

var dataFromArg = function(arg) {
  var dataArg = {}
  if (arg.prep) {
    dataArg.prep = arg.prep;
  }
  if (arg.noun_case) {
    dataArg.noun_case = arg.noun_case;
  }
  if (arg.example) {
    dataArg.example = arg.example;
  }
  return dataArg;
};

var writeFile = function() {
  console.log('Writing file...');
  fs.writeFile(program.jsonfile, JSON.stringify(data, null, 4), function (err) {
    if (err) throw err;
    console.log('It\'s saved!');
    process.exit();
  });
}

Arg.find({}, function(err, args) {
  //console.log(args);
  var unprocessed = args.length;
  for (var i = 0; i < args.length; ++i) {
    (function(arg) {
      Verb.findOne({_id: arg.verb_id}, function(err, verb) {
        Type.findOne({_id: verb.type_id}, function(err, type) {
          if (!data[type.name]) {
            data[type.name] = {};
          }
          if (!data[type.name][verb.inf]) {
            data[type.name][verb.inf] = [];
          }
          if (arg.type_arg_id) {
            TypeArg.findOne({_id: arg.type_arg_id}, function(err, typeArg) {
              var dataArg = dataFromArg(arg);
              dataArg["type_arg"] = typeArg.name;
              data[type.name][verb.inf].push(dataArg);
              console.log('Saved: ' + arg);
              unprocessed--;
              if (unprocessed == 0) {
                writeFile();
              }
            });
          } else {
            unprocessed--;
            if (unprocessed == 0) {
              writeFile();
            }
          }
        });
      });
    })(args[i]);
  }
});
{'fraction_non_alphanumeric': 0.179672131147541, 'fraction_numerical': 0.002622950819672131, 'mean_word_length': 2.3821138211382116, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5472566', 'n_tokens_mistral': 1584, 'n_tokens_neox': 1574, 'n_words': 325}