<commit_before>var Q = require('q');
var fs = require('fs');
var http = require('http');
var path = require('path');
var charSpinner = require('char-spinner');
var querystring = require('querystring');
var MIN = 19968, MAX = 40907, CONCURRENT = 5;
var Can2Latin = path.join(__dirname, 'dict', 'can2latin.js');

charSpinner();

var rangePromises = makeRanges(MIN, MAX, CONCURRENT).map(function(range) {
  var chars = charsRange(range.min, range.max);
  return request(chars);
});

Q.
all(rangePromises).
then(function(allContent) {
  var obj = {};
  for (var i = 0; i < allContent.length; i++) {
    parse(allContent[i], obj);
  }
  var chars = {};
  try { chars = require(Can2Latin); } catch(e) {}
  for (var ch in chars) {
    if (!obj[ch]) obj[ch] = chars[ch];
  }
  var str = stringify(obj);
  fs.writeFileSync(Can2Latin, str);
}).
catch(console.error);

function request(content) {
  var deferred = Q.defer();
  var query = querystring.stringify({
    mode: 'cantonese',
    q: content
  });
  var req = http.request({
    host: 'www.kawa.net',
    path: '/works/ajax/romanize/romanize.cgi',
    method: 'POST',
    headers: {
      'Content-Length': Buffer.byteLength(query)
    }
  }, function(res) {
    var body = '';
    res.on('data', function(data) {
      body += data;
    });<commit_msg>update make.js so that it can use options;<commit_after>#!/usr/bin/env node
var Q           = require('q');
var fs          = require('fs');
var http        = require('http');
var path        = require('path');
var charSpinner = require('char-spinner');
var querystring = require('querystring');

var MIN = 19968, MAX = 40907, CONCURRENT = 5;
var Can2Latin = path.join(__dirname, 'dict', 'can2latin.js');

var opts = process.argv.slice(2);
if (opts.length === 0) return getHelp();
var action = opts[0];

if (action === 'get-chars') {
  getChars();
} else {
  getHelp();
  process.exit(1);
}

function getHelp() {
  var l = console.log;
  l('Usage: ' + path.basename(process.argv[1]) + ' [ACTION]');
  l();
  l('Actions:');
  l('  get-chars        get cantonese to latins data from kawa.net');
}

function getChars() {
  charSpinner();
  var rangePromises = makeRanges(MIN, MAX, CONCURRENT).map(function(range) {
    var chars = charsRange(range.min, range.max);
    return request(chars);
  });
  Q.
  all(rangePromises).
  then(function(allContent) {
    var obj = {};
    for (var i = 0; i < allContent.length; i++) {
      parse(allContent[i], obj);
    }
    var chars = {};
    try { chars = require(Can2Latin); } catch(e) {}
    for (var ch in chars) {
      if (!obj[ch]) obj[ch] = chars[ch];
    }
    var str = stringify(obj);
    fs.writeFileSync(Can2Latin, str);
  }).
  catch(console.error);
}

function request(content) {
  var deferred = Q.defer();
  var query = querystring.stringify({
    mode: 'cantonese',
    q: content
  });
  var req = http.request({
    host: 'www.kawa.net',
    path: '/works/ajax/romanize/romanize.cgi',
    method: 'POST',
    headers: {
      'Content-Length': Buffer.byteLength(query)
    }
  }, function(res) {
    var body = '';
    res.on('data', function(data) {
      body += data;
    });
{'fraction_non_alphanumeric': 0.17871933736858872, 'fraction_numerical': 0.011787193373685887, 'mean_word_length': 3.307270233196159, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5113425', 'n_tokens_mistral': 1146, 'n_tokens_neox': 1092, 'n_words': 264}