<commit_before>// Imports the Google Cloud client library
const Vision = require('@google-cloud/vision');
const fs = require('fs');
const lineReader = require('line-reader');
var async = require("async");

// Instantiates a client
const credentials = [
  {
    projectId: 'react-google-maps',
    keyFilename: 'react-google-maps-476def2de5c2.json',
  },
  {
    projectId: 'nimble-answer-165506',
    keyFilename: 'Tutorial Project-e98b0d21b706.json',
  },
];

const vision = Vision(
  credentials[0]
  // credentials[1]
);

const options = {
  imageContext:
  {
    languageHints: [
      'zh-TW'
    ]
  }
};

// open OCR list file
const OCRList = 'units.list';
// Put images in this directory
const inputDir = 'input_images/';
const outputDir = 'output/';

function google_vision_api(imgFullname, callback) {
  var regex = /([^\/]+)$/;
  var match = imgFullname.match(regex);
  var filename = match[0];
  const inputFilename = inputDir + filename;
  const outputFilename = outputDir + filename + '.json';
  console.log('Detecting text on: ' + inputFilename);

  // Performs text detection on the local file
  vision.detectText(inputDir + imgFullname, options)
    .then((results) => {
      const detections = results[0];
      //detections.forEach((text) => console.log(text));
      //console.log(JSON.stringify(results));
      fs.writeFile(outputFilename, JSON.stringify(results), function (err) {
        if (err) {
          return console.log(err);
        }
        console.log('Write result to: ' + outputFilename);
        callback();
      });
    })
    .catch((err) => {
      console.error('Vision API ERROR:', err);
    });
}
var queue = async.queue(google_vision_api, 10);

lineReader.eachLine(OCRList, 'utf8', function (imgFullname, last) {
  queue.push(imgFullname);
});<commit_msg>Add async waterfall, check output file existence.<commit_after>// Imports the Google Cloud client library
const Vision = require('@google-cloud/vision');
const fs = require('fs');
const lineReader = require('line-reader');
var async = require("async");

// Instantiates a client
const credentials = [
  {
    projectId: 'react-google-maps',
    keyFilename: 'react-google-maps-476def2de5c2.json',
  },
  {
    projectId: 'nimble-answer-165506',
    keyFilename: 'Tutorial Project-e98b0d21b706.json',
  },
];

const vision = Vision(
  credentials[0]
  // credentials[1]
);

const options = {
  imageContext:
  {
    languageHints: [
      'zh-TW'
    ]
  }
};

// open OCR list file
const OCRList = 'units.list';
// Put images in this directory
const inputDir = 'input_images/';
const outputDir = 'output/';

function google_vision_api(imgFullname, vision_callback) {
  async.waterfall([
    // Check file existence
    (callback) => {
      var regex = /([^\/]+)$/;
      var match = imgFullname.match(regex);
      var filename = match[0];
      const inputFilename = inputDir + filename;
      const outputFilename = outputDir + filename + '.json';

      if (fs.existsSync(outputFilename)) {
        const err = '!!! Abort, file already exist ' + outputFilename;
        callback(err);
      }
      else {
        callback(null, inputFilename, outputFilename);
      }
    },
    // Do text detection
    function (inputFilename, outputFilename, callback) {
      console.log('<-- Detecting text on: ' + inputFilename);

      vision.detectText(inputFilename, options)
        .then((results) => {
          const detections = results[0];
          //detections.forEach((text) => console.log(text));
          //console.log(JSON.stringify(results));
          callback(null, results, outputFilename);
        })
        .catch((err) => {
          console.error('!!! ' + '\nVision API ERROR:', err);
          callback(err);
        })
    },

    // Write detection result to file
    (results, outputFilename, callback) => {
      console.log('--> Write result to: ' + outputFilename);

      fs.writeFile(outputFilename, JSON.stringify(results), function (err) {
        if (err) {
          callback(err);
        }
        else {
          callback(null, 'done');
        }
      });
    }
  ],

    (err, status) => {
      // if the above try/catche catches something, we will end up here
      // otherwise we will receive 'done' as the value of status
      // after the third function has finished
      if (err) {
        console.log(err);
      }
      else {
      console.log('status: ' + status);
      }
      vision_callback();
    }
  );
}

var queue = async.queue(google_vision_api, 10);

lineReader.eachLine(OCRList, 'utf8', function (imgFullname, last) {
  queue.push(imgFullname);
});
{'fraction_non_alphanumeric': 0.14480815087795362, 'fraction_numerical': 0.011706048124864513, 'mean_word_length': 2.810074318744839, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1703939', 'n_tokens_mistral': 1440, 'n_tokens_neox': 1401, 'n_words': 336}