<commit_before>'use strict';

const crusca = require('crusca');
const Promise = require('bluebird');
const fs = require('graceful-fs');
const path = require('path');
const recursive = require('recursive-readdir');
const readFile = Promise.promisify(fs.readFile);
const writeFile = Promise.promisify(fs.writeFile);

// Default should go in a separate place
const DEFAULT_CONFIG = './.cruscarc';
const DEFAULT_EXTENSIONS = ['.js'];
const DEFAULT_IGNORE = [];
const DEFAULT_VERBOSE = false;

const getConfig = (opts) => {
  const configFile = opts.config;

  try {
    fs.lstatSync(configFile);
  } catch(err) {
    return {};
  }

  return JSON.parse(
    fs.readFileSync(configFile, 'utf8')
  );
};

const ensureLeadingDot = (extArray) => {
  return extArray.map((ext) => {
    return ext.charAt(0) === '.' ? ext : `.${ext}`
  })
}

const processFile = (data, filePath, pKeys) => {
  const keys = crusca.extract(data);
  return crusca.tagKeys(keys, filePath, pKeys);
};

export default (argv) => {
  const filePath = argv.in;
  const outFile = argv.out;

  // Establishing optional parameters
  const opts = {
    config: argv.config || DEFAULT_CONFIG,
    extensions: argv.e
      ? Array.isArray(argv.e) ? argv.e : [argv.e]
      : DEFAULT_EXTENSIONS,
    ignore: argv.g
      ? Array.isArray(argv.g) ? argv.g : [argv.g]
      : DEFAULT_IGNORE,
    verbose: argv.v || DEFAULT_VERBOSE
  };

  const isFile = fs.lstatSync(filePath).isFile();

  if (isFile) {

    readFile(filePath, 'utf8').then((data) => {
      const taggedKeys = processFile(data, filePath);
      const keysCount = Object.keys(taggedKeys).length;

      writeFile(outFile, crusca.generatePot(taggedKeys)).then(() => {
        console.log(`${keysCount} strings extracted from ${filePath}`);
        process.exit(0);
      }, (err) => {
        throw new Error(err);
      });
    });
  } else {

    const config = getConfig(opts);

    const ignoreArr = config.ignore || opts.ignore;
    const extensions = config.extensions || opts.extensions;
    const whitelistExts = ensureLeadingDot(extensions);
    const whitelistFunc = (file, stats) => {
      return stats.isFile() ? whitelistExts.indexOf(path.extname(file)) === -1 : false;
    };

    const ignoreArguments = ignoreArr.length > 0 ? ignoreArr.concat(whitelistFunc) : [whitelistFunc];
    recursive(filePath, ignoreArguments, (err, files) => {
      const fileContentArray = files.map((file) => readFile(file, 'utf8'));
      const filesCount = fileContentArray.length;

      Promise.all(fileContentArray).then((data) => {
        const taggedKeys = data.reduce((keys, data, idx) => {
          if (opts.verbose) console.log(files[idx]);
          return processFile(data, files[idx], keys)
        }, {});

        const keysCount = Object.keys(taggedKeys).length;
        const potFileContent = crusca.generatePot(taggedKeys);

        writeFile(outFile, potFileContent).then(() => {
          console.log(`${keysCount} strings extracted from ${filesCount} files.`);
          process.exit(0);
        }, (err) => {
          throw new Error(err);
        });
      });
    });
  }
}<commit_msg>crusca-cli: entirely async, returning a Promise<commit_after>'use strict';

const crusca = require('crusca');
const Promise = require('bluebird');
const fs = require('graceful-fs');
const path = require('path');
const recursive = Promise.promisify(require('recursive-readdir'));
const readFile = Promise.promisify(fs.readFile);
const writeFile = Promise.promisify(fs.writeFile);

// Default should go in a separate place
const DEFAULT_CONFIG = './.cruscarc';
const DEFAULT_EXTENSIONS = ['.js'];
const DEFAULT_IGNORE = [];
const DEFAULT_VERBOSE = false;

const getConfig = (opts) => {
  const configFile = opts.config;

  try {
    fs.lstatSync(configFile);
  } catch(err) {
    return {};
  }

  return JSON.parse(
    fs.readFileSync(configFile, 'utf8')
  );
};

const ensureLeadingDot = (extArray) => {
  return extArray.map((ext) => {
    return ext.charAt(0) === '.' ? ext : `.${ext}`
  })
}

const processFile = (data, filePath, pKeys) => {
  const keys = crusca.extract(data);
  return crusca.tagKeys(keys, filePath, pKeys);
};

export default (argv) => {
  const filePath = argv.in;
  const outFile = argv.out;

  // Establishing optional parameters
  const opts = {
    config: argv.config || DEFAULT_CONFIG,
    extensions: argv.e
      ? Array.isArray(argv.e) ? argv.e : [argv.e]
      : DEFAULT_EXTENSIONS,
    ignore: argv.g
      ? Array.isArray(argv.g) ? argv.g : [argv.g]
      : DEFAULT_IGNORE,
    verbose: argv.v || DEFAULT_VERBOSE
  };

  const isFile = fs.lstatSync(filePath).isFile();

  if (isFile) {

    return readFile(filePath, 'utf8').then((data) => {
      const taggedKeys = processFile(data, filePath);
      const keysCount = Object.keys(taggedKeys).length;

      return writeFile(outFile, crusca.generatePot(taggedKeys)).then(() => {
        console.log(`${keysCount} strings extracted from ${filePath}`);
        process.exit(0);
      }, (err) => {
        throw new Error(err);
      });
    });
  } else {

    const config = getConfig(opts);

    const ignoreArr = config.ignore || opts.ignore;
    const extensions = config.extensions || opts.extensions;
    const whitelistExts = ensureLeadingDot(extensions);
    const whitelistFunc = (file, stats) => {
      return stats.isFile() ? whitelistExts.indexOf(path.extname(file)) === -1 : false;
    };

    const ignoreArguments = ignoreArr.length > 0 ? ignoreArr.concat(whitelistFunc) : [whitelistFunc];
    return recursive(filePath, ignoreArguments).then((files) => {
      const fileContentArray = files.map((file) => readFile(file, 'utf8'));
      const filesCount = fileContentArray.length;

      return Promise.all(fileContentArray).then((data) => {
        const taggedKeys = data.reduce((keys, data, idx) => {
          if (opts.verbose) console.log(files[idx]);
          return processFile(data, files[idx], keys)
        }, {});

        const keysCount = Object.keys(taggedKeys).length;
        const potFileContent = crusca.generatePot(taggedKeys);

        return writeFile(outFile, potFileContent).then(() => {
          console.log(`${keysCount} strings extracted from ${filesCount} files.`);
          process.exit(0);
        }, (err) => {
          throw new Error(err);
        });
      });
    });
  }
}
{'fraction_non_alphanumeric': 0.152160201829076, 'fraction_numerical': 0.002522863450015768, 'mean_word_length': 3.257046979865772, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '64745', 'n_tokens_mistral': 2040, 'n_tokens_neox': 1971, 'n_words': 424}