<commit_before>'use strict';

import path from 'path';
import fs from 'fs-extra';
import glob from 'glob';
import rl from 'readline';
import 'babel-polyfill';
const async = require('async');
const yaml = require('js-yaml');
const temp = require('temp').track();
const pathExists = require('path-exists');
const camelCase = require('camelcase');

const NEWLINE = '\n';
const ENCODING = 'utf8';
const INSERT_AT_BEGIN = 'begin';
const INSERT_AT_END = 'end';

export default class I18nPatch {
  constructor(src, options) {
    if (!src) {
      throw new Error('src is required');
    }
    this.src = src;
    this.options = options || {};
    this.options.dest = this.options.dest || this.src;
    this.options.config = this.options.config || 'config';
  }

  generate(config, localeConfig) {
    return new Promise((resolve, reject) => {
      this.setConfigs(config, localeConfig);
      this.buildPatterns();
      if (this.hasDest()) {
        fs.copySync(this.src, this.options.dest);
      }

      async.waterfall(this.config.translations.map((t) => {
        return (cb) => {
          this.processTranslation(t)
          .catch((err) => cb(err))
          .then(() => cb());
        };
      }), (err) => {
        if (err) {
          reject(err);
        } else {
          resolve();
        }
      });
    });
  }

  hasDest() {
    return this.src !== this.options.dest;
  }

  setConfigs(config, localeConfig) {
    this.config = config || this.readConfigFile('i18n');
    if (localeConfig) {
      this.localeConfig = localeConfig;
    } else {
      if (!this.options.locale) {
        throw new Error('Could not determine locale');
      }
      this.localeConfig = this.readConfigFile(this.options.locale);
    }

    for (let t of this.config.translations) {
      Object.keys(t).forEach((key) => {
        let converted = camelCase(key);
        if (key !== converted) {
          t[camelCase(key)] = t.key;
          delete t[key];
        }
      });
    }
  }

  readConfigFile(name) {
    let configPath = path.join(this.options.config, `${name}.yml`);
    if (pathExists.sync(configPath)) {
      try {
        return yaml.load(fs.readFileSync(configPath, ENCODING));
      } catch (err) {
        console.log(`Cannot read ${configPath}`);
        console.log(err.stack);
        process.exit(1);
      }
    } else {
      configPath = path.join(this.options.config, `${name}.json`);
      try {
        return JSON.parse(fs.readFileSync(configPath));
      } catch (err) {
        console.log(`Cannot read ${configPath}`);
        console.log(err.stack);
        process.exit(1);
      }
    }
  }

  buildPatterns() {
    for (let t of this.config.translations) {
      if (t.conditionals) {
        for (let c of t.conditionals) {
          if (!c.insert || !this.hasTranslationKey(c.insert.value)) {
            continue;
          }
          c.insert.resolved = this.localeConfig[c.insert.value];
          if (c.insert.resolved && !c.insert.resolved.endsWith(NEWLINE)) {
            c.insert.resolved += NEWLINE;
          }
        }
      }
      for (let p of t.patterns) {
        this.resolve(p);
        if (p.insert && this.hasTranslationKey(p.insert.value)) {
          p.insert.resolved = this.localeConfig[p.insert.value];
          if (p.insert.resolved && !p.insert.resolved.endsWith(NEWLINE)) {
            p.insert.resolved += NEWLINE;
          }
        }
      }
    }
  }

  processTranslation(t) {
    return new Promise((resolve, reject) => {
      let srcGlob = t.src || '**/*';
      let srcPaths = path.join(this.options.dest, srcGlob);
      glob(srcPaths, null, (err, files) => {
        if (err) {
          reject(err);
          return;
        }
        async.eachLimit(files, 100, (file, cb) => {
          this.processFile(t, file)
          .catch((err) => cb(err))
          .then(() => cb());<commit_msg>Add named-pattern to reuse pattern definition<commit_after>'use strict';

import path from 'path';
import fs from 'fs-extra';
import glob from 'glob';
import rl from 'readline';
import 'babel-polyfill';
const async = require('async');
const yaml = require('js-yaml');
const temp = require('temp').track();
const pathExists = require('path-exists');
const camelCase = require('camelcase');

const NEWLINE = '\n';
const ENCODING = 'utf8';
const INSERT_AT_BEGIN = 'begin';
const INSERT_AT_END = 'end';

export default class I18nPatch {
  constructor(src, options) {
    if (!src) {
      throw new Error('src is required');
    }
    this.src = src;
    this.options = options || {};
    this.options.dest = this.options.dest || this.src;
    this.options.config = this.options.config || 'config';
  }

  generate(config, localeConfig) {
    return new Promise((resolve, reject) => {
      this.setConfigs(config, localeConfig);
      this.buildPatterns();
      if (this.hasDest()) {
        fs.copySync(this.src, this.options.dest);
      }

      async.waterfall(this.config.translations.map((t) => {
        return (cb) => {
          this.processTranslation(t)
          .catch((err) => cb(err))
          .then(() => cb());
        };
      }), (err) => {
        if (err) {
          reject(err);
        } else {
          resolve();
        }
      });
    });
  }

  hasDest() {
    return this.src !== this.options.dest;
  }

  setConfigs(config, localeConfig) {
    this.config = config || this.readConfigFile('i18n');
    if (localeConfig) {
      this.localeConfig = localeConfig;
    } else {
      if (!this.options.locale) {
        throw new Error('Could not determine locale');
      }
      this.localeConfig = this.readConfigFile(this.options.locale);
    }

    for (let t of this.config.translations) {
      Object.keys(t).forEach((key) => {
        let converted = camelCase(key);
        if (key !== converted) {
          t[camelCase(key)] = t[key];
          delete t[key];
        }
      });
    }
  }

  readConfigFile(name) {
    let configPath = path.join(this.options.config, `${name}.yml`);
    if (pathExists.sync(configPath)) {
      try {
        return yaml.load(fs.readFileSync(configPath, ENCODING));
      } catch (err) {
        console.log(`Cannot read ${configPath}`);
        console.log(err.stack);
        process.exit(1);
      }
    } else {
      configPath = path.join(this.options.config, `${name}.json`);
      try {
        return JSON.parse(fs.readFileSync(configPath));
      } catch (err) {
        console.log(`Cannot read ${configPath}`);
        console.log(err.stack);
        process.exit(1);
      }
    }
  }

  buildPatterns() {
    for (let t of this.config.translations) {
      if (t.conditionals) {
        for (let c of t.conditionals) {
          if (!c.insert || !this.hasTranslationKey(c.insert.value)) {
            continue;
          }
          c.insert.resolved = this.localeConfig[c.insert.value];
          if (c.insert.resolved && !c.insert.resolved.endsWith(NEWLINE)) {
            c.insert.resolved += NEWLINE;
          }
        }
      }
      for (let p of t.patterns) {
        if (p.name) {
          if (t.namedPatterns) {
            let namedPattern;
            for (let np of t.namedPatterns) {
              if (p.name === np.name) {
                namedPattern = np;
                break;
              }
            }
            if (namedPattern) {
              let npPattern = namedPattern.pattern;
              let npReplace = namedPattern.replace;
              for (let npp of namedPattern.params) {
                if (p.params.hasOwnProperty(npp)) {
                  npPattern = npPattern.replace(`{${npp}}`, p.params[npp]);
                  npReplace = npReplace.replace(`{${npp}}`, p.params[npp]);
                }
              }
              p.pattern = new RegExp(npPattern);
              p.replace = npReplace;
            }
          }
        }

        this.resolve(p);
        if (p.insert && this.hasTranslationKey(p.insert.value)) {
          p.insert.resolved = this.localeConfig[p.insert.value];
          if (p.insert.resolved && !p.insert.resolved.endsWith(NEWLINE)) {
            p.insert.resolved += NEWLINE;
          }
        }
      }
    }
  }

  processTranslation(t) {
    return new Promise((resolve, reject) => {
      let srcGlob = t.src || '**/*';
      let srcPaths = path.join(this.options.dest, srcGlob);
      glob(srcPaths, null, (err, files) => {
        if (err) {
          reject(err);
          return;
        }
        async.eachLimit(files, 100, (file, cb) => {
          this.processFile(t, file)
          .catch((err) => cb(err))
          .then(() => cb());
{'fraction_non_alphanumeric': 0.15338923829489867, 'fraction_numerical': 0.0023293733985557887, 'mean_word_length': 2.0342756183745583, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1399417', 'n_tokens_mistral': 2692, 'n_tokens_neox': 2644, 'n_words': 525}