<commit_before>// Native
const {existsSync, statSync} = require('fs');

// External
const loadJsonFile = require('load-json-file');
const pify = require('pify');
const pump = require('pump');
const rsyncwrapper = require('rsyncwrapper');
const SSH = require('ssh2');
const through = require('through2');
const vfs = require('vinyl-fs');
const VTFP = require('vinyl-ftp');

// Ours
const {command} = require('../../cli');
const {packs, throws, unpack} = require('../../utils/async');
const {dry, info, spin, warn} = require('../../utils/logging');
const {DEFAULTS, OPTIONS, MESSAGES, REQUIRED_PROPERTIES, VALID_TYPES} = require('./constants');

// Wrapped
const getJSON = packs(loadJsonFile);

const ftp = packs(target => {
  const vftp = new VTFP({
    host: target.host,
    port: target.port || 21,
    user: target.username,
    password: <PASSWORD>,
    parallel: 10
  });

  return pify(pump)(
    vfs.src(target.files, {
      buffer: false,
      cwd: target.from
    }),
    vftp.dest(target.to),
    through.obj()
  );
});

const rsync = packs(target => {
  return pify(rsyncwrapper)(Object.assign(
    {},
    DEFAULTS.RSYNC,
    {
      src: `${target.from}/${Array.isArray(target.files) ?
        target.files[0] : target.files}`,
      dest: `${target.username}@${target.host}:${target.to}`
    }
  ));
});

const symlink = packs(async target => {
  const symlinkName = typeof target.symlink === 'string' ?
    target.symlink : DEFAULTS.SYMLINK.NAME;
  const symlinkPath = target.to.replace(target.id, symlinkName);

  if (symlinkPath === target.to) {
    warn(MESSAGES.NO_MAPPABLE_ID);
    return;
  }

  const ssh = new SSH();

  ssh.connect(target);

  await pify(ssh.on)('ready');

  const stream = await pify(ssh.exec)(
    `rm -rf ${symlinkPath} && ln -s ${target.to} ${symlinkPath}`
  );

  await pify(stream.on)('exit');

  ssh.end();
});

const deployToServer = packs(async target => {
  info(MESSAGES.deployment(target.type, target.from, target.to, target.host));

  const spinner = spin('Deploy');

  try {
    if (target.type === 'ftp') {
      await ftp(target);
    } else if (target.type === 'ssh') {
      await rsync(target);

      if (target.symlink) {
        await symlink(target);
      }
    }
  } catch (err) {
    spinner.fail();

    throw err;
  }

  spinner.succeed();

  if (target.publicURL) {
    info(MESSAGES.publicURL(target.publicURL));
  }
});

module.exports.deploy = command({
  name: 'deploy',
  options: OPTIONS,
  usage: MESSAGES.usage,
  isConfigRequired: true
}, async (argv, config) => {
  // 1) Get known/specified deployment target(s)

  let keys = Object.keys(config.deploy);

  if (argv.target) {
    keys = keys.filter(key => key === argv.target);

    if (keys.length < 1) {
      throw MESSAGES.targetDoesNotExist(argv.target);
    }
  }

  if (keys.length < 1) {
    throw MESSAGES.NO_TARGETS;
  }

  // 2) Create an array of config objects for each target we know about

  const credentials = unpack(await getJSON(argv.credentials));

  const targets = keys.map(key => Object.assign(
    {
      __key__: key,
      id: config.id,
      name: config.name,
      files: '**'
    },
    credentials[key],
    config.deploy[key],
    (argv.shouldRespectTargetSymlinks ? {} : {symlink: null})
  ));

  // 3) Validate & normalise those configs (in parallel)

  await Promise.all(targets.map(async target => {
    // 3.1) Check 'type' is valid
    if (!VALID_TYPES.has(target.type)) {
      throw MESSAGES.unrecognisedType(target.__key__, target.type);
    }

    // 3.2) Check all properties are present
    REQUIRED_PROPERTIES.forEach(prop => {
      if (target[prop] == null) {
        throw MESSAGES.targetNotConfigured(target.__key__, prop);
      }
    });

    // 3.3) Check 'from' directory exists
    if (!existsSync(target.from) || !statSync(target.from).isDirectory()) {
      throw (MESSAGES.sourceIsNotDirectory(target.from));
    }
  }));

  if (argv.dry) {
    return dry(targets.reduce((memo, target) => {
      memo[`Deployment to ${target.__key__}`] = target;

      return memo;
    }, {}));
  }

  for (const target of targets) {
    throws(await deployToServer(target));
  }
});<commit_msg>Remove temporary deployment target properties<commit_after>// Native
const {existsSync, statSync} = require('fs');

// External
const loadJsonFile = require('load-json-file');
const pify = require('pify');
const pump = require('pump');
const rsyncwrapper = require('rsyncwrapper');
const SSH = require('ssh2');
const through = require('through2');
const vfs = require('vinyl-fs');
const VTFP = require('vinyl-ftp');

// Ours
const {command} = require('../../cli');
const {packs, throws, unpack} = require('../../utils/async');
const {dry, info, spin, warn} = require('../../utils/logging');
const {DEFAULTS, OPTIONS, MESSAGES, REQUIRED_PROPERTIES, VALID_TYPES} = require('./constants');

// Wrapped
const getJSON = packs(loadJsonFile);

const ftp = packs(target => {
  const vftp = new VTFP({
    host: target.host,
    port: target.port || 21,
    user: target.username,
    password: <PASSWORD>,
    parallel: 10
  });

  return pify(pump)(
    vfs.src(target.files, {
      buffer: false,
      cwd: target.from
    }),
    vftp.dest(target.to),
    through.obj()
  );
});

const rsync = packs(target => {
  return pify(rsyncwrapper)(Object.assign(
    {},
    DEFAULTS.RSYNC,
    {
      src: `${target.from}/${Array.isArray(target.files) ?
        target.files[0] : target.files}`,
      dest: `${target.username}@${target.host}:${target.to}`
    }
  ));
});

const symlink = packs(async target => {
  const symlinkName = typeof target.symlink === 'string' ?
    target.symlink : DEFAULTS.SYMLINK.NAME;
  const symlinkPath = target.to.replace(target.id, symlinkName);

  if (symlinkPath === target.to) {
    warn(MESSAGES.NO_MAPPABLE_ID);
    return;
  }

  const ssh = new SSH();

  ssh.connect(target);

  await pify(ssh.on)('ready');

  const stream = await pify(ssh.exec)(
    `rm -rf ${symlinkPath} && ln -s ${target.to} ${symlinkPath}`
  );

  await pify(stream.on)('exit');

  ssh.end();
});

const deployToServer = packs(async target => {
  info(MESSAGES.deployment(target.type, target.from, target.to, target.host));

  const spinner = spin('Deploy');

  try {
    if (target.type === 'ftp') {
      await ftp(target);
    } else if (target.type === 'ssh') {
      await rsync(target);

      if (target.symlink) {
        await symlink(target);
      }
    }
  } catch (err) {
    spinner.fail();

    throw err;
  }

  spinner.succeed();

  if (target.publicURL) {
    info(MESSAGES.publicURL(target.publicURL));
  }
});

module.exports.deploy = command({
  name: 'deploy',
  options: OPTIONS,
  usage: MESSAGES.usage,
  isConfigRequired: true
}, async (argv, config) => {
  // 1) Get known/specified deployment target(s)

  let keys = Object.keys(config.deploy);

  if (argv.target) {
    keys = keys.filter(key => key === argv.target);

    if (keys.length < 1) {
      throw MESSAGES.targetDoesNotExist(argv.target);
    }
  }

  if (keys.length < 1) {
    throw MESSAGES.NO_TARGETS;
  }

  // 2) Create an array of config objects for each target we know about

  const credentials = unpack(await getJSON(argv.credentials));

  const targets = keys.map(key => Object.assign(
    {
      __key__: key,
      id: config.id,
      name: config.name,
      files: '**'
    },
    credentials[key],
    config.deploy[key],
    (argv.shouldRespectTargetSymlinks ? {} : {symlink: null})
  ));

  // 3) Validate & normalise those configs (in parallel)

  await Promise.all(targets.map(async target => {
    // 3.1) Check 'type' is valid
    if (!VALID_TYPES.has(target.type)) {
      throw MESSAGES.unrecognisedType(target.__key__, target.type);
    }

    // 3.2) Check all properties are present
    REQUIRED_PROPERTIES.forEach(prop => {
      if (target[prop] == null) {
        throw MESSAGES.targetNotConfigured(target.__key__, prop);
      }
    });

    // 3.3) Check 'from' directory exists
    if (!existsSync(target.from) || !statSync(target.from).isDirectory()) {
      throw (MESSAGES.sourceIsNotDirectory(target.from));
    }

    // 3.4) Remove temporary properties
    delete target._from;
    delete target._to;
  }));

  if (argv.dry) {
    return dry(targets.reduce((memo, target) => {
      memo[`Deployment to ${target.__key__}`] = target;

      return memo;
    }, {}));
  }

  for (const target of targets) {
    throws(await deployToServer(target));
  }
});
{'fraction_non_alphanumeric': 0.16613494032848872, 'fraction_numerical': 0.004490133522391587, 'mean_word_length': 3.1167315175097277, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2825080', 'n_tokens_mistral': 2971, 'n_tokens_neox': 2854, 'n_words': 620}