<commit_before>const {promisify} = require('util');
const fs = require('fs');
const path = require('path');
const symlink = promisify(fs.symlink);
const readFile = promisify(fs.readFile);
const writeFile = promisify(fs.writeFile);
const mkdirp = promisify(require('mkdirp'));
const chokidar = require('chokidar');
const chalk = require('chalk');
const log = require('../utils/log');
const globby = require('globby');
const timer = require('../utils/timer');
const ora = require('ora');
const sharp = require('sharp');
const config = require('../utils/config-store').getConfig();
const isProd = process.env.NODE_ENV === 'production';

// @todo Consider moving this to a place to share
const boltImageSizes = [
  50,
  100,
  200,
  320,
  480,
  640,
  800,
  1024,
  1366,
  1536,
  1920,
  2560,
  2880,
];

async function processImage(file, set) {
  if (config.verbosity > 2) {
    log.dim(`Processing image: ${file}`);
  }
  // If `set.base` is `images/` and `file` is `images/header/main.png`, then `fileId` is `header/main.png`
  const fileId = path.relative(set.base, file);
  const newPath = path.join(set.dist, fileId);
  const pathInfo = path.parse(newPath);
  // pathInfo.ext and pathInfo.name are ignored if pathInfo.base exists
  delete pathInfo.base;

  await mkdirp(pathInfo.dir);

  // We add `null` to beginning b/c we want the original file too
  const sizes = [null, ...boltImageSizes];

  let originalFileBuffer;
  if (isProd) {
    // we want to read the original file once, instead of reading for each size
    originalFileBuffer = await readFile(file);
  }

  return Promise.all(sizes.map(async (size) => {
    const isOrig = size === null;
    const sizeSuffix = size ? `-${size}` : '';
    const thisPathInfo = Object.assign({}, pathInfo, {
      name: `${pathInfo.name}${sizeSuffix}`,
    });
    const newSizedPath = path.format(thisPathInfo);

    if (isOrig) {// original file

    } else {// resized file
      // no need to resize these file extensions
      if (pathInfo.ext === '.svg' || pathInfo.ext === '.gif') {
        return;
      }
    }

    if (isProd) {
      if (isOrig) {
        await writeFile(newSizedPath, originalFileBuffer);
      } else {
        // http://sharp.pixelplumbing.com/en/stable/
        await sharp(originalFileBuffer)
          .resize(size)
          .toFile(newSizedPath);
      }
    } else {
      // Not prod, so let's be quick.
      // Symlinking works even if the original file is not served
      const symlinkPath = path.relative(thisPathInfo.dir, file);
      try {
        await symlink(symlinkPath, newSizedPath);
      } catch (error) {
        // If it's the error for symlink already exists, we don't care.
        if (error.code !== 'EEXIST') {
          log.errorAndExit(`Problem when attempting to symlink ${file} to ${newSizedPath}.`, error);
        }
      }
    }
  }));
}

async function processImages() {
  if (!config.images) {
    return;
  }
  const startMessage = chalk.blue('Processing images...');
  const startTime = timer.start();
  let spinner;
  if (config.verbosity > 2) {
    console.log(startMessage);
  } else {
    spinner = ora(startMessage).start();
  }

  return Promise.all(config.images.sets.map(async (set) => {
    const imagePaths = await globby(path.join(set.base, set.glob));
    return Promise.all(imagePaths.map(imagePath => processImage(imagePath, set)));
  })).then(() => {// When it's all done
    const endMessage = chalk.green(`Processed images in ${timer.end(startTime)}`);
    if (config.verbosity > 2) {
      console.log(endMessage);
    } else {
      spinner.succeed(endMessage);
    }
  });
}

module.exports = {
  processImages,
};
<commit_msg>Using config.prod flag in Image Tasks<commit_after>const {promisify} = require('util');
const fs = require('fs');
const path = require('path');
const symlink = promisify(fs.symlink);
const readFile = promisify(fs.readFile);
const writeFile = promisify(fs.writeFile);
const mkdirp = promisify(require('mkdirp'));
const chokidar = require('chokidar');
const chalk = require('chalk');
const log = require('../utils/log');
const globby = require('globby');
const timer = require('../utils/timer');
const ora = require('ora');
const sharp = require('sharp');
const config = require('../utils/config-store').getConfig();

// @todo Consider moving this to a place to share
const boltImageSizes = [
  50,
  100,
  200,
  320,
  480,
  640,
  800,
  1024,
  1366,
  1536,
  1920,
  2560,
  2880,
];

async function processImage(file, set) {
  if (config.verbosity > 2) {
    log.dim(`Processing image: ${file}`);
  }
  // If `set.base` is `images/` and `file` is `images/header/main.png`, then `fileId` is `header/main.png`
  const fileId = path.relative(set.base, file);
  const newPath = path.join(set.dist, fileId);
  const pathInfo = path.parse(newPath);
  // pathInfo.ext and pathInfo.name are ignored if pathInfo.base exists
  delete pathInfo.base;

  await mkdirp(pathInfo.dir);

  // We add `null` to beginning b/c we want the original file too
  const sizes = [null, ...boltImageSizes];

  let originalFileBuffer;
  if (config.prod) {
    // we want to read the original file once, instead of reading for each size
    originalFileBuffer = await readFile(file);
  }

  return Promise.all(sizes.map(async (size) => {
    const isOrig = size === null;
    const sizeSuffix = size ? `-${size}` : '';
    const thisPathInfo = Object.assign({}, pathInfo, {
      name: `${pathInfo.name}${sizeSuffix}`,
    });
    const newSizedPath = path.format(thisPathInfo);

    if (isOrig) {// original file

    } else {// resized file
      // no need to resize these file extensions
      if (pathInfo.ext === '.svg' || pathInfo.ext === '.gif') {
        return;
      }
    }

    if (config.prod) {
      if (isOrig) {
        await writeFile(newSizedPath, originalFileBuffer);
      } else {
        // http://sharp.pixelplumbing.com/en/stable/
        await sharp(originalFileBuffer)
          .resize(size)
          .toFile(newSizedPath);
      }
    } else {
      // Not prod, so let's be quick.
      // Symlinking works even if the original file is not served
      const symlinkPath = path.relative(thisPathInfo.dir, file);
      try {
        await symlink(symlinkPath, newSizedPath);
      } catch (error) {
        // If it's the error for symlink already exists, we don't care.
        if (error.code !== 'EEXIST') {
          log.errorAndExit(`Problem when attempting to symlink ${file} to ${newSizedPath}.`, error);
        }
      }
    }
  }));
}

async function processImages() {
  if (!config.images) {
    return;
  }
  const startMessage = chalk.blue('Processing images...');
  const startTime = timer.start();
  let spinner;
  if (config.verbosity > 2) {
    console.log(startMessage);
  } else {
    spinner = ora(startMessage).start();
  }

  return Promise.all(config.images.sets.map(async (set) => {
    const imagePaths = await globby(path.join(set.base, set.glob));
    return Promise.all(imagePaths.map(imagePath => processImage(imagePath, set)));
  })).then(() => {// When it's all done
    const endMessage = chalk.green(`Processed images in ${timer.end(startTime)}`);
    if (config.verbosity > 2) {
      console.log(endMessage);
    } else {
      spinner.succeed(endMessage);
    }
  });
}

module.exports = {
  processImages,
};

{'fraction_non_alphanumeric': 0.14819868995633187, 'fraction_numerical': 0.012827510917030568, 'mean_word_length': 3.033571821684095, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '920567', 'n_tokens_mistral': 2454, 'n_tokens_neox': 2379, 'n_words': 623}