<commit_before>import { get, set, template, last, isEmpty, dropRight } from 'lodash';
import { basename, join } from 'path';
const path = require('path');
const archiver = require('archiver');
const shortid = require('shortid');
const fs = require('fs-extra');
const Promise = require('bluebird');
const cpy = require('cpy');
const copy = Promise.promisify(fs.copy);
const move = Promise.promisify(fs.move);
const readFile = Promise.promisify(fs.readFile);
const writeFile = Promise.promisify(fs.writeFile);
const appendFile = Promise.promisify(fs.appendFile);
const remove = Promise.promisify(fs.remove);
const readJson = Promise.promisify(fs.readJson);
const writeJson = Promise.promisify(fs.writeJson);
const stat = Promise.promisify(fs.stat);
const mkdirs = Promise.promisify(fs.mkdirs);
const traverse = require('traverse');
const azure = require('azure-storage');
const stream = require('stream');

const npmDependencies = require('./npmDependencies.json');

export { cpy };
export { copy };
export { move };
export { remove };
export { mkdirs };
export { readFile };
export { writeFile };
export { appendFile };
export { readJson };
export { writeJson };

/**
 * @private
 * @param subStr {string} - what to indent
 * @param options {object} - how many levels (2 spaces per level) or how many spaces to indent
 * @returns {string}
 */
function indentCode(subStr, options) {
  const defaultIndentation = 2;
  let indent;

  if (options.indentLevel) {
    indent = ' '.repeat(options.indentLevel * defaultIndentation);
  } else if (options.indentSpaces) {
    indent = ' '.repeat(options.indentSpaces);
  }
  let array = subStr.toString().split('\n').filter(Boolean);
  array.forEach((line, index) => {
    array[index] = indent + line;
  });
  return array.join('\n');
}

/**
 * Traverse files and remove placeholder comments
 * @param params
 */
export function walkAndRemoveComments(params) {
  const build = path.join(__base, 'build', params.uuid);

  return new Promise((resolve, reject) => {
    fs.walk(build)
      .on('data', (item) => {
        return stat(item.path).then((stats) => {
          if (stats.isFile()) {
            return removeCode(item.path, '//=');
          }
        });
      })
      .on('error', (err) => {
        reject(err);
      })
      .on('end', () => {
        resolve();
      });
  });
}

export function walkAndRemoveCommentsMemory(params) {
  traverse(params.build).forEach(function() {
    if (Buffer.isBuffer(this.node)) {
      const buf = removeCodeMemory(this.node, '//=');
      this.update(this.node, true);
      set(params, ['build'].concat(this.path), buf);
    }
  });
}


export async function exists(filepath) {
  try {
    await stat(filepath);
  } catch (err) {
    if (err.code === 'ENOENT') {
      return false;
    }
  }
  return true;
}

function uploadAndReturnDownloadLink(archive) {
  const blobService = azure.createBlobService();

  return new Promise((resolve, reject) => {
    blobService.createContainerIfNotExists('megaboilerplate', { publicAccessLevel: 'blob' }, function(error, result, response) {
      if (error) {
        return reject(error)
      }
      const container = 'archive';
      const blobName = `megaboilerplate-${shortid.generate()}.zip`;

      const writeStream = blobService.createWriteStreamToBlockBlob(container, blobName, archive.pointer(), function(error, result, response) {
        if (error) {
          return reject(error);
        }
        resolve(`https://megaboilerplate.blob.core.windows.net/${container}/${blobName}`)
      });

      archive.pipe(writeStream);
    });
  });
}

export function createZipArchive(res, params) {<commit_msg>Schedule azure files for deletion after 24 hours<commit_after>import { get, set, template, last, isEmpty, dropRight } from 'lodash';
import { basename, join } from 'path';
const path = require('path');
const archiver = require('archiver');
const shortid = require('shortid');
const fs = require('fs-extra');
const Promise = require('bluebird');
const cpy = require('cpy');
const copy = Promise.promisify(fs.copy);
const move = Promise.promisify(fs.move);
const readFile = Promise.promisify(fs.readFile);
const writeFile = Promise.promisify(fs.writeFile);
const appendFile = Promise.promisify(fs.appendFile);
const remove = Promise.promisify(fs.remove);
const readJson = Promise.promisify(fs.readJson);
const writeJson = Promise.promisify(fs.writeJson);
const stat = Promise.promisify(fs.stat);
const mkdirs = Promise.promisify(fs.mkdirs);
const traverse = require('traverse');
const azure = require('azure-storage');
const stream = require('stream');
const schedule = require('node-schedule');
const moment = require('moment');

const npmDependencies = require('./npmDependencies.json');

export { cpy };
export { copy };
export { move };
export { remove };
export { mkdirs };
export { readFile };
export { writeFile };
export { appendFile };
export { readJson };
export { writeJson };

/**
 * @private
 * @param subStr {string} - what to indent
 * @param options {object} - how many levels (2 spaces per level) or how many spaces to indent
 * @returns {string}
 */
function indentCode(subStr, options) {
  const defaultIndentation = 2;
  let indent;

  if (options.indentLevel) {
    indent = ' '.repeat(options.indentLevel * defaultIndentation);
  } else if (options.indentSpaces) {
    indent = ' '.repeat(options.indentSpaces);
  }
  let array = subStr.toString().split('\n').filter(Boolean);
  array.forEach((line, index) => {
    array[index] = indent + line;
  });
  return array.join('\n');
}

/**
 * Traverse files and remove placeholder comments
 * @param params
 */
export function walkAndRemoveComments(params) {
  const build = path.join(__base, 'build', params.uuid);

  return new Promise((resolve, reject) => {
    fs.walk(build)
      .on('data', (item) => {
        return stat(item.path).then((stats) => {
          if (stats.isFile()) {
            return removeCode(item.path, '//=');
          }
        });
      })
      .on('error', (err) => {
        reject(err);
      })
      .on('end', () => {
        resolve();
      });
  });
}

export function walkAndRemoveCommentsMemory(params) {
  traverse(params.build).forEach(function() {
    if (Buffer.isBuffer(this.node)) {
      const buf = removeCodeMemory(this.node, '//=');
      this.update(this.node, true);
      set(params, ['build'].concat(this.path), buf);
    }
  });
}


export async function exists(filepath) {
  try {
    await stat(filepath);
  } catch (err) {
    if (err.code === 'ENOENT') {
      return false;
    }
  }
  return true;
}

function uploadAndReturnDownloadLink(archive) {
  const blobService = azure.createBlobService();

  return new Promise((resolve, reject) => {
    blobService.createContainerIfNotExists('megaboilerplate', { publicAccessLevel: 'blob' }, function(error, result, response) {
      if (error) {
        return reject(error)
      }
      const container = 'archive';
      const blobName = `megaboilerplate-${shortid.generate()}.zip`;

      const writeStream = blobService.createWriteStreamToBlockBlob(container, blobName, archive.pointer(), function(error, result, response) {
        if (error) {
          return reject(error);
        }

        resolve(`https://megaboilerplate.blob.core.windows.net/${container}/${blobName}`);

        // schedule for deletion
        const date = moment().add(24, 'hours').toDate();
        schedule.scheduleJob(date, function(){
          blobService.deleteBlobIfExists(container, blobName, (errorOrResult) => {
            // do nothing
          });
        });
      });

      archive.pipe(writeStream);
    });
  });
}

export function createZipArchive(res, params) {
{'fraction_non_alphanumeric': 0.15378620600077933, 'fraction_numerical': 0.001039095986491752, 'mean_word_length': 3.5724465558194773, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1261257', 'n_tokens_mistral': 2352, 'n_tokens_neox': 2298, 'n_words': 535}