<commit_before>const sh = require('shelljs');
const fs = require('fs');
const path = require('path');

const pkg = require('../package.json');

const dirs = [
  '.',
  'blog',
  'blog/p',
  'p',
  'projects',
  'tools',
]

const build = () => {
  checkDeps();
  doSass();
  doBeautify();
}

const checkDeps = () => {
  if (!sh.which('sass')) {
    console.error('building requires Sass, please install Sass');
    sh.exit(1);
  }
  if (!sh.which('node_modules/purify-css/bin/purifycss')) {
    console.error('Try `npm install` first');
    sh.exit(1);
  }
}

const doSass = () => {
  compileSass();
  removeSassMap();
}

const compileSass = () => {
  sh.exec('sass assets/style/main.scss assets/style/main.css');
}

const removeSassMap = () => {
  sh.rm('assets/style/main.css.map');
}

const doBeautify = () => {
  prepStyle();
  try {
    beautifyAndMinifyStyle();
  } catch (e) {
    console.error(e);
    sh.exit(1);
  } finally {
    removeTmp();
  }
}

const prepStyle = () => {
  sh.mkdir('tmp');
  sh.mv('assets/style/main.css', 'tmp/m.css');
}

const beautifyAndMinifyStyle = async () => {
  const whitelist = await getHtmlFiles();
  console.log(whitelist);
  const cmd = `node_modules/purify-css/bin/purifycss --min --info --rejected --out assets/style/main.css tmp/m.css ${whitelist}`;
  sh.exec(cmd);
}

const getHtmlFiles = async () => {
  let fileList = [];
  let files;
  for (let d of dirs) {
    files = await getFiles(d);
    files = files
      .filter(f => path.extname(f) === '.html')
      .map((f) => `${d}/${f}`);
    fileList.push(files);
  }
  return stringifySpaces(fileList);
}

const getFiles = (dir) => new Promise((resolve, reject) => {
  console.log(`reading ${dir} for HTML files...`);
  fs.readdir(dir, (err, files) => {
    if (err) {
      reject(err);
    } else {
      resolve(files);
    }
  });
});

const stringifySpaces = (arr) => {
  let str = '';
  let i = 0;
  while (i < arr.length) {
    const val = arr[i];
    if (Array.isArray(val)) {
      str += `${stringifySpaces(val)} `;
    } else {
      str += `${val} `;
    }
    i++;
  }
  return str;
}

const removeTmp = () => {
  sh.rm('-rf', 'tmp');
}

module.exports = build;

/* Now time for the fun */
build();<commit_msg>fix build bugs<commit_after>const sh = require('shelljs');
const fs = require('fs');
const path = require('path');

const dirs = [
  '.',
  'blog',
  'blog/p',
  'p',
  'projects',
  'tools',
]

const build = () => {
  checkDeps();
  compileSass();
  doBeautify();
}

const checkDeps = () => {
  if (!sh.which('sass')) {
    console.error('building requires Sass, please install Sass');
    sh.exit(1);
  }
  if (!sh.which('node_modules/purify-css/bin/purifycss')) {
    console.error('Try `npm install` first');
    sh.exit(1);
  }
}

const compileSass = () => {
  sh.exec('sass assets/style/main.scss assets/style/main.css');
}

const doBeautify = () => {
  prepStyle();
  purifyStyle()
    .then((success) => {
      console.log('Purification complete');
      removeTmp();
    }, (err) => {
      console.error(err);
      removeTmp();
    });
}

const prepStyle = () => {
  sh.mkdir('tmp');
  sh.mv('assets/style/main.css', 'tmp/m.css');
}

const purifyStyle = async () => {
  const whitelist = await getHtmlFiles();
  const cmd = `node_modules/purify-css/bin/purifycss --min --info --out assets/style/main.css tmp/m.css ${whitelist}`;
  console.log(cmd);
  sh.exec(cmd);
}

const getHtmlFiles = async () => {
  let fileList = [];
  let files;
  for (let d of dirs) {
    files = await getFiles(d);
    files = files
      .filter(f => path.extname(f) === '.html')
      .map((f) => `${d}/${f}`);
    fileList.push(files);
  }
  return stringifySpaces(fileList);
}

const getFiles = (dir) => new Promise((resolve, reject) => {
  console.log(`reading ${dir} for HTML files...`);
  fs.readdir(dir, (err, files) => {
    if (err) {
      reject(err);
    } else {
      resolve(files);
    }
  });
});

const stringifySpaces = (arr) => {
  let str = '';
  let i = 0;
  while (i < arr.length) {
    const val = arr[i];
    if (Array.isArray(val)) {
      str += `${stringifySpaces(val)} `;
    } else {
      str += `${val} `;
    }
    i++;
  }
  return str;
}

const removeTmp = () => {
  sh.rm('-rf', './tmp');
}

module.exports = build;

/* Now time for the fun */
build();
{'fraction_non_alphanumeric': 0.21036727103672712, 'fraction_numerical': 0.0016271501627150163, 'mean_word_length': 2.8696043165467624, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5150292', 'n_tokens_mistral': 1620, 'n_tokens_neox': 1622, 'n_words': 328}