<commit_before>import path = require('path');
import glob = require('glob');
import fs = require('fs-extra');
import childProcess = require('child_process');
import sortPackageJson = require('sort-package-json');


/**
 * Get all of the lerna package paths.
 */
export
function getLernaPaths(): string[] {
  let basePath = path.resolve('.');
  let lernaConfig = require(path.join(basePath, 'lerna.json'));
  let paths: string[] = [];
  for (let config of lernaConfig.packages) {
    paths = paths.concat(glob.sync(path.join(basePath, config)));
  }
  return paths;
}

/**
 * Get all of the core package paths.
 */
export
function getCorePaths(): string[] {
  let spec = path.resolve(path.join('.', 'packages', '*'));
  return glob.sync(spec);
}


/**
 * Write a package.json if necessary.
 *
 * @param data - The package data.
 *
 * @oaram pkgJsonPath - The path to the package.json file.
 *
 * @returns Whether the file has changed.
 */
export
function ensurePackageData(data: any, pkgJsonPath: string): boolean {
  let text = JSON.stringify(sortPackageJson(data), null, 2) + '\n';
  let orig = fs.readFileSync(pkgJsonPath, 'utf8').toString();
  if (text !== orig) {
    fs.writeFileSync(pkgJsonPath, text, 'utf8');
    return true;
  }
  return false;
}

/**
 * Run a command with terminal output.
 *
 * @param cmd - The command to run.
 */
export
function run(cmd: string, options: childProcess.ExecSyncOptions = {}): void {
  options = options || {};
  options['stdio'] = [1, 2, 3];
  console.log('>', cmd);
  childProcess.execSync(cmd, options);
}<commit_msg>Only store if the contents change<commit_after>import path = require('path');
import glob = require('glob');
import fs = require('fs-extra');
import childProcess = require('child_process');
import sortPackageJson = require('sort-package-json');


/**
 * Get all of the lerna package paths.
 */
export
function getLernaPaths(): string[] {
  let basePath = path.resolve('.');
  let lernaConfig = require(path.join(basePath, 'lerna.json'));
  let paths: string[] = [];
  for (let config of lernaConfig.packages) {
    paths = paths.concat(glob.sync(path.join(basePath, config)));
  }
  return paths;
}


/**
 * Get all of the core package paths.
 */
export
function getCorePaths(): string[] {
  let spec = path.resolve(path.join('.', 'packages', '*'));
  return glob.sync(spec);
}


/**
 * Write a package.json if necessary.
 *
 * @param data - The package data.
 *
 * @oaram pkgJsonPath - The path to the package.json file.
 *
 * @returns Whether the file has changed.
 */
export
function ensurePackageData(data: any, pkgJsonPath: string): boolean {
  let text = JSON.stringify(sortPackageJson(data), null, 2) + '\n';
  let origData = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf8').toString());
  let orig = JSON.stringify(sortPackageJson(origData), null, 2) + '\n';
  if (text !== orig) {
    fs.writeFileSync(pkgJsonPath, text, 'utf8');
    return true;
  }
  return false;
}

/**
 * Run a command with terminal output.
 *
 * @param cmd - The command to run.
 */
export
function run(cmd: string, options: childProcess.ExecSyncOptions = {}): void {
  options = options || {};
  options['stdio'] = [1, 2, 3];
  console.log('>', cmd);
  childProcess.execSync(cmd, options);
}
{'fraction_non_alphanumeric': 0.16249613839975285, 'fraction_numerical': 0.004016064257028112, 'mean_word_length': 4.043613707165109, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '618242', 'n_tokens_mistral': 1031, 'n_tokens_neox': 1013, 'n_words': 297}