<commit_before>// Copyright 2017, University of Colorado Boulder

/**
 * Generates the lists under perennial/data/, and if there were changes, will commit and push.
 *
 * Ideally, this grunt task is never run itself, but instead from `/bin/generate-data.sh` from bayes.colorado.edu
 * under the phet-admin user from /data/share/phet/generate-data/perennial:
 * `pm2 start bin/generate-data.sh`
 *
 * The task will show up in output of `pm2 list` as "generate-data"
 *
 * If you are curious about the state of this process, you can see logs with `pm2 logs generate-data`
 *
 * See https://github.com/phetsims/perennial/issues/66
 *
 * @author <NAME> <<EMAIL>>
 */

'use strict';

const execute = require( '../common/execute' );
const getActiveRepos = require( '../common/getActiveRepos' );
const getBranch = require( '../common/getBranch' );
const gitAdd = require( '../common/gitAdd' );
const gitCommit = require( '../common/gitCommit' );
const gitIsClean = require( '../common/gitIsClean' );
const gitPull = require( '../common/gitPull' );
const gitPush = require( '../common/gitPush' );
const assert = require( 'assert' );
const fs = require( 'fs' );
const grunt = require( 'grunt' );
const os = require( 'os' );
const winston = require( 'winston' );

/**
 * Generates the lists under perennial/data/, and if there were changes, will commit and push.
 * @public
 */
module.exports = async function() {
  if ( await getBranch( 'perennial' ) !== 'master' || !await gitIsClean( 'perennial' ) ) {
    grunt.fail.fatal( 'Data will only be generated if perennial is on master with no working-copy changes.' );
  }

  // Make sure to clone anything we are missing
  await execute( 'bash', [ 'perennial/bin/clone-missing-repos.sh' ], '..' );

  const activeRepos = getActiveRepos();

  for ( const repo of activeRepos ) {
    await gitPull( repo );
  }

  function writeList( name, packageFilter ) {
    const repos = activeRepos.filter( repo => {
      // Make sure that if someone doesn't have all repositories checked out that this will FAIL. Otherwise bad things.
      assert( grunt.file.exists( `../${repo}` ) );

      let packageObject;
      try {
        packageObject = JSON.parse( fs.readFileSync( `../${repo}/package.json`, 'utf8' ) );
      }
      catch( e ) {
        return false;
      }
      return packageObject.phet && packageFilter( packageObject.phet );
    } );

    grunt.log.writeln( `Writing to data/${name}` );
    fs.writeFileSync( `data/${name}`, repos.join( os.EOL ) + os.EOL );
  }

  writeList( 'interactive-description', phet => phet.supportsInteractiveDescription );
  writeList( 'active-runnables', phet => phet.runnable );<commit_msg>nest supportsInteractiveDescription under "features" in package.json, https://github.com/phetsims/chipper/issues/1031<commit_after>// Copyright 2017, University of Colorado Boulder

/**
 * Generates the lists under perennial/data/, and if there were changes, will commit and push.
 *
 * Ideally, this grunt task is never run itself, but instead from `/bin/generate-data.sh` from bayes.colorado.edu
 * under the phet-admin user from /data/share/phet/generate-data/perennial:
 * `pm2 start bin/generate-data.sh`
 *
 * The task will show up in output of `pm2 list` as "generate-data"
 *
 * If you are curious about the state of this process, you can see logs with `pm2 logs generate-data`
 *
 * See https://github.com/phetsims/perennial/issues/66
 *
 * @author <NAME> <<EMAIL>>
 */

'use strict';

const execute = require( '../common/execute' );
const getActiveRepos = require( '../common/getActiveRepos' );
const getBranch = require( '../common/getBranch' );
const gitAdd = require( '../common/gitAdd' );
const gitCommit = require( '../common/gitCommit' );
const gitIsClean = require( '../common/gitIsClean' );
const gitPull = require( '../common/gitPull' );
const gitPush = require( '../common/gitPush' );
const assert = require( 'assert' );
const fs = require( 'fs' );
const grunt = require( 'grunt' );
const os = require( 'os' );
const winston = require( 'winston' );

/**
 * Generates the lists under perennial/data/, and if there were changes, will commit and push.
 * @public
 */
module.exports = async function() {
  if ( await getBranch( 'perennial' ) !== 'master' || !await gitIsClean( 'perennial' ) ) {
    grunt.fail.fatal( 'Data will only be generated if perennial is on master with no working-copy changes.' );
  }

  // Make sure to clone anything we are missing
  await execute( 'bash', [ 'perennial/bin/clone-missing-repos.sh' ], '..' );

  const activeRepos = getActiveRepos();

  for ( const repo of activeRepos ) {
    await gitPull( repo );
  }

  function writeList( name, packageFilter ) {
    const repos = activeRepos.filter( repo => {
      // Make sure that if someone doesn't have all repositories checked out that this will FAIL. Otherwise bad things.
      assert( grunt.file.exists( `../${repo}` ) );

      let packageObject;
      try {
        packageObject = JSON.parse( fs.readFileSync( `../${repo}/package.json`, 'utf8' ) );
      }
      catch( e ) {
        return false;
      }
      return packageObject.phet && packageFilter( packageObject.phet );
    } );

    grunt.log.writeln( `Writing to data/${name}` );
    fs.writeFileSync( `data/${name}`, repos.join( os.EOL ) + os.EOL );
  }

  writeList( 'interactive-description', phet => phet.features && phet.features.supportsInteractiveDescription );
  writeList( 'active-runnables', phet => phet.runnable );
{'fraction_non_alphanumeric': 0.14463566459172464, 'fraction_numerical': 0.004393994873672647, 'mean_word_length': 3.437855402112104, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2816350', 'n_tokens_mistral': 1706, 'n_tokens_neox': 1669, 'n_words': 525}