<commit_before>// Copyright 2002-2015, University of Colorado Boulder

/**
 * This grunt task iterates over all of the license.json files and reports any images or audio that have any of the following
 * problems:
 *
 * missing-license.json    The resource file has no associated license.json file
 * incompatible-license    Known to be from an unapproved source outside of PhET
 * not-annotated           There is a license.json file but the asset is not in there
 * missing-file            There is an entry in the license.json but no corresponding file
 *
 * This can be run from any simulation directory with `grunt create-image-and-audio-license-report` and it reports for
 * all directories (not just the simulation at hand).
 *
 * Note that this program relies on numerous heuristics for determining the output, such as allowed entries that
 * determine if a file originates from PhET.
 *
 * See https://github.com/phetsims/tasks/issues/274
 *
 * @author <NAME>
 */

/**
 * @param grunt the grunt instance
 */
module.exports = function( grunt ) {
  // The following comment permits node-specific globals (such as process.cwd()) to pass jshint
  /* jslint node: true */
  'use strict';

  // modules
  var getLicenseInfo = require( '../../../chipper/js/grunt/getLicenseInfo' );

  // constants
  var ACTIVE_REPOS_FILENAME = 'chipper/data/active-repos';  // The relative path to the list of active repos

  // Start in the github checkout dir (above one of the sibling directories)
  var directory = process.cwd();
  var rootdir = directory + '/../';

  // Iterate over all active-repos
  var repos = grunt.file.read( rootdir + '/' + ACTIVE_REPOS_FILENAME ).trim();
  var reposByLine = repos.split( /\r?\n/ );

  /**
   * Create a fast report based on the license.json files for the specified repository and directory (images or audio)
   * @param {string} repo - the name of the repository, such as 'balancing-act'
   * @param {string} directory - the name of the directory to search such as 'images'
   * @private
   */
  var reportForDirectory = function( repo, directory ) {

    var searchDir = rootdir + repo + '/' + directory;

    // Not all projects have an audio/ directory
    if ( grunt.file.exists( searchDir ) ) {

      // Iterate over all images and audio directories recursively
      grunt.file.recurse( searchDir, function( abspath, rootdir, subdir, filename ) {

        // Some files don't need to be attributed in the license.json
        if ( abspath.indexOf( 'README.md' ) < 0 &&
             filename.indexOf( 'license.json' ) !== 0 ) {

          // Classify the resource
          var result = getLicenseInfo( abspath, abspath );

          // Report if it is a problem
          if ( result.isProblematic === true ) {
            grunt.log.writeln( 'incompatible-license: ' + repo + '/' + directory + '/' + filename );
          }
        }

        // Now iterate through the license.json entries and see which are missing files
        // This helps to identify stale entries in the license.json files.
        if ( filename === 'license.json' ) {

          var file = grunt.file.read( abspath, 'utf8' );
          var json = JSON.parse( file );

          // For each key in the json file, make sure that file exists in the directory
          for ( var key in json ) {
            if ( json.hasOwnProperty( key ) ) {
              var resourceFilename = searchDir + '/' + key;
              var exists = grunt.file.exists( resourceFilename );
              if ( !exists ) {
                grunt.log.writeln( 'missing-file: ' + repo + '/' + directory + '/' + key );
              }
            }
          }
        }
      } );
    }
  };
  for ( var i = 0; i < reposByLine.length; i++ ) {
    reportForDirectory( reposByLine[ i ], 'images' );
    reportForDirectory( reposByLine[ i ], 'audio' );
  }
};<commit_msg>#198 assert that the repo exists<commit_after>// Copyright 2002-2015, University of Colorado Boulder

/**
 * This grunt task iterates over all of the license.json files and reports any images or audio that have any of the following
 * problems:
 *
 * missing-license.json    The resource file has no associated license.json file
 * incompatible-license    Known to be from an unapproved source outside of PhET
 * not-annotated           There is a license.json file but the asset is not in there
 * missing-file            There is an entry in the license.json but no corresponding file
 *
 * This can be run from any simulation directory with `grunt create-image-and-audio-license-report` and it reports for
 * all directories (not just the simulation at hand).
 *
 * Note that this program relies on numerous heuristics for determining the output, such as allowed entries that
 * determine if a file originates from PhET.
 *
 * See https://github.com/phetsims/tasks/issues/274
 *
 * @author <NAME>
 */

var assert = require( 'assert' );

/**
 * @param grunt the grunt instance
 */
module.exports = function( grunt ) {
  // The following comment permits node-specific globals (such as process.cwd()) to pass jshint
  /* jslint node: true */
  'use strict';

  // modules
  var getLicenseInfo = require( '../../../chipper/js/grunt/getLicenseInfo' );

  // constants
  var ACTIVE_REPOS_FILENAME = 'chipper/data/active-repos';  // The relative path to the list of active repos

  // Start in the github checkout dir (above one of the sibling directories)
  var directory = process.cwd();
  var rootdir = directory + '/../';

  // Iterate over all active-repos
  var repos = grunt.file.read( rootdir + '/' + ACTIVE_REPOS_FILENAME ).trim();
  var reposByLine = repos.split( /\r?\n/ );

  /**
   * Create a fast report based on the license.json files for the specified repository and directory (images or audio)
   * @param {string} repo - the name of the repository, such as 'balancing-act'
   * @param {string} directory - the name of the directory to search such as 'images'
   * @private
   */
  var reportForDirectory = function( repo, directory ) {

    assert( grunt.file.exists( repo ), 'missing required repo: ' + repo );

    var searchDir = rootdir + repo + '/' + directory;

    // Not all projects have an audio/ directory
    if ( grunt.file.exists( searchDir ) ) {

      // Iterate over all images and audio directories recursively
      grunt.file.recurse( searchDir, function( abspath, rootdir, subdir, filename ) {

        // Some files don't need to be attributed in the license.json
        if ( abspath.indexOf( 'README.md' ) < 0 &&
             filename.indexOf( 'license.json' ) !== 0 ) {

          // Classify the resource
          var result = getLicenseInfo( abspath, abspath );

          // Report if it is a problem
          if ( result.isProblematic === true ) {
            grunt.log.writeln( 'incompatible-license: ' + repo + '/' + directory + '/' + filename );
          }
        }

        // Now iterate through the license.json entries and see which are missing files
        // This helps to identify stale entries in the license.json files.
        if ( filename === 'license.json' ) {

          var file = grunt.file.read( abspath, 'utf8' );
          var json = JSON.parse( file );

          // For each key in the json file, make sure that file exists in the directory
          for ( var key in json ) {
            if ( json.hasOwnProperty( key ) ) {
              var resourceFilename = searchDir + '/' + key;
              var exists = grunt.file.exists( resourceFilename );
              if ( !exists ) {
                grunt.log.writeln( 'missing-file: ' + repo + '/' + directory + '/' + key );
              }
            }
          }
        }
      } );
    }
  };
  for ( var i = 0; i < reposByLine.length; i++ ) {
    reportForDirectory( reposByLine[ i ], 'images' );
    reportForDirectory( reposByLine[ i ], 'audio' );
  }
};
{'fraction_non_alphanumeric': 0.10678744577698393, 'fraction_numerical': 0.004210257718805818, 'mean_word_length': 2.439666520403686, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2756753', 'n_tokens_mistral': 2259, 'n_tokens_neox': 2143, 'n_words': 826}