<commit_before>/* ************************************************************************
 *
 *    qooxdoo-compiler - node.js based replacement for the Qooxdoo python
 *    toolchain
 *
 *    https://github.com/qooxdoo/qooxdoo-compiler
 *
 *    Copyright:
 *      2011-2017 Zenesis Limited, http://www.zenesis.com
 *
 *    License:
 *      MIT: https://opensource.org/licenses/MIT
 *
 *      This software is provided under the same licensing terms as Qooxdoo,
 *      please see the LICENSE file in the Qooxdoo project's top-level directory
 *      for details.
 *
 *    Authors:
 *      * <NAME> (<EMAIL>, @johnspackman)
 *
 * *********************************************************************** */

/* eslint-disable @qooxdoo/qx/no-illegal-private-usage */

var path = require("upath");

var log = require("../util").createLog("resource-manager");

/**
 * Analyses library resources, collecting information into a cached database
 * file
 */
qx.Class.define("qx.tool.compiler.resources.Manager", {
  extend: qx.core.Object,

  /**
   * Constructor
   *
   * @param dbFilename
   *          {String?} database filename, default is "resource-db.json"
   */
  construct: function(analyser) {
    this.base(arguments);
    this.__analyser = analyser;
    this.__dbFilename = analyser.getResDbFilename() || "resource-db.json";
    this.__loaders = [
      new qx.tool.compiler.resources.ImageLoader(),
      new qx.tool.compiler.resources.MetaLoader()
    ];
    this.__converters = [
      new qx.tool.compiler.resources.ScssConverter()
    ];
  },

  members: {
    /** {String} filename of database */
    __dbFilename: null,

    /** {Object} Database */
    __db: null,

    /** the used analyser */
    __analyser: null,

    /** {Map{String,Library}} Lookup of libraries, indexed by resource URI */
    __librariesByResourceUri: null,

    /** {String[]} Array of all resource URIs, sorted alphabetically (ie these are the keys in __librariesByResourceUri) */
    __allResourceUris: null,

    /** {ResourceLoader[]} list of resource loaders, used to add info to the database */
    __loaders: null,

    /** {ResourceConverter[]} list of resource converters, used to copy resources to the target */
    __converters: null,

    /**
     * Loads the cached database
     */
    async loadDatabase() {
      this.__db = (await qx.tool.utils.Json.loadJsonAsync(this.__dbFilename))||{};
    },

    /**
     * Saves the database
     */
    async saveDatabase() {
      log.debug("saving resource manager database");
      return qx.tool.utils.Json.saveJsonAsync(this.__dbFilename, this.__db);
    },

    /**
     * Returns the loaded database
     *
     * @returns
     */
    getDatabase: function() {
      return this.__db;
    },

    /**
     * Finds the library needed for a resource; this depends on `findAllResources` having
     * already been called.  `uri` can include optional explicit namespace (eg "qx:blah/blah.png"),
     * otherwise the library resource lookups are examined to find the library.
     *
     * @param uri {String} URI
     * @return {Library?} the library, null if not found
     */
    findLibraryForResource: function(uri) {
      // check for absolute path first, in windows c:/ is a valid absolute name
      if (path.isAbsolute(uri)) {
        let library = this.__analyser.getLibraries().find(lib => uri.startsWith(path.resolve(lib.getRootDir())));
        return library || null;
      }

      // Explicit library?
      var pos = uri.indexOf(":");
      if (pos !== -1) {
        var ns = uri.substring(0, pos);
        var library = this.__analyser.findLibrary(ns);
        return library || null;
      }
      
      // Non-wildcards are a direct lookup
      // check for $ and *. less pos wins
      // fix for https://github.com/qooxdoo/qooxdoo-compiler/issues/260
      var pos1 = uri.indexOf("$"); // Variable references are effectively a wildcard lookup
      var pos2 = uri.indexOf("*");
      if (pos1 === -1) {
        pos = pos2;
      } else if (pos2 === -1) {
        pos = pos1;
      } else {
        pos = Math.min(pos1, pos2);
      }
      if (pos === -1) {
        library = this.__librariesByResourceUri[uri] || null;
        return library;
      }

      // Strip wildcard
      var isFolderMatch = uri[pos - 1] === "/";
      uri = uri.substring(0, pos - 1);

      // Fast folder match
      if (isFolderMatch) {
        library = this.__librariesByResourceUri[uri] || null;
        return library;
      }

      // Slow scan
      if (!this.__allResourceUris) {
        this.__allResourceUris = Object.keys(this.__librariesByResourceUri).sort();
      }
      pos = qx.tool.utils.Values.binaryStartsWith(this.__allResourceUris, uri);
      if (pos > -1) {
        var firstUri = this.__allResourceUris[pos];
        library = this.__librariesByResourceUri[firstUri] || null;
        return library;
      }

      return null;
    },

    /**
     * Scans all libraries looking for resources; this does not analyse the
     * files, simply compiles the list
     *
     * @param callback
     */
    async findAllResources() {
      var t = this;
      var db = this.__db;
      if (!db.resources) {
        db.resources = {};
      }
      t.__librariesByResourceUri = {};
      this.__allResourceUris = null;
      this.__assets = {};
      
      await qx.Promise.all(t.__analyser.getLibraries().map(async library => {
        var resources = db.resources[library.getNamespace()];
        if (!resources) {
          db.resources[library.getNamespace()] = resources = {};
        }
        var unconfirmed = {};
        for (let relFile in resources) {
          unconfirmed[relFile] = true;
        }
        
        let resPath = path.join(library.getRootDir(), library.get("resourcePath"));
        const scanResources = async resourcePath => {
          // If the root folder exists, scan it
          var rootDir = path.join(library.getRootDir(), library.get(resourcePath));
          await qx.tool.utils.files.Utils.findAllFiles(rootDir, async filename => {
            var relFile = filename.substring(rootDir.length + 1).replace(/\\/g, "/");
            var fileInfo = resources[relFile];
            delete unconfirmed[relFile];
            if (!fileInfo) {
              fileInfo = resources[relFile] = {};
            }
            fileInfo.resourcePath = resourcePath;
            fileInfo.mtime = await qx.tool.utils.files.Utils.safeStat(filename).mtime;
            
            let asset = new qx.tool.compiler.resources.Asset(library, relFile, fileInfo);
            this.__addAsset(asset);
          });
        };
        
        await scanResources("resourcePath");
        await scanResources("themePath");
        <commit_msg>fix lint<commit_after>/* ************************************************************************
 *
 *    qooxdoo-compiler - node.js based replacement for the Qooxdoo python
 *    toolchain
 *
 *    https://github.com/qooxdoo/qooxdoo-compiler
 *
 *    Copyright:
 *      2011-2017 Zenesis Limited, http://www.zenesis.com
 *
 *    License:
 *      MIT: https://opensource.org/licenses/MIT
 *
 *      This software is provided under the same licensing terms as Qooxdoo,
 *      please see the LICENSE file in the Qooxdoo project's top-level directory
 *      for details.
 *
 *    Authors:
 *      * <NAME> (<EMAIL>, @johnspackman)
 *
 * *********************************************************************** */

/* eslint-disable @qooxdoo/qx/no-illegal-private-usage */

var path = require("upath");

var log = require("../util").createLog("resource-manager");

/**
 * Analyses library resources, collecting information into a cached database
 * file
 */
qx.Class.define("qx.tool.compiler.resources.Manager", {
  extend: qx.core.Object,

  /**
   * Constructor
   *
   * @param dbFilename
   *          {String?} database filename, default is "resource-db.json"
   */
  construct: function(analyser) {
    this.base(arguments);
    this.__analyser = analyser;
    this.__dbFilename = analyser.getResDbFilename() || "resource-db.json";
    this.__loaders = [
      new qx.tool.compiler.resources.ImageLoader(),
      new qx.tool.compiler.resources.MetaLoader()
    ];
    this.__converters = [
      new qx.tool.compiler.resources.ScssConverter()
    ];
  },

  members: {
    /** {String} filename of database */
    __dbFilename: null,

    /** {Object} Database */
    __db: null,

    /** the used analyser */
    __analyser: null,

    /** {Map{String,Library}} Lookup of libraries, indexed by resource URI */
    __librariesByResourceUri: null,

    /** {String[]} Array of all resource URIs, sorted alphabetically (ie these are the keys in __librariesByResourceUri) */
    __allResourceUris: null,

    /** {ResourceLoader[]} list of resource loaders, used to add info to the database */
    __loaders: null,

    /** {ResourceConverter[]} list of resource converters, used to copy resources to the target */
    __converters: null,

    /**
     * Loads the cached database
     */
    async loadDatabase() {
      this.__db = (await qx.tool.utils.Json.loadJsonAsync(this.__dbFilename))||{};
    },

    /**
     * Saves the database
     */
    async saveDatabase() {
      log.debug("saving resource manager database");
      return qx.tool.utils.Json.saveJsonAsync(this.__dbFilename, this.__db);
    },

    /**
     * Returns the loaded database
     *
     * @returns
     */
    getDatabase: function() {
      return this.__db;
    },

    /**
     * Finds the library needed for a resource; this depends on `findAllResources` having
     * already been called.  `uri` can include optional explicit namespace (eg "qx:blah/blah.png"),
     * otherwise the library resource lookups are examined to find the library.
     *
     * @param uri {String} URI
     * @return {Library?} the library, null if not found
     */
    findLibraryForResource: function(uri) {
      // check for absolute path first, in windows c:/ is a valid absolute name
      if (path.isAbsolute(uri)) {
        let library = this.__analyser.getLibraries().find(lib => uri.startsWith(path.resolve(lib.getRootDir())));
        return library || null;
      }

      // Explicit library?
      var pos = uri.indexOf(":");
      if (pos !== -1) {
        var ns = uri.substring(0, pos);
        var library = this.__analyser.findLibrary(ns);
        return library || null;
      }
      
      // Non-wildcards are a direct lookup
      // check for $ and *. less pos wins
      // fix for https://github.com/qooxdoo/qooxdoo-compiler/issues/260
      var pos1 = uri.indexOf("$"); // Variable references are effectively a wildcard lookup
      var pos2 = uri.indexOf("*");
      if (pos1 === -1) {
        pos = pos2;
      } else if (pos2 === -1) {
        pos = pos1;
      } else {
        pos = Math.min(pos1, pos2);
      }
      if (pos === -1) {
        library = this.__librariesByResourceUri[uri] || null;
        return library;
      }

      // Strip wildcard
      var isFolderMatch = uri[pos - 1] === "/";
      uri = uri.substring(0, pos - 1);

      // Fast folder match
      if (isFolderMatch) {
        library = this.__librariesByResourceUri[uri] || null;
        return library;
      }

      // Slow scan
      if (!this.__allResourceUris) {
        this.__allResourceUris = Object.keys(this.__librariesByResourceUri).sort();
      }
      pos = qx.tool.utils.Values.binaryStartsWith(this.__allResourceUris, uri);
      if (pos > -1) {
        var firstUri = this.__allResourceUris[pos];
        library = this.__librariesByResourceUri[firstUri] || null;
        return library;
      }

      return null;
    },

    /**
     * Scans all libraries looking for resources; this does not analyse the
     * files, simply compiles the list
     *
     * @param callback
     */
    async findAllResources() {
      var t = this;
      var db = this.__db;
      if (!db.resources) {
        db.resources = {};
      }
      t.__librariesByResourceUri = {};
      this.__allResourceUris = null;
      this.__assets = {};
      
      await qx.Promise.all(t.__analyser.getLibraries().map(async library => {
        var resources = db.resources[library.getNamespace()];
        if (!resources) {
          db.resources[library.getNamespace()] = resources = {};
        }
        var unconfirmed = {};
        for (let relFile in resources) {
          unconfirmed[relFile] = true;
        }
        
        const scanResources = async resourcePath => {
          // If the root folder exists, scan it
          var rootDir = path.join(library.getRootDir(), library.get(resourcePath));
          await qx.tool.utils.files.Utils.findAllFiles(rootDir, async filename => {
            var relFile = filename.substring(rootDir.length + 1).replace(/\\/g, "/");
            var fileInfo = resources[relFile];
            delete unconfirmed[relFile];
            if (!fileInfo) {
              fileInfo = resources[relFile] = {};
            }
            fileInfo.resourcePath = resourcePath;
            fileInfo.mtime = await qx.tool.utils.files.Utils.safeStat(filename).mtime;
            
            let asset = new qx.tool.compiler.resources.Asset(library, relFile, fileInfo);
            this.__addAsset(asset);
          });
        };
        
        await scanResources("resourcePath");
        await scanResources("themePath");
        
{'fraction_non_alphanumeric': 0.14447020115157408, 'fraction_numerical': 0.00433709713602034, 'mean_word_length': 2.3285216525634644, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4817578', 'n_tokens_mistral': 3924, 'n_tokens_neox': 3841, 'n_words': 953}