<commit_before>/* global zip */

import angular from 'angular';
import _ from 'underscore';
import idbKeyval from 'idb-keyval';

import sqlWasmPath from 'file-loader?name=[name]-[hash:6].[ext]!sql.js/js/sql-debug-wasm.js';
import sqlWasmBinaryPath from 'file-loader?name=[name]-[hash:6].[ext]!sql.js/js/sql-debug-wasm-raw.wasm';

// For zip
import 'imports-loader?this=>window!zip-js/WebContent/zip.js';

// Dynamic import splits up the sql library so the user only loads it
// if they need it. So we can minify sql.js specifically (as explained
// in the Webpack config, we need to explicitly name this chunk, which
// can only be done using the dynamic import method.
function requireSqlLib() {
  if (typeof WebAssembly === 'object') {
    return new Promise((resolve) => {
      let loaded = false;

      window.Module = {
        wasmBinaryFile: sqlWasmBinaryPath
      };
      window.SQL = {
        onRuntimeInitialized: function() {
          if (!loaded) {
            console.log("Using WASM SQLite");
            loaded = true;
            resolve(window.SQL);
            delete window.SQL;
          }
        }
      };

      // Give it 3 seconds to load
      setTimeout(() => {
        if (!loaded) {
          loaded = true;

          // Fall back to the old one
          import(/* webpackChunkName: "sqlLib" */ 'sql.js').then(resolve);
        }
      }, 3000);

      var head = document.getElementsByTagName('head')[0];
      var script = document.createElement('script');
      script.type = 'text/javascript';
      script.src = sqlWasmPath;
      script.async = true;
      head.appendChild(script);
    });
  } else {
    return import(/* webpackChunkName: "sqlLib" */ 'sql.js');
  }
}

angular.module('dimApp')
  .factory('dimManifestService', ManifestService);


function ManifestService($q, dimBungieService, $http, toaster, dimSettingsService, $translate, $rootScope) {
  // Testing flags
  const alwaysLoadRemote = false;

  let manifestPromise = null;

  const makeStatement = _.memoize(function(table, db) {
    return db.prepare(`select json from ${table} where id = ?`);
  });

  const service = {
    isLoaded: true,
    isError: false,
    statusText: null,
    version: null,

    // This tells users to reload the extension. It fires no more
    // often than every 10 seconds, and only warns if the manifest
    // version has actually changed.
    warnMissingDefinition: _.debounce(function() {
      dimBungieService.getManifest()
        .then(function(data) {
          const language = dimSettingsService.language;
          const path = data.mobileWorldContentPaths[language] || data.mobileWorldContentPaths.en;

          // The manifest has updated!
          if (path !== service.version) {
            toaster.pop('error',
                        $translate.instant('Manifest.Outdated'),
                        $translate.instant('Manifest.OutdatedExplanation'));
          }
        });
    }, 10000, true),

    getManifest: function() {
      if (manifestPromise) {
        return manifestPromise;
      }

      console.time('manifest');

      service.isLoaded = false;

      manifestPromise = Promise
        .all([
          requireSqlLib(), // load in the sql.js library
          dimBungieService.getManifest()
            .then(function(data) {
              const language = dimSettingsService.language;
              const path = data.mobileWorldContentPaths[language] || data.mobileWorldContentPaths.en;

              // Use the path as the version, rather than the "version" field, because
              // Bungie can update the manifest file without changing that version.
              const version = path;
              service.version = version;

              return loadManifestFromCache(version)
                .catch(function(e) {
                  return loadManifestRemote(version, language, path);
                });
            })
        ])
        .then(function([SQLLib, typedArray]) {
          service.statusText = $translate.instant('Manifest.Build') + '...';
          const db = new SQLLib.Database(typedArray);
          // do a small request, just to test it out
          service.getAllRecords(db, 'DestinyRaceDefinition');
          console.timeEnd('manifest');
          return db;
        })
        .catch((e) => {
          let message = e.message || e;
          service.statusText = $translate.instant('Manifest.Error', { error: message });

          if (e.status === -1) {
            message = $translate.instant('BungieService.NotConnected');
          } else if (e.status === 503 || e.status === 522 /* cloudflare */) {
            message = $translate.instant('BungieService.Down');
          } else if (e.status < 200 || e.status >= 400) {
            message = $translate.instant('BungieService.NetworkError', {
              status: e.status,
              statusText: e.statusText
            });
          } else {
            // Something may be wrong with the manifest
            //deleteManifestFile();
          }

          manifestPromise = null;
          service.isError = true;
          throw new Error(message);
        });

      return manifestPromise;
    },

    getRecord: function(db, table, id) {
      const statement = makeStatement(table, db);
      // The ID in sqlite is a signed 32-bit int, while the id we
      // use is unsigned, so we must convert
      const sqlId = new Int32Array([id])[0];
      const result = statement.get([sqlId]);
      statement.reset();
      if (result.length) {
        return JSON.parse(result[0]);
      }
      return null;
    },

    getAllRecords: function(db, table) {
      const rows = db.exec(`SELECT json FROM ${table}`);
      const result = {};
      rows[0].values.forEach((row) => {
        const obj = JSON.parse(row);
        result[obj.hash] = obj;<commit_msg>Optimized version<commit_after>/* global zip */

import angular from 'angular';
import _ from 'underscore';
import idbKeyval from 'idb-keyval';

import sqlWasmPath from 'file-loader?name=[name]-[hash:6].[ext]!sql.js/js/sql-wasm.js';
import sqlWasmBinaryPath from 'file-loader?name=[name]-[hash:6].[ext]!sql.js/js/sql-optimized-wasm-raw.wasm';

// For zip
import 'imports-loader?this=>window!zip-js/WebContent/zip.js';

// Dynamic import splits up the sql library so the user only loads it
// if they need it. So we can minify sql.js specifically (as explained
// in the Webpack config, we need to explicitly name this chunk, which
// can only be done using the dynamic import method.
function requireSqlLib() {
  if (typeof WebAssembly === 'object') {
    return new Promise((resolve, reject) => {
      let loaded = false;

      window.Module = {
        wasmBinaryFile: sqlWasmBinaryPath
      };
      window.SQL = {
        onRuntimeInitialized: function() {
          if (!loaded) {
            console.info("Using WASM SQLite");
            loaded = true;
            resolve(window.SQL);
            delete window.SQL;
          }
        }
      };

      // Give it 10 seconds to load
      setTimeout(() => {
        if (!loaded) {
          loaded = true;

          // Fall back to the old one
          import(/* webpackChunkName: "sqlLib" */ 'sql.js').then(resolve, reject);
        }
      }, 10000);

      var head = document.getElementsByTagName('head')[0];
      var script = document.createElement('script');
      script.type = 'text/javascript';
      script.src = sqlWasmPath;
      script.async = true;
      head.appendChild(script);
    });
  } else {
    return import(/* webpackChunkName: "sqlLib" */ 'sql.js');
  }
}

angular.module('dimApp')
  .factory('dimManifestService', ManifestService);


function ManifestService($q, dimBungieService, $http, toaster, dimSettingsService, $translate, $rootScope) {
  // Testing flags
  const alwaysLoadRemote = false;

  let manifestPromise = null;

  const makeStatement = _.memoize(function(table, db) {
    return db.prepare(`select json from ${table} where id = ?`);
  });

  const service = {
    isLoaded: true,
    isError: false,
    statusText: null,
    version: null,

    // This tells users to reload the extension. It fires no more
    // often than every 10 seconds, and only warns if the manifest
    // version has actually changed.
    warnMissingDefinition: _.debounce(function() {
      dimBungieService.getManifest()
        .then(function(data) {
          const language = dimSettingsService.language;
          const path = data.mobileWorldContentPaths[language] || data.mobileWorldContentPaths.en;

          // The manifest has updated!
          if (path !== service.version) {
            toaster.pop('error',
                        $translate.instant('Manifest.Outdated'),
                        $translate.instant('Manifest.OutdatedExplanation'));
          }
        });
    }, 10000, true),

    getManifest: function() {
      if (manifestPromise) {
        return manifestPromise;
      }

      service.isLoaded = false;

      manifestPromise = Promise
        .all([
          requireSqlLib(), // load in the sql.js library
          dimBungieService.getManifest()
            .then(function(data) {
              const language = dimSettingsService.language;
              const path = data.mobileWorldContentPaths[language] || data.mobileWorldContentPaths.en;

              // Use the path as the version, rather than the "version" field, because
              // Bungie can update the manifest file without changing that version.
              const version = path;
              service.version = version;

              return loadManifestFromCache(version)
                .catch(function(e) {
                  return loadManifestRemote(version, language, path);
                });
            })
        ])
        .then(function([SQLLib, typedArray]) {
          service.statusText = $translate.instant('Manifest.Build') + '...';
          const db = new SQLLib.Database(typedArray);
          // do a small request, just to test it out
          service.getAllRecords(db, 'DestinyRaceDefinition');
          return db;
        })
        .catch((e) => {
          let message = e.message || e;
          service.statusText = $translate.instant('Manifest.Error', { error: message });

          if (e.status === -1) {
            message = $translate.instant('BungieService.NotConnected');
          } else if (e.status === 503 || e.status === 522 /* cloudflare */) {
            message = $translate.instant('BungieService.Down');
          } else if (e.status < 200 || e.status >= 400) {
            message = $translate.instant('BungieService.NetworkError', {
              status: e.status,
              statusText: e.statusText
            });
          } else {
            // Something may be wrong with the manifest
            deleteManifestFile();
          }

          manifestPromise = null;
          service.isError = true;
          throw new Error(message);
        });

      return manifestPromise;
    },

    getRecord: function(db, table, id) {
      const statement = makeStatement(table, db);
      // The ID in sqlite is a signed 32-bit int, while the id we
      // use is unsigned, so we must convert
      const sqlId = new Int32Array([id])[0];
      const result = statement.get([sqlId]);
      statement.reset();
      if (result.length) {
        return JSON.parse(result[0]);
      }
      return null;
    },

    getAllRecords: function(db, table) {
      const rows = db.exec(`SELECT json FROM ${table}`);
      const result = {};
      rows[0].values.forEach((row) => {
        const obj = JSON.parse(row);
        result[obj.hash] = obj;
{'fraction_non_alphanumeric': 0.12118869707120158, 'fraction_numerical': 0.006183973202782788, 'mean_word_length': 2.1727520435967302, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5930502', 'n_tokens_mistral': 3280, 'n_tokens_neox': 3151, 'n_words': 850}