<commit_before>/**
 * WebExtension driver
 */

/* eslint-env browser */
/* global browser, chrome, fetch, Wappalyzer */

/** global: browser */
/** global: chrome */
/** global: fetch */
/** global: Wappalyzer */

const wappalyzer = new Wappalyzer();

const tabCache = {};
let categoryOrder = [];
const options = {};
const robotsTxtQueue = {};

browser.tabs.onRemoved.addListener((tabId) => {
  tabCache[tabId] = null;
});

/**
 * Get a value from localStorage
 */
function getOption(name, defaultValue = null) {
  return new Promise((resolve, reject) => {
    const callback = (item) => {
      options[name] = item[name] ? item[name] : defaultValue;

      resolve(options[name]);
    };

    browser.storage.local.get(name)
      .then(callback)
      .catch((error) => {
        wappalyzer.log(error, 'driver', 'error');

        reject();
      });
  });
}

/**
 * Set a value in localStorage
 */
function setOption(name, value) {
  const option = {};

  option[name] = value;

  browser.storage.local.set(option);

  options[name] = value;
}

/**
 * Open a tab
 */
function openTab(args) {
  browser.tabs.create({
    url: args.url,
    active: args.background === undefined || !args.background,
  });
}

/**
 * Make a POST request
 */
function post(url, body) {
  fetch(url, {
    method: 'POST',
    body: JSON.stringify(body),
  })
    .then(response => wappalyzer.log(`POST ${url}: ${response.status}`, 'driver'))
    .catch(error => wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error'));
}

// Prod config
// fetch('https://raw.githubusercontent.com/AliasIO/Wappalyzer/master/src/apps.json')
//   .then(response => response.json())
//   .then((json) => {
//     wappalyzer.apps = json.apps;
//     wappalyzer.categories = json.categories;
    
//     fetch('https://raw.githubusercontent.com/philkrie/ampbench/master/readiness-tool/extended_apps.json')
//       .then(response_ext => response_ext.json())
//       .then((json_ext) => {
// Prod config

// We always fetch directly from Wappalyzer's raw apps.json content to get the latest stuff
fetch('https://raw.githubusercontent.com/AliasIO/Wappalyzer/master/src/apps.json')
  .then(response => response.json())
  .then((json) => {
    wappalyzer.apps = json.apps;
    wappalyzer.categories = json.categories;
    
    fetch('../extended_apps.json')
      .then(response_ext => response_ext.json())
      .then((json_ext) => {

        wappalyzer.apps = Object.assign({}, wappalyzer.apps, json_ext.apps);

        wappalyzer.parseJsPatterns();

        categoryOrder = Object.keys(wappalyzer.categories)
          .map(categoryId => parseInt(categoryId, 10))
          .sort((a, b) => wappalyzer.categories[a].priority - wappalyzer.categories[b].priority);

        wappalyzer.supported_apps = json_ext.supported;
        wappalyzer.incompatible_apps = json_ext.incompatible;
        wappalyzer.conv_cat_tooltips = json_ext.conversionCategoryTooltips;
        wappalyzer.incom_cat_tooltips = json_ext.incompatibleCategoryTooltips;
        wappalyzer.tech_tooltips = json_ext.technologyTooltips;        
        wappalyzer.convertable_apps = json_ext.conversionPatterns;  
		    wappalyzer.tracked_urls = {};      
    })
    .catch(error => wappalyzer.log(`GET extended_apps.json: ${error}`, 'driver', 'error'));
  })
  .catch(error => wappalyzer.log(`GET apps.json: ${error}`, 'driver', 'error'));

// Version check
const { version } = browser.runtime.getManifest();

getOption('version')
  .then((previousVersion) => {
    if (previousVersion === null) {
      
    } else if (version !== previousVersion) {
      getOption('upgradeMessage', true)
        .then((upgradeMessage) => {
          if (upgradeMessage) {
           
          }
        });
    }

    setOption('version', version);
  });

getOption('dynamicIcon', false);

<commit_msg>added local fallback in case of apps.json retrieval failure<commit_after>/**
 * WebExtension driver
 */

/* eslint-env browser */
/* global browser, chrome, fetch, Wappalyzer */

/** global: browser */
/** global: chrome */
/** global: fetch */
/** global: Wappalyzer */

const wappalyzer = new Wappalyzer();

const tabCache = {};
let categoryOrder = [];
const options = {};
const robotsTxtQueue = {};

browser.tabs.onRemoved.addListener((tabId) => {
  tabCache[tabId] = null;
});

/**
 * Get a value from localStorage
 */
function getOption(name, defaultValue = null) {
  return new Promise((resolve, reject) => {
    const callback = (item) => {
      options[name] = item[name] ? item[name] : defaultValue;

      resolve(options[name]);
    };

    browser.storage.local.get(name)
      .then(callback)
      .catch((error) => {
        wappalyzer.log(error, 'driver', 'error');

        reject();
      });
  });
}

/**
 * Set a value in localStorage
 */
function setOption(name, value) {
  const option = {};

  option[name] = value;

  browser.storage.local.set(option);

  options[name] = value;
}

/**
 * Open a tab
 */
function openTab(args) {
  browser.tabs.create({
    url: args.url,
    active: args.background === undefined || !args.background,
  });
}

/**
 * Make a POST request
 */
function post(url, body) {
  fetch(url, {
    method: 'POST',
    body: JSON.stringify(body),
  })
    .then(response => wappalyzer.log(`POST ${url}: ${response.status}`, 'driver'))
    .catch(error => wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error'));
}

function readApps(json) {
  wappalyzer.apps = json.apps;
  wappalyzer.categories = json.categories;
  
  fetch('../extended_apps.json')
    .then(response_ext => response_ext.json())
    .then((json_ext) => {

      wappalyzer.apps = Object.assign({}, wappalyzer.apps, json_ext.apps);

      wappalyzer.parseJsPatterns();

      categoryOrder = Object.keys(wappalyzer.categories)
        .map(categoryId => parseInt(categoryId, 10))
        .sort((a, b) => wappalyzer.categories[a].priority - wappalyzer.categories[b].priority);

      wappalyzer.supported_apps = json_ext.supported;
      wappalyzer.incompatible_apps = json_ext.incompatible;
      wappalyzer.conv_cat_tooltips = json_ext.conversionCategoryTooltips;
      wappalyzer.incom_cat_tooltips = json_ext.incompatibleCategoryTooltips;
      wappalyzer.tech_tooltips = json_ext.technologyTooltips;        
      wappalyzer.convertable_apps = json_ext.conversionPatterns;  
      wappalyzer.tracked_urls = {};      
  })
  .catch(error => wappalyzer.log(`GET extended_apps.json: ${error}`, 'driver', 'error'));
}

// We always fetch directly from Wappalyzer's raw apps.json content to get the latest content
fetch('https://raw.githubusercontent.com/AliasIO/Wappalyzer/master/src/apps.json')
  .then((response) => {
    //If we don't get a 200, we access the local version
    if(!response.ok) {
      console.log("Failed to retrive apps.json, falling back to local version");
      fetch('../apps.json')
      .then(response => response.json())
      .then(json => readApps(json))
    } else {
      response.json()
      .then(json => readApps(json))
    }
  })
  .catch(error => wappalyzer.log(`GET apps.json: ${error}`, 'driver', 'error'));

// Version check
const { version } = browser.runtime.getManifest();

getOption('version')
  .then((previousVersion) => {
    if (previousVersion === null) {
      
    } else if (version !== previousVersion) {
      getOption('upgradeMessage', true)
        .then((upgradeMessage) => {
          if (upgradeMessage) {
           
          }
        });
    }

    setOption('version', version);
  });

getOption('dynamicIcon', false);


{'fraction_non_alphanumeric': 0.16307814992025518, 'fraction_numerical': 0.0009303561935140882, 'mean_word_length': 3.1712860310421287, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 41, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5746134', 'n_tokens_mistral': 2465, 'n_tokens_neox': 2405, 'n_words': 452}