<commit_before>"use strict";

const MAX_ITEMS = 10;

const downloads = new Map();
const currentRequests = new Map();

const defaultOptions = {
  doubleQuotes: false,
  excludeHeaders: "Accept-Encoding Connection",
  command: "curl",
  curlOptions: "",
  wgetOptions: "",
  aria2Options: ""
};

function getOptions() {
  return new Promise(resolve => {
    browser.storage.local.get().then(res => {
      res = Object.assign({}, defaultOptions, res);
      resolve(res);
    });
  });
}

function setOptions(values) {
  new Promise(resolve => {
    browser.storage.local.set(values).then(() =>
      getOptions().then(c => {
        resolve(c);
      })
    );
  });
}

function resetOptions() {
  new Promise(resolve => {
    browser.storage.local.clear().then(() =>
      getOptions().then(c => {
        resolve(c);
      })
    );
  });
}

function clear() {
  downloads.clear();
}

function getDownloadList() {
  const list = [];
  for (let [reqId, req] of downloads)
    list.push({
      id: reqId,
      url: req.url,
      filename: req.filename,
      size: req.size
    });

  return list;
}

function generateCommand(reqId, options) {
  const request = downloads.get(reqId);
  if (!request) throw new Error("Request not found");

  let excludeHeaders = options.excludeHeaders
    .split(" ")
    .map(h => h.toLowerCase());

  let headers = request.headers.filter(
    h => excludeHeaders.indexOf(h.name.toLowerCase()) === -1
  );

  const cmd = window[options.command](
    request.url,
    request.method,
    headers,
    request.payload,
    request.filename,
    options
  );

  return cmd;
}

function handleMessage(msg) {
  const name = msg[0];
  const args = msg.slice(1);

  if (name === "getOptions") return getOptions();
  else if (name === "setOptions") return setOptions(...args);
  else if (name === "resetOptions") return resetOptions();
  else if (name === "getDownloadList")
    return new Promise(resolve => resolve(getDownloadList()));
  else if (name === "clear") return clear(...args);
  else if (name === "generateCommand")
    return new Promise(resolve => {
      try {
        resolve(generateCommand(...args));
      } catch (err) {
        resolve(err.message);
      }
    });
}

browser.runtime.onMessage.addListener(handleMessage);

function onBeforeRequest(details) {
  if (
    (details.type === "main_frame" || details.type === "sub_frame") &&
    details.tabId >= 0
  ) {
    const now = Date.now();

    // Just in case of a leak
    currentRequests.forEach((req, reqId) => {
      if (req.timestamp + 10000 < now) currentRequests.delete(reqId);
    });

    const req = {
      id: details.requestId,
      method: details.method,
      url: details.url,
      timestamp: now,
      payload: details.requestBody
    };
    currentRequests.set(details.requestId, req);
  }
}

function onSendHeaders(details) {
  const req = currentRequests.get(details.requestId);
  if (req) {
    req.headers = details.requestHeaders;
  } else if (
    (details.type === "main_frame" || details.type === "sub_frame") &&
    details.tabId >= 0 &&
    details.method === "GET"
  ) {
    // Firefox 52 (ESR) doesn't call "onBeforeRequest" because requestBody
    // property isn't supported
    const now = Date.now();

    // Just in case of a leak
    currentRequests.forEach((r, reqId) => {
      if (r.timestamp + 10000 < now) currentRequests.delete(reqId);
    });

    currentRequests.set(details.requestId, {
      id: details.requestId,
      method: details.method,
      url: details.url,
      timestamp: now,
      headers: details.requestHeaders
    });
  }
}

function onResponseStarted(details) {
  const request = currentRequests.get(details.requestId);

  if (!request) return;

  currentRequests.delete(details.requestId);

  if (details.statusCode !== 200 || details.fromCache) return;

  let contentType, contentDisposition;

  for (let header of details.responseHeaders) {
    let headerName = header.name.toLowerCase();
    if (headerName === "content-type") {
      contentType = header.value.toLowerCase();
    } else if (headerName === "content-disposition") {
      contentDisposition = header.value.toLowerCase();
      request.filename = window.getFilenameFromContentDisposition(header.value);
    } else if (headerName === "content-length") {
      request.size = +header.value;
    }
  }

  if (!contentDisposition || !contentDisposition.startsWith("attachment"))
    if (
      contentType.startsWith("text/html") ||
      contentType.startsWith("text/plain") ||
      contentType.startsWith("image/") ||
      contentType.startsWith("application/xml")
    )
      return;

  if (!request.filename)
    request.filename = window.getFilenameFromUrl(request.url);

  downloads.set(details.requestId, request);

  browser.browserAction.getBadgeText({}).then(txt => {
    browser.browserAction.setBadgeText({ text: `${+txt + 1}` });
  });

  if (downloads.size > MAX_ITEMS) {
    let keys = Array.from(downloads.keys());
    keys.slice(0, keys.length - MAX_ITEMS).forEach(k => downloads.delete(k));
  }
}

function onBeforeRedirect() {
  // Need to listen to this event otherwise the new request will include
  // the old URL. This is possibly a bug.
}

function onErrorOccurred(details) {
  currentRequests.delete(details.requestId);
}<commit_msg>Stop erroneously tracking xhtml pages as downloads<commit_after>"use strict";

const MAX_ITEMS = 10;

const downloads = new Map();
const currentRequests = new Map();

const defaultOptions = {
  doubleQuotes: false,
  excludeHeaders: "Accept-Encoding Connection",
  command: "curl",
  curlOptions: "",
  wgetOptions: "",
  aria2Options: ""
};

function getOptions() {
  return new Promise(resolve => {
    browser.storage.local.get().then(res => {
      res = Object.assign({}, defaultOptions, res);
      resolve(res);
    });
  });
}

function setOptions(values) {
  new Promise(resolve => {
    browser.storage.local.set(values).then(() =>
      getOptions().then(c => {
        resolve(c);
      })
    );
  });
}

function resetOptions() {
  new Promise(resolve => {
    browser.storage.local.clear().then(() =>
      getOptions().then(c => {
        resolve(c);
      })
    );
  });
}

function clear() {
  downloads.clear();
}

function getDownloadList() {
  const list = [];
  for (let [reqId, req] of downloads)
    list.push({
      id: reqId,
      url: req.url,
      filename: req.filename,
      size: req.size
    });

  return list;
}

function generateCommand(reqId, options) {
  const request = downloads.get(reqId);
  if (!request) throw new Error("Request not found");

  let excludeHeaders = options.excludeHeaders
    .split(" ")
    .map(h => h.toLowerCase());

  let headers = request.headers.filter(
    h => excludeHeaders.indexOf(h.name.toLowerCase()) === -1
  );

  const cmd = window[options.command](
    request.url,
    request.method,
    headers,
    request.payload,
    request.filename,
    options
  );

  return cmd;
}

function handleMessage(msg) {
  const name = msg[0];
  const args = msg.slice(1);

  if (name === "getOptions") return getOptions();
  else if (name === "setOptions") return setOptions(...args);
  else if (name === "resetOptions") return resetOptions();
  else if (name === "getDownloadList")
    return new Promise(resolve => resolve(getDownloadList()));
  else if (name === "clear") return clear(...args);
  else if (name === "generateCommand")
    return new Promise(resolve => {
      try {
        resolve(generateCommand(...args));
      } catch (err) {
        resolve(err.message);
      }
    });
}

browser.runtime.onMessage.addListener(handleMessage);

function onBeforeRequest(details) {
  if (
    (details.type === "main_frame" || details.type === "sub_frame") &&
    details.tabId >= 0
  ) {
    const now = Date.now();

    // Just in case of a leak
    currentRequests.forEach((req, reqId) => {
      if (req.timestamp + 10000 < now) currentRequests.delete(reqId);
    });

    const req = {
      id: details.requestId,
      method: details.method,
      url: details.url,
      timestamp: now,
      payload: details.requestBody
    };
    currentRequests.set(details.requestId, req);
  }
}

function onSendHeaders(details) {
  const req = currentRequests.get(details.requestId);
  if (req) {
    req.headers = details.requestHeaders;
  } else if (
    (details.type === "main_frame" || details.type === "sub_frame") &&
    details.tabId >= 0 &&
    details.method === "GET"
  ) {
    // Firefox 52 (ESR) doesn't call "onBeforeRequest" because requestBody
    // property isn't supported
    const now = Date.now();

    // Just in case of a leak
    currentRequests.forEach((r, reqId) => {
      if (r.timestamp + 10000 < now) currentRequests.delete(reqId);
    });

    currentRequests.set(details.requestId, {
      id: details.requestId,
      method: details.method,
      url: details.url,
      timestamp: now,
      headers: details.requestHeaders
    });
  }
}

function onResponseStarted(details) {
  const request = currentRequests.get(details.requestId);

  if (!request) return;

  currentRequests.delete(details.requestId);

  if (details.statusCode !== 200 || details.fromCache) return;

  let contentType, contentDisposition;

  for (let header of details.responseHeaders) {
    let headerName = header.name.toLowerCase();
    if (headerName === "content-type") {
      contentType = header.value.toLowerCase();
    } else if (headerName === "content-disposition") {
      contentDisposition = header.value.toLowerCase();
      request.filename = window.getFilenameFromContentDisposition(header.value);
    } else if (headerName === "content-length") {
      request.size = +header.value;
    }
  }

  if (!contentDisposition || !contentDisposition.startsWith("attachment"))
    if (
      contentType.startsWith("text/html") ||
      contentType.startsWith("text/plain") ||
      contentType.startsWith("image/") ||
      contentType.startsWith("application/xhtml") ||
      contentType.startsWith("application/xml")
    )
      return;

  if (!request.filename)
    request.filename = window.getFilenameFromUrl(request.url);

  downloads.set(details.requestId, request);

  browser.browserAction.getBadgeText({}).then(txt => {
    browser.browserAction.setBadgeText({ text: `${+txt + 1}` });
  });

  if (downloads.size > MAX_ITEMS) {
    let keys = Array.from(downloads.keys());
    keys.slice(0, keys.length - MAX_ITEMS).forEach(k => downloads.delete(k));
  }
}

function onBeforeRedirect() {
  // Need to listen to this event otherwise the new request will include
  // the old URL. This is possibly a bug.
}

function onErrorOccurred(details) {
  currentRequests.delete(details.requestId);
}
{'fraction_non_alphanumeric': 0.14905554516551336, 'fraction_numerical': 0.0046755189826070695, 'mean_word_length': 3.1957630443311102, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 23, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4065129', 'n_tokens_mistral': 3299, 'n_tokens_neox': 3250, 'n_words': 721}