<commit_before>// This function cleans all the elements that could leak user data
// before sending to the server. It also removes any attributes that
// could have personal data so we end up with a clean dom tree.
const selectors = [
  'video',
  'input',
  'button',
  'iframe',
  'a[href=""]',
  '.accessible_elem',
  '.uiLikePagebutton',
  '.uiPopOver',
  '.uiCloseButton',
  '.uiChevronSelectorButton'
].join(', ');

const TIMELINE_SELECTOR = '.fbUserContent, .fbUserPost, ._5pcr';
const SIDEBAR_SELECTOR = '.ego_unit';

const cleanAd = (html) => {
  let node = document.createElement("div");
  node.innerHTML = html;

  // We're not saving video ads for now, we don't need buttons, hidden forms,
  // or like links
  Array.from(node.querySelectorAll(selectors))
    .forEach((i) => i.parentElement.removeChild(i));

  // remove attributes
  const killAttrs = (node) => {
    Array.from(node.attributes).forEach(attr => {
      if(attr.name !== "id" &&
         attr.name !== "class" &&
         attr.name !== "src" &&
         attr.name !== "href")
        node.removeAttribute(attr.name);
      // remove tracking get variables from links and l.facebook.com
      if(attr.name === "href") {
        try {
          let url = new URL(attr.value);
          if(url.host === 'l.facebook.com')
            url = new URL(new URLSearchParams(url.search).get("u"));
          if(url.origin && url.pathname) {
            node.setAttribute(attr.name, url.origin + url.pathname);
          } else {
            node.removeAttribute(attr.name);
          }
        } catch(e) {
          node.removeAttribute(attr.name);
        }
      }

    });
    // walk through the tree.
    Array.from(node.children).forEach(killAttrs);
  };
  Array.from(node.children).forEach(killAttrs);
  return node.innerHTML.replace(/&amp;/g, '&');
};

const checkSponsor = (node) => {
  return Array.from(node.querySelectorAll("a")).some((a) => {
    return ['Sponsored','Gesponsert'].some((sponsor) => a.textContent === sponsor);
  });
};

// We have to do this because content scripts can't read window variables
const grabVariable = (fn, args)  => {
  let script = document.createElement("script");
  script.textContent = 'localStorage.setItem("pageVariable", (' + fn + ').apply(this, ' + JSON.stringify(args) + '));';
  script.setAttribute('id', 'pageVariable');
  (document.head||document.documentElement).appendChild(script);
  script.remove();
  return localStorage.getItem("pageVariable");
};

// Getting the targeting routine. We use XMLHttpRequest here because facebook
// bans fetch.
let targetingBlocked = false;
let targetingCache = new Map();
const getTargeting = (ad) => {
  if(ad.targeting) {
    if(targetingCache.has(ad.targeting)) return {
      ...ad,
      targeting: targetingCache.get(ad.targeting)
    };
    const url = ad.targeting;
    delete ad.targeting;
    if(targetingBlocked) return ad;
    return new Promise((resolve, reject) => {
      let req = new XMLHttpRequest();

      req.onreadystatechange = function() {
        if(req.readyState === 4) {
          try {
            const targeting = JSON.parse(req.response.replace('for (;;);', ''))["jsmods"]["markup"][0][1]["__html"];
            if(!targeting) {
              return resolve(ad);
            }
            targetingCache.set(url, targeting);
            resolve({
              ...ad,
              targeting
            });
          } catch(e) {
            targetingBlocked = true;
            setTimeout(() => targetingBlocked = false, 15 * 60 * 100);
            reject(ad);
          }
        }
      };

      let built = grabVariable((url) => {
        let parsed = new (window.require('URI'))(url);
        localStorage.setItem('url', url);
        let req = new window.AsyncRequest()
          .setURI(url)
          .setData(parsed)
          .setMethod('GET')
          .setRelativeTo(document.body)
          .setNectarModuleDataSafe(document.body)
          .setReadOnly(true);<commit_msg>capture all the ads<commit_after>// This function cleans all the elements that could leak user data
// before sending to the server. It also removes any attributes that
// could have personal data so we end up with a clean dom tree.
const selectors = [
  'video',
  'input',
  'button',
  'iframe',
  'a[href=""]',
  '.accessible_elem',
  '.uiLikePagebutton',
  '.uiPopOver',
  '.uiCloseButton',
  '.uiChevronSelectorButton'
].join(', ');

const TIMELINE_SELECTOR = '.fbUserContent, .fbUserPost, ._5pcr';
const SIDEBAR_SELECTOR = '.ego_unit';

const cleanAd = (html) => {
  let node = document.createElement("div");
  node.innerHTML = html;

  // We're not saving video ads for now, we don't need buttons, hidden forms,
  // or like links
  Array.from(node.querySelectorAll(selectors))
    .forEach((i) => i.parentElement.removeChild(i));

  // remove attributes
  const killAttrs = (node) => {
    Array.from(node.attributes).forEach(attr => {
      if(attr.name !== "id" &&
         attr.name !== "class" &&
         attr.name !== "src" &&
         attr.name !== "href")
        node.removeAttribute(attr.name);
      // remove tracking get variables from links and l.facebook.com
      if(attr.name === "href") {
        try {
          let url = new URL(attr.value);
          if(url.host === 'l.facebook.com')
            url = new URL(new URLSearchParams(url.search).get("u"));
          if(url.origin && url.pathname) {
            node.setAttribute(attr.name, url.origin + url.pathname);
          } else {
            node.removeAttribute(attr.name);
          }
        } catch(e) {
          node.removeAttribute(attr.name);
        }
      }

    });
    // walk through the tree.
    Array.from(node.children).forEach(killAttrs);
  };
  Array.from(node.children).forEach(killAttrs);
  return node.innerHTML.replace(/&amp;/g, '&');
};

const checkSponsor = (node) => {
  return Array.from(node.querySelectorAll("a")).some((a) => {
    return ['Sponsored','Gesponsert'].some((sponsor) => a.textContent === sponsor);
  });
};

// We have to do this because content scripts can't read window variables
const grabVariable = (fn, args)  => {
  let script = document.createElement("script");
  script.textContent = 'localStorage.setItem("pageVariable", (' + fn + ').apply(this, ' + JSON.stringify(args) + '));';
  script.setAttribute('id', 'pageVariable');
  (document.head||document.documentElement).appendChild(script);
  script.remove();
  return localStorage.getItem("pageVariable");
};

// Getting the targeting routine. We use XMLHttpRequest here because facebook
// bans fetch.
let targetingBlocked = false;
let targetingCache = new Map();
const getTargeting = (ad) => {
  if(ad.targeting) {
    if(targetingCache.has(ad.targeting)) return {
      ...ad,
      targeting: targetingCache.get(ad.targeting)
    };
    const url = ad.targeting;
    delete ad.targeting;
    if(targetingBlocked) return ad;
    return new Promise((resolve) => {
      let req = new XMLHttpRequest();

      req.onreadystatechange = function() {
        if(req.readyState === 4) {
          try {
            const targeting = JSON.parse(req.response.replace('for (;;);', ''))["jsmods"]["markup"][0][1]["__html"];
            if(!targeting) {
              return resolve(ad);
            }
            targetingCache.set(url, targeting);
            resolve({
              ...ad,
              targeting
            });
          } catch(e) {
            targetingBlocked = true;
            setTimeout(() => targetingBlocked = false, 15 * 60 * 100);
            resolve(ad);
          }
        }
      };

      let built = grabVariable((url) => {
        let parsed = new (window.require('URI'))(url);
        localStorage.setItem('url', url);
        let req = new window.AsyncRequest()
          .setURI(url)
          .setData(parsed)
          .setMethod('GET')
          .setRelativeTo(document.body)
          .setNectarModuleDataSafe(document.body)
          .setReadOnly(true);
{'fraction_non_alphanumeric': 0.15383649378374983, 'fraction_numerical': 0.002762777847544895, 'mean_word_length': 2.560125167635226, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '81637', 'n_tokens_mistral': 2418, 'n_tokens_neox': 2373, 'n_words': 533}