<commit_before>(function() {
  const Cc = Components.classes;
  const Ci = Components.interfaces;

  const ScriptableInputStream = Cc["@mozilla.org/scriptableinputstream;1"];
  const nsIScriptableInputStream = Ci.nsIScriptableInputStream;

  const ios = Cc["@mozilla.org/network/io-service;1"].getService(Ci.nsIIOService)

  const DiffModule = FBTest.FirebugWindow.Firebug.DiffModule,
        FireDiff = FBTest.FirebugWindow.FireDiff,
        FBTrace = FBTest.FirebugWindow.FBTrace;

  FBTestFireDiff = {
    isFirefox30: function() {
      return navigator.userAgent.indexOf("Firefox/3.0") >= 0;
    },

    enableDiffPanel: function(callback) {
        FBTestFirebug.setPanelState(FW.Firebug.DiffModule, "firediff", callback, true);
    },
    disableDiffPanel: function(callback) {
        FBTestFirebug.setPanelState(FW.Firebug.DiffModule, "firediff", callback, false);
    },

    compareChangeList: function(expected, actual, msg) {
      FBTest.FirebugWindow.FBTrace.sysout(msg);
      FBTest.compare(expected.length, actual.length, msg + " length");
      for (var i = 0; i < actual.length; i++) {
        this.compareObjects(
            expected[i],
            actual[i],
            msg + " value " + i,
            {date: true});
      }
    },
    
    compareObjects: function(expected, actual, msg, excluded) {
      var tested = {};
      excluded = excluded || {};
      
      if (expected) {
        for (var i in expected) {
          if (i in excluded || (typeof expected[i] == "function"))    continue;
          
          tested[i] = true;
          
          if (expected[i] && expected[i].isEqualNode) {
            var equal = actual[i] && expected[i].isEqualNode(actual[i]);
            FBTest.ok(equal, msg + " " + i);
            if (!equal) {
              FBTest.sysout(msg + " " + i + " expected", expected[i]);
              FBTest.sysout(msg + " " + i + " actual", actual[i]);
            }
          } else if (expected[i] && expected[i].equals) {
            var equal = actual[i] && expected[i].equals(actual[i]);
            FBTest.ok(equal, msg + " " + i);
            if (!equal) {
              FBTest.sysout(msg + " " + i + " expected", expected[i]);
              FBTest.sysout(msg + " " + i + " actual", actual[i]);
            }
          } else {
            FBTest.compare(expected[i], actual && actual[i], msg + " " + i);
          }
        }
      }
      if (actual) {
        for (var i in actual) {
          // These are all errors, but run through compare anyway
          if (!(i in excluded) && !tested[i] && !(typeof actual[i] == "function")) {
            FBTest.compare(expected && expected[i], actual[i], msg + " " + i);
          }
        }
      }
    },
    
    executeModuleTests: function(tests, win, finalizer) {
      var running = true, setup = false;
      var curTest = -1;
      var changeNum = 0;
      var timeout;
      
      var listener = {
        onDiffChange: cleanupWrapper(function(change) {
          if (timeout) {
            clearTimeout(timeout);  timeout = undefined;
          }
          if (!running || setup) {
            return;
          }
          
          if (tests[curTest].verify) {
            tests[curTest].verify(win, changeNum, change);
          }
          
          changeNum++;
          if (tests[curTest].eventCount == changeNum) {
            tests[curTest].verified = true;
            setCatchTimeout(executeTest, 0);
          } else if (tests[curTest].eventCount < changeNum) {
            FBTest.compare(tests[curTest].eventCount, changeNum, "Unexpected number of events");
          } else {
            timeout = setCatchTimeout(cancelTest, 5000);<commit_msg>Additional logging info<commit_after>(function() {
  const Cc = Components.classes;
  const Ci = Components.interfaces;

  const ScriptableInputStream = Cc["@mozilla.org/scriptableinputstream;1"];
  const nsIScriptableInputStream = Ci.nsIScriptableInputStream;

  const ios = Cc["@mozilla.org/network/io-service;1"].getService(Ci.nsIIOService)

  const DiffModule = FBTest.FirebugWindow.Firebug.DiffModule,
        FireDiff = FBTest.FirebugWindow.FireDiff,
        FBTrace = FBTest.FirebugWindow.FBTrace;

  FBTestFireDiff = {
    isFirefox30: function() {
      return navigator.userAgent.indexOf("Firefox/3.0") >= 0;
    },

    enableDiffPanel: function(callback) {
        FBTestFirebug.setPanelState(FW.Firebug.DiffModule, "firediff", callback, true);
    },
    disableDiffPanel: function(callback) {
        FBTestFirebug.setPanelState(FW.Firebug.DiffModule, "firediff", callback, false);
    },

    compareChangeList: function(expected, actual, msg) {
      FBTest.FirebugWindow.FBTrace.sysout(msg);
      FBTest.compare(expected.length, actual.length, msg + " length");
      for (var i = 0; i < actual.length; i++) {
        this.compareObjects(
            expected[i],
            actual[i],
            msg + " value " + i,
            {date: true});
      }
    },
    
    compareObjects: function(expected, actual, msg, excluded) {
      var tested = {};
      excluded = excluded || {};
      
      if (expected) {
        for (var i in expected) {
          if (i in excluded || (typeof expected[i] == "function"))    continue;
          
          tested[i] = true;
          
          if (expected[i] && expected[i].isEqualNode) {
            var equal = actual[i] && expected[i].isEqualNode(actual[i]);
            FBTest.ok(equal, msg + " " + i);
            if (!equal) {
              FBTest.sysout(msg + " " + i + " expected", expected[i]);
              FBTest.sysout(msg + " " + i + " actual", actual[i]);
            }
          } else if (expected[i] && expected[i].equals) {
            var equal = actual[i] && expected[i].equals(actual[i]);
            FBTest.ok(equal, msg + " " + i);
            if (!equal) {
              FBTest.sysout(msg + " " + i + " expected", expected[i]);
              FBTest.sysout(msg + " " + i + " actual", actual[i]);
            }
          } else {
            FBTest.compare(expected[i], actual && actual[i], msg + " " + i);
          }
        }
      }
      if (actual) {
        for (var i in actual) {
          // These are all errors, but run through compare anyway
          if (!(i in excluded) && !tested[i] && !(typeof actual[i] == "function")) {
            FBTest.compare(expected && expected[i], actual[i], msg + " " + i);
          }
        }
      }
    },
    
    executeModuleTests: function(tests, win, finalizer) {
      var running = true, setup = false;
      var curTest = -1;
      var changeNum = 0;
      var timeout;
      
      var listener = {
        onDiffChange: cleanupWrapper(function(change) {
          if (timeout) {
            clearTimeout(timeout);  timeout = undefined;
          }
          if (!running || setup) {
            return;
          }
          
          if (tests[curTest].verify) {
            tests[curTest].verify(win, changeNum, change);
          }
          
          changeNum++;
          if (tests[curTest].eventCount == changeNum) {
            tests[curTest].verified = true;
            setCatchTimeout(executeTest, 0);
          } else if (tests[curTest].eventCount < changeNum) {
            FBTest.compare(tests[curTest].eventCount, changeNum, "Unexpected number of events " + JSON.stringify(change));
          } else {
            timeout = setCatchTimeout(cancelTest, 5000);
{'fraction_non_alphanumeric': 0.13711201079622132, 'fraction_numerical': 0.004048582995951417, 'mean_word_length': 2.008932196508323, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4475448', 'n_tokens_mistral': 2235, 'n_tokens_neox': 2159, 'n_words': 450}