<commit_before>(function() {
  // TODO(sissel): Write something that will use history.pushState and fall back
  // to document.location.hash madness.

  var logstash = {
    params: { 
      offset: 0,
      count: 50,
    },

    search: function(query, options) {
      if (query == undefined || query == "") {
        return;
      }

      /* Default options */
      if (typeof(options) == 'undefined') {
        options = { graph: true };
      }

      var display_query = query.replace("<", "&lt;").replace(">", "&gt;")
      $("#querystatus, #results h1").html("Loading query '" + display_query + "' (offset:" + logstash.params.offset + ", count:" + logstash.params.count + ") <img class='throbber' src='/media/construction.gif'>")
      //console.log(logstash.params)
      logstash.params.q = query;
      document.location.hash = escape(JSON.stringify(logstash.params));

      /* Load the search results */
      $("#results").load("/api/search?format=html", logstash.params);

      if (options.graph != false) {
        /* Load the default histogram graph */
        logstash.params.interval = 3600000; /* 1 hour, default */
        logstash.histogram();
      } /* if options.graph != false */
      $("#query").val(logstash.params.q);
    }, /* search */

    histogram: function(tries) {
      if (typeof(tries) == 'undefined') {
        tries = 7;
      }

      /* GeoCities mode on the graph while waiting ...
       * This won't likely survive 1.0, but it's fun for now... */
      $("#visual").html("<center><img src='/media/truckconstruction.gif'><center>");

      jQuery.getJSON("/api/histogram", logstash.params, function(histogram, text, jqxhr) {
        /* Load the data into the graph */
        flot_data = [];
        // histogram is an array of { "key": ..., "count": ... }
        for (var i in histogram) {
          flot_data.push([parseInt(histogram[i]["key"]), histogram[i]["count"]])
        }
        //console.log(histogram);

        /* Try to be intelligent about how we choose the histogram interval.
         * If there are too few data points, try a smaller interval.
         * If there are too many data points, try a larger interval.
         * Give up after a few tries and go with the last result. 
         *
         * This queries the backend several times, but should be reasonably
         * speedy as this behaves roughly as a binary search. */
        if (flot_data.length < 6 && tries > 0) {
          //console.log("Histogram bucket " + logstash.params.interval + " has only " + flot_data.length + " data points, trying smaller...");
          logstash.params.interval /= 2;
          if (logstash.params.interval < 1000) {
            tries = 0; /* stop trying, too small... */
            logstash.plot(flot_data, logstash.params.interval);
            return;
          }
          logstash.histogram(tries - 1);
        } else if (flot_data.length > 50 && tries > 0) {
          //console.log("Histogram bucket " + logstash.params.interval + " too many (" + flot_data.length + ") data points, trying larger interval...");
          logstash.params.interval *= 2;<commit_msg>- Don't bother retrying if we got 0 results for our histogram.<commit_after>(function() {
  // TODO(sissel): Write something that will use history.pushState and fall back
  // to document.location.hash madness.

  var logstash = {
    params: { 
      offset: 0,
      count: 50,
    },

    search: function(query, options) {
      if (query == undefined || query == "") {
        return;
      }

      /* Default options */
      if (typeof(options) == 'undefined') {
        options = { graph: true };
      }

      var display_query = query.replace("<", "&lt;").replace(">", "&gt;")
      $("#querystatus, #results h1").html("Loading query '" + display_query + "' (offset:" + logstash.params.offset + ", count:" + logstash.params.count + ") <img class='throbber' src='/media/construction.gif'>")
      //console.log(logstash.params)
      logstash.params.q = query;
      document.location.hash = escape(JSON.stringify(logstash.params));

      /* Load the search results */
      $("#results").load("/api/search?format=html", logstash.params);

      if (options.graph != false) {
        /* Load the default histogram graph */
        logstash.params.interval = 3600000; /* 1 hour, default */
        logstash.histogram();
      } /* if options.graph != false */
      $("#query").val(logstash.params.q);
    }, /* search */

    histogram: function(tries) {
      if (typeof(tries) == 'undefined') {
        tries = 7;
      }

      /* GeoCities mode on the graph while waiting ...
       * This won't likely survive 1.0, but it's fun for now... */
      $("#visual").html("<center><img src='/media/truckconstruction.gif'><center>");

      jQuery.getJSON("/api/histogram", logstash.params, function(histogram, text, jqxhr) {
        /* Load the data into the graph */
        flot_data = [];
        // histogram is an array of { "key": ..., "count": ... }
        for (var i in histogram) {
          flot_data.push([parseInt(histogram[i]["key"]), histogram[i]["count"]])
        }
        //console.log(histogram);

        /* Try to be intelligent about how we choose the histogram interval.
         * If there are too few data points, try a smaller interval.
         * If there are too many data points, try a larger interval.
         * Give up after a few tries and go with the last result. 
         *
         * This queries the backend several times, but should be reasonably
         * speedy as this behaves roughly as a binary search. */
        if (flot_data.length < 6 && flot_data.length > 0 && tries > 0) {
          //console.log("Histogram bucket " + logstash.params.interval + " has only " + flot_data.length + " data points, trying smaller...");
          logstash.params.interval /= 2;
          if (logstash.params.interval < 1000) {
            tries = 0; /* stop trying, too small... */
            logstash.plot(flot_data, logstash.params.interval);
            return;
          }
          logstash.histogram(tries - 1);
        } else if (flot_data.length > 50 && tries > 0) {
          //console.log("Histogram bucket " + logstash.params.interval + " too many (" + flot_data.length + ") data points, trying larger interval...");
          logstash.params.interval *= 2;
{'fraction_non_alphanumeric': 0.15010237832729564, 'fraction_numerical': 0.009135296897149158, 'mean_word_length': 2.2816537467700257, 'pattern_counts': {'":': 4, '<': 17, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2477622', 'n_tokens_mistral': 1919, 'n_tokens_neox': 1826, 'n_words': 514}