<commit_before><!doctype html>
<html lang="en">
  <head>
    <!-- Required meta tags -->
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">

    <!-- Bootstrap CSS -->
    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/css/bootstrap.min.css" integrity="<KEY>" crossorigin="anonymous">

    <title>viadown</title>
  </head>
  <body>
      <main id="app">
          <div class="container">
              <h1>viadown administration</h1>
              <h2>Stats</h2>
              <div class="row">
                  <div class="row col-md-auto">
                      <table id="stats" class="table table-borderless">
                          <tbody>
                              <tr>
                                  <td>Files</td><td>{{ cache.stats.count }} </td>
                              </tr>
                              <tr>
                                  <td>Data (MiB)</td><td>{{ cache.stats.size }}</td>
                              </tr>
                              <tr>
                                  <td>Cache Hits</td><td>{{ cache.stats.hits }}</td>
                              </tr>
                              <tr>
                                  <td>Cache Misses</td><td>{{ cache.stats.misses }}</td>
                              </tr>
                          </tbody>
                      </table>
                  </div>
              </div>
              <h2>Cache Control</h2>
              <div class="row">
                  <div class="col-md-auto"><button type="button" class="btn btn-danger" v-on:click="clearCache">{{ cache.statusString }}</button></div>
                  <div class="col-md-auto"><div id="clear-status">{{ cache.lastClearStatus }}</div></div>
              </div>
              <div class="row pt-3">
                  <div class="col-md-auto">
                      <h4>Cache clear history</h4>
                      <table class="table table-borderless">
                          <tbody>
                              <tr v-for="entry in cache.history">
                                  <td>{{ entry.when }}</td><td>Removed {{ entry.removed }} items</td>
                              </tr>
                          </tbody>
                      </table>
                  </div>
              </div>
          </div>
      </main>
      <script src="https://cdn.jsdelivr.net/npm/vue@2.6.0"></script>
      <script src="https://cdn.jsdelivr.net/npm/vue-resource@1.5.1"></script>
      <script>
       let statusStrings = {
           CLEAR: "Clear now",
           REFRESHING: "Refreshing...",
           WAITING: "...",
           ERROR: "Error"
       }
       new Vue({
           el: "#app",
           data: {
               cache: {
                   purgeOlderThanDays: 30,
                   stats: {
                       hits: 0,
                       misses: 0,
                       size: 0,
                       count: 0
                   },
                   history: [],
                   statusString: statusStrings.CLEAR,
                   clearStatus: "",
               }
           },
           methods: {
               clearCache: function() {
                   console.log("clear clicked");
                   this.$data.cache.statusString = statusStrings.REFRESHING;
                   this.$data.cache.lastClearStatus = "";
                   /* how old files to remove */
                   let olderThan = this.$data.cache.purgeOlderThanDays;

                   this.$http.delete("data?older-than-days=" + olderThan).then(
                       successResponse => {
                           this.$data.cache.statusString = statusStrings.CLEAR;
                           let removedStats = successResponse.body;
                           /* cache clearned, reload all stats */
                           this.reloadStats()
                       },
                       errorResponse => {
                           this.$data.cache.statusString = statusStrings.CLEAR;
                           this.$data.cache.lastClearStatus = statusStrings.ERROR;
                           let err = errorResponse.body;
                           if (err.Error != "") {
                               this.$data.cache.lastClearStatus = "ERROR: " + err.Error;
                           }
                       }
                   );
               },
               reloadStats: function() {
                   for (let prop in this.$data.cache.stats) {
                       this.$data.cache.stats[prop] = statusStrings.WAITING;
                   }

                   this.$http.get("stats").then(
                       successResponse => {
                           console.log("stats done, got:");
                           console.log(successResponse);
                           let stats = successResponse.body;
                           /* fill trivial stats */
                           this.$data.cache.stats.hits = stats.Hit;
                           this.$data.cache.stats.misses = stats.Miss;
                           /* update cache clear history */
                           this.$data.cache.history = this.$data.cache.history.splice(0, this.$data.cache.history.splice.length);
                           for (let i in stats.PurgeHistory) {
                               let historyEvent = stats.PurgeHistory[i];
                               console.debug("history event:");
                               console.debug(historyEvent);
                               let d = new Date(historyEvent.When);
                               if (d == 0) {
                                   continue;
                               }
                               let event = {
                                   when: d.toLocaleString(),
                                   removed: historyEvent.Removed
                               }
                               console.log(event);
                               this.$data.cache.history.push(event);
                           }
                       },
                       errorResponse => {
                           console.log("stats error");
                       }
                   );
                   this.$http.get("count").then(
                       successResponse => {
                           console.log("count done, got:");
                           console.log(successResponse);
                           let count = successResponse.body;
                           this.$data.cache.stats.size = count.TotalSize / 1024 / 1024;
                           this.$data.cache.stats.count = count.Items;
                       },
                       errorResponse => {
                           console.log("count error");
                       }
                   );
               }
           },
           mounted: function() {
               this.reloadStats();
           }
       });
      </script>
  </body>
</html><commit_msg>data/assets/index: use fixed precision<commit_after><!doctype html>
<html lang="en">
  <head>
    <!-- Required meta tags -->
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">

    <!-- Bootstrap CSS -->
    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/css/bootstrap.min.css" integrity="<KEY>" crossorigin="anonymous">

    <title>viadown</title>
  </head>
  <body>
      <main id="app">
          <div class="container">
              <h1>viadown administration</h1>
              <h2>Stats</h2>
              <div class="row">
                  <div class="row col-md-auto">
                      <table id="stats" class="table table-borderless">
                          <tbody>
                              <tr>
                                  <td>Files</td><td>{{ cache.stats.count }} </td>
                              </tr>
                              <tr>
                                  <td>Data (MiB)</td><td>{{ cache.stats.size }}</td>
                              </tr>
                              <tr>
                                  <td>Cache Hits</td><td>{{ cache.stats.hits }}</td>
                              </tr>
                              <tr>
                                  <td>Cache Misses</td><td>{{ cache.stats.misses }}</td>
                              </tr>
                          </tbody>
                      </table>
                  </div>
              </div>
              <h2>Cache Control</h2>
              <div class="row">
                  <div class="col-md-auto"><button type="button" class="btn btn-danger" v-on:click="clearCache">{{ cache.statusString }}</button></div>
                  <div class="col-md-auto"><div id="clear-status">{{ cache.lastClearStatus }}</div></div>
              </div>
              <div class="row pt-3">
                  <div class="col-md-auto">
                      <h4>Cache clear history</h4>
                      <table class="table table-borderless">
                          <tbody>
                              <tr v-for="entry in cache.history">
                                  <td>{{ entry.when }}</td><td>Removed {{ entry.removed }} items</td>
                              </tr>
                          </tbody>
                      </table>
                  </div>
              </div>
          </div>
      </main>
      <script src="https://cdn.jsdelivr.net/npm/vue@2.6.0"></script>
      <script src="https://cdn.jsdelivr.net/npm/vue-resource@1.5.1"></script>
      <script>
       let statusStrings = {
           CLEAR: "Clear now",
           REFRESHING: "Refreshing...",
           WAITING: "...",
           ERROR: "Error"
       }
       new Vue({
           el: "#app",
           data: {
               cache: {
                   purgeOlderThanDays: 30,
                   stats: {
                       hits: 0,
                       misses: 0,
                       size: 0,
                       count: 0
                   },
                   history: [],
                   statusString: statusStrings.CLEAR,
                   clearStatus: "",
               }
           },
           methods: {
               clearCache: function() {
                   console.log("clear clicked");
                   this.$data.cache.statusString = statusStrings.REFRESHING;
                   this.$data.cache.lastClearStatus = "";
                   /* how old files to remove */
                   let olderThan = this.$data.cache.purgeOlderThanDays;

                   this.$http.delete("data?older-than-days=" + olderThan).then(
                       successResponse => {
                           this.$data.cache.statusString = statusStrings.CLEAR;
                           let removedStats = successResponse.body;
                           /* cache clearned, reload all stats */
                           this.reloadStats()
                       },
                       errorResponse => {
                           this.$data.cache.statusString = statusStrings.CLEAR;
                           this.$data.cache.lastClearStatus = statusStrings.ERROR;
                           let err = errorResponse.body;
                           if (err.Error != "") {
                               this.$data.cache.lastClearStatus = "ERROR: " + err.Error;
                           }
                       }
                   );
               },
               reloadStats: function() {
                   for (let prop in this.$data.cache.stats) {
                       this.$data.cache.stats[prop] = statusStrings.WAITING;
                   }

                   this.$http.get("stats").then(
                       successResponse => {
                           console.log("stats done, got:");
                           console.log(successResponse);
                           let stats = successResponse.body;
                           /* fill trivial stats */
                           this.$data.cache.stats.hits = stats.Hit;
                           this.$data.cache.stats.misses = stats.Miss;
                           /* update cache clear history */
                           this.$data.cache.history = this.$data.cache.history.splice(0, this.$data.cache.history.splice.length);
                           for (let i in stats.PurgeHistory) {
                               let historyEvent = stats.PurgeHistory[i];
                               console.debug("history event:");
                               console.debug(historyEvent);
                               let d = new Date(historyEvent.When);
                               if (d == 0) {
                                   continue;
                               }
                               let event = {
                                   when: d.toLocaleString(),
                                   removed: historyEvent.Removed
                               }
                               console.log(event);
                               this.$data.cache.history.push(event);
                           }
                       },
                       errorResponse => {
                           console.log("stats error");
                       }
                   );
                   this.$http.get("count").then(
                       successResponse => {
                           console.log("count done, got:");
                           console.log(successResponse);
                           let count = successResponse.body;
                           let sizeMB = (count.TotalSize / 1024 / 1024).toFixed(2);
                           this.$data.cache.stats.size = sizeMB;
                           this.$data.cache.stats.count = count.Items;
                       },
                       errorResponse => {
                           console.log("count error");
                       }
                   );
               }
           },
           mounted: function() {
               this.reloadStats();
           }
       });
      </script>
  </body>
</html>
{'fraction_non_alphanumeric': 0.11854146135435731, 'fraction_numerical': 0.0051386737998029, 'mean_word_length': 0.8752639915522703, 'pattern_counts': {'":': 0, '<': 181, '<?xml version=': 0, '>': 193, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '475575', 'n_tokens_mistral': 3436, 'n_tokens_neox': 3243, 'n_words': 565}