<commit_before><!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="UTF-8" />
    <title>Crawling Monitor — GBIF</title>
    <link rel="icon" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMAAAADACAMAAABlApw1AAAACXBIWXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH4QQTECQwbfdOTAAAAuhQTFRFAAAAoj3KqlDRu3HZvnjbwoXe0Jvk48Xv6dLz8eT49uv5+/X7/Pv+/fz+/vz+/v3+/v7+//7/////o0fLrVTQwHzbwH7cxIHf1Kfn2rLp+vz8/fv+/fz+/f7+/vz+/v3+/v3/////tWvVuW3XvJDcxonexonf+vf7/fv+/vv+/vz+////sV3Tsl3TtXHcunLWvn7XwIje3sDu473v5Mnw5svw9fT6+O35////oj/KsVzTtGbX////oj/LrFjQrVXRs17Uvnnawo7h7tv18N72////woDcw4TdzZfj0qDm7tz08+f5vXHZ0J/k1qjlp1jRsF/TyIbe587wsWDTxozf1qvn4MHtqlPRr13Ss2PUy5Hhoj7KpELOpUjMqFDPqkzOq1LRrFLPr1zS373s6tHz////o0HLpEPMpkbMsFnRunHX3LTr5cnvqkzNrljRr1rPum3Yvnfa+fP9/v//iw6/p0bMs1/SwoDbxITdo0DKqlHQxIbeyo/gy5Xj1Kfm6NHyp0nNr1jR16no7NfzqU3Pt2fWt2jYpEHLs2LUunHYXIterVbTp0rOsV7T48XvW3lkqk3OiQ/A3brto0LLvXnb1qzpqk/Pzpvk2avo4Lztoj7Lr1jSpUHMrlXQuGrWpULMz5zkqlHPq0/QwHzdy5XizZfjoj/Koj7JqEzPqUvOiAu+ig6+o0HMrlnSgmCZpUTMsVvToz/LyI/hqEzOr1fRrVXRpULMVJNTiQu8qk/Psl3SrljSXYpgqEzNqk/Pp0rNoz/KpkfNjA++pEHLpUPLUpZPgSqoV5BWWI9Yo0DLb1OFiQm7gyStVJRSZGpyV5BXbXd5bnZ5Z2V2XIpeUJVMa1t/UpZPZYBsYm9uY2txTJxGXIlfTptITppJT5lKUpZOUZJPUpVPU5VQU5VQXHtjVJRRUo9RU49RVZJUVohYTZtHTZxHTZtHTppJUJhLUJdMUJRNUZdNUJRNUZJPUpBQU49SU45TTJ5FP8Oy9QAAAPd0Uk5TAAEBAQEBAQEBAQEBAQEBAQEBAQICAgICAgICAgICAgICAgMDAwMDAwMDAwMEBAQEBAQEBAQEBAQEBQUFBQYGBgYGBgYGBgcHBwcHBwgICAkJCQkKCgoKCwsLCwwMDAwMDAwMDAwMDQ0NDQ0NDQ4ODg4ODg4PDw8PDxAQEBAQEBAREREREhISExQUFRUWFhYXFxgYGRkZGhocHB0dHh4eHx8hISUlJSYnKCorKy0tLi8xMjIzMzQ3OTk8PT9AR0dNT09SV1tcXmJkZW11foSHioyOkJOjpKaqtLu9v8DBw8rMzMzNzc/R0dPf4eHi5Onr7e3u8vX4+qjtfskAAAABYktHRBJ7vGwAAAAF8UlEQVR42u2bVZgkRQzHg+uw0Dh7cDc4LK4LHHK4u7u7ux9yuLu7u7u7u7vD4c7/lYe9uW92d6YrlUpVV75v87bTSSq/Srq6Kt1LpCZPkHHBFtYBYBxgF/MEwMXWAeynAOYBTBMU1gEI1gn2A4ATzadgHusAMA9gmOCzgRREkWn5qh9lSfA+P54j80zBf6v71hByq2xvgEusEjQAMF9mAHVfAL5JZhv9JgLQxFyrazcoOmICHMQnQC+5NZdtrBSAaxd96QKACWUA8LDK4agCiBCSEGwlBeAEFpvAcyb7ytJcww1jAXT7TaQgCaPUpo6aguXjAyAqgNP7YxATIDLBZX5RtJCNs0jB/HKAqlPwe4/38ct0ti8FQFEpAcf7iHIAVApQZ7h/FyEESJQC1DtrklvAGdnVkQE2bY7kRhlAJiloO8pQJ8BrvAGKJACYVTkFT8VNQY0RjhsAY7JmKE0GAEznDVAywM8VAGClXirD3QB/MRsC+nJn64Ae8E3BYlXVEKsoIC8iQS/AS9blRFRwAArO/BybLgG9CCBMwVmydowWQD0UQNbK0AP41y8Fd3B8r5ESoGm+RCkQtmI0AUYP95WAoKXKCYkBPvbJwL0Mz2cmBoC8hoR9GG0A+ADs0+R47zY6q6QG2EuWgkLcCtMGwBSyGsoH4FcRwFVtdb5LDdB4IPsByDth+gDga97TcLtiidIhqgC/qQKwVt27kmcA/jUU0k2NAHAyERFtzb9fEgKsxgH4mg27st8eMd8aSgjwlirBcz2K+wY15KtMwWZem/SEAFyCHXv0Xs4PgPk43inbDCzptftOCsAjuNzrBOfQ2qG6m8ClvTDL5XVVAlD4YwAPU5UEiygcQJUBDlRLwRHVAPietvYP7mFUDEB7unQeLHd1N1VB0Kx/QGuVoxvXLyh3dXrlAM6+p3YFTaJA4LaYKx7Ajy6btd0AF7kINqFoAG6bidwADznStlbzpWfVARBcRI+3eEm+3eir9xeS1w4+AJ3Bd0GZnfC9iQ9AEZyCdoaTeXuTrZKha+k1agvzUOEy79I6itfy4Um36ioKjRSoPRuflAIUYQRqAC+IdwoTKL23Z8kCbR0NEQN87tb8W69yNffSDdMl3Kr/tBv2OC0AyWZzVw9j7aVD5zCDYIL1tTbpsv3+mw3zKRnKV2iN+0MLPzeFnli2lM2c2jlJGH/R38MpvwDA4cyRZcPOqHeab/IxHlGt3vT3fTSDC2EmrbPqeiqnXuYEqzRBFuw1zDYxj+0trc6+smwL18UpDq1u0ItOgqn8HNZX5db3Oz2XF4rfOQl21zZdu61QSe+nvTyaqmcY0j9sL7WEXc+A9wDt5VtUQwAlAod9V+UE05Q/1N/jfi+hL8OYBMsGzsIZVacAmye4keISINDD9bEACi7ARtFmIEzGoMAAUDUBP4SxAgGiEWwbFABgKAehAMCi1d7ILWwf8QPASBrUx0Nnx+DOUIJDucOP08/0D3jL201biwu/7/nt9cqeBhDKSx988mefn5ZLAdClBaB9k4vHADJByARAjtAlHEAdALXIKZg7NsDNY8clOC02AH6K/DiLDiC/D8wDLCVwj6wIKBeAS6UAi2cCgHmTNUwjAciLqPD0/UpuAL7vDYD8CLp9XD+TIYDXJx6vZgng8YlHtBI6L9YZbXLZcdpbPgw9J0/f2u/BiRKg0oNhuP0ma4D+DGpn+nRdsCHFbCNHeTys/xdqb+QPoNQLyBRgZ+sAMA4A4wAwDnCMcYDI4eO2yPF/CdsJuAW2AW6AbYBPjcdPMA6QIP5ZjMd/+0ABDcQfIoVxAOsr6MzGAYxvQveo2Y7fzCnsaQB4vs+P59K40eMPboXuXu/o9XFQbfbBnXMURHPWBiXpA1HuFRITgLIQafQnFWQagLIRUfinkm0AItsARU7xF8bnX5CBzOKnL4zH75sCMg5wvHWADOP3WocoSzEPQNbjJ+vx6/0HZmWyjPH4ida0toUTFBFlLuYBHATnkwEpiX8dItME55AVMVr+jp3dpGRLkk3//2wnurFjC45eAAAAAElFTkSuQmCC">
    <link rel="stylesheet" type="text/css" href="css/bootstrap.min.css">
    <link href="css/bootstrap-responsive.min.css" rel="stylesheet">
    <link rel="stylesheet" type="text/css" href="css/monitor.css">
    <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
    <script type="text/javascript" language="javascript" src="js/jquery-1.8.3.min.js"></script>
    <script type="text/javascript" language="javascript" src="js/bootstrap.min.js"></script>
    <script type="text/javascript" language="javascript" src="js/jquery.dataTables-1.9.4.min.js"></script>
    <script type="text/javascript" language="javascript" src="js/datatables-bootstrap-pagation.js"></script>
    <script type="text/javascript" language="javascript" src="js/datatables-reload-ajax.js"></script>
    <script type="text/javascript" language="javascript" src="js/safebase64.js"></script>
    <script type="text/javascript" charset="utf-8">
      // overwrites for bootstrap
      $.extend( $.fn.dataTableExt.oStdClasses, {
        "sWrapper": "dataTables_wrapper form-inline"
      } );

      // WS locations to get the data
      var runningContentUrl = "dataset/process/running";
      // var pendingXmlContentUrl = "dataset/process/xml/pending";
      // var pendingDwcaContentUrl = "dataset/process/dwca/pending";
      var detailUrl = "dataset/process/detail/";

      // Environment, used to construct the Kibana URL
      var env = 'prod';
      var domain = 'gbif';
      if (window.location.hostname.match(/uat/)) {
        env = 'uat';
        domain = domain + '-uat';
      } else if (window.location.hostname.match(/dev/)) {
        env = 'dev';
        domain = domain + '-dev';
      }
      domain = domain + '.org';

      // to aid development of this UI
      // runningXmlContentUrl = "./running.json";

      var oTable; // The main data table

      // utility to trigger live view class
      var click = function() {
        // live view is monitored on the polling, the second is bootstrap styling only
        $(this).toggleClass('row_liveview').toggleClass('success');
      }

      // decorates rows with missing pieces
      var decorate = function(nRow, aData) {
        // URL
        $('td:eq(0)', nRow).addClass("url");
        $('td:eq(0)', nRow).css("width","80px");

        // colorize the columns based on the rules to showcase errors
        $('td:eq(2)', nRow).addClass("text-info"); // pages crawled

        if (aData.pagesCrawled == aData.pagesFragmentedSuccessful) $('td:eq(3)', nRow).removeClass("text-error").addClass("text-success");
        else $('td:eq(3)', nRow).removeClass("text-success").addClass("text-error");

        if (aData.pagesFragmentedError > 0) $('td:eq(4)', nRow).removeClass("text-success").addClass("text-error");
        else $('td:eq(4)', nRow).removeClass("text-error").addClass("text-success");

        $('td:eq(5)', nRow).addClass("text-info");  // fragments emitted

        if (aData.fragmentsEmitted == aData.fragmentsReceived) $('td:eq(6)', nRow).removeClass("text-error").addClass("text-success");
        else $('td:eq(6)', nRow).removeClass("text-success").addClass("text-error");

        $('td:eq(7)', nRow).addClass("text-info"); // new records
        $('td:eq(8)', nRow).addClass("text-info"); // modified records
        $('td:eq(9)', nRow).addClass("text-info"); // unchanged records

        if (aData.rawOccurrencesPersistedError > 0) $('td:eq(10)', nRow).removeClass("text-success").addClass("text-error");
        else $('td:eq(10)', nRow).removeClass("text-error").addClass("text-success");

        if (aData.fragmentsReceived == aData.fragmentsProcessed) $('td:eq(11)', nRow).removeClass("text-error").addClass("text-success");
        else $('td:eq(11)', nRow).removeClass("text-success").addClass("text-error");

        if (aData.verbatimOccurrencesPersistedSuccessful ==
          aData.rawOccurrencesPersistedNew + aData.rawOccurrencesPersistedUpdated) $('td:eq(12)', nRow).removeClass("text-error").addClass("text-success");
        else $('td:eq(12)', nRow).removeClass("text-success").addClass("text-error");

        if (aData.verbatimOccurrencesPersistedError > 0) $('td:eq(13)', nRow).removeClass("text-success").addClass("text-error");
        else $('td:eq(13)', nRow).removeClass("text-error").addClass("text-success");

        if (aData.verbatimOccurrencesPersistedSuccessful == aData.interpretedOccurrencesPersistedSuccessful)
          $('td:eq(14)', nRow).removeClass("text-error").addClass("text-success");
        else $('td:eq(14)', nRow).removeClass("text-success").addClass("text-error");

        if (aData.interpretedOccurrencesPersistedError > 0) $('td:eq(15)', nRow).removeClass("text-success").addClass("text-error");
        else $('td:eq(15)', nRow).removeClass("text-error").addClass("text-success");

        // if new, updated and not modified = declared count then show it
        if (aData.declaredCount ==
          aData.rawOccurrencesPersistedNew + aData.rawOccurrencesPersistedUpdated + aData.rawOccurrencesPersistedUnchanged)
          $('td:eq(1)', nRow).removeClass("text-error").addClass("text-success");
        else $('td:eq(1)', nRow).removeClass("text-success").addClass("text-error");

        // The final log | detail section
        var url = "http://elk.gbif.org:5601/app/kibana#/discover?_a=(filters:!((meta:(alias:!n,disabled:!f,index:'" + env + "-crawl-*',key:datasetKey,negate:!f,value:'" + aData.datasetKey + "'),query:(match:(datasetKey:(query:'" + aData.datasetKey + "',type:phrase))))),index:'" + env + "-crawl-*')&_g=(time:(from:now-7d,mode:quick,to:now))"

        var $log = $("<a href='"+url+"'>Logs</a>").bind('click', function() {
          // open a tab to Kibana
          window.open(url,'Logs');
          return false;
        });

        var $details = $("<a href='#'>Details</a>").bind('click', function() {
          if ( oTable.fnIsOpen(nRow) ) {
            oTable.fnClose( nRow );
          } else {
            var detailRow = oTable.fnOpen( nRow, "", "info_row" );
            $.ajax({
              url: detailUrl + aData.datasetKey
            }).done(function ( data ) {
              $('td', detailRow).html( "<textarea class='detail'>" + JSON.stringify(data, undefined, 2) + "</textarea>" );
            });
          }
          return false;
        });

        var portalUrl = "http://www."+domain+"/dataset/"+aData.datasetKey;
        var $portal = $("<a href='"+portalUrl+"'>Portal</a>").bind('click', function() {
          // open a tab to the dataset page in the portal
          window.open(portalUrl,'Portal');
          return false;
        });

        $('td:eq(16)', nRow).html("");
        $('td:eq(16)', nRow).append($log).append(" | ").append($details).append(" | ").append($portal);
      }


      $(document).ready(function() {
        /*
         * Set up the data table
         */
        oTable = $('#datasetCrawls')
          .dataTable( {
            "sDom": "<'row'<'span6'f><'span6'p>r>t<'row'<'span6'i><'span6'l>>",
            "aLengthMenu": [[10, 25, 50, 100, 200, -1], [10, 25, 50, 100, 200, "All"]],
            "sAjaxSource": runningContentUrl,
            "sAjaxDataProp": "",  // There is no field name - e.g. {[i1,i2]} not {"data":[i1,i2]}
            "aoColumns": [
              { "mData": "crawlJob.targetUrl", sDefaultContent: "" },  // defaults to be fail safe when omitted in JSON
              { "mData": "declaredCount", sDefaultContent: "" },
              { "mData": "pagesCrawled", sDefaultContent: 0 },
              { "mData": "pagesFragmentedSuccessful", sDefaultContent: 0 },
              { "mData": "pagesFragmentedError", sDefaultContent: 0 },
              { "mData": "fragmentsEmitted", sDefaultContent: 0 },
              { "mData": "fragmentsReceived", sDefaultContent: 0 },
              { "mData": "rawOccurrencesPersistedNew", sDefaultContent: 0 },
              { "mData": "rawOccurrencesPersistedUpdated", sDefaultContent: 0 },
              { "mData": "rawOccurrencesPersistedUnchanged", sDefaultContent: 0 },
              { "mData": "rawOccurrencesPersistedError", sDefaultContent: 0 },
              { "mData": "fragmentsProcessed", sDefaultContent: 0 },
              { "mData": "verbatimOccurrencesPersistedSuccessful", sDefaultContent: 0 },
              { "mData": "verbatimOccurrencesPersistedError", sDefaultContent: 0 },
              { "mData": "interpretedOccurrencesPersistedSuccessful", sDefaultContent: 0 },
              { "mData": "interpretedOccurrencesPersistedError", sDefaultContent: 0 },
              { sDefaultContent: " ", 'bSortable': false },  // crawl logs
              { "mData": "datasetKey", 'bVisible': false, bSearchable: true }
            ],
            //"fnInfoCallback": function( oSettings, iStart, iEnd, iMax, iTotal, sPre ) {
              //return iStart +" to "+ iEnd;
            //},
            "fnRowCallback" :  function(nRow, aData, iDisplayIndex) { // inject the missing fields
              decorate(nRow, aData);
              // row click handling by toggling a class, which enables live view
              // we unbind to ensure only 1 event logs
              $(nRow).unbind('click').bind('click', click);
            },
            "bProcessing": true, // Show processing indicator when busy
            "bDeferRender": true, // delay rendering for performance
            "bStateSave" : true, // save current state in cookie to help reload
            "fnInitComplete": function() {
              // restores saved filters on refresh
              // http://datatables.net/forums/discussion/2864/x
              var oSettings = $('#datasetCrawls').dataTable().fnSettings();
              for ( var i=0 ; i<oSettings.aoPreSearchCols.length ; i++ ) {
                if(oSettings.aoPreSearchCols[i].sSearch.length>0){
                  $("tfoot input")[i].value = oSettings.aoPreSearchCols[i].sSearch;
                  $("tfoot input")[i].className = "";
                }
              }

            },
            // There appears a bug that DT is disabling the tabs incorrectly - reenable here
            "fnDrawCallback": function( oSettings ) {
              $('#tabs').find('li').removeClass("disabled");
            },
            "oLanguage" : { // custom labels
              "sSearch": "Search: _INPUT_   <button class='btn allLiveView'>LiveView</button>" +
                  "   <button class='btn hideHelp'>Toggle help</button>"
            },
            "sPaginationType": "bootstrap"
        });

        /*
         * Support functions to provide a little bit of 'user friendliness' to the textboxes in
         * the footer
         */
        var asInitVals = [];
        $("tfoot input").each( function (i) {
          asInitVals[i] = this.value;
        });
        $("tfoot input").focus( function () {
          if ( this.className == "search_init" ) {
          this.className = "";
          this.value = "";
          }
        });
        $("tfoot input").blur( function (i) {
          if ( this.value == "" ) {
          this.className = "search_init";
          this.value = asInitVals[$("tfoot input").index(this)];
          }
        });
        $("tfoot input").keyup( function () {
          /* Filter on the column (the index) of this element */
          oTable.fnFilter( this.value, $("tfoot input").index(this) );
        });

        /*
         * Tab interceptor to set url of data based on the tab
         */
        $("#runningTab").click(function() {
          console.log("Switching to running: " + runningContentUrl);
          oTable.fnReloadAjax(runningContentUrl);
        });
        // $("#pendingXmlTab").click(function() {
        //   console.log("Switching to pending XML: " + pendingXmlContentUrl);
        //   oTable.fnReloadAjax(pendingXmlContentUrl);
        // });
        // $("#pendingDwcaTab").click(function() {
        //   console.log("Switching to pending DwC-A: " + pendingDwcaContentUrl);
        //   oTable.fnReloadAjax(pendingDwcaContentUrl);
        // });

        $(function () {
          $('#tabs').find('a:first').tab('show');
        });

        /*
         * Live view polling
         */
        function liveView() {
          //console.log("Live viewing " +oTable.$('tr.row_liveview').length + " records");

          var ajaxRequests = [];
          $.each(oTable.$('tr.row_liveview'), function(index, nRow) {
            var aData = oTable.fnGetData(nRow);
            ajaxRequests.push(
              $.ajax({
                url: detailUrl + aData.datasetKey,
                success: function(json, textStatus, jqXHR) {
                  if (jqXHR.status == "200") {
                    oTable.fnUpdate(json, nRow, undefined, false, false); // don't repaint the table
                    // we just refreshed it
                    aData = oTable.fnGetData(nRow);
                    decorate(nRow, aData);

                    // if the detail tab, is open, refresh that too
                    if($('textarea', $(nRow).next()).length>0) {
                      $('textarea', $(nRow).next()).text(JSON.stringify(json, undefined, 2));
                    }
                  }
                }
              })
            );
          });

          var defer = $.when.apply($, ajaxRequests);
          // Only when all ajax requests are complete schedule a new one
          defer.done(function(args) {
            //console.log("Polling");
            setTimeout(function() {
              liveView();
            }, 2000); // liveview update period msecs
          });
        }
        // Bulk update
        $(".allLiveView").click(function() {
          // If the first is set, clear them all, otherwise set them all
          var b = oTable.$('tbody tr').length && $(oTable.$('tbody tr')[0]).hasClass('row_liveview');
          if (b) oTable.$('tbody tr').removeClass('row_liveview success');
          else oTable.$('tbody tr').addClass('row_liveview success');
        });

        // start the live view polling
        liveView();

        // Hide the help
        $(".hideHelp").click(function() {
          $('#full').toggleClass("disabled");
          if ($('#full').hasClass("disabled")) {
            localStorage.setItem('helpHidden', true);
          } else {
            localStorage.removeItem('helpHidden');
          }
        });

        if (localStorage.getItem('helpHidden')) {
          $('#full').toggleClass("disabled");
        }
      });
    </script>
  </head>
  <body>
    <!-- Page layout is using bootstrap -->
    <div class="container">
      <div>
        <h3>Crawling Monitor <small> ...filter, sort, click for <em>live view</em></small></h3>
      </div>
      <ul id="tabs" class="nav nav-tabs">
        <li><a href="#data" data-toggle="tab" id="runningTab">Running crawls</a></li>
        <!--li><a href="#data" data-toggle="tab" id="pendingXmlTab">Pending (XML)</a></li-->
        <!--li><a href="#data" data-toggle="tab" id="pendingDwcaTab">Pending (DwC-A)</a></li-->
      </ul>
      <div class="tab-content">
        <div class="tab-pane active" id="data">
          <div id="tabTable">
            <table class="table table-striped table-bordered table-condensed" id="datasetCrawls">
              <thead>
                <tr id="full">
                  <th><div>URL</div></th>
                  <th><div>Declared count</div></th>
                  <th><div>Pages crawled</div></th>
                  <th><div>Pages fragmented successfully</div></th>
                  <th><div>Pages not fragmented (error)</div></th>
                  <th><div>Fragments emitted</div></th>
                  <th><div>Fragments received</div></th>
                  <th><div>Raw occurrence records<br/>persisted (new)</div></th>
                  <th><div>Raw occurrence records<br/>persisted (modified)</div></th>
                  <th><div>Raw occurrence records<br/>persisted (unchanged)</div></th>
                  <th><div>Raw occurrence records<br/>persisted (error)</div></th>
                  <th><div>Fragments processed</div></th>
                  <th><div>Verbatim occurrence records<br/>persisted successfully</div></th>
                  <th><div>Verbatim occurrence records<br/>not persisted (error)</div></th>
                  <th><div>Occurrence records interpreted</div></th>
                  <th><div>Occurrence records error<br/>on interpretation</div></th>
                  <th style="width:150px" colspan="2"><div>Miscellaneous<!-- colspan 2 hides the UUID --></div></th>
                </tr>
                <tr id="abbr">
                  <th>URL</th>
                  <th title="Declared count">DC</th>
                  <th title="Pages crawled">PC</th>
                  <th title="Pages fragmented successfully">PFS</th>
                  <th title="Pages not fragmented (error)">PFE</th>
                  <th title="Fragments emitted">FE</th>
                  <th title="Fragments received">FR</th>
                  <th title="Raw occurrence records persisted (new)">RON</th>
                  <th title="Raw occurrence records persisted (modified)">ROM</th>
                  <th title="Raw occurrence records persisted (unchanged)">ROU</th>
                  <th title="Raw occurrence records persisted (error)">ROE</th>
                  <th title="Fragments processed">FP</th>
                  <th title="Verbatim occurrence records persisted successfully">VOP</th>
                  <th title="Verbatim occurrence records not persisted (error)">VOE</th>
                  <th title="Occurrence records interpreted">OI</th>
                  <th title="Occurrence records error on interpretation">OE</th>
                  <th title="Miscellaneous" style="width:150px" colspan="2"><!-- colspan 2 hides the UUID --></th>
                </tr>
              </thead>
              <tbody/>
              <tfoot>
                <tr>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th>
                    <input type="text" name="search_ds" value="..." class="search_init" style="width:0px;visibility:hidden" colspan="2"/>
                  </th>
                </tr>
              </tfoot>
            </table>
            <p/>
          </div>
        </div>
      </div>
  </body>
</html>
<commit_msg>Update URL to logs.<commit_after><!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="UTF-8" />
    <title>Crawling Monitor — GBIF</title>
    <link rel="icon" href="data:image/png;base64,<KEY>">
    <link rel="stylesheet" type="text/css" href="css/bootstrap.min.css">
    <link href="css/bootstrap-responsive.min.css" rel="stylesheet">
    <link rel="stylesheet" type="text/css" href="css/monitor.css">
    <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
    <script type="text/javascript" language="javascript" src="js/jquery-1.8.3.min.js"></script>
    <script type="text/javascript" language="javascript" src="js/bootstrap.min.js"></script>
    <script type="text/javascript" language="javascript" src="js/jquery.dataTables-1.9.4.min.js"></script>
    <script type="text/javascript" language="javascript" src="js/datatables-bootstrap-pagation.js"></script>
    <script type="text/javascript" language="javascript" src="js/datatables-reload-ajax.js"></script>
    <script type="text/javascript" language="javascript" src="js/safebase64.js"></script>
    <script type="text/javascript" charset="utf-8">
      // overwrites for bootstrap
      $.extend( $.fn.dataTableExt.oStdClasses, {
        "sWrapper": "dataTables_wrapper form-inline"
      } );

      // WS locations to get the data
      var runningContentUrl = "dataset/process/running";
      // var pendingXmlContentUrl = "dataset/process/xml/pending";
      // var pendingDwcaContentUrl = "dataset/process/dwca/pending";
      var detailUrl = "dataset/process/detail/";

      // Environment, used to construct the Kibana URL
      var env = 'prod';
      var domain = 'gbif';
      if (window.location.hostname.match(/uat/)) {
        env = 'uat';
        domain = domain + '-uat';
      } else if (window.location.hostname.match(/dev/)) {
        env = 'dev';
        domain = domain + '-dev';
      }
      domain = domain + '.org';

      // to aid development of this UI
      // runningXmlContentUrl = "./running.json";

      var oTable; // The main data table

      // utility to trigger live view class
      var click = function() {
        // live view is monitored on the polling, the second is bootstrap styling only
        $(this).toggleClass('row_liveview').toggleClass('success');
      }

      // decorates rows with missing pieces
      var decorate = function(nRow, aData) {
        // URL
        $('td:eq(0)', nRow).addClass("url");
        $('td:eq(0)', nRow).css("width","80px");

        // colorize the columns based on the rules to showcase errors
        $('td:eq(2)', nRow).addClass("text-info"); // pages crawled

        if (aData.pagesCrawled == aData.pagesFragmentedSuccessful) $('td:eq(3)', nRow).removeClass("text-error").addClass("text-success");
        else $('td:eq(3)', nRow).removeClass("text-success").addClass("text-error");

        if (aData.pagesFragmentedError > 0) $('td:eq(4)', nRow).removeClass("text-success").addClass("text-error");
        else $('td:eq(4)', nRow).removeClass("text-error").addClass("text-success");

        $('td:eq(5)', nRow).addClass("text-info");  // fragments emitted

        if (aData.fragmentsEmitted == aData.fragmentsReceived) $('td:eq(6)', nRow).removeClass("text-error").addClass("text-success");
        else $('td:eq(6)', nRow).removeClass("text-success").addClass("text-error");

        $('td:eq(7)', nRow).addClass("text-info"); // new records
        $('td:eq(8)', nRow).addClass("text-info"); // modified records
        $('td:eq(9)', nRow).addClass("text-info"); // unchanged records

        if (aData.rawOccurrencesPersistedError > 0) $('td:eq(10)', nRow).removeClass("text-success").addClass("text-error");
        else $('td:eq(10)', nRow).removeClass("text-error").addClass("text-success");

        if (aData.fragmentsReceived == aData.fragmentsProcessed) $('td:eq(11)', nRow).removeClass("text-error").addClass("text-success");
        else $('td:eq(11)', nRow).removeClass("text-success").addClass("text-error");

        if (aData.verbatimOccurrencesPersistedSuccessful ==
          aData.rawOccurrencesPersistedNew + aData.rawOccurrencesPersistedUpdated) $('td:eq(12)', nRow).removeClass("text-error").addClass("text-success");
        else $('td:eq(12)', nRow).removeClass("text-success").addClass("text-error");

        if (aData.verbatimOccurrencesPersistedError > 0) $('td:eq(13)', nRow).removeClass("text-success").addClass("text-error");
        else $('td:eq(13)', nRow).removeClass("text-error").addClass("text-success");

        if (aData.verbatimOccurrencesPersistedSuccessful == aData.interpretedOccurrencesPersistedSuccessful)
          $('td:eq(14)', nRow).removeClass("text-error").addClass("text-success");
        else $('td:eq(14)', nRow).removeClass("text-success").addClass("text-error");

        if (aData.interpretedOccurrencesPersistedError > 0) $('td:eq(15)', nRow).removeClass("text-success").addClass("text-error");
        else $('td:eq(15)', nRow).removeClass("text-error").addClass("text-success");

        // if new, updated and not modified = declared count then show it
        if (aData.declaredCount ==
          aData.rawOccurrencesPersistedNew + aData.rawOccurrencesPersistedUpdated + aData.rawOccurrencesPersistedUnchanged)
          $('td:eq(1)', nRow).removeClass("text-error").addClass("text-success");
        else $('td:eq(1)', nRow).removeClass("text-success").addClass("text-error");

        // The final log | detail section
        var index = 'AWBqgxPaf8lu3pmE7fpp';
        if (env == 'uat') index = 'AWDqIcB5f8lu3pmEHgcY';
        if (env == 'prod') index = 'AWDqIeCpf8lu3pmEHglJ';
        var url = "https://logs.gbif.org/app/kibana#/discover?_g=(time:(from:now-7d,mode:quick,to:now))&_a=(filters:!((meta:(alias:!n,disabled:!f,index:"+index+",key:datasetKey,negate:!f,type:phrase,value:'"+aData.datasetKey+"'),query:(match:(datasetKey:(query:'"+aData.datasetKey+"',type:phrase))))),index:"+index+")";

        var $log = $("<a href='"+url+"'>Logs</a>").bind('click', function() {
          // open a tab to Kibana
          window.open(url,'Logs');
          return false;
        });

        var $details = $("<a href='#'>Details</a>").bind('click', function() {
          if ( oTable.fnIsOpen(nRow) ) {
            oTable.fnClose( nRow );
          } else {
            var detailRow = oTable.fnOpen( nRow, "", "info_row" );
            $.ajax({
              url: detailUrl + aData.datasetKey
            }).done(function ( data ) {
              $('td', detailRow).html( "<textarea class='detail'>" + JSON.stringify(data, undefined, 2) + "</textarea>" );
            });
          }
          return false;
        });

        var portalUrl = "http://www."+domain+"/dataset/"+aData.datasetKey;
        var $portal = $("<a href='"+portalUrl+"'>Portal</a>").bind('click', function() {
          // open a tab to the dataset page in the portal
          window.open(portalUrl,'Portal');
          return false;
        });

        $('td:eq(16)', nRow).html("");
        $('td:eq(16)', nRow).append($log).append(" | ").append($details).append(" | ").append($portal);
      }


      $(document).ready(function() {
        /*
         * Set up the data table
         */
        oTable = $('#datasetCrawls')
          .dataTable( {
            "sDom": "<'row'<'span6'f><'span6'p>r>t<'row'<'span6'i><'span6'l>>",
            "aLengthMenu": [[10, 25, 50, 100, 200, -1], [10, 25, 50, 100, 200, "All"]],
            "sAjaxSource": runningContentUrl,
            "sAjaxDataProp": "",  // There is no field name - e.g. {[i1,i2]} not {"data":[i1,i2]}
            "aoColumns": [
              { "mData": "crawlJob.targetUrl", sDefaultContent: "" },  // defaults to be fail safe when omitted in JSON
              { "mData": "declaredCount", sDefaultContent: "" },
              { "mData": "pagesCrawled", sDefaultContent: 0 },
              { "mData": "pagesFragmentedSuccessful", sDefaultContent: 0 },
              { "mData": "pagesFragmentedError", sDefaultContent: 0 },
              { "mData": "fragmentsEmitted", sDefaultContent: 0 },
              { "mData": "fragmentsReceived", sDefaultContent: 0 },
              { "mData": "rawOccurrencesPersistedNew", sDefaultContent: 0 },
              { "mData": "rawOccurrencesPersistedUpdated", sDefaultContent: 0 },
              { "mData": "rawOccurrencesPersistedUnchanged", sDefaultContent: 0 },
              { "mData": "rawOccurrencesPersistedError", sDefaultContent: 0 },
              { "mData": "fragmentsProcessed", sDefaultContent: 0 },
              { "mData": "verbatimOccurrencesPersistedSuccessful", sDefaultContent: 0 },
              { "mData": "verbatimOccurrencesPersistedError", sDefaultContent: 0 },
              { "mData": "interpretedOccurrencesPersistedSuccessful", sDefaultContent: 0 },
              { "mData": "interpretedOccurrencesPersistedError", sDefaultContent: 0 },
              { sDefaultContent: " ", 'bSortable': false },  // crawl logs
              { "mData": "datasetKey", 'bVisible': false, bSearchable: true }
            ],
            //"fnInfoCallback": function( oSettings, iStart, iEnd, iMax, iTotal, sPre ) {
              //return iStart +" to "+ iEnd;
            //},
            "fnRowCallback" :  function(nRow, aData, iDisplayIndex) { // inject the missing fields
              decorate(nRow, aData);
              // row click handling by toggling a class, which enables live view
              // we unbind to ensure only 1 event logs
              $(nRow).unbind('click').bind('click', click);
            },
            "bProcessing": true, // Show processing indicator when busy
            "bDeferRender": true, // delay rendering for performance
            "bStateSave" : true, // save current state in cookie to help reload
            "fnInitComplete": function() {
              // restores saved filters on refresh
              // http://datatables.net/forums/discussion/2864/x
              var oSettings = $('#datasetCrawls').dataTable().fnSettings();
              for ( var i=0 ; i<oSettings.aoPreSearchCols.length ; i++ ) {
                if(oSettings.aoPreSearchCols[i].sSearch.length>0){
                  $("tfoot input")[i].value = oSettings.aoPreSearchCols[i].sSearch;
                  $("tfoot input")[i].className = "";
                }
              }

            },
            // There appears a bug that DT is disabling the tabs incorrectly - reenable here
            "fnDrawCallback": function( oSettings ) {
              $('#tabs').find('li').removeClass("disabled");
            },
            "oLanguage" : { // custom labels
              "sSearch": "Search: _INPUT_   <button class='btn allLiveView'>LiveView</button>" +
                  "   <button class='btn hideHelp'>Toggle help</button>"
            },
            "sPaginationType": "bootstrap"
        });

        /*
         * Support functions to provide a little bit of 'user friendliness' to the textboxes in
         * the footer
         */
        var asInitVals = [];
        $("tfoot input").each( function (i) {
          asInitVals[i] = this.value;
        });
        $("tfoot input").focus( function () {
          if ( this.className == "search_init" ) {
          this.className = "";
          this.value = "";
          }
        });
        $("tfoot input").blur( function (i) {
          if ( this.value == "" ) {
          this.className = "search_init";
          this.value = asInitVals[$("tfoot input").index(this)];
          }
        });
        $("tfoot input").keyup( function () {
          /* Filter on the column (the index) of this element */
          oTable.fnFilter( this.value, $("tfoot input").index(this) );
        });

        /*
         * Tab interceptor to set url of data based on the tab
         */
        $("#runningTab").click(function() {
          console.log("Switching to running: " + runningContentUrl);
          oTable.fnReloadAjax(runningContentUrl);
        });
        // $("#pendingXmlTab").click(function() {
        //   console.log("Switching to pending XML: " + pendingXmlContentUrl);
        //   oTable.fnReloadAjax(pendingXmlContentUrl);
        // });
        // $("#pendingDwcaTab").click(function() {
        //   console.log("Switching to pending DwC-A: " + pendingDwcaContentUrl);
        //   oTable.fnReloadAjax(pendingDwcaContentUrl);
        // });

        $(function () {
          $('#tabs').find('a:first').tab('show');
        });

        /*
         * Live view polling
         */
        function liveView() {
          //console.log("Live viewing " +oTable.$('tr.row_liveview').length + " records");

          var ajaxRequests = [];
          $.each(oTable.$('tr.row_liveview'), function(index, nRow) {
            var aData = oTable.fnGetData(nRow);
            ajaxRequests.push(
              $.ajax({
                url: detailUrl + aData.datasetKey,
                success: function(json, textStatus, jqXHR) {
                  if (jqXHR.status == "200") {
                    oTable.fnUpdate(json, nRow, undefined, false, false); // don't repaint the table
                    // we just refreshed it
                    aData = oTable.fnGetData(nRow);
                    decorate(nRow, aData);

                    // if the detail tab, is open, refresh that too
                    if($('textarea', $(nRow).next()).length>0) {
                      $('textarea', $(nRow).next()).text(JSON.stringify(json, undefined, 2));
                    }
                  }
                }
              })
            );
          });

          var defer = $.when.apply($, ajaxRequests);
          // Only when all ajax requests are complete schedule a new one
          defer.done(function(args) {
            //console.log("Polling");
            setTimeout(function() {
              liveView();
            }, 2000); // liveview update period msecs
          });
        }
        // Bulk update
        $(".allLiveView").click(function() {
          // If the first is set, clear them all, otherwise set them all
          var b = oTable.$('tbody tr').length && $(oTable.$('tbody tr')[0]).hasClass('row_liveview');
          if (b) oTable.$('tbody tr').removeClass('row_liveview success');
          else oTable.$('tbody tr').addClass('row_liveview success');
        });

        // start the live view polling
        liveView();

        // Hide the help
        $(".hideHelp").click(function() {
          $('#full').toggleClass("disabled");
          if ($('#full').hasClass("disabled")) {
            localStorage.setItem('helpHidden', true);
          } else {
            localStorage.removeItem('helpHidden');
          }
        });

        if (localStorage.getItem('helpHidden')) {
          $('#full').toggleClass("disabled");
        }
      });
    </script>
  </head>
  <body>
    <!-- Page layout is using bootstrap -->
    <div class="container">
      <div>
        <h3>Crawling Monitor <small> ...filter, sort, click for <em>live view</em></small></h3>
      </div>
      <ul id="tabs" class="nav nav-tabs">
        <li><a href="#data" data-toggle="tab" id="runningTab">Running crawls</a></li>
        <!--li><a href="#data" data-toggle="tab" id="pendingXmlTab">Pending (XML)</a></li-->
        <!--li><a href="#data" data-toggle="tab" id="pendingDwcaTab">Pending (DwC-A)</a></li-->
      </ul>
      <div class="tab-content">
        <div class="tab-pane active" id="data">
          <div id="tabTable">
            <table class="table table-striped table-bordered table-condensed" id="datasetCrawls">
              <thead>
                <tr id="full">
                  <th><div>URL</div></th>
                  <th><div>Declared count</div></th>
                  <th><div>Pages crawled</div></th>
                  <th><div>Pages fragmented successfully</div></th>
                  <th><div>Pages not fragmented (error)</div></th>
                  <th><div>Fragments emitted</div></th>
                  <th><div>Fragments received</div></th>
                  <th><div>Raw occurrence records<br/>persisted (new)</div></th>
                  <th><div>Raw occurrence records<br/>persisted (modified)</div></th>
                  <th><div>Raw occurrence records<br/>persisted (unchanged)</div></th>
                  <th><div>Raw occurrence records<br/>persisted (error)</div></th>
                  <th><div>Fragments processed</div></th>
                  <th><div>Verbatim occurrence records<br/>persisted successfully</div></th>
                  <th><div>Verbatim occurrence records<br/>not persisted (error)</div></th>
                  <th><div>Occurrence records interpreted</div></th>
                  <th><div>Occurrence records error<br/>on interpretation</div></th>
                  <th style="width:150px" colspan="2"><div>Miscellaneous<!-- colspan 2 hides the UUID --></div></th>
                </tr>
                <tr id="abbr">
                  <th>URL</th>
                  <th title="Declared count">DC</th>
                  <th title="Pages crawled">PC</th>
                  <th title="Pages fragmented successfully">PFS</th>
                  <th title="Pages not fragmented (error)">PFE</th>
                  <th title="Fragments emitted">FE</th>
                  <th title="Fragments received">FR</th>
                  <th title="Raw occurrence records persisted (new)">RON</th>
                  <th title="Raw occurrence records persisted (modified)">ROM</th>
                  <th title="Raw occurrence records persisted (unchanged)">ROU</th>
                  <th title="Raw occurrence records persisted (error)">ROE</th>
                  <th title="Fragments processed">FP</th>
                  <th title="Verbatim occurrence records persisted successfully">VOP</th>
                  <th title="Verbatim occurrence records not persisted (error)">VOE</th>
                  <th title="Occurrence records interpreted">OI</th>
                  <th title="Occurrence records error on interpretation">OE</th>
                  <th title="Miscellaneous" style="width:150px" colspan="2"><!-- colspan 2 hides the UUID --></th>
                </tr>
              </thead>
              <tbody/>
              <tfoot>
                <tr>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th><input type="text" name="search_ds" value="..." class="search_init" style="width:80%"/></th>
                  <th>
                    <input type="text" name="search_ds" value="..." class="search_init" style="width:0px;visibility:hidden" colspan="2"/>
                  </th>
                </tr>
              </tfoot>
            </table>
            <p/>
          </div>
        </div>
      </div>
  </body>
</html>

{'fraction_non_alphanumeric': 0.16567900117615128, 'fraction_numerical': 0.019089839862480775, 'mean_word_length': 2.548394863563403, 'pattern_counts': {'":': 62, '<': 512, '<?xml version=': 0, '>': 522, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '350564', 'n_tokens_mistral': 15089, 'n_tokens_neox': 14226, 'n_words': 2332}