<commit_before>(ns backtype.storm.ui.core
  (:use compojure.core)
  (:use [hiccup core page-helpers])
  (:use [backtype.storm config util])
  (:use [backtype.storm.ui helpers])
  (:use [backtype.storm.daemon [common :only [ACKER-COMPONENT-ID]]])
  (:use [clojure.contrib.def :only [defnk]])
  (:use [clojure.contrib.seq-utils :only [find-first]])
  (:use [ring.adapter.jetty :only [run-jetty]])
  (:use [clojure.string :only [trim]])
  (:import [backtype.storm.generated TaskSpecificStats
            TaskStats TaskSummary TopologyInfo SpoutStats BoltStats
            ErrorInfo ClusterSummary SupervisorSummary TopologySummary
            Nimbus$Client StormTopology GlobalStreamId])
  (:import [java.io File])
  (:require [compojure.route :as route]
            [compojure.handler :as handler]
            [backtype.storm [thrift :as thrift]])
  (:gen-class))

(def *STORM-CONF* (read-storm-config))

(defmacro with-nimbus [nimbus-sym & body]
  `(thrift/with-nimbus-connection [~nimbus-sym "localhost" (*STORM-CONF* NIMBUS-THRIFT-PORT)]
     ~@body
     ))

(defn get-filled-stats [summs]
  (->> summs
       (map #(.get_stats ^TaskSummary %))
       (filter not-nil?)))

(defn ui-template [body]
  (html
   [:head
    [:title "Storm UI"]
    (include-css "/css/bootstrap-1.1.0.css")
    (include-js "/js/jquery-1.6.2.min.js")
    (include-js "/js/jquery.tablesorter.min.js")
    ]
   [:script "$.tablesorter.addParser({ 
        id: 'stormtimestr', 
        is: function(s) { 
            return false; 
        }, 
        format: function(s) {
            if(s.search('All time')!=-1) {
              return 1000000000;
            }
            var total = 0;
            $.each(s.split(' '), function(i, v) {
              var amt = parseInt(v);
              if(v.search('ms')!=-1) {
                total += amt;
              } else if (v.search('s')!=-1) {
                total += amt * 1000;
              } else if (v.search('m')!=-1) {
                total += amt * 1000 * 60;
              } else if (v.search('h')!=-1) {
                total += amt * 1000 * 60 * 60;
              } else if (v.search('d')!=-1) {
                total += amt * 1000 * 60 * 60 * 24;
              }
            });
            return total;
        }, 
        type: 'numeric' 
    }); "]
   [:body
    [:h1 (link-to "/" "Storm UI")]
    (seq body)
    ]))

(defn read-storm-version []
  (let [storm-home (System/getProperty "storm.home")
        release-path (format "%s/RELEASE" storm-home)
        release-file (File. release-path)]
    (if (and (.exists release-file) (.isFile release-file))
      (trim (slurp release-path))
      "Unknown")))

(defn cluster-summary-table [^ClusterSummary summ]
  (let [sups (.get_supervisors summ)
        used-slots (reduce + (map #(.get_num_used_workers ^SupervisorSummary %) sups))
        total-slots (reduce + (map #(.get_num_workers ^SupervisorSummary %) sups))
        free-slots (- total-slots used-slots)
        total-tasks (->> (.get_topologies summ)
                         (map #(.get_num_tasks ^TopologySummary %))
                         (reduce +))]
    (table ["Storm version" "Nimbus uptime" "Supervisors" "Used slots" "Free slots" "Total slots" "Running tasks"]
           [[(read-storm-version)
             (pretty-uptime-sec (.get_nimbus_uptime_secs summ))
             (count sups)
             used-slots
             free-slots
             total-slots
             total-tasks]])
    ))

(defn topology-link
  ([id] (topology-link id id))
  ([id content]
     (link-to (url-format "/topology/%s" id) content)))<commit_msg>change heading for Storm version<commit_after>(ns backtype.storm.ui.core
  (:use compojure.core)
  (:use [hiccup core page-helpers])
  (:use [backtype.storm config util])
  (:use [backtype.storm.ui helpers])
  (:use [backtype.storm.daemon [common :only [ACKER-COMPONENT-ID]]])
  (:use [clojure.contrib.def :only [defnk]])
  (:use [clojure.contrib.seq-utils :only [find-first]])
  (:use [ring.adapter.jetty :only [run-jetty]])
  (:use [clojure.string :only [trim]])
  (:import [backtype.storm.generated TaskSpecificStats
            TaskStats TaskSummary TopologyInfo SpoutStats BoltStats
            ErrorInfo ClusterSummary SupervisorSummary TopologySummary
            Nimbus$Client StormTopology GlobalStreamId])
  (:import [java.io File])
  (:require [compojure.route :as route]
            [compojure.handler :as handler]
            [backtype.storm [thrift :as thrift]])
  (:gen-class))

(def *STORM-CONF* (read-storm-config))

(defmacro with-nimbus [nimbus-sym & body]
  `(thrift/with-nimbus-connection [~nimbus-sym "localhost" (*STORM-CONF* NIMBUS-THRIFT-PORT)]
     ~@body
     ))

(defn get-filled-stats [summs]
  (->> summs
       (map #(.get_stats ^TaskSummary %))
       (filter not-nil?)))

(defn ui-template [body]
  (html
   [:head
    [:title "Storm UI"]
    (include-css "/css/bootstrap-1.1.0.css")
    (include-js "/js/jquery-1.6.2.min.js")
    (include-js "/js/jquery.tablesorter.min.js")
    ]
   [:script "$.tablesorter.addParser({ 
        id: 'stormtimestr', 
        is: function(s) { 
            return false; 
        }, 
        format: function(s) {
            if(s.search('All time')!=-1) {
              return 1000000000;
            }
            var total = 0;
            $.each(s.split(' '), function(i, v) {
              var amt = parseInt(v);
              if(v.search('ms')!=-1) {
                total += amt;
              } else if (v.search('s')!=-1) {
                total += amt * 1000;
              } else if (v.search('m')!=-1) {
                total += amt * 1000 * 60;
              } else if (v.search('h')!=-1) {
                total += amt * 1000 * 60 * 60;
              } else if (v.search('d')!=-1) {
                total += amt * 1000 * 60 * 60 * 24;
              }
            });
            return total;
        }, 
        type: 'numeric' 
    }); "]
   [:body
    [:h1 (link-to "/" "Storm UI")]
    (seq body)
    ]))

(defn read-storm-version []
  (let [storm-home (System/getProperty "storm.home")
        release-path (format "%s/RELEASE" storm-home)
        release-file (File. release-path)]
    (if (and (.exists release-file) (.isFile release-file))
      (trim (slurp release-path))
      "Unknown")))

(defn cluster-summary-table [^ClusterSummary summ]
  (let [sups (.get_supervisors summ)
        used-slots (reduce + (map #(.get_num_used_workers ^SupervisorSummary %) sups))
        total-slots (reduce + (map #(.get_num_workers ^SupervisorSummary %) sups))
        free-slots (- total-slots used-slots)
        total-tasks (->> (.get_topologies summ)
                         (map #(.get_num_tasks ^TopologySummary %))
                         (reduce +))]
    (table ["Version" "Nimbus uptime" "Supervisors" "Used slots" "Free slots" "Total slots" "Running tasks"]
           [[(read-storm-version)
             (pretty-uptime-sec (.get_nimbus_uptime_secs summ))
             (count sups)
             used-slots
             free-slots
             total-slots
             total-tasks]])
    ))

(defn topology-link
  ([id] (topology-link id id))
  ([id content]
     (link-to (url-format "/topology/%s" id) content)))
{'fraction_non_alphanumeric': 0.16038915913829047, 'fraction_numerical': 0.014454482279360668, 'mean_word_length': 2.0752136752136754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '504448', 'n_tokens_mistral': 2545, 'n_tokens_neox': 2376, 'n_words': 520}