<commit_before>(ns backtype.storm.daemon.logviewer
  (:use compojure.core)
  (:use [hiccup core page-helpers])
  (:use [backtype.storm config util log])
  (:use [ring.adapter.jetty :only [run-jetty]])
  (:import [org.apache.commons.logging LogFactory])
  (:import [org.apache.commons.logging.impl Log4JLogger])
  (:import [org.apache.log4j Level])
  (:require [compojure.route :as route]
            [compojure.handler :as handler])
  (:gen-class))

(defn tail-file [path tail]
  (let [flen (.length (clojure.java.io/file path))
        skip (- flen tail)]
    (with-open [input (clojure.java.io/input-stream path)
                output (java.io.ByteArrayOutputStream.)]
      (if (> skip 0) (.skip input skip))
      (let [buffer (make-array Byte/TYPE 1024)]
        (loop []
          (let [size (.read input buffer)]
            (when (and (pos? size) (< (.size output) tail))
              (do (.write output buffer 0 size)
                  (recur))))))
      (.toString output))
    ))

(defn log-page [file tail grep]
  (let [path (str (System/getProperty "storm.home") "/logs/" file)
        tail (if tail
               (min 10485760 (Integer/parseInt tail))
               10240)
        tail-string (tail-file path tail)]
    (if grep
       (clojure.string/join "\n<br>"
         (filter #(.contains % grep) (.split tail-string "\n")))
       (.replaceAll tail-string "\n" "\n<br>"))))

(defn log-level-page [name level]
  (let [log (LogFactory/getLog name)]
    (if level<commit_msg>logviewer tries to determine log directory via logback configuration, falls back to storm.home/logs<commit_after>(ns backtype.storm.daemon.logviewer
  (:use compojure.core)
  (:use [hiccup core page-helpers])
  (:use [backtype.storm config util log])
  (:use [ring.adapter.jetty :only [run-jetty]])
  (:import [org.slf4j LoggerFactory])
  (:import [ch.qos.logback.classic Logger])
  (:import [org.apache.commons.logging LogFactory])
  (:import [org.apache.commons.logging.impl Log4JLogger])
  (:import [ch.qos.logback.core FileAppender])
  (:import [org.apache.log4j Level])
  (:require [compojure.route :as route]
            [compojure.handler :as handler]
            [clojure.string :as string])
  (:gen-class))

(defn tail-file [path tail]
  (let [flen (.length (clojure.java.io/file path))
        skip (- flen tail)]
    (with-open [input (clojure.java.io/input-stream path)
                output (java.io.ByteArrayOutputStream.)]
      (if (> skip 0) (.skip input skip))
      (let [buffer (make-array Byte/TYPE 1024)]
        (loop []
          (let [size (.read input buffer)]
            (when (and (pos? size) (< (.size output) tail))
              (do (.write output buffer 0 size)
                  (recur))))))
      (.toString output))
    ))

(defn log-root-dir
  []
  (let [appender (first (iterator-seq (.iteratorForAppenders (LoggerFactory/getLogger Logger/ROOT_LOGGER_NAME))))]
    (if (and appender (instance? FileAppender appender))
      (string/join
       "/" (butlast
            (string/split (.getFile appender) #"/")))
      (str (System/getProperty "storm.home") "/logs/"))))

(defn log-page [file tail grep]
  (let [path (str (log-root-dir) "/" file)
        tail (if tail
               (min 10485760 (Integer/parseInt tail))
               10240)
        tail-string (tail-file path tail)]
    (if grep
       (clojure.string/join "\n<br>"
         (filter #(.contains % grep) (.split tail-string "\n")))
       (.replaceAll tail-string "\n" "\n<br>"))))

(defn log-level-page [name level]
  (let [log (LogFactory/getLog name)]
    (if level
{'fraction_non_alphanumeric': 0.15869017632241814, 'fraction_numerical': 0.012034704729918836, 'mean_word_length': 2.599194360523666, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1082977', 'n_tokens_mistral': 1246, 'n_tokens_neox': 1161, 'n_words': 311}