<commit_before>(ns angelic.scripts.experiments-11icaps
 (:require [edu.berkeley.ai [util :as util]] 
	   [edu.berkeley.ai.util [charts :as charts] [datasets :as datasets] [experiments :as experiments]]
           [angelic [env :as env] [hierarchy :as hierarchy] [sas :as sas]]
           [angelic.domains.sas-problems :as sas-problems]
           [angelic.search.textbook :as textbook]
           [angelic.search.action-set.asplan :as asplan]
	   [edu.berkeley.ai.util.experiments :as experiments]))

(def *exp-result* ::ExpResult)

(defstruct exp-result 
  :class :experiment :commit-id :timeout? :memout? :output :printed :init-ms :ms :mb
  :next-count :add-count)

(defmethod experiments/setup-experiment-result ::ExpResult [experiment]
           (env/reset-next-counter)
           (util/sref-set! asplan/*add-count* 0))


(defmethod experiments/make-experiment-result ::ExpResult 
  [experiment setup-info timeout? memout? output printed init-ms ms mb]
  (struct exp-result ::ExpResult
          experiment (util/git-commit-id) timeout? memout? output printed init-ms ms mb
          (util/sref-get env/*next-counter*)
          (util/sref-get asplan/*add-count*)
          ))

;;;;;;; logistics

(defn make-exp-set []
  (experiments/make-experiment-set
   "logistics"
   [:product
    [:instance (vec (range 27))]
    [:alg
      [[:baseline]
       [:asplan]
       [:asplan :directed? false]
       [:asplan :greedy? false]
       [:asplan :deadlock? false]
       [:asplan :dead-vars? false]]]]
   (fn [m]
     (let [{[type & args] :alg instance :instance} m
           instance-form `(force (nth sas-problems/ipc2-logistics ~instance))]
       (if (= type :baseline)
         instance-form
         `(asplan/make-asplan-env ~instance-form ~@args))))
   (fn [m] `(textbook/uniform-cost-search ~'init))
   'angelic.scripts.experiments-11icaps 10 600 512 false ::ExpResult))

(defonce *results* nil)
(defn read-results []
  (def *results* 
       (experiments/experiment-set-results->dataset 
        (experiments/read-experiment-set-results (make-exp-set)))))


(comment

;;;;;;;;;; Table

  (defn pad-right [x n]  
    (let [xs (str x) 
          s #^String (str "                               " xs)]
                                        ;    (assert (<= (.length xs) n))
      (.substring s (- (.length s) n)))) 

  (def *cw* 8)
  (def *w* (+ 4 (* 3 *cw*)))
  (defn make-table []
    (let [results (group-by #(get % :alg) *results*)]
      (doseq [alg (map first his/aaai-algs)]
        (let [alg-results (results alg)
              obj-map (group-by #(get % :objects) alg-results)
              objects    (sort (keys obj-map))]
          (println (apply str (pad-right alg 9) "|" (for [o ["s" "progs" "refs"]] (str (pad-right o *w*) "|"))))
          (println (apply str (repeat (+ 10 (* 3 (inc *w*))) "-")))
          (doseq [[no n-res] (map (fn [x] [x (obj-map x)]) objects)]
            (println (apply str (pad-right no 9) "|"
                            (for [angelic n-res]
                              (if (:ms exp)
                                (str (pad-right (/ (int (:ms exp)) 1000.0) *cw*) ", " (pad-right (:next-count exp) *cw*) ", " (pad-right (second (:output exp)) #_(:plan-count exp) *cw*) "|")
                                (str (pad-right (cond (:memout? exp) "mem" (:timeout? exp) "time") *w*) "|")
                                ))))))    
        (println "\n\n"))))



;;;;;; charts


  (def *alg-order* ["H-UCS" "DH-UCS" "DSH-UCS" "AHA*" "DAHA*" "DASHA*"])
  (defn order [things key-fn desired-order]
                                        ;  (println things)
    (let [m (group-by key-fn things)]
      (map #(first (util/safe-get m %)) desired-order)))

  (defn make-charts
    ([] (make-charts "/Users/jawolfe/Projects/reports/10-aaai-btamp/poster/figs/" ))
    ([dir]
       (charts/plot 
        (datasets/ds->chart 
         (datasets/ds-derive #(/ (:ms %) 1000) (filter (datasets/ds-fn [ms] (and ms)) *results*) :secs) 
         [:alg] :objects :secs 
         {:term "solid dashed size 2.4,1.3"   :ylog nil :xrange "[1:6]" :yrange "[0:60]"
                                        ;        :title "discrete manipulation problems" :xlabel "# of objects" ;:ylabel "runtime (s)"
          :key "at 4.8, 67 height 3"                    
          :extra-commands ["set ylabel \"runtime(s)\" 1,0"
                           "set xlabel \"# of objects\" 0,.5"]} 
         (let [c (util/counter-from 0)] (fn [& args] (let [v ([2 3 1 5 4.8 6] (c))]  {:lw 3 :pt v :lt v})))
         #(let [n (first %)] (if (= (last n) \U) (apply str (concat (drop-last n) "-UCS")) (str n "*")))
         #(order % :title *alg-order*))
        (str dir "dm-time.pdf") false)
       #_ (charts/plot 
           (datasets/ds->chart 
            (datasets/ds-derive #(/ (:ms %) 1000) (filter (datasets/ds-fn [ms] (and ms)) *results*) :secs) 
            [:alg] :objects :next-count 
            {:term "solid dashed size 3,1.7"   :ylog nil :xrange "[1:6]" :yrange "[0:60000]"  :key "at 3.6,57000"
                                        ;   :title "discrete manipulation problems"
             :extra-commands ["set ylabel \"# of primitive action evaluations(s)\" 1,0"
                              "set xlabel \"# of objects\" 0,.5"]} 
            (let [c (util/counter-from 0)] (fn [& args] (let [v ([1 2 3 6 5 4] (c))]  {:lw 3 :pt v :lt v})))
            first #(order % :title *alg-order*))
           (str dir "dm-prim.pdf") false)
       ))
<commit_msg>Start ot adjust miconic<commit_after>(ns angelic.scripts.experiments-11icaps
 (:require [edu.berkeley.ai [util :as util]] 
	   [edu.berkeley.ai.util [charts :as charts] [datasets :as datasets] [experiments :as experiments]]
           [angelic [env :as env] [hierarchy :as hierarchy] [sas :as sas]]
           [angelic.domains.sas-problems :as sas-problems]
           [angelic.search.textbook :as textbook]
           [angelic.search.action-set.asplan :as asplan]
	   [edu.berkeley.ai.util.experiments :as experiments]))

(def *exp-result* ::ExpResult)

(defstruct exp-result 
  :class :experiment :commit-id :timeout? :memout? :output :printed :init-ms :ms :mb
  :next-count :add-count)

(defmethod experiments/setup-experiment-result ::ExpResult [experiment]
           (env/reset-next-counter)
           (util/sref-set! asplan/*add-count* 0))


(defmethod experiments/make-experiment-result ::ExpResult 
  [experiment setup-info timeout? memout? output printed init-ms ms mb]
  (struct exp-result ::ExpResult
          experiment (util/git-commit-id) timeout? memout? output printed init-ms ms mb
          (util/sref-get env/*next-counter*)
          (util/sref-get asplan/*add-count*)
          ))

;;;;;;; logistics

(def algs
  [[[:baseline] "base"]
   [[:asplan] "bi"]
   [[:asplan :directed? false] "bi-d"]
   [[:asplan :greedy? false] "bi-g"]
   [[:asplan :deadlock? false] "bi-dl"]
   [[:asplan :dead-vars? false] "bi-dv"]])

(defn make-exp-set []
  (experiments/make-experiment-set
   "logistics"
   [:product
    [:instance (vec (range 27))]
    [:alg (map first algs)]]
   (fn [m]
     (let [{[type & args] :alg instance :instance} m
           instance-form `(force (nth sas-problems/ipc2-logistics ~instance))]
       (if (= type :baseline)
         instance-form
         `(asplan/make-asplan-env ~instance-form ~@args))))
   (fn [m] `(textbook/uniform-cost-search ~'init))
   'angelic.scripts.experiments-11icaps 10 600 512 false ::ExpResult))

(defonce *results* nil)
(defn read-results []
  (def *results* 
       (experiments/experiment-set-results->dataset 
        (experiments/read-experiment-set-results (make-exp-set)))))

;;;;;;; table

(defn pad-right [x n]  
    (let [xs (str x) 
          s #^String (str "                               " xs)]
      (assert (<= (.length xs) n))
      (.substring s (- (.length s) n)))) 

(defn prob-name [^String x]
  (.substring x 14 (- (count x) 5)))



(def *cw* 8)
(def *w* (+ 4 (* 3 *cw*)))
(defn make-full-table []
  (println (apply str (pad-right " " 9) "|" (for [[_ alg] algs] (str (pad-right alg *w*) "|"))))
  (doseq [i (range 25)]
    (print (str (prob-name (nth sas-problems/ipc2-logistics-names i)) "   "  " " "|"))
    (doseq [[algg name] algs
            :let [result (util/find-first (datasets/ds-fn [instance alg] (and (= instance i) (= alg algg))) *results*)]]
      (print (str (pad-right (if (:output result) (str " " (pad-right (:next-count result) *cw*) (pad-right  (int (:ms result)) *cw*)) (str ">" (:next-count result))) *w*) "|" )))
    (println) ))

(defn make-table []
  (println (apply str (pad-right " " 9) "|" (for [[_ alg] algs] (str (pad-right alg *w*) "|"))))
  (doseq [[i reward] (sort-by second (map (juxt identity #(- (second (:output (util/find-first (datasets/ds-fn [instance] (= instance %)) *results*))))) (range 10)))]
    (print (str (prob-name (nth sas-problems/ipc2-logistics-names i)) "   " (pad-right reward 2) " " "|"))
    (doseq [[algg name] algs
            :let [result (util/find-first (datasets/ds-fn [instance alg] (and (= instance i) (= alg algg))) *results*)]]
      (print (str (pad-right (if (:output result) (str " " (pad-right (:next-count result) *cw*) (pad-right  (int (:ms result)) *cw*)) (str ">" (:next-count result))) *w*) "|" )))
    (println) ))

(defn make-latex-table []
  (println (apply str (pad-right " " 9) "&" (for [[_ alg] algs] (str (pad-right alg (/ *w* 2)) " &"))))
  (doseq [[i reward] (sort-by second (map (juxt identity #(- (second (:output (util/find-first (datasets/ds-fn [instance] (= instance %)) *results*))))) (range 10)))]
    (print (str (prob-name (nth sas-problems/ipc2-logistics-names i)) " & " (pad-right reward 2) " " "&"))
    (doseq [[algg name] algs
            :let [result (util/find-first (datasets/ds-fn [instance alg] (and (= instance i) (= alg algg))) *results*)]]
      (print (str (str " " (pad-right (:next-count result) *cw*) " & " (pad-right  (int (:ms result)) *cw*)) " & " )))
    (println) ))





(comment
;;;;;;; miconic

 (def miconic-algs
      [[[:baseline] "base"]
       [[:asplan] "bi"]])

 (defn make-exp-set []
   (experiments/make-experiment-set
    "miconic"
    [:product
     [:instance (vec (range 150))]
     [:alg (map first algs)]]
    (fn [m]
      (let [{[type & args] :alg instance :instance} m
            instance-form `(force (nth sas-problems/ipc2-logistics ~instance))]
        (if (= type :baseline)
          instance-form
          `(asplan/make-asplan-env ~instance-form ~@args))))
    (fn [m] `(textbook/uniform-cost-search ~'init))
    'angelic.scripts.experiments-11icaps 10 600 512 false ::ExpResult))

 (defonce *results* nil)
 (defn read-results []
   (def *results* 
        (experiments/experiment-set-results->dataset 
         (experiments/read-experiment-set-results (make-exp-set))))))





;;;;;; charts

(comment
 (def *alg-order* ["H-UCS" "DH-UCS" "DSH-UCS" "AHA*" "DAHA*" "DASHA*"])
 (defn order [things key-fn desired-order]
                                        ;  (println things)
   (let [m (group-by key-fn things)]
     (map #(first (util/safe-get m %)) desired-order)))

 (defn make-charts
   ([] (make-charts "/Users/jawolfe/Projects/reports/10-aaai-btamp/poster/figs/" ))
   ([dir]
      (charts/plot 
       (datasets/ds->chart 
        (datasets/ds-derive #(/ (:ms %) 1000) (filter (datasets/ds-fn [ms] (and ms)) *results*) :secs) 
        [:alg] :objects :secs 
        {:term "solid dashed size 2.4,1.3"   :ylog nil :xrange "[1:6]" :yrange "[0:60]"
                                        ;        :title "discrete manipulation problems" :xlabel "# of objects" ;:ylabel "runtime (s)"
         :key "at 4.8, 67 height 3"                    
         :extra-commands ["set ylabel \"runtime(s)\" 1,0"
                          "set xlabel \"# of objects\" 0,.5"]} 
        (let [c (util/counter-from 0)] (fn [& args] (let [v ([2 3 1 5 4.8 6] (c))]  {:lw 3 :pt v :lt v})))
        #(let [n (first %)] (if (= (last n) \U) (apply str (concat (drop-last n) "-UCS")) (str n "*")))
        #(order % :title *alg-order*))
       (str dir "dm-time.pdf") false)
      #_ (charts/plot 
          (datasets/ds->chart 
           (datasets/ds-derive #(/ (:ms %) 1000) (filter (datasets/ds-fn [ms] (and ms)) *results*) :secs) 
           [:alg] :objects :next-count 
           {:term "solid dashed size 3,1.7"   :ylog nil :xrange "[1:6]" :yrange "[0:60000]"  :key "at 3.6,57000"
                                        ;   :title "discrete manipulation problems"
            :extra-commands ["set ylabel \"# of primitive action evaluations(s)\" 1,0"
                             "set xlabel \"# of objects\" 0,.5"]} 
           (let [c (util/counter-from 0)] (fn [& args] (let [v ([1 2 3 6 5 4] (c))]  {:lw 3 :pt v :lt v})))
           first #(order % :title *alg-order*))
          (str dir "dm-prim.pdf") false)
      ))

{'fraction_non_alphanumeric': 0.18902914134330684, 'fraction_numerical': 0.016674458469689887, 'mean_word_length': 2.321687370600414, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5890039', 'n_tokens_mistral': 4807, 'n_tokens_neox': 4438, 'n_words': 1175}