<commit_before>(ns distillery.xmlresult
  (:import [java.util Locale])
  (:require [clojure.java.io :refer (writer)])
  (:require [clojure.data.xml :as xml]))

(defn- tag-list
  [tag-name f xs]
  (let [tags (map f xs)]
    (vec (concat [tag-name {:length (count tags)}] tags))))

(defn- format-invariant
  [fmt & args]
  (String/format Locale/US fmt (to-array args)))

(defn- category-resource-tag
  [job category resource]
  [:Resource {:type (name (:type resource))} (:url resource)])

(defn- category-match-tag
  [job category [video-id match]]
  [:Match {:video-id video-id
           :score (format-invariant "%.6f" (:score match))}])

(defn- category-tag
  [job category]
  [:Category {:id (:id category)}
   [:Name {} (:name category)]
   (tag-list :MatchList
             (partial category-match-tag job category)
             (filter
              (fn [[id match]] (>= (:score match) (get-in job [:configuration :min-match-score])))
              (:matches category)))
   (tag-list :ResourceList
             (partial category-resource-tag job category)
             (:resources category))])

(defn- video-match-tag
  [job video [category-id match]]
  [:Match {:category-id category-id
           :score (format-invariant "%.6f" (:score match))}])

(defn- recognized-word
  [job phrase word]
  [:Word {:no (:no word)
          :confidence (:confidence word)}
   (:text word)])

(defn- recognized-phrase
  [job result]
  [:Phrase {:no (:no result)
            :start (:start result)
            :duration (:duration result)
            :confidence (:confidence result)}
   [:Text {} (:text result)]
   ;(tag-list :WordList
   ;          (partial recognized-word job result)
   ;          (:words result))
   ])

(defn- speech-recognition-tag
  [job video]
  [:SpeechRecognition {:profile (:recognition-profile video)
                       :profile-name (:recognition-profile-name video)}
   (tag-list :PhraseList
             (partial recognized-phrase job)
             (:results video))])

(defn- video-tag
  [job video]
  [:Media {:id (:id video)}
   [:Name {} (:name video)]
   (tag-list :MatchList
             (partial video-match-tag job video)
             (filter
              (fn [[id match]] (>= (:score match) (get-in job [:configuration :min-match-score])))
              (:matches video)))
   [:Source {} (:video-file video)]
   (speech-recognition-tag job video)])

(defn- job-result-tags
  [job]
  (xml/sexp-as-element
   [:MediaCategorizerResults {:tool-version (:media-categorizer-version job)}
    [:JobInfo {}
     [:Name {} (:job-name job)]
     [:Description {} (:job-description job)]]
    (tag-list :CategoryList
              (partial category-tag job)
              (:categories job))
    (tag-list :MediaList
              (partial video-tag job)
              (:videos job))]))

(defn save-result
  [path job]<commit_msg>added glossary to xml formatted result<commit_after>(ns distillery.xmlresult
  (:import [java.util Locale])
  (:require [clojure.java.io :refer (writer)])
  (:require [clojure.data.xml :as xml]))

(defn- tag-list
  [tag-name f xs]
  (let [tags (map f xs)]
    (vec (concat [tag-name {:length (count tags)}] tags))))

(defn- format-invariant
  [fmt & args]
  (String/format Locale/US fmt (to-array args)))

(defn- category-resource-tag
  [job category resource]
  [:Resource {:type (name (:type resource))} (:url resource)])

(defn- category-match-tag
  [job category [video-id match]]
  [:Match {:video-id video-id
           :score (format-invariant "%.6f" (:score match))}])

(defn- category-tag
  [job category]
  [:Category {:id (:id category)}
   [:Name {} (:name category)]
   (tag-list :MatchList
             (partial category-match-tag job category)
             (filter
              (fn [[id match]] (>= (:score match) (get-in job [:configuration :min-match-score])))
              (:matches category)))
   (tag-list :ResourceList
             (partial category-resource-tag job category)
             (:resources category))])

(defn- video-match-tag
  [job video [category-id match]]
  [:Match {:category-id category-id
           :score (format-invariant "%.6f" (:score match))}])

(defn- recognized-word
  [job phrase word]
  [:Word {:no (:no word)
          :confidence (:confidence word)}
   (:text word)])

(defn- recognized-phrase
  [job result]
  [:Phrase {:no (:no result)
            :start (:start result)
            :duration (:duration result)
            :confidence (:confidence result)}
   [:Text {} (:text result)]
   ;(tag-list :WordList
   ;          (partial recognized-word job result)
   ;          (:words result))
   ])

(defn- speech-recognition-tag
  [job video]
  [:SpeechRecognition {:profile (:recognition-profile video)
                       :profile-name (:recognition-profile-name video)}
   (tag-list :PhraseList
             (partial recognized-phrase job)
             (:results video))])

(defn- video-tag
  [job video]
  [:Media {:id (:id video)}
   [:Name {} (:name video)]
   (tag-list :MatchList
             (partial video-match-tag job video)
             (filter
              (fn [[id match]] (>= (:score match) (get-in job [:configuration :min-match-score])))
              (:matches video)))
   [:Source {} (:video-file video)]
   (speech-recognition-tag job video)])

(defn- occurrence-tag
  [job word {:keys [video-id result-no word-no] :as occurrence}]
  (let [video (first (filter #(= video-id (:id %)) (:videos job)))
        phrase-start (get-in video [:results result-no :start])]
    [:Occurrence {:video video-id
                  :phrase result-no
                  :no word-no
                  :phrase-start phrase-start}]))

(defn- word-tag
  [job {:keys [lexical-form pronunciation occurrence-count occurrences] :as word}]
  [:Word {:lexical-form lexical-form
          :pronunciation pronunciation}
   (tag-list :OccurrenceList
             (partial occurrence-tag job word)
             occurrences)])

(defn- job-result-tags
  [job]
  (xml/sexp-as-element
   [:MediaCategorizerResults {:tool-version (:media-categorizer-version job)}
    [:JobInfo {}
     [:Name {} (:job-name job)]
     [:Description {} (:job-description job)]]
    (tag-list :CategoryList
              (partial category-tag job)
              (:categories job))
    (tag-list :MediaList
              (partial video-tag job)
              (:videos job))
    (tag-list :Glossary
              (partial word-tag job)
              (map val (:words job)))]))

(defn save-result
  [path job]
{'fraction_non_alphanumeric': 0.15804066543438078, 'fraction_numerical': 0.0006161429451632779, 'mean_word_length': 2.511627906976744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 9}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3282183', 'n_tokens_mistral': 2147, 'n_tokens_neox': 1976, 'n_words': 556}