<commit_before>(ns bzzz.queries.term-payload-clj-score
  (use bzzz.util)
  (:require [bzzz.index-stat :as index-stat])
  (:import (org.apache.lucene.index Term)
           (org.apache.lucene.search MatchAllDocsQuery)
           (com.googlecode.concurrentlinkedhashmap ConcurrentLinkedHashMap ConcurrentLinkedHashMap$Builder)
           (bzzz.java.query TermPayloadClojureScoreQuery NoZeroQuery ExpressionContext Helper)))

(defn fixed-bucket-aggregation-result [^TermPayloadClojureScoreQuery query]
  (let [fba (.fba_get_results ^ExpressionContext (.clj_context query))]
    (zipmap (.keySet fba)
            (map (fn [f]
                   (map (fn [^java.util.HashMap v]
                          {:count (.get v "count")
                           :label (.get v "label")})
                        (into [] f)))
                 (.values fba)))))

(defn extract-result-state [^TermPayloadClojureScoreQuery query doc-id]
  (.result_state_get_for_doc ^ExpressionContext (.clj_context query) doc-id))

(def expr-cache ^java.util.Map
  (let [b ^java.util.Map (ConcurrentLinkedHashMap$Builder.)]
    (.maximumWeightedCapacity b 1000)
    (.build b)))

(defn get-or-eval [expr]
  (if-let [v (.get ^java.util.Map expr-cache expr)]
    v
    (let [t0 (time-ms)
          evaluated-expr (eval (read-string expr))]
      (index-stat/update-took-count index-stat/total
                                    "eval"
                                    (time-took t0))
      (.put ^java.util.Map expr-cache expr evaluated-expr)
      evaluated-expr)))

(defn parse<commit_msg>add giant lock around eval expressions<commit_after>(ns bzzz.queries.term-payload-clj-score
  (use bzzz.util)
  (:require [bzzz.index-stat :as index-stat])
  (:import (org.apache.lucene.index Term)
           (org.apache.lucene.search MatchAllDocsQuery)
           (com.googlecode.concurrentlinkedhashmap ConcurrentLinkedHashMap ConcurrentLinkedHashMap$Builder)
           (bzzz.java.query TermPayloadClojureScoreQuery NoZeroQuery ExpressionContext Helper)))

(defn fixed-bucket-aggregation-result [^TermPayloadClojureScoreQuery query]
  (let [fba (.fba_get_results ^ExpressionContext (.clj_context query))]
    (zipmap (.keySet fba)
            (map (fn [f]
                   (map (fn [^java.util.HashMap v]
                          {:count (.get v "count")
                           :label (.get v "label")})
                        (into [] f)))
                 (.values fba)))))

(defn extract-result-state [^TermPayloadClojureScoreQuery query doc-id]
  (.result_state_get_for_doc ^ExpressionContext (.clj_context query) doc-id))

(def expr-cache ^java.util.Map
  (let [b ^java.util.Map (ConcurrentLinkedHashMap$Builder.)]
    (.maximumWeightedCapacity b 1000)
    (.build b)))

(def giant (Object.))

(defn get-or-eval [expr]
  (if-let [v (.get ^java.util.Map expr-cache expr)]
    v
    (locking giant
      (if-let [v (.get ^java.util.Map expr-cache expr)]
        v
        (let [t0 (time-ms)
              evaluated-expr (eval (read-string expr))]
          (index-stat/update-took-count index-stat/total
                                        "eval"
                                        (time-took t0))
          (.put ^java.util.Map expr-cache expr evaluated-expr)
          evaluated-expr)))))

(defn parse
{'fraction_non_alphanumeric': 0.12557008209182122, 'fraction_numerical': 0.0036485253876558225, 'mean_word_length': 2.2606541129831514, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5926299', 'n_tokens_mistral': 1063, 'n_tokens_neox': 1008, 'n_words': 226}