<commit_before>#! /usr/bin/env hy
"""
    latexonhttp.caching.resources
    ~~~~~~~~~~~~~~~~~~~~~
    Managing input resources caching.

    :copyright: (c) 2019 <NAME>.
    :license: AGPL, see LICENSE for more details.
"""
(import logging)
(import datetime)
(import [
    latexonhttp.utils.fun [get-default fun-merge-dicts fun-dict-update fun-dict-remove-key fun-sort]
])
(import [
    latexonhttp.caching.store [get-cache-metadata persist-cache-metadata]
])
(import [
    latexonhttp.caching.filesystem [apply-cache-action apply-sanity-check MAX-RESOURCES-CACHE-SIZE ENABLE-SANITY-CHECKS]
])

(setv logger (.getLogger logging __name__))

; 3 Ko.
(setv MIN-FILE-SIZE-CACHE-THRESHOLD (* 3 1000))

; # Cache module:
; # - Create directory for caching files (delete on start / if no metadata);
; # - Put input files on cache directory, with metadata (size, cache id: flat directory);
; # - Check input cache metadata for enforcing constraints (max size cache), act if needed (delete files on overflow);
; # - Actually take the metadata, the resource forwared and return the actions: add/rm, rm --all;
; # - From time to time, as a sanity check, get directory size to check that it matches our metadata.

; TODO /caches/resources endpoint

; --------------------------------
; External API.
; --------------------------------

; TODO Actually the caching must be forwarded to a decicated process
; for the whole node to ensure consistency.
; Also will avoid cache management overhead in main process.
; --> Uses a zeroMQ socket as the API.
; The cache layer could then be 100% independent.
; For eg. could be implemented in Go, with a mixed
; in-memory and on-disk cache.
; There could be a memcached adapter, to rely on existing
; caching technology.
; With enough data, there could be neural-network trained
; to optimized bandwidth-saving cache hits.

(defn forward-resource-to-cache [resource data]
    (setv cache-metadata (get-cache-metadata))
    ; TODO Maintain metrics on resource usage, for eg:
    ; - resources inputs for last 24 hours;
    ; - last N (10000) resources inputs.
    ; With this data + hit logs, we could process cache efficiency metrics.
    ; TODO Logs this data and create datasets, so we can try strategies and
    ; see which ones work best on past-data.
    (setv actions (+
        ; - Check/Init resource cache -> actions;
        (prepare-cache cache-metadata)
        ; - Decide what to do with current resource -> actions;
        (process-resource-caching-decision cache-metadata resource data)
    ))
    ; (logger.debug "Cache actions: %s" actions)
    (for [action actions]
        ; - Apply actions;
        ; TODO Normalize action using map-resource-for-cache-metadata?
        (apply-cache-action action)
        ; - Update cache metadata after actions.
        ; Update cache metadata after each action,
        ; so we have an as much as possible up-to-date cache
        ; for multi-threading/process context.
        ; (See comment above on making it a dedicated and concurrent-safe process)
        (setv cache-metadata (persist-cache-metadata (update-cache-metadata-for-action cache-metadata action))))
        (if ENABLE-SANITY-CHECKS
          (apply-sanity-check))
    )

; TODO Cache usage.
; Maintain statistics on cache hits,
; with volumes of input resource bandwitdh hit/missed.

; --------------------------------
; Decision making.
; --------------------------------

(defn prepare-cache [cache-metadata]
    (if (get-default cache-metadata "last_updated_at")
        []
        [
            {
                "name" "reset_cache"
            }
        ]))

(defn is-resource-cached [cache-metadata resource]
  (in (get resource "data_spec" "hash") (get cache-metadata "cached_resources")))

(defn map-add-resource-action [resource data]
  {
    "name" "add_resource"
    "resource" resource
    "data" data
  })

(defn map-remove-resource-action [resource]
  {
    "name" "remove_resource"
    "resource" resource
  })

(defn free-space-from-first-rec [resources size-to-free actions]
  (if (> size-to-free 0)
    (free-space-from-first-rec
      (list (drop 1 resources))
      (- size-to-free (get (first resources) "size"))
      (+ actions [(map-remove-resource-action (first resources))]))
    actions))

(defn free-space-from-old-entries [cache-metadata size-to-free]
  (setv ordered-resources (fun-sort (.values (get cache-metadata "cached_resources")) (fn [resource] (get resource "added_at"))))
  ; (logger.debug "Ordered resources: %s" ordered-resources)
  (if (> size-to-free 0)
    ; Order resources by timestamp.
    ; Remove until we have freed the specified size
    (free-space-from-first-rec
      ordered-resources
      size-to-free
      [])
    []))

(defn process-resource-caching-decision [cache-metadata resource data]
    ; Naive FIFO cache:
    ; TODO Uses resources stats to cache the most used resources.
    (if (and
      ; - Check for a caching size threshold;
      (>= (get resource "data_spec" "size") MIN-FILE-SIZE-CACHE-THRESHOLD)
      ; - Check that not more than half the max cache size;
      (< (get resource "data_spec" "size") MAX-RESOURCES-CACHE-SIZE)
      ; - Check if already cached;
      (not (is-resource-cached cache-metadata resource))
    )
      (+
        ; - (If threshold passed) add to cache;
        [<commit_msg>Use half max size as max cachable resource size<commit_after>#! /usr/bin/env hy
"""
    latexonhttp.caching.resources
    ~~~~~~~~~~~~~~~~~~~~~
    Managing input resources caching.

    :copyright: (c) 2019 <NAME>.
    :license: AGPL, see LICENSE for more details.
"""
(import logging)
(import datetime)
(import [
    latexonhttp.utils.fun [get-default fun-merge-dicts fun-dict-update fun-dict-remove-key fun-sort]
])
(import [
    latexonhttp.caching.store [get-cache-metadata persist-cache-metadata]
])
(import [
    latexonhttp.caching.filesystem [apply-cache-action apply-sanity-check MAX-RESOURCES-CACHE-SIZE ENABLE-SANITY-CHECKS]
])

(setv logger (.getLogger logging __name__))

; 3 Ko.
(setv MIN-FILE-SIZE-CACHE-THRESHOLD (* 3 1000))

; # Cache module:
; # - Create directory for caching files (delete on start / if no metadata);
; # - Put input files on cache directory, with metadata (size, cache id: flat directory);
; # - Check input cache metadata for enforcing constraints (max size cache), act if needed (delete files on overflow);
; # - Actually take the metadata, the resource forwared and return the actions: add/rm, rm --all;
; # - From time to time, as a sanity check, get directory size to check that it matches our metadata.

; TODO /caches/resources endpoint

; --------------------------------
; External API.
; --------------------------------

; TODO Actually the caching must be forwarded to a decicated process
; for the whole node to ensure consistency.
; Also will avoid cache management overhead in main process.
; --> Uses a zeroMQ socket as the API.
; The cache layer could then be 100% independent.
; For eg. could be implemented in Go, with a mixed
; in-memory and on-disk cache.
; There could be a memcached adapter, to rely on existing
; caching technology.
; With enough data, there could be neural-network trained
; to optimized bandwidth-saving cache hits.

(defn forward-resource-to-cache [resource data]
    (setv cache-metadata (get-cache-metadata))
    ; TODO Maintain metrics on resource usage, for eg:
    ; - resources inputs for last 24 hours;
    ; - last N (10000) resources inputs.
    ; With this data + hit logs, we could process cache efficiency metrics.
    ; TODO Logs this data and create datasets, so we can try strategies and
    ; see which ones work best on past-data.
    (setv actions (+
        ; - Check/Init resource cache -> actions;
        (prepare-cache cache-metadata)
        ; - Decide what to do with current resource -> actions;
        (process-resource-caching-decision cache-metadata resource data)
    ))
    ; (logger.debug "Cache actions: %s" actions)
    (for [action actions]
        ; - Apply actions;
        ; TODO Normalize action using map-resource-for-cache-metadata?
        (apply-cache-action action)
        ; - Update cache metadata after actions.
        ; Update cache metadata after each action,
        ; so we have an as much as possible up-to-date cache
        ; for multi-threading/process context.
        ; (See comment above on making it a dedicated and concurrent-safe process)
        (setv cache-metadata (persist-cache-metadata (update-cache-metadata-for-action cache-metadata action))))
        (if ENABLE-SANITY-CHECKS
          (apply-sanity-check))
    )

; TODO Cache usage.
; Maintain statistics on cache hits,
; with volumes of input resource bandwitdh hit/missed.

; --------------------------------
; Decision making.
; --------------------------------

(defn prepare-cache [cache-metadata]
    (if (get-default cache-metadata "last_updated_at")
        []
        [
            {
                "name" "reset_cache"
            }
        ]))

(defn is-resource-cached [cache-metadata resource]
  (in (get resource "data_spec" "hash") (get cache-metadata "cached_resources")))

(defn map-add-resource-action [resource data]
  {
    "name" "add_resource"
    "resource" resource
    "data" data
  })

(defn map-remove-resource-action [resource]
  {
    "name" "remove_resource"
    "resource" resource
  })

(defn free-space-from-first-rec [resources size-to-free actions]
  (if (> size-to-free 0)
    (free-space-from-first-rec
      (list (drop 1 resources))
      (- size-to-free (get (first resources) "size"))
      (+ actions [(map-remove-resource-action (first resources))]))
    actions))

(defn free-space-from-old-entries [cache-metadata size-to-free]
  (setv ordered-resources (fun-sort (.values (get cache-metadata "cached_resources")) (fn [resource] (get resource "added_at"))))
  ; (logger.debug "Ordered resources: %s" ordered-resources)
  (if (> size-to-free 0)
    ; Order resources by timestamp.
    ; Remove until we have freed the specified size
    (free-space-from-first-rec
      ordered-resources
      size-to-free
      [])
    []))

(defn process-resource-caching-decision [cache-metadata resource data]
    ; Naive FIFO cache:
    ; TODO Uses resources stats to cache the most used resources.
    (if (and
      ; - Check for a caching size threshold;
      (>= (get resource "data_spec" "size") MIN-FILE-SIZE-CACHE-THRESHOLD)
      ; - Check that not more than half the max cache size;
      (< (get resource "data_spec" "size") (/ MAX-RESOURCES-CACHE-SIZE 2))
      ; - Check if already cached;
      (not (is-resource-cached cache-metadata resource))
    )
      (+
        ; - (If threshold passed) add to cache;
        [
{'fraction_non_alphanumeric': 0.1335014947683109, 'fraction_numerical': 0.004390881913303438, 'mean_word_length': 2.7351709699930216, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6865711', 'n_tokens_mistral': 3157, 'n_tokens_neox': 3032, 'n_words': 1063}