<commit_before>(ns vip.data-processor.validation.zip
  (:require [clojure.tools.logging :as log])
  (:import [net.lingala.zip4j.core ZipFile]
           [java.nio.file Files]
           [java.nio.file.attribute FileAttribute]))

(defn xml-file? [path]
  (-> path
      str
      (.endsWith ".xml")))

(defn zip-file? [path]
  (-> path
      str
      (.endsWith ".zip")))

(defn too-big-msg
  [size max-zipfile-size]
  (str "Zip file size is " size
       ", which is greater than the max-zipfile-size " max-zipfile-size))

(defn get-uncompressed-size
  "Given a net.lingala.zip4j.core.ZipFile returns the uncompressed
  size as provided by the first (and presumably only) header in the
  file."
  [zip-file]
  (.getUncompressedSize (first (.getFileHeaders zip-file))))

(defn unzip-file [path max-zipfile-size]
  (let [zip-file (ZipFile. (str path))
        size (get-uncompressed-size zip-file)
        tmp-dir (Files/createTempDirectory "vip-extracted-feed"
                                           (into-array FileAttribute []))]
    (log/info "zipfile size: " size " type? " (type size))
    (log/info "max-zipfile-size: " max-zipfile-size " type? " (type max-zipfile-size))
    (if (> size max-zipfile-size)
      (-> (too-big-msg size max-zipfile-size)
          (ex-info {:msg (too-big-msg size max-zipfile-size)
                    :max-zipfile-size-exceeded true})<commit_msg>let's try coercing the `max-zipfile-size` config value and see if that fixes this<commit_after>(ns vip.data-processor.validation.zip
  (:require [clojure.tools.logging :as log])
  (:import [net.lingala.zip4j.core ZipFile]
           [java.nio.file Files]
           [java.nio.file.attribute FileAttribute]))

(defn xml-file? [path]
  (-> path
      str
      (.endsWith ".xml")))

(defn zip-file? [path]
  (-> path
      str
      (.endsWith ".zip")))

(defn too-big-msg
  [size max-zipfile-size]
  (str "Zip file size is " size
       ", which is greater than the max-zipfile-size " max-zipfile-size))

(defn get-uncompressed-size
  "Given a net.lingala.zip4j.core.ZipFile returns the uncompressed
  size as provided by the first (and presumably only) header in the
  file."
  [zip-file]
  (.getUncompressedSize (first (.getFileHeaders zip-file))))

(defn unzip-file [path max-zipfile-size]
  (let [zip-file (ZipFile. (str path))
        size (get-uncompressed-size zip-file)
        tmp-dir (Files/createTempDirectory "vip-extracted-feed"
                                           (into-array FileAttribute []))]
    (log/info "zipfile size: " size " type? " (type size))
    (log/info "max-zipfile-size: " max-zipfile-size " type? " (type max-zipfile-size))
    (if (> size (Long. max-zipfile-size))
      (-> (too-big-msg size max-zipfile-size)
          (ex-info {:msg (too-big-msg size max-zipfile-size)
                    :max-zipfile-size-exceeded true})
{'fraction_non_alphanumeric': 0.14446002805049088, 'fraction_numerical': 0.001402524544179523, 'mean_word_length': 2.561797752808989, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '357276', 'n_tokens_mistral': 996, 'n_tokens_neox': 960, 'n_words': 238}