<commit_before>(ns cljam.io.bigwig
  "Functions to read the bigWig format. See https://genome.ucsc.edu/goldenpath/help/bigWig.html
  and https://github.com/ucscGenomeBrowser/kent for the detail bigWig
  specifications."
  (:require [clojure.java.io :as cio]
            [cljam.io.protocols :as protocols]
            [cljam.io.util.lsb :as lsb]
            [cljam.util :as util])
  (:import [java.net URL]
           [java.io Closeable IOException RandomAccessFile]
           [java.nio ByteBuffer ByteOrder]
           [java.util.zip Inflater]))

(def ^:private bigwig-magic 0x888ffc26)

(def ^:private bpt-magic 0x78CA8C91)

(def ^:private cir-tree-magic 0x2468ACE0)

(declare read-tracks read-all-headers read-bbi-chrom-info read-cir-tree)

(defrecord FixedWidthHeader [magic version zoom-levels chromosome-tree-offset
                             full-data-offset full-index-offset
                             total-summary-offset uncompress-buf-size
                             extension-offset])

(defrecord ZoomHeader [reduction-level data-offset index-offset])

(defrecord TotalSummary [bases-covered min-val max-val sum-data sum-squared])

(defrecord ExtendedHeader [extension-size extra-index-count
                           extra-index-list-offset])

(defrecord BptHeader [block-size key-size val-size item-count root-offset])

(defrecord BbiChromInfo [name id size])

(defrecord CirTree [block-size item-count start-chrom-ix start-base
                    end-chrom-ix end-base file-size items-per-slot
                    root-offset])

; Currently, max number of zoom levels (i.e. max number of zoom headers) is 10,
; the numberais small. Therefore, we choose a record containing vector of zoom
; header instead of a flat record.
; Cf. https://github.com/ucscGenomeBrowser/kent/blob/3a0198acd1f859a603f5aad90188bee2d82efe0c/src/inc/bbiFile.h#L384
(defrecord BigWigHeaders [^FixedWidthHeader fixed-width-header
                          zoom-headers
                          ^TotalSummary total-summary
                          ^ExtendedHeader extended-header
                          ^BptHeader bpt-header
                          bbi-chrom-info
                          ^CirTree cir-tree])

(defrecord BIGWIGReader [^RandomAccessFile reader ^URL url ^BigWigHeaders headers]
  Closeable
  (close [this]
    (.close ^Closeable (.reader this)))
  protocols/IReader
  (reader-url [this] (.url this))
  (read [this] (read-tracks this))
  (read [this option] (read-tracks this))
  (indexed? [_] false))

(defn ^BIGWIGReader reader
  "Returns an open cljam.io.bigwig.BIGWIGReader of f. Should be used inside with-open
  to ensure the reader is properly closed."
  [f]
  (let [f (.getAbsolutePath (cio/file f))
        reader (RandomAccessFile. f "r")
        headers (read-all-headers reader)]
    (BIGWIGReader. reader (util/as-url f) headers)))

(defn- check-bigwig-magic
  "Checks if the magic is right for bigWig format. Otherwise, throws IOException."
  [uint]
  (when-not (= uint bigwig-magic)
    (throw (IOException. "Invalid bigWig magic"))))

(defn- check-version
  "Ranged from [1,4]. Throws IOException if the version is out of range."
  [ushort]
  (when-not (and (<= 1 ushort) (<= ushort 4))
    (throw (IOException. "Invalid bigWig version"))))

(defn- check-field-count
  "For bigWig 0. Throws IOException if the fieldCount is invalid."
  [ushort]
  (when-not (zero? ushort)
    (throw (IOException. "Invalid bigWig fieldCount"))))

(defn- check-defined-field-count
  "For bigWig 0. Throws IOException if the definedFieldCount is invalid."
  [ushort]
  (when-not (zero? ushort)
    (throw (IOException. "Invalid bigWig definedFieldCount"))))

(defn- check-auto-sql-offset
  "For bigWig 0. Throws IOException if the autoSqlOffset is invalid."
  [^long long]
  (when-not (zero? long)
    (throw (IOException. "Invalid bigWig autoSqlOffset"))))

(defn- read-fixed-width-header
  "Returns a FixedWidthHeader."
  [^RandomAccessFile r]
  (let [magic (lsb/read-uint r)<commit_msg>Pass multiple arguments to function<commit_after>(ns cljam.io.bigwig
  "Functions to read the bigWig format. See https://genome.ucsc.edu/goldenpath/help/bigWig.html
  and https://github.com/ucscGenomeBrowser/kent for the detail bigWig
  specifications."
  (:require [clojure.java.io :as cio]
            [cljam.io.protocols :as protocols]
            [cljam.io.util.lsb :as lsb]
            [cljam.util :as util])
  (:import [java.net URL]
           [java.io Closeable IOException RandomAccessFile]
           [java.nio ByteBuffer ByteOrder]
           [java.util.zip Inflater]))

(def ^:private bigwig-magic 0x888ffc26)

(def ^:private bpt-magic 0x78CA8C91)

(def ^:private cir-tree-magic 0x2468ACE0)

(declare read-tracks read-all-headers read-bbi-chrom-info read-cir-tree)

(defrecord FixedWidthHeader [magic version zoom-levels chromosome-tree-offset
                             full-data-offset full-index-offset
                             total-summary-offset uncompress-buf-size
                             extension-offset])

(defrecord ZoomHeader [reduction-level data-offset index-offset])

(defrecord TotalSummary [bases-covered min-val max-val sum-data sum-squared])

(defrecord ExtendedHeader [extension-size extra-index-count
                           extra-index-list-offset])

(defrecord BptHeader [block-size key-size val-size item-count root-offset])

(defrecord BbiChromInfo [name id size])

(defrecord CirTree [block-size item-count start-chrom-ix start-base
                    end-chrom-ix end-base file-size items-per-slot
                    root-offset])

; Currently, max number of zoom levels (i.e. max number of zoom headers) is 10,
; the numberais small. Therefore, we choose a record containing vector of zoom
; header instead of a flat record.
; Cf. https://github.com/ucscGenomeBrowser/kent/blob/3a0198acd1f859a603f5aad90188bee2d82efe0c/src/inc/bbiFile.h#L384
(defrecord BigWigHeaders [^FixedWidthHeader fixed-width-header
                          zoom-headers
                          ^TotalSummary total-summary
                          ^ExtendedHeader extended-header
                          ^BptHeader bpt-header
                          bbi-chrom-info
                          ^CirTree cir-tree])

(defrecord BIGWIGReader [^RandomAccessFile reader ^URL url ^BigWigHeaders headers]
  Closeable
  (close [this]
    (.close ^Closeable (.reader this)))
  protocols/IReader
  (reader-url [this] (.url this))
  (read [this] (read-tracks this))
  (read [this option] (read-tracks this))
  (indexed? [_] false))

(defn ^BIGWIGReader reader
  "Returns an open cljam.io.bigwig.BIGWIGReader of f. Should be used inside with-open
  to ensure the reader is properly closed."
  [f]
  (let [f (.getAbsolutePath (cio/file f))
        reader (RandomAccessFile. f "r")
        headers (read-all-headers reader)]
    (BIGWIGReader. reader (util/as-url f) headers)))

(defn- check-bigwig-magic
  "Checks if the magic is right for bigWig format. Otherwise, throws IOException."
  [uint]
  (when-not (= uint bigwig-magic)
    (throw (IOException. "Invalid bigWig magic"))))

(defn- check-version
  "Ranged from [1,4]. Throws IOException if the version is out of range."
  [ushort]
  (when-not (<= 1 ushort 4)
    (throw (IOException. "Invalid bigWig version"))))

(defn- check-field-count
  "For bigWig 0. Throws IOException if the fieldCount is invalid."
  [ushort]
  (when-not (zero? ushort)
    (throw (IOException. "Invalid bigWig fieldCount"))))

(defn- check-defined-field-count
  "For bigWig 0. Throws IOException if the definedFieldCount is invalid."
  [ushort]
  (when-not (zero? ushort)
    (throw (IOException. "Invalid bigWig definedFieldCount"))))

(defn- check-auto-sql-offset
  "For bigWig 0. Throws IOException if the autoSqlOffset is invalid."
  [^long long]
  (when-not (zero? long)
    (throw (IOException. "Invalid bigWig autoSqlOffset"))))

(defn- read-fixed-width-header
  "Returns a FixedWidthHeader."
  [^RandomAccessFile r]
  (let [magic (lsb/read-uint r)
{'fraction_non_alphanumeric': 0.10983524712930604, 'fraction_numerical': 0.012980529206190713, 'mean_word_length': 2.8303059273422564, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4590814', 'n_tokens_mistral': 2665, 'n_tokens_neox': 2513, 'n_words': 705}