<commit_before>(ns setbang.core
  (:require [clojure.java.io :as io]
            [clojure.set :as clj-set]
            [clojure.string :as clj-string])
  (:gen-class))

;; Only exists for testing. Generates a non-normalized integer-equivalent set.
(defn int-to-set [n]
  (loop [i 0 acc #{}]
    (if (>= i n) acc (recur (inc i) (conj acc acc)))))

(defn normalize [x]
  (cond (number? x) x
        (set? x)
          (let [s (set (map normalize x))]
            (if (= s (into #{} (range (count x))))
              (count x)
              s))
        :else (map normalize x)))

(defn is-mumber? [x]
  (number? (normalize x)))

(defn as-set [x]
  (cond
   (number? x) (into #{} (range x))
   (set?    x) x
   :else       (into #{} x)))

(def omega (range))

;; bottom : extreme pathological set. Empty but yields no information. Only used
;; for testing.
(def bottom (lazy-seq (loop [] (recur))))

;; we should be able to show "there are at least 20 primes"
;; e.g. (primes)#(20)?


(declare member-check)

(defn equality-check [x1 x2]
  (if (and (number? x1) (number? x2))
    (= x1 x2)
    (let [s1 (if (number? x1) (into #{} (range x1)) x1)
          s2 (if (number? x2) (into #{} (range x2)) x2)]
      (if (set? s1)
        (and (every? #(member-check s1 %) s2)
             (every? #(member-check s2 %) s1))
        (and (every? #(member-check s2 %) s1)
             (every? #(member-check s1 %) s2))))))

(defn member-check [x e]
  (cond (number? x) (and (number? e) (< e x))
        (set? x)    (if (or (number? e) (set? e)) (x e)
                        (some #(equality-check % e) x))
        :else       (some #(equality-check % e) x)))

(defn pair [x1 x2]
  (normalize (hash-set x1 x2)))

(defn choice [x]
  (cond
   (= x 0) [0 0]
   (number? x) (choice (into #{} (range x)))
   (set? x) (let [elt (first x)
                  remt (if (empty? (rest x)) 0 (into #{} (rest x)))]
              [(normalize remt) elt])
   :else (if (empty? x)
           [0 0]
           (let [elt (first x)]
             [(remove #(equality-check % elt) (rest x)) elt]))))

(defn is-empty-set [x]
  (cond
   (number? x) (= x 0)
   (set?    x) (empty? x)
   :else       (empty? x)))

(defn choose-many [x n]
  (let [s (if (number? x) (into #{} (range x)) x)]
    (loop [remt s out #{} i 0]
      (if (or (>= i n) (is-empty-set remt))
        [(normalize remt) (normalize out)]
        (let [[remt-1 e] (choice remt)]
          (recur remt-1 (conj out e) (inc i)))))))

;; FIXME: insert should reorder inf. sets to avoid long equality-checks.
(defn insert [x e]
  (cond
   (and (number? x) (number? e) (< e x)) x
   (and (number? x) (number? e) (= e x)) (+ x 1)
   (or (number? x) (set? x))
     (let [s (if (number? x) (into #{} (range x)) x)]
       (normalize (conj s e)))
   :else (cons e (remove #(equality-check e (second %)) x))))

(defn has-definite-size? [x]
  (or (number? x) (set? x)))

;; Non-terminating on an infinite set.
(defn get-size-up-to [x n]
  (cond (number? x) x
        (set?    x) (count x)
        :else       (count (take n x))))

(defn gen-union-2 [x1 x2]
  (cond
   (and (number? x1) (number? x2)) (max x1 x2)
   (and (has-definite-size? x1) (has-definite-size? x2))
     (normalize (clj-set/union (as-set x1) (as-set x2)))
     :else
       (if (is-empty-set x1)
         x2
         (lazy-seq
          (let [[x' e] (choice x1)]
            (cons e (remove #(equality-check % e) (gen-union-2 x2 x'))))))))

(defn as-seq [x]
  (cond
   (number? x) (range x)
   (set? x)    (seq x)
   :else       x))

;; [x1 x2 x3 ...] [y1 y2 y3 ... ] [z1 z2 z3 ...] [w1 ...] ->
;; x1 x2 y1 x3 y2 z1 x4 y3 z2 w1 ...
;; "diagonal" approach to make sure everything gets picked.
(defn stagger [xs & [n]]
  (if (empty? xs) ()
      (let [n (or n 1)
            choices (map choice (take n (remove is-empty-set xs)))]
        (lazy-seq
         (concat (map second choices)
                 (stagger (concat (map first choices)
                                  (drop n xs))
                          (inc n)))))))

(defn staggered-union [xs]
  (lazy-seq
   (let [ys (stagger xs)
         y1 (first ys)]
     (cons y1 (remove #(equality-check % y1) ys)))))

(defn gen-union [x-set]
  (cond
   (number? x-set) x-set
   (set?    x-set) (reduce gen-union-2 #{} x-set)
   :else (staggered-union x-set)))

(defn gen-filter [f x]
  (cond
   (number? x) (normalize (into #{} (filter f (range x))))
   (set?    x) (normalize (into #{} (filter f x)))
   :else       (filter f x)))
<commit_msg>Fix bugs in staggered union empty cases.<commit_after>(ns setbang.core
  (:require [clojure.java.io :as io]
            [clojure.set :as clj-set]
            [clojure.string :as clj-string])
  (:gen-class))

;; Only exists for testing. Generates a non-normalized integer-equivalent set.
(defn int-to-set [n]
  (loop [i 0 acc #{}]
    (if (>= i n) acc (recur (inc i) (conj acc acc)))))

(defn normalize [x]
  (cond (number? x) x
        (set? x)
          (let [s (set (map normalize x))]
            (if (= s (into #{} (range (count x))))
              (count x)
              s))
        (empty? x) 0
        :else (map normalize x)))

(defn is-mumber? [x]
  (number? (normalize x)))

(defn as-set [x]
  (cond
   (number? x) (into #{} (range x))
   (set?    x) x
   :else       (into #{} x)))

(def omega (range))

;; bottom : extreme pathological set. Empty but yields no information. Only used
;; for testing.
(def bottom (lazy-seq (loop [] (recur))))

;; we should be able to show "there are at least 20 primes"
;; e.g. (primes)#(20)?


(declare member-check)

(defn equality-check [x1 x2]
  (if (and (number? x1) (number? x2))
    (= x1 x2)
    (let [s1 (if (number? x1) (into #{} (range x1)) x1)
          s2 (if (number? x2) (into #{} (range x2)) x2)]
      (if (set? s1)
        (and (every? #(member-check s1 %) s2)
             (every? #(member-check s2 %) s1))
        (and (every? #(member-check s2 %) s1)
             (every? #(member-check s1 %) s2))))))

(defn member-check [x e]
  (cond (number? x) (and (number? e) (< e x))
        (set? x)    (if (or (number? e) (set? e)) (x e)
                        (some #(equality-check % e) x))
        :else       (some #(equality-check % e) x)))

(defn pair [x1 x2]
  (normalize (hash-set x1 x2)))

(defn choice [x]
  (cond
   (= x 0) [0 0]
   (number? x) (choice (into #{} (range x)))
   (set? x) (let [elt (first x)
                  remt (if (empty? (rest x)) 0 (into #{} (rest x)))]
              [(normalize remt) elt])
   :else (if (empty? x)
           [0 0]
           (let [elt (first x)]
             [(remove #(equality-check % elt) (rest x)) elt]))))

(defn is-empty-set [x]
  (cond
   (number? x) (= x 0)
   (set?    x) (empty? x)
   :else       (empty? x)))

(defn choose-many [x n]
  (let [s (if (number? x) (into #{} (range x)) x)]
    (loop [remt s out #{} i 0]
      (if (or (>= i n) (is-empty-set remt))
        [(normalize remt) (normalize out)]
        (let [[remt-1 e] (choice remt)]
          (recur remt-1 (conj out e) (inc i)))))))

;; FIXME: insert should reorder inf. sets to avoid long equality-checks.
(defn insert [x e]
  (cond
   (and (number? x) (number? e) (< e x)) x
   (and (number? x) (number? e) (= e x)) (+ x 1)
   (or (number? x) (set? x))
     (let [s (if (number? x) (into #{} (range x)) x)]
       (normalize (conj s e)))
   :else (cons e (remove #(equality-check e (second %)) x))))

(defn has-definite-size? [x]
  (or (number? x) (set? x)))

;; Non-terminating on an infinite set.
(defn get-size-up-to [x n]
  (cond (number? x) x
        (set?    x) (count x)
        :else       (count (take n x))))

(defn gen-union-2 [x1 x2]
  (cond
   (and (number? x1) (number? x2)) (max x1 x2)
   (and (has-definite-size? x1) (has-definite-size? x2))
     (normalize (clj-set/union (as-set x1) (as-set x2)))
     :else
       (if (is-empty-set x1)
         x2
         (lazy-seq
          (let [[x' e] (choice x1)]
            (cons e (remove #(equality-check % e) (gen-union-2 x2 x'))))))))

(defn as-seq [x]
  (cond
   (number? x) (range x)
   (set? x)    (seq x)
   :else       x))

;; [x1 x2 x3 ...] [y1 y2 y3 ... ] [z1 z2 z3 ...] [w1 ...] ->
;; x1 x2 y1 x3 y2 z1 x4 y3 z2 w1 ...
;; "diagonal" approach to make sure everything gets picked.
(defn stagger [xs & [n]]
  (if (empty? xs) ()
      (let [n (or n 1)
            choices (map choice (take n (remove is-empty-set xs)))]
        (lazy-seq
         (concat (map second choices)
                 (stagger (concat (map first choices)
                                  (drop n xs))
                          (inc n)))))))

(defn staggered-union [xs]
  (if (empty? xs) ()
      (lazy-seq
       (let [ys (stagger xs)]
         (if-let [y1 (first ys)]
           (cons y1 (remove #(equality-check % y1) ys))
           ())))))

(defn gen-union [x-set]
  (cond
   (number? x-set) x-set
   (set?    x-set) (reduce gen-union-2 #{} x-set)
   :else (staggered-union x-set)))

(defn gen-filter [f x]
  (cond
   (number? x) (normalize (into #{} (filter f (range x))))
   (set?    x) (normalize (into #{} (filter f x)))
   :else       (filter f x)))

{'fraction_non_alphanumeric': 0.1918623883559378, 'fraction_numerical': 0.018634910133421546, 'mean_word_length': 2.0631543397500844, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '728601', 'n_tokens_mistral': 3669, 'n_tokens_neox': 3568, 'n_words': 1177}