<commit_before>;; Copyright (c) <NAME>. All rights reserved.
;; The use and distribution terms for this software are covered by the
;; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
;; which can be found in the file LICENSE at the root of this distribution.
;; By using this software in any fashion, you are agreeing to be bound by
;; the terms of this license.
;; You must not remove this notice, or any other, from this software.

(in-ns 'conexp.contrib.algorithms)

(use '[conexp.base :only (illegal-argument, improve-basic-order)]
     'conexp.contrib.algorithms.generators
     'conexp.contrib.algorithms.bitwise
     '[conexp.contrib.algorithms.next-closure :only (next-closed-set bitwise-context-attribute-closure)])
(use '[conexp.fca.contexts :only (context?, objects, attributes,
                                            incidence, attribute-derivation,
                                            context-attribute-closure)])
(require '[conexp.contrib.algorithms.close-by-one :as cbo])
(import '[java.util BitSet List ArrayList])
(import '[java.util.concurrent SynchronousQueue])

;;; Concept Calculation Multi-Method

(defmulti concepts
  "Computes concepts with various algorithms, given as first
  argument. Default is :next-closure."
  {:arglists '([algorithm context] [context])}
  (fn [& args]
    (when-not (<= 1 (count args) 2)
      (illegal-argument "Wrong number of args passed to c.c.algorithms.concepts."))
    (when (and (= 1 (count args))
               (not (context? (first args))))
      (illegal-argument "Argument of concepts is not a context."))
    (when (and (= 2 (count args))
               (or (not (keyword? (first args)))
                   (not (context? (second args)))))
      (illegal-argument "First argument to concepts must be a keyword and the second argument must be a context."))
    (cond
     (= 1 (count args)) ::default-concepts
     (= 2 (count args)) (first args))))

(defmethod concepts ::default-concepts [context]
  (concepts :next-closure context))


;;; NextClosure (:next-closure)

(defmethod concepts :next-closure
  [_ context]
  (let [object-vector    (vec (objects context)),
        attribute-vector (vec (improve-basic-order (attributes context)
                                                   #(context-attribute-closure context %))),
        object-count     (count object-vector),
        attribute-count  (count attribute-vector),
        incidence-matrix (to-binary-matrix object-vector attribute-vector (incidence context)),

        o-prime (partial bitwise-object-derivation incidence-matrix object-count attribute-count),
        a-prime (partial bitwise-attribute-derivation incidence-matrix object-count attribute-count),
        start   (o-prime (a-prime (BitSet.))),
        intents (take-while identity
                            (iterate #(next-closed-set attribute-count
                                                       (partial bitwise-context-attribute-closure
                                                                object-count
                                                                attribute-count
                                                                incidence-matrix)
                                                       %)
                                     start))]
    (map (fn [bitset]
           [(to-hashset object-vector (a-prime bitset)),
            (to-hashset attribute-vector bitset)])
         intents)))


;;; Vychodil (:vychodil)

(defn- compute-closure
  [object-count, attribute-count, incidence-matrix, rows, ^BitSet A, ^BitSet B, y]
  (let [^BitSet C (BitSet.),
        ^BitSet D (BitSet.),
        ^BitSet E (.clone A),
        y (int y)]
    (.set D 0 (int attribute-count))
    (.and E (aget ^objects rows y))
    (dobits [i E]
      (.set C i)
      (dotimes [j attribute-count]
        (if (== 0 (deep-aget ints incidence-matrix i j))
          (.set D j false))))
    [C, D]))

(defg vychodil-generate-from
  [object-count, attribute-count, incidence-matrix, rows, ^BitSet A, ^BitSet B, y]
  (yield [A, B])
  (when (and (not (== attribute-count (.cardinality B)))
             (< (int y) (int attribute-count)))
    (doseq [j (range (int y) (int attribute-count))
            :when (not (.get B j))
            :let [[^BitSet C, ^BitSet D] (compute-closure object-count attribute-count,
                                                          incidence-matrix rows,
                                                          A B j)
                  skip (loop [k (int 0)]
                         (cond
                           (== k j) false,
                           (not= (.get D k) (.get B k)) true,
                           :else (recur (inc k))))]
            :when (not skip)]
      (vychodil-generate-from object-count attribute-count,
                              incidence-matrix rows,
                              C D (inc j)))))

(alter-meta! (var vychodil-generate-from) assoc :private true)

(defmethod concepts :vychodil
  [_ context]
  (with-binary-context context
    (let [rows          (into-array (map (fn [y]
                                           (let [^BitSet bs (BitSet.)]
                                             (.set bs y)
                                             (bitwise-attribute-derivation incidence-matrix
                                                                           object-count
                                                                           attribute-count
                                                                           bs)))
                                         (range attribute-count)))
          empty-down    (bitwise-attribute-derivation incidence-matrix
                                                      object-count
                                                      attribute-count
                                                      (BitSet.))
          empty-down-up (bitwise-object-derivation incidence-matrix
                                                   object-count
                                                   attribute-count
                                                   empty-down)]
      (map (fn [pair]
             [(to-hashset object-vector (first pair))
              (to-hashset attribute-vector (second pair))])
           (generate (vychodil-generate-from object-count
                                             attribute-count
                                             incidence-matrix
                                             rows
                                             empty-down
                                             empty-down-up
                                             0))))))


;;; In-Close (:in-close)<commit_msg>(concepts :vychodil ctx) does not use generators anymore<commit_after>;; Copyright (c) <NAME>. All rights reserved.
;; The use and distribution terms for this software are covered by the
;; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
;; which can be found in the file LICENSE at the root of this distribution.
;; By using this software in any fashion, you are agreeing to be bound by
;; the terms of this license.
;; You must not remove this notice, or any other, from this software.

(in-ns 'conexp.contrib.algorithms)

(use '[conexp.base :only (illegal-argument, improve-basic-order)]
     'conexp.contrib.algorithms.generators
     'conexp.contrib.algorithms.bitwise
     '[conexp.contrib.algorithms.next-closure :only (next-closed-set bitwise-context-attribute-closure)])
(use '[conexp.fca.contexts :only (context?, objects, attributes,
                                            incidence, attribute-derivation,
                                            context-attribute-closure)])
(require '[conexp.contrib.algorithms.close-by-one :as cbo])
(require '[clojure.core.async :refer (thread >!! <!! chan close!)])
(import '[java.util BitSet List ArrayList])
(import '[java.util.concurrent SynchronousQueue])

;;; Concept Calculation Multi-Method

(defmulti concepts
  "Computes concepts with various algorithms, given as first
  argument. Default is :next-closure."
  {:arglists '([algorithm context] [context])}
  (fn [& args]
    (when-not (<= 1 (count args) 2)
      (illegal-argument "Wrong number of args passed to c.c.algorithms.concepts."))
    (when (and (= 1 (count args))
               (not (context? (first args))))
      (illegal-argument "Argument of concepts is not a context."))
    (when (and (= 2 (count args))
               (or (not (keyword? (first args)))
                   (not (context? (second args)))))
      (illegal-argument "First argument to concepts must be a keyword and the second argument must be a context."))
    (cond
     (= 1 (count args)) ::default-concepts
     (= 2 (count args)) (first args))))

(defmethod concepts ::default-concepts [context]
  (concepts :next-closure context))


;;; NextClosure (:next-closure)

(defmethod concepts :next-closure
  [_ context]
  (let [object-vector    (vec (objects context)),
        attribute-vector (vec (improve-basic-order (attributes context)
                                                   #(context-attribute-closure context %))),
        object-count     (count object-vector),
        attribute-count  (count attribute-vector),
        incidence-matrix (to-binary-matrix object-vector attribute-vector (incidence context)),

        o-prime (partial bitwise-object-derivation incidence-matrix object-count attribute-count),
        a-prime (partial bitwise-attribute-derivation incidence-matrix object-count attribute-count),
        start   (o-prime (a-prime (BitSet.))),
        intents (take-while identity
                            (iterate #(next-closed-set attribute-count
                                                       (partial bitwise-context-attribute-closure
                                                                object-count
                                                                attribute-count
                                                                incidence-matrix)
                                                       %)
                                     start))]
    (map (fn [bitset]
           [(to-hashset object-vector (a-prime bitset)),
            (to-hashset attribute-vector bitset)])
         intents)))


;;; Vychodil (:vychodil)

(defn- compute-closure
  [object-count, attribute-count, incidence-matrix, rows, ^BitSet A, ^BitSet B, y]
  (let [^BitSet C (BitSet.),
        ^BitSet D (BitSet.),
        ^BitSet E (.clone A),
        y (int y)]
    (.set D 0 (int attribute-count))
    (.and E (aget ^objects rows y))
    (dobits [i E]
      (.set C i)
      (dotimes [j attribute-count]
        (if (== 0 (deep-aget ints incidence-matrix i j))
          (.set D j false))))
    [C, D]))

(defn vychodil-generate-from
  [output, object-count, attribute-count, incidence-matrix, rows, ^BitSet A, ^BitSet B, y]
  (>!! output [A, B])
  (when (and (not (== attribute-count (.cardinality B)))
             (< (int y) (int attribute-count)))
    (doseq [j (range (int y) (int attribute-count))
            :when (not (.get B j))
            :let [[^BitSet C, ^BitSet D] (compute-closure object-count attribute-count,
                                                          incidence-matrix rows,
                                                          A B j)
                  skip (loop [k (int 0)]
                         (cond
                           (== k j) false,
                           (not= (.get D k) (.get B k)) true,
                           :else (recur (inc k))))]
            :when (not skip)]
      (vychodil-generate-from output
                              object-count attribute-count,
                              incidence-matrix rows,
                              C D (inc j)))))

(alter-meta! (var vychodil-generate-from) assoc :private true)

(defmethod concepts :vychodil
  [_ context]
  (with-binary-context context
    (let [rows          (into-array (map (fn [y]
                                           (let [^BitSet bs (BitSet.)]
                                             (.set bs y)
                                             (bitwise-attribute-derivation incidence-matrix
                                                                           object-count
                                                                           attribute-count
                                                                           bs)))
                                         (range attribute-count)))
          empty-down    (bitwise-attribute-derivation incidence-matrix
                                                      object-count
                                                      attribute-count
                                                      (BitSet.))
          empty-down-up (bitwise-object-derivation incidence-matrix
                                                   object-count
                                                   attribute-count
                                                   empty-down)
          output        (chan)]
      (thread (vychodil-generate-from output
                                      object-count
                                      attribute-count
                                      incidence-matrix
                                      rows
                                      empty-down
                                      empty-down-up
                                      0)
              (close! output))
      (map (fn [pair]
             [(to-hashset object-vector (first pair))
              (to-hashset attribute-vector (second pair))])
           (take-while (comp not nil?)
                       (repeatedly #(<!! output)))))))


;;; In-Close (:in-close)
{'fraction_non_alphanumeric': 0.10343087790111, 'fraction_numerical': 0.0020181634712411706, 'mean_word_length': 1.2979463398476316, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 121, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7388082', 'n_tokens_mistral': 3353, 'n_tokens_neox': 3176, 'n_words': 987}