<commit_before>(ns forumdb.core
  (:import [org.hypergraphdb HGEnvironment HGHandle HGPlainLink HGQuery$hg HGQuery HGValueLink]
           (java.util Locale TimeZone)
           (java.text SimpleDateFormat)
           (java.io File)
           (model User ForumThread Post)
           (org.joda.time DateTime)
           (org.hypergraphdb.atom HGRel HGRelType)
           (org.hypergraphdb.query HGQueryCondition AtomPartRegExPredicate)
           (java.util.regex Pattern))
  (:require [clojure.string :as string]
            [clojure.xml :as xml]
            [clj-time.format :as f]
            [clj-time.core :as t]
  )
  (:gen-class :main true)
  (:import (model ForumThread))
  )

(def database (atom nil))
(def db-name "db")
(def data-files-directory "data")
(def xml-filename "tolkien.xml")

(def data-tag "rule")

(def thread-title-name "thread-title")
(def user-data-name "user-data")
(def user-login-name "user-login")
(def post-content-name "post-content")
(def post-details-name "post-details")

(defn create-database
  "
  Creates a database or opens existing one from the folder specified by argument
  "
  [path]
  (let [dbinstance (HGEnvironment/get path)]
    (reset! database dbinstance)))

(defn close-database
  "
  Closes the existing database
  "
  []
  (. @database close)
)

(defn delete-database
  "
  Deletes the database from the filesystem
  "
  []
  (def db-folder (File. db-name))
  (doseq [file (. db-folder listFiles)] (. file delete))
  (. db-folder delete)
)

(defn -main
  [& args]
  (do

    (if (. (File. db-name) exists)
      (do
        (println "Database already exists")
        (def database-exists 1)
        (println)
      )
      (do
        (println "Reading xml file...")
        (def parsed (xml/parse (string/join "/" [data-files-directory xml-filename])))
        (def database-exists 0)
        (println "Xml file loaded")
        (println)
      )
    )

    (println "Opening database...")
    (create-database db-name)
    (println "Database opened")
    (println)

    (def topType (. (. @database getTypeSystem) getTop))
    (def userThreadRelType (HGQuery$hg/assertAtom @database (HGRelType. "user-thread" (into-array HGHandle [topType topType]))))
    ;(def userPostRelType (HGQuery$hg/assertAtom @database (HGRelType. "user-post" (into-array HGHandle [topType topType]))))

    (if (= database-exists 0)
      (do
        (def saveTime (Long. 0))

        (def locale (Locale. "pl" "PL"))
        (println "Parsing and saving data...")
        (def start (System/currentTimeMillis))

        (doseq [task-result (:content parsed)]
          (let [user (User.) thread (ForumThread.) post (Post.)]
            (doseq [rule (filter (fn [x] (= (name (:tag x)) data-tag)) (:content task-result))]
              (cond
                (= (:name (:attrs rule)) thread-title-name)
                (do<commit_msg>Added DBMS configuration.<commit_after>(ns forumdb.core
  (:import [org.hypergraphdb HGEnvironment HGHandle HGPlainLink HGQuery$hg HGQuery HGValueLink HGConfiguration]
           (java.util Locale TimeZone)
           (java.text SimpleDateFormat)
           (java.io File)
           (model User ForumThread Post)
           (org.joda.time DateTime)
           (org.hypergraphdb.atom HGRel HGRelType)
           (org.hypergraphdb.query HGQueryCondition AtomPartRegExPredicate)
           (java.util.regex Pattern)
           (org.hypergraphdb.indexing ByPartIndexer)
           (org.hypergraphdb.handle SequentialUUIDHandleFactory))
  (:require [clojure.string :as string]
            [clojure.xml :as xml]
            [clj-time.format :as f]
            [clj-time.core :as t]
  )
  (:gen-class :main true)
  (:import (model ForumThread))
  )

(def database (atom nil))
(def db-name "db")
(def data-files-directory "data")
(def xml-filename "tolkien.xml")

(def data-tag "rule")

(def thread-title-name "thread-title")
(def user-data-name "user-data")
(def user-login-name "user-login")
(def post-content-name "post-content")
(def post-details-name "post-details")

(defn create-database
  "
  Creates a database or opens existing one from the folder specified by argument
  "
  [path]
  (def config (HGConfiguration.))
  (def handleFactory (SequentialUUIDHandleFactory. (System/currentTimeMillis) 0))
  (. config setHandleFactory handleFactory)
  (. config setTransactional false)
  (def storeConfig (. (. config getStoreImplementation) getConfiguration))
  (. (. storeConfig getEnvironmentConfig) setCacheSize (* (* 1024 1024) 500))
  (let [dbinstance (HGEnvironment/get path config)]
    (reset! database dbinstance)))

(defn close-database
  "
  Closes the existing database
  "
  []
  (. @database close)
)

(defn delete-database
  "
  Deletes the database from the filesystem
  "
  []
  (def db-folder (File. db-name))
  (doseq [file (. db-folder listFiles)] (. file delete))
  (. db-folder delete)
)

(defn -main
  [& args]
  (do

    (if (. (File. db-name) exists)
      (do
        (println "Database already exists")
        (def database-exists 1)
        (println)
      )
      (do
        (println "Reading xml file...")
        (def parsed (xml/parse (string/join "/" [data-files-directory xml-filename])))
        (def database-exists 0)
        (println "Xml file loaded")
        (println)
      )
    )

    (println "Opening database...")
    (create-database db-name)
    (println "Database opened")
    (println)

    (def topType (. (. @database getTypeSystem) getTop))
    (def userThreadRelType (HGQuery$hg/assertAtom @database (HGRelType. "user-thread" (into-array HGHandle [topType topType]))))
    ;(def userPostRelType (HGQuery$hg/assertAtom @database (HGRelType. "user-post" (into-array HGHandle [topType topType]))))

    ;(def postTypeHandle (. (. @database getTypeSystem) getTypeHandle Post))
    ;(. (. @database getIndexManager) register (ByPartIndexer. postTypeHandle, "content"))
    ;(. @database runMaintenance)

    (if (= database-exists 0)
      (do
        (def saveTime (Long. 0))

        (def locale (Locale. "pl" "PL"))
        (println "Parsing and saving data...")
        (def start (System/currentTimeMillis))

        (doseq [task-result (:content parsed)]
          (let [user (User.) thread (ForumThread.) post (Post.)]
            (doseq [rule (filter (fn [x] (= (name (:tag x)) data-tag)) (:content task-result))]
              (cond
                (= (:name (:attrs rule)) thread-title-name)
                (do
{'fraction_non_alphanumeric': 0.13570651329084407, 'fraction_numerical': 0.00310896937665164, 'mean_word_length': 2.6849942726231384, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2841422', 'n_tokens_mistral': 2133, 'n_tokens_neox': 2015, 'n_words': 548}