<commit_before>(in-package #:cliki2)
(in-readtable cliki2)

(defstruct (wiki (:conc-name nil) (:constructor make-wiki-struct))
  ;; config
  home-directory
  wiki-name
  description
  password-reminder-email-address

  ;; locks
  (update-lock     (make-lock "update lock"))
  (data-lock       (make-lock "data lock"))
  (index-lock      (make-lock "index lock"))
  (session-lock    (make-lock "session lock"))

  ;; data
  (accounts        (make-hash-table :test 'equal))
  (articles        (make-hash-table :test 'equal))
  (blacklist       (make-hash-table :test 'equal))

  ;; cache
  (article-cache   (make-hash-table :test 'equal))
  (last-timestamp  0)

  ;; indexes
  (topic-index     (make-hash-table :test 'equal)) ;; article name
  (search-index    (make-hash-table :test 'equal)) ;; article name
  (author-index    (make-hash-table :test 'equal)) ;; revision obj
  (recent-changes  ())                             ;; revision obj

  ;; sessions
  (sessions        (make-hash-table :test 'equal)))

;;; access

(defun find-account (name)
  (with-lock-held ((data-lock *wiki*))
    (gethash (cut-whitespace name) (accounts *wiki*))))

(defun banned? (account-name)
  (with-lock-held ((data-lock *wiki*))
    (gethash account-name (blacklist *wiki*))))

(defun get-blacklist ()
  (with-lock-held ((data-lock *wiki*))
    (alexandria:hash-table-keys (blacklist *wiki*))))

(define-condition cannot-find-article-error (error)
  ((title   :initarg :title)
   (link    :initarg :link)
   (referer :initarg :referer))
  (:report (lambda (e stream)
             (format stream
                     "Cannot find article entitled '~A' (link ~A refered by ~A)"
                     (slot-value e 'title)
                     (slot-value e 'link)
                     (slot-value e 'referer)))))

(defun find-article (title &key error)
  (let ((article (with-lock-held ((data-lock *wiki*))
                   (gethash (canonicalize title) (articles *wiki*)))))
    (if (and error (not article))
        (error 'cannot-find-article-error
               :title   title
               :link    (request-uri*)
               :referer (referer))
        article)))

(defun get-all-articles (filter)
  (mapcar #'article-title
          (remove-if-not
           filter
           (with-lock-held ((data-lock *wiki*))
             (alexandria:hash-table-values (articles *wiki*))))))

(defun edits-by-author (name)
  (with-lock-held ((index-lock *wiki*))
    (gethash name (author-index *wiki*))))

(defun cached-content (title)
  (with-lock-held ((data-lock *wiki*))
    (gethash (canonicalize title) (article-cache *wiki*))))

(defun get-recent-changes ()
  (with-lock-held ((index-lock *wiki*))
    (recent-changes *wiki*)))

;;; paths

(defun wiki-path (path)
  (merge-pathnames path (home-directory *wiki*)))

(defun file-path (type id)
  (let ((id (uri-encode (if (eq type 'article) (canonicalize id) id))))
    (wiki-path (ecase type
                 (account  #?"accounts/${ id }")
                 (article  #?"articles/${ id }/article")))))

;;; storage

(defun write-to-file (to-file obj
                      &optional (tmpdir (wiki-path "tmp/")))
  (let* ((content (if (stringp obj)
                      obj
                      (with-standard-io-syntax<commit_msg>Sorted tools articles pages (deleted/uncategorized/all) alphabetically<commit_after>(in-package #:cliki2)
(in-readtable cliki2)

(defstruct (wiki (:conc-name nil) (:constructor make-wiki-struct))
  ;; config
  home-directory
  wiki-name
  description
  password-reminder-email-address

  ;; locks
  (update-lock     (make-lock "update lock"))
  (data-lock       (make-lock "data lock"))
  (index-lock      (make-lock "index lock"))
  (session-lock    (make-lock "session lock"))

  ;; data
  (accounts        (make-hash-table :test 'equal))
  (articles        (make-hash-table :test 'equal))
  (blacklist       (make-hash-table :test 'equal))

  ;; cache
  (article-cache   (make-hash-table :test 'equal))
  (last-timestamp  0)

  ;; indexes
  (topic-index     (make-hash-table :test 'equal)) ;; article name
  (search-index    (make-hash-table :test 'equal)) ;; article name
  (author-index    (make-hash-table :test 'equal)) ;; revision obj
  (recent-changes  ())                             ;; revision obj

  ;; sessions
  (sessions        (make-hash-table :test 'equal)))

;;; access

(defun find-account (name)
  (with-lock-held ((data-lock *wiki*))
    (gethash (cut-whitespace name) (accounts *wiki*))))

(defun banned? (account-name)
  (with-lock-held ((data-lock *wiki*))
    (gethash account-name (blacklist *wiki*))))

(defun get-blacklist ()
  (with-lock-held ((data-lock *wiki*))
    (alexandria:hash-table-keys (blacklist *wiki*))))

(define-condition cannot-find-article-error (error)
  ((title   :initarg :title)
   (link    :initarg :link)
   (referer :initarg :referer))
  (:report (lambda (e stream)
             (format stream
                     "Cannot find article entitled '~A' (link ~A refered by ~A)"
                     (slot-value e 'title)
                     (slot-value e 'link)
                     (slot-value e 'referer)))))

(defun find-article (title &key error)
  (let ((article (with-lock-held ((data-lock *wiki*))
                   (gethash (canonicalize title) (articles *wiki*)))))
    (if (and error (not article))
        (error 'cannot-find-article-error
               :title   title
               :link    (request-uri*)
               :referer (referer))
        article)))

(defun get-all-articles (filter)
  (sort
   (mapcar #'article-title
           (remove-if-not
            filter
            (with-lock-held ((data-lock *wiki*))
              (alexandria:hash-table-values (articles *wiki*)))))
   #'string-lessp))

(defun edits-by-author (name)
  (with-lock-held ((index-lock *wiki*))
    (gethash name (author-index *wiki*))))

(defun cached-content (title)
  (with-lock-held ((data-lock *wiki*))
    (gethash (canonicalize title) (article-cache *wiki*))))

(defun get-recent-changes ()
  (with-lock-held ((index-lock *wiki*))
    (recent-changes *wiki*)))

;;; paths

(defun wiki-path (path)
  (merge-pathnames path (home-directory *wiki*)))

(defun file-path (type id)
  (let ((id (uri-encode (if (eq type 'article) (canonicalize id) id))))
    (wiki-path (ecase type
                 (account  #?"accounts/${ id }")
                 (article  #?"articles/${ id }/article")))))

;;; storage

(defun write-to-file (to-file obj
                      &optional (tmpdir (wiki-path "tmp/")))
  (let* ((content (if (stringp obj)
                      obj
                      (with-standard-io-syntax
{'fraction_non_alphanumeric': 0.16302216860202082, 'fraction_numerical': 0.0009048408988086261, 'mean_word_length': 2.4559666492965087, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4460443', 'n_tokens_mistral': 2222, 'n_tokens_neox': 2156, 'n_words': 536}