<commit_before>(in-package :manifest)

(defvar *manifest-server* nil)

(defparameter *categories* '(:function :generic-function :slot-accessor :variable :class :condition :constant))

(defun start (&key (port 0))
  "Start the manifest server and return the URL to browse. By default
picks a random unused port or you can specify a port with the :port
keyword argument."
  (cond
    ((and *manifest-server* (not (toot::shutdown-p *manifest-server*)))
     (warn "Manifest server already running."))
    (*manifest-server*
     (start-acceptor *manifest-server*))
    (t
     (setf *manifest-server*
           (start-server :handler (make-handler) :port port :access-logger nil :message-logger nil))))
  (format nil "http://localhost:~d/" (port *manifest-server*)))

(defun stop (&optional (server *manifest-server*))
  "Stop the manifest server, defaulting to *manifest-server*."
  (stop-acceptor server))

(defun make-handler (&optional (root-dir (asdf:system-relative-pathname :manifest nil)))
  (let ((static-files (make-instance 'static-file-handler :root root-dir)))
    (lambda (request)
      (let ((result (manifest request)))
        (case result
          (not-handled (handle-request static-files request))
          (t result))))))

(defun manifest (request)
  (cond
    ((string= (uri-path (request-uri request)) "/") (index-page request))
    (t (package-page request))))

(defun package-page (request)
  (destructuring-bind (package-name &rest rest)
      (split-sequence #\/ (subseq (uri-path (request-uri request)) 1))
    (declare (ignore rest))

    (let ((package (find-package (string-upcase package-name)))
          (some-docs-p nil))
      (cond
        (package
          (let ((s (send-headers request)))
            (format s "<html><head><title>Package: ~a</title><link rel='stylesheet' type='text/css' href='manifest.css'></head>" (package-name package))
            (format s "<body><h1>Package: ~a</h1>" (package-name package))

            (let ((readme (readme-text package-name)))
              (when readme
                (setf some-docs-p t)
                (format s "<pre>~a</pre>" (escape-for-html readme))))

            (loop for what in *categories*
               for names = (names package what)
               when names do
                 (setf some-docs-p t)
                 (format s "~&<h2>~:(~a~)</h2>~&<table>" (pluralization what))
                 (loop for sym in names do
                      (format s "~&<tr><td class='symbol'>~(~a~)</td>
<td class='docs'>~a</td></tr>"
                              (escape-for-html (princ-to-string sym))
                              (escape-for-html (or (docs-for sym what) "NO DOCS!"))))
                 (format s "~&</table>"))

            (unless some-docs-p
              (format s "<p>Uh oh! No docs at all.</p>"))


            (format s "~&</body></html>")))
        (t 'not-handled)))))

(defun index-page (request)
  (let ((s (send-headers request)))
    (format s "<html><head><title>Manifest: all packages</title><link rel='stylesheet' type='text/css' href='manifest.css'></head>")
    (format s "<body><h1>All Packages</h1>")
    (format s "~&<ul>")
    (loop for pkg in (sort (mapcar #'package-name (public-packages)) #'string<) do
         (format s "<li><a href='~a'>~a</a></li>" (string-downcase pkg) pkg))
    (format s "~&</ul>")
    (format s "~&</body></html>")))

(defun public-packages ()
  (loop for p in (list-all-packages)
     when (and (has-exported-symbols-p p) (not (eql p (find-package :keyword))))
     collect p))
<commit_msg>More docs about package itself.<commit_after>(in-package :manifest)

(defvar *manifest-server* nil)

(defparameter *categories* '(:function :generic-function :slot-accessor :variable :class :condition :constant))

(defun start (&key (port 0))
  "Start the manifest server and return the URL to browse. By default
picks a random unused port or you can specify a port with the :port
keyword argument."
  (cond
    ((and *manifest-server* (not (toot::shutdown-p *manifest-server*)))
     (warn "Manifest server already running."))
    (*manifest-server*
     (start-acceptor *manifest-server*))
    (t
     (setf *manifest-server*
           (start-server :handler (make-handler) :port port :access-logger nil :message-logger nil))))
  (format nil "http://localhost:~d/" (port *manifest-server*)))

(defun stop (&optional (server *manifest-server*))
  "Stop the manifest server, defaulting to *manifest-server*."
  (stop-acceptor server))

(defun make-handler (&optional (root-dir (asdf:system-relative-pathname :manifest nil)))
  (let ((static-files (make-instance 'static-file-handler :root root-dir)))
    (lambda (request)
      (let ((result (manifest request)))
        (case result
          (not-handled (handle-request static-files request))
          (t result))))))

(defun manifest (request)
  (cond
    ((string= (uri-path (request-uri request)) "/") (index-page request))
    (t (package-page request))))

(defun package-page (request)
  (destructuring-bind (package-name &rest rest)
      (split-sequence #\/ (subseq (uri-path (request-uri request)) 1))
    (declare (ignore rest))

    (let ((package (find-package (string-upcase package-name)))
          (some-docs-p nil))
      (cond
        (package
          (let ((s (send-headers request)))
            (format s "<html><head><title>Package: ~a</title><link rel='stylesheet' type='text/css' href='manifest.css'></head>" (package-name package))
            (format s "<body><h1>~a</h1>" (package-name package))
            (when (documentation package t)
              (format s "<body><p class='package-desc'>~a</p>" (documentation package t)))

            (let ((readme (readme-text package-name)))
              (when readme
                (setf some-docs-p t)
                (format s "<pre>~a</pre>" (escape-for-html readme))))

            (loop for what in *categories*
               for names = (names package what)
               when names do
                 (setf some-docs-p t)
                 (format s "~&<h2>~:(~a~)</h2>~&<table>" (pluralization what))
                 (loop for sym in names do
                      (format s "~&<tr><td class='symbol'>~(~a~)</td>
<td class='docs'>~a</td></tr>"
                              (escape-for-html (princ-to-string sym))
                              (escape-for-html (or (docs-for sym what) "NO DOCS!"))))
                 (format s "~&</table>"))

            (let ((used-by (sort (package-used-by-list package) #'string< :key #'package-name)))
              (when used-by
                (format s "~&<h2>Used by:</h2>~&<ul>")
                (loop for p in used-by do
                     (format s "~&<li><a href='./~(~a~)'>~:*~a</a></li>" (package-name p)))
                (format s "~&</ul>")))

            (let ((uses (sort (package-use-list package) #'string< :key #'package-name)))
              (when uses
                (format s "~&<h2>Uses:</h2>~&<ul>")
                (loop for p in uses do
                     (format s "~&<li><a href='./~(~a~)'>~:*~a</a></li>" (package-name p)))
                (format s "~&</ul>")))

            (unless some-docs-p
              (format s "<p>Uh oh! No docs at all.</p>"))


            (format s "~&</body></html>")))
        (t 'not-handled)))))

(defun index-page (request)
  (let ((s (send-headers request)))
    (format s "<html><head><title>Manifest: all packages</title><link rel='stylesheet' type='text/css' href='manifest.css'></head>")
    (format s "<body><h1>All Packages</h1>")
    (format s "~&<ul>")
    (loop for pkg in (sort (mapcar #'package-name (public-packages)) #'string<) do
         (format s "<li><a href='~a'>~a</a></li>" (string-downcase pkg) pkg))
    (format s "~&</ul>")
    (format s "~&</body></html>")))

(defun public-packages ()
  (loop for p in (list-all-packages)
     when (and (has-exported-symbols-p p) (not (eql p (find-package :keyword))))
     collect p))

{'fraction_non_alphanumeric': 0.18108651911468812, 'fraction_numerical': 0.0025150905432595573, 'mean_word_length': 2.5488621151271755, 'pattern_counts': {'":': 0, '<': 110, '<?xml version=': 0, '>': 106, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '599503', 'n_tokens_mistral': 2560, 'n_tokens_neox': 2454, 'n_words': 699}