<commit_before>;; -*- no-byte-compile: t; -*-
;;; core/cli/packages.el

(defcli! (update u)
  ((discard-p ["--discard"] "All local changes to packages are discarded"))
  "Updates packages.

This works by fetching all installed package repos and checking the distance
between HEAD and FETCH_HEAD. This can take a while.

This excludes packages whose `package!' declaration contains a non-nil :freeze
or :ignore property."
  (straight-check-all)
  (let ((doom-auto-discard discard-p))
    (doom-cli-reload-core-autoloads)
    (when (doom-cli-packages-update)
      (doom-cli-reload-package-autoloads))
    t))

(defcli! (build b)
    ((rebuild-p ["-r"] "Only rebuild packages that need rebuilding"))
  "Byte-compiles & symlinks installed packages.

This ensures that all needed files are symlinked from their package repo and
their elisp files are byte-compiled. This is especially necessary if you upgrade
Emacs (as byte-code is generally not forward-compatible)."
  (when (doom-cli-packages-build (not rebuild-p))
    (doom-cli-reload-package-autoloads))
  t)

(defcli! (purge p)
    ((nobuilds-p ["-b" "--no-builds"] "Don't purge unneeded (built) packages")
     (noelpa-p   ["-p" "--no-elpa"]   "Don't purge ELPA packages")
     (norepos-p  ["-r" "--no-repos"]  "Don't purge unused straight repos")
     (regraft-p  ["-g" "--regraft"]   "Regraft git repos (ie. compact them)"))
  "Deletes orphaned packages & repos, and compacts them.

Purges all installed ELPA packages (as they are considered temporary). Purges
all orphaned package repos and builds. If -g/--regraft is supplied, the git
repos among them will be regrafted and compacted to ensure they are as small as
possible.

It is a good idea to occasionally run this doom purge -g to ensure your package
list remains lean."
  (straight-check-all)
  (when (doom-cli-packages-purge
         (not noelpa-p)
         (not norepos-p)
         (not nobuilds-p)
         regraft-p)
    (doom-cli-reload-package-autoloads))
  t)

;; (defcli! rollback () ; TODO doom rollback
;;   "<Not implemented yet>"
;;   (user-error "Not implemented yet, sorry!"))


;;
;;; Library

(defun doom--same-commit-p (abbrev-ref ref)
  (and (stringp abbrev-ref)
       (stringp ref)
       (string-match-p (concat "^" (regexp-quote abbrev-ref))
                       ref)))

(defun doom--abbrev-commit (commit &optional full)
  (if full commit (substring commit 0 7)))

(defun doom--commit-log-between (start-ref end-ref)
  (and (straight--call
        "git" "log" "--oneline" "--no-merges"
        "-n" "25" end-ref (concat "^" (regexp-quote start-ref)))
       (straight--process-get-output)))

(defun doom-cli-packages-install ()
  "Installs missing packages.

This function will install any primary package (i.e. a package with a `package!'
declaration) or dependency thereof that hasn't already been."
  (straight--transaction-finalize)
  (print! (start "Installing packages..."))
  (print-group!
   (if-let (built
            (doom-with-package-recipes (doom-package-recipe-list)
                (recipe package type local-repo)
              (condition-case-unless-debug e
                  (progn
                    (straight-use-package (intern package))
                    (when-let* ((target-ref (cdr (assoc local-repo doom-pinned-packages)))
                                (ref (straight-vc-get-commit type local-repo)))
                      (unless (doom--same-commit target-ref ref)
                        (unless (straight-vc-commit-present-p recipe target-ref)
                          (straight-vc-fetch-from-remote recipe))
                        (if (straight-vc-commit-present-p recipe target-ref)
                            (progn
                              (print! (success "Checking out %s to %s")
                                      package (doom--abbrev-commit target-ref))
                              (straight-vc-check-out-commit recipe target-ref)
                              (straight-rebuild-package package t))
                          (ignore-errors
                            (delete-directory (straight--repos-dir package) 'recursive))
                          (straight-use-package (intern package))))))
                (error
                 (signal 'doom-package-error
                         (list package e (straight--process-get-output)))))))
       (print! (success "Installed %d packages")
               (length built))
     (print! (info "No packages need to be installed"))
     nil)))


(defun doom-cli-packages-build (&optional force-p)
  "(Re)build all packages."
  (straight--transaction-finalize)
  (print! (start "(Re)building %spackages...") (if force-p "all " ""))
  (print-group!
   (let ((straight-check-for-modifications
          (when (file-directory-p (straight--modified-dir))
            '(find-when-checking)))
         (straight--allow-find
          (and straight-check-for-modifications
               (executable-find straight-find-executable)
               t))
         (straight--packages-not-to-rebuild
          (or straight--packages-not-to-rebuild (make-hash-table :test #'equal)))
         (straight--packages-to-rebuild
          (or (if force-p :all straight--packages-to-rebuild)
              (make-hash-table :test #'equal))))
     (unless force-p
       (straight--make-package-modifications-available))
     (if-let (built
              (doom-with-package-recipes (doom-package-recipe-list) (package)
                (straight-use-package (intern package))))
         (print! (success "Rebuilt %d package(s)") (length built))
       (print! (success "No packages need rebuilding"))
       nil))))


(defun doom-cli-packages-update ()
  "Updates packages."
  (straight--transaction-finalize)
  (print! (start "Updating packages (this may take a while)..."))
  (let* ((repo-dir (straight--repos-dir))
         (packages-to-rebuild (make-hash-table :test 'equal))
         (repos-to-rebuild (make-hash-table :test 'equal))
         (recipes (doom-package-recipe-list))
         (total (length recipes))
         (esc (unless doom-debug-mode "\033[1A"))
         (i 0)
         errors)
    (doom-with-package-recipes recipes (recipe package type local-repo)
      (cl-incf i)
      (print-group!
       (unless (straight--repository-is-available-p recipe)
         (print! (error "(%d/%d) Couldn't find local repo for %s") i total package)
         (cl-return))
       (when (gethash local-repo repos-to-rebuild)
         (puthash package t packages-to-rebuild)
         (print! (success "(%d/%d) %s was updated indirectly (with %s)") i total package local-repo)
         (cl-return))
       (let ((default-directory (straight--repos-dir local-repo)))
         (unless (file-in-directory-p default-directory repo-dir)
           (print! (warn "(%d/%d) Skipping %s because it is local") i total package)
           (cl-return))
         (condition-case-unless-debug e
             (let ((ref (straight-vc-get-commit type local-repo))
                   (target-ref (cdr (assoc local-repo doom-pinned-packages)))
                   output)
               (or (cond
                    ((not (stringp target-ref))
                     (print! (start "\033[K(%d/%d) Fetching %s...%s") i total package esc)
                     (when (straight-vc-fetch-from-remote recipe)
                       (setq output (straight--process-get-output))
                       (straight-merge-package package)
                       (setq target-ref (straight-vc-get-commit type local-repo))
                       (when (doom--same-commit-p target-ref ref)
                         (cl-return))))

                    ((doom--same-commit-p target-ref ref)
                     (print! (info "\033[K(%d/%d) %s is up-to-date...%s") i total package esc)
                     (cl-return))

                    ((straight-vc-commit-present-p recipe target-ref)
                     (print! (start "\033[K(%d/%d) Checking out %s (%s)...%s")
                             i total package (doom--abbrev-commit target-ref) esc)
                     (straight-vc-check-out-commit recipe target-ref)
                     (or (not (eq type 'git))
                         (setq output (doom--commit-log-between ref target-ref)))
                     (doom--same-commit-p target-ref (straight-vc-get-commit type local-repo)))

                    ((print! (start "\033[K(%d/%d) Re-cloning %s...%s") i total local-repo esc)
                     (ignore-errors (delete-directory (straight--repos-dir package) 'recursive))
                     (straight-use-package (intern package) nil 'no-build)
                     (prog1 (file-directory-p (straight--repos-dir package))
                       (or (not (eq type 'git))
                           (setq output (doom--commit-log-between ref target-ref))))))
                   (progn
                     (print! (warn "\033[K(%d/%d) Failed to fetch %s") i total local-repo)
                     (cl-return)))
               (puthash local-repo t repos-to-rebuild)
               (puthash package t packages-to-rebuild)
               (unless (string-empty-p output)
                 (print-group! (print! (info "%s" output))))
               (print! (success "(%d/%d) %s updated (%s -> %s)")
                       i total local-repo
                       (doom--abbrev-commit ref)
                       (doom--abbrev-commit target-ref)))
           (user-error
            (signal 'user-error (error-message-string e)))
           (error
            (print! (warn "\033[K(%d/%d) Encountered error with %s" i total package))
            (print-group!
             (print! (error "%s") e)
             (print-group! (print! (info "%s" (straight--process-get-output)))))
            (push package errors))))))
    (princ "\033[K")
    (when errors<commit_msg>Fix some packages failing to fetch on 'doom update'<commit_after>;; -*- no-byte-compile: t; -*-
;;; core/cli/packages.el

(defcli! (update u)
  ((discard-p ["--discard"] "All local changes to packages are discarded"))
  "Updates packages.

This works by fetching all installed package repos and checking the distance
between HEAD and FETCH_HEAD. This can take a while.

This excludes packages whose `package!' declaration contains a non-nil :freeze
or :ignore property."
  (straight-check-all)
  (let ((doom-auto-discard discard-p))
    (doom-cli-reload-core-autoloads)
    (when (doom-cli-packages-update)
      (doom-cli-reload-package-autoloads))
    t))

(defcli! (build b)
    ((rebuild-p ["-r"] "Only rebuild packages that need rebuilding"))
  "Byte-compiles & symlinks installed packages.

This ensures that all needed files are symlinked from their package repo and
their elisp files are byte-compiled. This is especially necessary if you upgrade
Emacs (as byte-code is generally not forward-compatible)."
  (when (doom-cli-packages-build (not rebuild-p))
    (doom-cli-reload-package-autoloads))
  t)

(defcli! (purge p)
    ((nobuilds-p ["-b" "--no-builds"] "Don't purge unneeded (built) packages")
     (noelpa-p   ["-p" "--no-elpa"]   "Don't purge ELPA packages")
     (norepos-p  ["-r" "--no-repos"]  "Don't purge unused straight repos")
     (regraft-p  ["-g" "--regraft"]   "Regraft git repos (ie. compact them)"))
  "Deletes orphaned packages & repos, and compacts them.

Purges all installed ELPA packages (as they are considered temporary). Purges
all orphaned package repos and builds. If -g/--regraft is supplied, the git
repos among them will be regrafted and compacted to ensure they are as small as
possible.

It is a good idea to occasionally run this doom purge -g to ensure your package
list remains lean."
  (straight-check-all)
  (when (doom-cli-packages-purge
         (not noelpa-p)
         (not norepos-p)
         (not nobuilds-p)
         regraft-p)
    (doom-cli-reload-package-autoloads))
  t)

;; (defcli! rollback () ; TODO doom rollback
;;   "<Not implemented yet>"
;;   (user-error "Not implemented yet, sorry!"))


;;
;;; Library

(defun doom--same-commit-p (abbrev-ref ref)
  (and (stringp abbrev-ref)
       (stringp ref)
       (string-match-p (concat "^" (regexp-quote abbrev-ref))
                       ref)))

(defun doom--abbrev-commit (commit &optional full)
  (if full commit (substring commit 0 7)))

(defun doom--commit-log-between (start-ref end-ref)
  (and (straight--call
        "git" "log" "--oneline" "--no-merges"
        "-n" "25" end-ref (concat "^" (regexp-quote start-ref)))
       (straight--process-get-output)))

(defun doom-cli-packages-install ()
  "Installs missing packages.

This function will install any primary package (i.e. a package with a `package!'
declaration) or dependency thereof that hasn't already been."
  (straight--transaction-finalize)
  (print! (start "Installing packages..."))
  (print-group!
   (if-let (built
            (doom-with-package-recipes (doom-package-recipe-list)
                (recipe package type local-repo)
              (condition-case-unless-debug e
                  (progn
                    (straight-use-package (intern package))
                    (when-let* ((target-ref (cdr (assoc local-repo doom-pinned-packages)))
                                (ref (straight-vc-get-commit type local-repo)))
                      (unless (doom--same-commit target-ref ref)
                        (unless (straight-vc-commit-present-p recipe target-ref)
                          (straight-vc-fetch-from-remote recipe))
                        (if (straight-vc-commit-present-p recipe target-ref)
                            (progn
                              (print! (success "Checking out %s to %s")
                                      package (doom--abbrev-commit target-ref))
                              (straight-vc-check-out-commit recipe target-ref)
                              (straight-rebuild-package package t))
                          (ignore-errors
                            (delete-directory (straight--repos-dir local-repo) 'recursive))
                          (straight-use-package (intern package))))))
                (error
                 (signal 'doom-package-error
                         (list package e (straight--process-get-output)))))))
       (print! (success "Installed %d packages")
               (length built))
     (print! (info "No packages need to be installed"))
     nil)))


(defun doom-cli-packages-build (&optional force-p)
  "(Re)build all packages."
  (straight--transaction-finalize)
  (print! (start "(Re)building %spackages...") (if force-p "all " ""))
  (print-group!
   (let ((straight-check-for-modifications
          (when (file-directory-p (straight--modified-dir))
            '(find-when-checking)))
         (straight--allow-find
          (and straight-check-for-modifications
               (executable-find straight-find-executable)
               t))
         (straight--packages-not-to-rebuild
          (or straight--packages-not-to-rebuild (make-hash-table :test #'equal)))
         (straight--packages-to-rebuild
          (or (if force-p :all straight--packages-to-rebuild)
              (make-hash-table :test #'equal))))
     (unless force-p
       (straight--make-package-modifications-available))
     (if-let (built
              (doom-with-package-recipes (doom-package-recipe-list) (package)
                (straight-use-package (intern package))))
         (print! (success "Rebuilt %d package(s)") (length built))
       (print! (success "No packages need rebuilding"))
       nil))))


(defun doom-cli-packages-update ()
  "Updates packages."
  (straight--transaction-finalize)
  (print! (start "Updating packages (this may take a while)..."))
  (let* ((repo-dir (straight--repos-dir))
         (packages-to-rebuild (make-hash-table :test 'equal))
         (repos-to-rebuild (make-hash-table :test 'equal))
         (recipes (doom-package-recipe-list))
         (total (length recipes))
         (esc (unless doom-debug-mode "\033[1A"))
         (i 0)
         errors)
    (doom-with-package-recipes recipes (recipe package type local-repo)
      (cl-incf i)
      (print-group!
       (unless (straight--repository-is-available-p recipe)
         (print! (error "(%d/%d) Couldn't find local repo for %s") i total package)
         (cl-return))
       (when (gethash local-repo repos-to-rebuild)
         (puthash package t packages-to-rebuild)
         (print! (success "(%d/%d) %s was updated indirectly (with %s)") i total package local-repo)
         (cl-return))
       (let ((default-directory (straight--repos-dir local-repo)))
         (unless (file-in-directory-p default-directory repo-dir)
           (print! (warn "(%d/%d) Skipping %s because it is local") i total package)
           (cl-return))
         (condition-case-unless-debug e
             (let ((ref (straight-vc-get-commit type local-repo))
                   (target-ref (cdr (assoc local-repo doom-pinned-packages)))
                   output)
               (or (cond
                    ((not (stringp target-ref))
                     (print! (start "\033[K(%d/%d) Fetching %s...%s") i total package esc)
                     (when (straight-vc-fetch-from-remote recipe)
                       (setq output (straight--process-get-output))
                       (straight-merge-package package)
                       (setq target-ref (straight-vc-get-commit type local-repo))
                       (when (doom--same-commit-p target-ref ref)
                         (cl-return))))

                    ((doom--same-commit-p target-ref ref)
                     (print! (info "\033[K(%d/%d) %s is up-to-date...%s") i total package esc)
                     (cl-return))

                    ((straight-vc-commit-present-p recipe target-ref)
                     (print! (start "\033[K(%d/%d) Checking out %s (%s)...%s")
                             i total package (doom--abbrev-commit target-ref) esc)
                     (straight-vc-check-out-commit recipe target-ref)
                     (or (not (eq type 'git))
                         (setq output (doom--commit-log-between ref target-ref)))
                     (doom--same-commit-p target-ref (straight-vc-get-commit type local-repo)))

                    ((print! (start "\033[K(%d/%d) Re-cloning %s...%s") i total local-repo esc)
                     (let ((repo (straight--repos-dir local-repo)))
                       (ignore-errors
                         (delete-directory repo 'recursive))
                       (print-group!
                        (straight-use-package (intern package) nil 'no-build))
                       (prog1 (file-directory-p repo)
                         (or (not (eq type 'git))
                             (setq output (doom--commit-log-between ref target-ref)))))))
                   (progn
                     (print! (warn "\033[K(%d/%d) Failed to fetch %s")
                             i total local-repo)
                     (unless (string-empty-p output)
                       (print-group! (print! (info "%s" output))))
                     (cl-return)))
               (puthash local-repo t repos-to-rebuild)
               (puthash package t packages-to-rebuild)
               (unless (string-empty-p output)
                 (print-group! (print! (info "%s" output))))
               (print! (success "(%d/%d) %s updated (%s -> %s)")
                       i total local-repo
                       (doom--abbrev-commit ref)
                       (doom--abbrev-commit target-ref)))
           (user-error
            (signal 'user-error (error-message-string e)))
           (error
            (print! (warn "\033[K(%d/%d) Encountered error with %s" i total package))
            (print-group!
             (print! (error "%s") e)
             (print-group! (print! (info "%s" (straight--process-get-output)))))
            (push package errors))))))
    (princ "\033[K")
    (when errors
{'fraction_non_alphanumeric': 0.1378184193337688, 'fraction_numerical': 0.003115108275134402, 'mean_word_length': 1.8913422428820452, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '193111', 'n_tokens_mistral': 6064, 'n_tokens_neox': 5762, 'n_words': 1568}