<commit_before>;;; futhark-mode.el --- major mode for editing Futhark source files

;; Copyright (C) DIKU 2013-2017, University of Copenhagen

;;; Commentary:
;; This mode provides syntax highlighting and automatic indentation for
;; Futhark source files.  There is sadly yet no automatic recompilation
;; or interactive environment, mostly because there is no good futharki
;; yet.
;;
;; This mode provides the following features for Futhark source files:
;;
;;   + syntax highlighting
;;   + automatic indentation
;;
;; To load futhark-mode automatically on Emacs startup, put this file in
;; your load path and require the mode, e.g. something like this:
;;
;;   (add-to-list 'load-path "~/.emacs.d/futhark-mode")
;;   (require 'futhark-mode)
;;
;; In this case, you have to create the directory
;; "~/.emacs.d/futhark-mode" and store this file in that directory.
;;
;; This will also tell your Emacs that ".fut" files are to be handled by
;; futhark-mode.
;;
;; Define your local keybindings in `futhark-mode-map', and add startup
;; functions to `futhark-mode-hook'.


;;; Code:

(require 'cl) ; `some'

;;;###autoload
(add-to-list 'auto-mode-alist '("\\.fut\\'" . futhark-mode))

(defvar futhark-mode-hook nil
  "Hook for futhark-mode.  Is run whenever the mode is entered.")

(defvar futhark-mode-map
  (make-keymap)
  "Keymap for futhark-mode.")


;;; Highlighting

(let (
      (ws "[[:space:]\n]*")
      (ws1 "[[:space:]\n]+")
      )

  ;; FIXME: Backslash should also be a keyword (for anonymous functions), but
  ;; Emacs Lisp is stupid.
  (defconst futhark-keywords
    '("if" "then" "else" "let" "loop" "in" "with" "type"
      "fun" "val" "entry" "for" "while" "do"
      "empty" "unsafe" "default" "include" "import" "module" "open")
    "All Futhark keywords.")

  (defconst futhark-builtin-functions
    '("pow" "iota" "shape" "replicate" "reshape" "rotate" "transpose" "map"
      "reduce" "reduceComm" "zip" "unzip" "zipWith" "scan" "split"
      "concat" "filter" "partition" "redomap" "empty" "copy" "size"
      "write")
    "All Futhark builtin SOACs, functions, and non-symbolic operators.")

  (defconst futhark-builtin-types
    '("i8" "i16" "i32" "i64"
      "u8" "u16" "u32" "u64"
      "f32" "f64"
      "int" "real" "bool")
    "A list of Futhark types.")

  (defconst futhark-booleans
    '("true" "false")
    "All Futhark booleans.")

  (defconst futhark-var
    (concat "\\(?:" "[_'[:alnum:]]+" "\\)")
    "A regex describing a Futhark variable.")

  (defconst futhark-operator
    (concat "["
            "+*\\-/%!<>=&|@"
            "]" "+"))

  (defconst futhark-non-tuple-type
    (concat "\\(?:"
            "\\*" "?"
            "\\(?:"
            "\\["
            "\\(?:"
            ""
            "\\|"
            futhark-var
            "\\)"
            "\\]"
            "\\)" "*"
            futhark-var
            "\\)"
            )
    "A regex describing a Futhark type which is not a tuple")

  ;; This does not work with nested tuple types.
  (defconst futhark-tuple-type
    (concat "\\(?:"
            "("
            "\\(?:" ws futhark-non-tuple-type ws "," "\\)" "*"
            ws futhark-non-tuple-type ws
            ")"
            "\\)"
            )
    "A regex describing a Futhark type which is a tuple")

  (defconst futhark-type
    (concat "\\(?:"
            futhark-non-tuple-type
            "\\|"
            futhark-tuple-type
            "\\)"
            )
    "A regex describing a Futhark type")


  (defvar futhark-font-lock
    `(

      ;; Function declarations.
      (,(concat "\\(?:" "fun" "\\|" "val" "\\|" "entry" "\\)"
                ws1 "\\(" futhark-var "\\)")
       . '(1 font-lock-function-name-face))

      ;; Variable and tuple declarations.
      ;;; Lets.
      ;;;; Primitive values.
      (,(concat "let" ws1
                "\\(" futhark-var "\\)")
       . '(1 font-lock-variable-name-face))
      ;;;; Tuples.  FIXME: It would be nice to highlight only the variable names
      ;;;; inside the parantheses, and not also the commas.
      (,(concat "let" ws1 "("
                "\\(" "[^)]+" "\\)")
       . '(1 font-lock-variable-name-face))
      ;;; Function parameters.
      (,(concat "\\(" futhark-var "\\)" ws ":")
       . '(1 font-lock-variable-name-face))

      ;; Keywords.
      (,(regexp-opt futhark-keywords 'words)
       . font-lock-keyword-face)

      ;; Types.
      ;;; Type aliases.  FIXME: It would be nice to highlight also the right
      ;;; hand side.
      (,(concat "type" ws1 "\\(" futhark-type "\\)")
       . '(1 font-lock-type-face))
      ;;; Function parameters types and return type.
      (,(concat ":" ws "\\(" "[^=,)]+" "\\)")
       . '(1 font-lock-type-face))
      ;;; Builtin types.
      (,(regexp-opt futhark-builtin-types 'words)
       . font-lock-type-face)

      ;; Builtins.
      ;;; Functions.
      ;;;; Builtin functions.
      (,(regexp-opt futhark-builtin-functions 'words)
       . font-lock-builtin-face)
      ;;;; Tuple accessors.
      (,(concat "#" "[[:digit:]]+")
       . font-lock-builtin-face)
      ;;; Operators.
      (,futhark-operator
       . font-lock-builtin-face)

      ;; Constants.
      ;;; Booleans.
      (,(regexp-opt futhark-booleans 'words)
       . font-lock-constant-face)

      )
    "Highlighting expressions for Futhark.")
  )

(defvar futhark-mode-syntax-table
  (let ((st (make-syntax-table)))
    ;; Define the -- line comment syntax.
    (modify-syntax-entry ?- ". 123" st)
    (modify-syntax-entry ?\n ">" st)
    ;; Make apostrophe and underscore be part of variable names.
    ;; Technically, they should probably be part of the symbol class,
    ;; but it works out better for some of the regexpes if they are part
    ;; of the word class.
    (modify-syntax-entry ?' "w" st)
    (modify-syntax-entry ?_ "w" st)
    (modify-syntax-entry ?\\ "_" st)
    st)
  "Syntax table used in `futhark-mode'.")


;;; Indentation

(defvar futhark-indent-level 2
  "The basic indent level for futhark-mode.")

(defun futhark-indent-line ()
  "Indent current line as Futhark code."
  (let ((savep (> (current-column) (current-indentation)))
        (indent (or (futhark-calculate-indentation)
                    (current-indentation))))
    (if savep ; The cursor is beyond leading whitespace.
        (save-excursion (indent-line-to indent))
      (indent-line-to indent))))

(defun futhark-calculate-indentation ()
  "Calculate the indentation for the current line.
In general, prefer as little indentation as possible."
  (let ((parse-sexp-lookup-properties t)
        (parse-sexp-ignore-comments t))

    (save-excursion
      (futhark-beginning-of-line-text)

      ;; The following code is fickle and deceptive.  Don't change it
      ;; unless you kind of know what you're doing!
      (or

       ;; Align comment to next non-comment line.
       (and (looking-at comment-start)
            (forward-comment (count-lines (point-min) (point)))
            (current-column))

       ;; Align global definitions and headers to nearest module definition or
       ;; column 0.
       (and (or (futhark-looking-at-word "fun")
                (futhark-looking-at-word "entry")
                (futhark-looking-at-word "type")
                (futhark-looking-at-word "val")
                (futhark-looking-at-word "module")
                (futhark-looking-at-word "include")
                (futhark-looking-at-word "import")
                (futhark-looking-at-word "default"))
            (or
             (save-excursion
               (and
                (ignore-errors (backward-up-list 1) t)
                (looking-at "{")
                (or
                 (futhark-keyword-backward "module")
                 (futhark-keyword-backward "open")
                 (and
                  (ignore-errors (backward-up-list 1) t)
                  (or
                   (futhark-keyword-backward "module")
                   (futhark-keyword-backward "open"))))
                (+ futhark-indent-level (current-column))))
             0))

       ;; Align closing parentheses and commas to the matching opening
       ;; parenthesis.
       (save-excursion
         (and (looking-at (regexp-opt '(")" "]" ",")))
              (ignore-errors
                (backward-up-list 1)
                (current-column))))

       ;; Align closing curly brackets to the matching opening 'module'
       ;; keyword.
       (save-excursion
         (and (looking-at "}")
              (ignore-errors
                (backward-up-list 1)
                (or
                 (save-excursion
                   (ignore-errors
                     (and
                      (backward-up-list 1)
                      (looking-at "(")
                      (futhark-keyword-backward "module")
                      (current-column))))
                 (and
                  (futhark-keyword-backward "module")
                  (current-column))))))

       ;; Align "in", "let", or "loop" to the closest previous "let" or "loop".
       (save-excursion
         (and (or (futhark-looking-at-word "in")
                  (futhark-looking-at-word "let")
                  (futhark-looking-at-word "loop"))
              (let ((m
                     (futhark-max
                      (save-excursion
                        (futhark-keyword-backward "let"))
                      (save-excursion
                        (futhark-keyword-backward "loop")))))
                (and (not (eq nil m))
                     (goto-char m)
                     (current-column)))))

       ;; Otherwise, if the previous code line ends with "in" align to
       ;; the matching "let" or "loop" column.
       (save-excursion
         (and (futhark-backward-part)
              (looking-at "\\<in[[:space:]]*$")
              (let ((m
                     (futhark-max
                      (save-excursion
                        (futhark-keyword-backward "let"))
                      (save-excursion
                        (futhark-keyword-backward "loop")))))
                (and (not (eq nil m))
                     (goto-char m)
                     (current-column)))))

       ;; If the previous code line ends with "=", align to the matching "fun"
       ;; or "let" or "loop" column plus one indent level.
       (save-excursion
         (and (futhark-backward-part)
              (looking-at "=[[:space:]]*$")
              (let ((m
                     (futhark-max
                      (futhark-max
                       (save-excursion
                         (futhark-keyword-backward "fun"))
                       (save-excursion
                         (futhark-keyword-backward "let")))
                      (save-excursion
                        (futhark-keyword-backward "loop")))))
                (and (not (eq nil m))
                     (goto-char m)
                     (+ (current-column) futhark-indent-level)))))

       ;; Align "then" to nearest "else if" or "if".
       (save-excursion
         (and (futhark-looking-at-word "then")
              (futhark-keyword-backward "if")
              (or
               (let ((curline (line-number-at-pos)))
                 (save-excursion
                   (and (futhark-backward-part)
                        (= (line-number-at-pos) curline)
                        (futhark-looking-at-word "else")
                        (current-column))))
               (current-column))))

       ;; Align "else" to nearest "then" or "else if" or "if".
       (save-excursion
         (and (futhark-looking-at-word "else")
              (let ((m
                     (futhark-max
                      (save-excursion
                        (and
                         (futhark-keyword-backward "then")
                         (futhark-is-beginning-of-line-text)
                         (point)))
                      (save-excursion
                        (let ((pos0 (futhark-keyword-backward "if")))
                          (or
                           (let ((curline (line-number-at-pos)))
                             (and (futhark-backward-part)
                                  (= (line-number-at-pos) curline)
                                  (futhark-looking-at-word "else")
                                  (point)))
                           pos0))))))
                (and (not (eq nil m))
                     (goto-char m)
                     (current-column)))))

       ;; Align general content inside parentheses to the first general
       ;; non-space content.
       (save-excursion
         (when (ignore-errors (backward-up-list 1) t)
              (forward-char 1)
              (futhark-goto-first-text)
              (and
               (not (futhark-is-looking-at-keyword))
               (current-column))))

       ;; Otherwise, keep the user-specified indentation level.
       ))))

(defun futhark-min (a b)
  "Like `min', but more accepting."
  (or (and (eq nil a) b)
      (and (eq nil b) a)
      (and (not (eq nil a))
           (not (eq nil b))
           (min a b))))

(defun futhark-max (a b)
  "Like `max', but more accepting."
  (or (and (eq nil a) b)
      (and (eq nil b) a)
      (and (not (eq nil a))
           (not (eq nil b))
           (max a b))))

(defun futhark-beginning-of-line-text ()
  "Move to the beginning of the non-whitespace text on this line."
  (beginning-of-line)
  (futhark-goto-first-text))
<commit_msg>futhark-mode.el: Satisfy Emacs' internal checkdoc style.<commit_after>;;; futhark-mode.el --- major mode for editing Futhark source files

;; Copyright (C) DIKU 2013-2017, University of Copenhagen

;;; Commentary:
;; This mode provides syntax highlighting and automatic indentation for
;; Futhark source files.  There is sadly yet no automatic recompilation
;; or interactive environment, mostly because there is no good futharki
;; yet.
;;
;; This mode provides the following features for Futhark source files:
;;
;;   + syntax highlighting
;;   + automatic indentation
;;
;; To load futhark-mode automatically on Emacs startup, put this file in
;; your load path and require the mode, e.g. something like this:
;;
;;   (add-to-list 'load-path "~/.emacs.d/futhark-mode")
;;   (require 'futhark-mode)
;;
;; In this case, you have to create the directory
;; "~/.emacs.d/futhark-mode" and store this file in that directory.
;;
;; This will also tell your Emacs that ".fut" files are to be handled by
;; futhark-mode.
;;
;; Define your local keybindings in `futhark-mode-map', and add startup
;; functions to `futhark-mode-hook'.


;;; Code:

(require 'cl) ; `some'

;;;###autoload
(add-to-list 'auto-mode-alist '("\\.fut\\'" . futhark-mode))

(defvar futhark-mode-hook nil
  "Hook for `futhark-mode'.  Is run whenever the mode is entered.")

(defvar futhark-mode-map
  (make-keymap)
  "Keymap for `futhark-mode'.")


;;; Highlighting

(let (
      (ws "[[:space:]\n]*")
      (ws1 "[[:space:]\n]+")
      )

  ;; FIXME: Backslash should also be a keyword (for anonymous functions), but
  ;; Emacs Lisp is stupid.
  (defconst futhark-keywords
    '("if" "then" "else" "let" "loop" "in" "with" "type"
      "fun" "val" "entry" "for" "while" "do"
      "empty" "unsafe" "default" "include" "import" "module" "open")
    "All Futhark keywords.")

  (defconst futhark-builtin-functions
    '("pow" "iota" "shape" "replicate" "reshape" "rotate" "transpose" "map"
      "reduce" "reduceComm" "zip" "unzip" "zipWith" "scan" "split"
      "concat" "filter" "partition" "redomap" "empty" "copy" "size"
      "write")
    "All Futhark builtin SOACs, functions, and non-symbolic operators.")

  (defconst futhark-builtin-types
    '("i8" "i16" "i32" "i64"
      "u8" "u16" "u32" "u64"
      "f32" "f64"
      "int" "real" "bool")
    "A list of Futhark types.")

  (defconst futhark-booleans
    '("true" "false")
    "All Futhark booleans.")

  (defconst futhark-var
    (concat "\\(?:" "[_'[:alnum:]]+" "\\)")
    "A regex describing a Futhark variable.")

  (defconst futhark-operator
    (concat "["
            "+*\\-/%!<>=&|@"
            "]" "+"))

  (defconst futhark-non-tuple-type
    (concat "\\(?:"
            "\\*" "?"
            "\\(?:"
            "\\["
            "\\(?:"
            ""
            "\\|"
            futhark-var
            "\\)"
            "\\]"
            "\\)" "*"
            futhark-var
            "\\)"
            )
    "A regex describing a Futhark type which is not a tuple")

  ;; This does not work with nested tuple types.
  (defconst futhark-tuple-type
    (concat "\\(?:"
            "("
            "\\(?:" ws futhark-non-tuple-type ws "," "\\)" "*"
            ws futhark-non-tuple-type ws
            ")"
            "\\)"
            )
    "A regex describing a Futhark type which is a tuple")

  (defconst futhark-type
    (concat "\\(?:"
            futhark-non-tuple-type
            "\\|"
            futhark-tuple-type
            "\\)"
            )
    "A regex describing a Futhark type")


  (defvar futhark-font-lock
    `(

      ;; Function declarations.
      (,(concat "\\(?:" "fun" "\\|" "val" "\\|" "entry" "\\)"
                ws1 "\\(" futhark-var "\\)")
       . '(1 font-lock-function-name-face))

      ;; Variable and tuple declarations.
      ;;; Lets.
      ;;;; Primitive values.
      (,(concat "let" ws1
                "\\(" futhark-var "\\)")
       . '(1 font-lock-variable-name-face))
      ;;;; Tuples.  FIXME: It would be nice to highlight only the variable names
      ;;;; inside the parantheses, and not also the commas.
      (,(concat "let" ws1 "("
                "\\(" "[^)]+" "\\)")
       . '(1 font-lock-variable-name-face))
      ;;; Function parameters.
      (,(concat "\\(" futhark-var "\\)" ws ":")
       . '(1 font-lock-variable-name-face))

      ;; Keywords.
      (,(regexp-opt futhark-keywords 'words)
       . font-lock-keyword-face)

      ;; Types.
      ;;; Type aliases.  FIXME: It would be nice to highlight also the right
      ;;; hand side.
      (,(concat "type" ws1 "\\(" futhark-type "\\)")
       . '(1 font-lock-type-face))
      ;;; Function parameters types and return type.
      (,(concat ":" ws "\\(" "[^=,)]+" "\\)")
       . '(1 font-lock-type-face))
      ;;; Builtin types.
      (,(regexp-opt futhark-builtin-types 'words)
       . font-lock-type-face)

      ;; Builtins.
      ;;; Functions.
      ;;;; Builtin functions.
      (,(regexp-opt futhark-builtin-functions 'words)
       . font-lock-builtin-face)
      ;;;; Tuple accessors.
      (,(concat "#" "[[:digit:]]+")
       . font-lock-builtin-face)
      ;;; Operators.
      (,futhark-operator
       . font-lock-builtin-face)

      ;; Constants.
      ;;; Booleans.
      (,(regexp-opt futhark-booleans 'words)
       . font-lock-constant-face)

      )
    "Highlighting expressions for Futhark.")
  )

(defvar futhark-mode-syntax-table
  (let ((st (make-syntax-table)))
    ;; Define the -- line comment syntax.
    (modify-syntax-entry ?- ". 123" st)
    (modify-syntax-entry ?\n ">" st)
    ;; Make apostrophe and underscore be part of variable names.
    ;; Technically, they should probably be part of the symbol class,
    ;; but it works out better for some of the regexpes if they are part
    ;; of the word class.
    (modify-syntax-entry ?' "w" st)
    (modify-syntax-entry ?_ "w" st)
    (modify-syntax-entry ?\\ "_" st)
    st)
  "Syntax table used in `futhark-mode'.")


;;; Indentation

(defvar futhark-indent-level 2
  "The basic indent level for `futhark-mode'.")

(defun futhark-indent-line ()
  "Indent current line as Futhark code."
  (let ((savep (> (current-column) (current-indentation)))
        (indent (or (futhark-calculate-indentation)
                    (current-indentation))))
    (if savep ; The cursor is beyond leading whitespace.
        (save-excursion (indent-line-to indent))
      (indent-line-to indent))))

(defun futhark-calculate-indentation ()
  "Calculate the indentation for the current line.
In general, prefer as little indentation as possible."
  (let ((parse-sexp-lookup-properties t)
        (parse-sexp-ignore-comments t))

    (save-excursion
      (futhark-beginning-of-line-text)

      ;; The following code is fickle and deceptive.  Don't change it
      ;; unless you kind of know what you're doing!
      (or

       ;; Align comment to next non-comment line.
       (and (looking-at comment-start)
            (forward-comment (count-lines (point-min) (point)))
            (current-column))

       ;; Align global definitions and headers to nearest module definition or
       ;; column 0.
       (and (or (futhark-looking-at-word "fun")
                (futhark-looking-at-word "entry")
                (futhark-looking-at-word "type")
                (futhark-looking-at-word "val")
                (futhark-looking-at-word "module")
                (futhark-looking-at-word "include")
                (futhark-looking-at-word "import")
                (futhark-looking-at-word "default"))
            (or
             (save-excursion
               (and
                (ignore-errors (backward-up-list 1) t)
                (looking-at "{")
                (or
                 (futhark-keyword-backward "module")
                 (futhark-keyword-backward "open")
                 (and
                  (ignore-errors (backward-up-list 1) t)
                  (or
                   (futhark-keyword-backward "module")
                   (futhark-keyword-backward "open"))))
                (+ futhark-indent-level (current-column))))
             0))

       ;; Align closing parentheses and commas to the matching opening
       ;; parenthesis.
       (save-excursion
         (and (looking-at (regexp-opt '(")" "]" ",")))
              (ignore-errors
                (backward-up-list 1)
                (current-column))))

       ;; Align closing curly brackets to the matching opening 'module'
       ;; keyword.
       (save-excursion
         (and (looking-at "}")
              (ignore-errors
                (backward-up-list 1)
                (or
                 (save-excursion
                   (ignore-errors
                     (and
                      (backward-up-list 1)
                      (looking-at "(")
                      (futhark-keyword-backward "module")
                      (current-column))))
                 (and
                  (futhark-keyword-backward "module")
                  (current-column))))))

       ;; Align "in", "let", or "loop" to the closest previous "let" or "loop".
       (save-excursion
         (and (or (futhark-looking-at-word "in")
                  (futhark-looking-at-word "let")
                  (futhark-looking-at-word "loop"))
              (let ((m
                     (futhark-max
                      (save-excursion
                        (futhark-keyword-backward "let"))
                      (save-excursion
                        (futhark-keyword-backward "loop")))))
                (and (not (eq nil m))
                     (goto-char m)
                     (current-column)))))

       ;; Otherwise, if the previous code line ends with "in" align to
       ;; the matching "let" or "loop" column.
       (save-excursion
         (and (futhark-backward-part)
              (looking-at "\\<in[[:space:]]*$")
              (let ((m
                     (futhark-max
                      (save-excursion
                        (futhark-keyword-backward "let"))
                      (save-excursion
                        (futhark-keyword-backward "loop")))))
                (and (not (eq nil m))
                     (goto-char m)
                     (current-column)))))

       ;; If the previous code line ends with "=", align to the matching "fun"
       ;; or "let" or "loop" column plus one indent level.
       (save-excursion
         (and (futhark-backward-part)
              (looking-at "=[[:space:]]*$")
              (let ((m
                     (futhark-max
                      (futhark-max
                       (save-excursion
                         (futhark-keyword-backward "fun"))
                       (save-excursion
                         (futhark-keyword-backward "let")))
                      (save-excursion
                        (futhark-keyword-backward "loop")))))
                (and (not (eq nil m))
                     (goto-char m)
                     (+ (current-column) futhark-indent-level)))))

       ;; Align "then" to nearest "else if" or "if".
       (save-excursion
         (and (futhark-looking-at-word "then")
              (futhark-keyword-backward "if")
              (or
               (let ((curline (line-number-at-pos)))
                 (save-excursion
                   (and (futhark-backward-part)
                        (= (line-number-at-pos) curline)
                        (futhark-looking-at-word "else")
                        (current-column))))
               (current-column))))

       ;; Align "else" to nearest "then" or "else if" or "if".
       (save-excursion
         (and (futhark-looking-at-word "else")
              (let ((m
                     (futhark-max
                      (save-excursion
                        (and
                         (futhark-keyword-backward "then")
                         (futhark-is-beginning-of-line-text)
                         (point)))
                      (save-excursion
                        (let ((pos0 (futhark-keyword-backward "if")))
                          (or
                           (let ((curline (line-number-at-pos)))
                             (and (futhark-backward-part)
                                  (= (line-number-at-pos) curline)
                                  (futhark-looking-at-word "else")
                                  (point)))
                           pos0))))))
                (and (not (eq nil m))
                     (goto-char m)
                     (current-column)))))

       ;; Align general content inside parentheses to the first general
       ;; non-space content.
       (save-excursion
         (when (ignore-errors (backward-up-list 1) t)
              (forward-char 1)
              (futhark-goto-first-text)
              (and
               (not (futhark-is-looking-at-keyword))
               (current-column))))

       ;; Otherwise, keep the user-specified indentation level.
       ))))

(defun futhark-min (a b)
  "Like `min', but also accepts nil values in A and B."
  (or (and (eq nil a) b)
      (and (eq nil b) a)
      (and (not (eq nil a))
           (not (eq nil b))
           (min a b))))

(defun futhark-max (a b)
  "Like `max', but also accepts nil values in A and B."
  (or (and (eq nil a) b)
      (and (eq nil b) a)
      (and (not (eq nil a))
           (not (eq nil b))
           (max a b))))

(defun futhark-beginning-of-line-text ()
  "Move to the beginning of the non-whitespace text on this line."
  (beginning-of-line)
  (futhark-goto-first-text))

{'fraction_non_alphanumeric': 0.14689202713221394, 'fraction_numerical': 0.003854850068571852, 'mean_word_length': 1.5959780621572213, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '77670', 'n_tokens_mistral': 8619, 'n_tokens_neox': 8132, 'n_words': 2087}