<commit_before>(defconst p2l-editor-macros
  '("apl" "ab" "sd" "dc" "md" "on" "damien" "lr" "orla" "alex" "alx" "dr" "ja" "jr" "jb" "fp" "michael" "ew" "mb" "hw" "ben" "hjo" "ml"))

(defconst p2l-misc-macros-to-remove
  '("clsindexmain" "needlines"))

(defun p2l--setup-buffer ()
  (goto-char (point-min))
  (fundamental-mode)
  (font-lock-mode -1))

(defun p2l-remove-latex-comments ()
  (interactive)
  (p2l--setup-buffer)
  (delete-matching-lines "^%.*$"))

(defun p2l-remove-useless-macros ()
  (interactive)
  (p2l--setup-buffer)
  (while (re-search-forward
          (concat "\\\\"
                  (regexp-opt 
                   (append p2l-editor-macros
                           p2l-misc-macros-to-remove))
                  "{")
          nil t)
    (backward-char)
    (let ((before-curly (point))
          start)
      (setq start (match-beginning 0))
      (goto-char before-curly)
      (forward-sexp)
      (delete-region start (point)))))

(defun p2l-remove-header ()
  (interactive)
  (p2l--setup-buffer)
  (let ((start (point)))
    (when (search-forward "\\chapter{" nil t)
      (beginning-of-line)
      (delete-region start (point)))))

(defun p2l-remove-footer ()
  (interactive)
  (goto-char (point-max))
  (let ((end (point)))
    (when (search-backward "\\ifx\\wholebook\\relax" nil t)
     (beginning-of-line)
     (delete-region (point) end))))

(defun p2l--convert-command0-once (latex pillar &optional newline)
  (when (re-search-forward
         (concat "\\\\" latex) nil t)
    (let ((start (match-beginning 0))
          end)
      (replace-match pillar)
      (setq end (point))
      (goto-char start)
      (when (and newline (not (zerop (current-column))))
        (open-line 1)
        (setq end (1+ end)))
      (goto-char end)
      (when (looking-at "{}")
        (replace-match ""))
      t)))

(defun p2l--convert-command0 (latex pillar &optional newline)
  (p2l--setup-buffer)
  (while (p2l--convert-command0-once latex pillar newline)))

(defun p2l--convert-command1-once (latex pillar-begin &optional pillar-end newline)
  (when (p2l--convert-command0-once latex pillar-begin newline)
    (let ((start (point))
          end)
      (forward-sexp)
      (delete-backward-char 1)
      (insert pillar-end)
      (setq end (point))
      (goto-char start)
      (delete-forward-char 1)
      (goto-char (1- end))
      t)))

(defun p2l--convert-command1 (latex pillar-begin &optional pillar-end newline)
  (p2l--setup-buffer)
  (while (p2l--convert-command1-once latex pillar-begin pillar-end newline)))

(defun p2l--convert-command2-once (latex pillar-begin pillar-middle &optional pillar-end newline)
  (when (p2l--convert-command1-once latex pillar-begin pillar-middle newline)
    (let ((start (point)))
      (forward-sexp)
      (delete-backward-char 1)
      (insert pillar-end)
      (goto-char start)
      (delete-forward-char 1)
      t)))

(defun p2l--convert-command2 (latex pillar-begin pillar-middle &optional pillar-end newline)
  (p2l--setup-buffer)
  (while (p2l--convert-command2-once latex pillar-begin pillar-middle pillar-end newline)))

(defconst p2l--command0-conversion-table
  '(("ie" "''i.e.'',")))

(defconst p2l--command1-conversion-table
  '(("chapter" "!" "\n" t)
    ("section" "!!" "\n" t)
    ("subsection" "!!!" "\n" t)
    ("subsubsection" "!!!!" "\n" t)
    ("paragraph" "!!!!!" "\n" t)
    ("chalabel" "@cha:" "\n" t)
    ("seclabel" "@sec:" "\n" t)
    ("clsind" "==" "==")
    ("figref" "*fig:" "*")
    ("ct" "==" "==")<commit_msg>Add some more simple macros<commit_after>(defconst p2l-editor-macros
  '("apl" "ab" "sd" "dc" "md" "on" "damien" "lr" "orla" "alex" "alx" "dr" "ja" "jr" "jb" "fp" "michael" "ew" "mb" "hw" "ben" "hjo" "ml"))

(defconst p2l-misc-macros-to-remove
  '("clsindexmain" "needlines"))

(defun p2l--setup-buffer ()
  (goto-char (point-min))
  (fundamental-mode)
  (font-lock-mode -1))

(defun p2l-remove-latex-comments ()
  (interactive)
  (p2l--setup-buffer)
  (delete-matching-lines "^%.*$"))

(defun p2l-remove-useless-macros ()
  (interactive)
  (p2l--setup-buffer)
  (while (re-search-forward
          (concat "\\\\"
                  (regexp-opt 
                   (append p2l-editor-macros
                           p2l-misc-macros-to-remove))
                  "{")
          nil t)
    (backward-char)
    (let ((before-curly (point))
          start)
      (setq start (match-beginning 0))
      (goto-char before-curly)
      (forward-sexp)
      (delete-region start (point)))))

(defun p2l-remove-header ()
  (interactive)
  (p2l--setup-buffer)
  (let ((start (point)))
    (when (search-forward "\\chapter{" nil t)
      (beginning-of-line)
      (delete-region start (point)))))

(defun p2l-remove-footer ()
  (interactive)
  (goto-char (point-max))
  (let ((end (point)))
    (when (search-backward "\\ifx\\wholebook\\relax" nil t)
     (beginning-of-line)
     (delete-region (point) end))))

(defun p2l--convert-command0-once (latex pillar &optional newline)
  (when (re-search-forward
         (concat "\\\\" latex) nil t)
    (let ((start (match-beginning 0))
          end)
      (replace-match pillar)
      (setq end (point))
      (goto-char start)
      (when (and newline (not (zerop (current-column))))
        (open-line 1)
        (setq end (1+ end)))
      (goto-char end)
      (when (looking-at "{}")
        (replace-match ""))
      t)))

(defun p2l--convert-command0 (latex pillar &optional newline)
  (p2l--setup-buffer)
  (while (p2l--convert-command0-once latex pillar newline)))

(defun p2l--convert-command1-once (latex pillar-begin &optional pillar-end newline)
  (when (p2l--convert-command0-once latex pillar-begin newline)
    (let ((start (point))
          end)
      (forward-sexp)
      (delete-backward-char 1)
      (insert pillar-end)
      (setq end (point))
      (goto-char start)
      (delete-forward-char 1)
      (goto-char (1- end))
      t)))

(defun p2l--convert-command1 (latex pillar-begin &optional pillar-end newline)
  (p2l--setup-buffer)
  (while (p2l--convert-command1-once latex pillar-begin pillar-end newline)))

(defun p2l--convert-command2-once (latex pillar-begin pillar-middle &optional pillar-end newline)
  (when (p2l--convert-command1-once latex pillar-begin pillar-middle newline)
    (let ((start (point)))
      (forward-sexp)
      (delete-backward-char 1)
      (insert pillar-end)
      (goto-char start)
      (delete-forward-char 1)
      t)))

(defun p2l--convert-command2 (latex pillar-begin pillar-middle &optional pillar-end newline)
  (p2l--setup-buffer)
  (while (p2l--convert-command2-once latex pillar-begin pillar-middle pillar-end newline)))

(defconst p2l--command0-conversion-table
  '(("ie" "''i.e.'',")
    ("eg" "''e.g.'',")
    ("etc" "etc.")))

(defconst p2l--command1-conversion-table
  '(("chapter" "!" "\n" t)
    ("section" "!!" "\n" t)
    ("subsection" "!!!" "\n" t)
    ("subsubsection" "!!!!" "\n" t)
    ("paragraph" "!!!!!" "\n" t)
    ("chalabel" "@cha:" "\n" t)
    ("seclabel" "@sec:" "\n" t)
    ("clsind" "==" "==")
    ("figref" "*fig:" "*")
    ("ct" "==" "==")
    ("lct" "==" "==")
{'fraction_non_alphanumeric': 0.1925238898257448, 'fraction_numerical': 0.01433389544688027, 'mean_word_length': 2.6553672316384183, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5167534', 'n_tokens_mistral': 2749, 'n_tokens_neox': 2630, 'n_words': 565}