<commit_before>(in-package :lem)

(export '(raw-forward-sexp
          forward-sexp
          backward-sexp
          scan-lists
          forward-list
          backward-list
          down-list
          up-list
          top-of-defun
          mark-sexp
          kill-sexp
          transpose-sexps
          beginning-of-defun-abstract))

#|
(defun skip-block-comment-forward ()
  (loop
    for c1 = (following-char)
    for c2 = (char-after 1)
    do (if (and (syntax-end-block-comment-p c1 c2)
                (not (sexp-escape-p t)))
           (progn
             (shift-position 2)
             (return))
           (unless (shift-position 1)
             (return)))))

(defun skip-space-forward ()
  (loop
    for c1 = (following-char)
    for c2 = (char-after 1)
    do (cond ((syntax-space-char-p c1)
              (unless (shift-position 1)
                (return nil)))
             ((and (syntax-line-comment-p c1 c2)
                   (not (sexp-escape-p t)))
              (unless (forward-line 1)
                (return nil))
              (beginning-of-line))
             ((and (syntax-start-block-comment-p c1 c2)
                   (not (sexp-escape-p t)))
              (skip-block-comment-forward))
             (t
              (return t)))))

(defun line-comment-charpos ()
  (labels ((f (str i in-string-p)
              (if (>= i (length str))
                  nil
                  (let ((c (schar str i)))
                    (cond ((syntax-escape-char-p c)
                           (f str (+ i 2) in-string-p))
                          (in-string-p
                           (f str (1+ i) t))
                          ((syntax-string-quote-char-p c)
                           (f str (1+ i) (not in-string-p)))
                          ((syntax-line-comment-p c
                                                  (safe-aref str
                                                             (1+ i)
                                                             #\newline))
                           i)
                          (t
                           (f str (1+ i) in-string-p)))))))
    (f (current-line-string)
       0
       nil)))

(defun skip-block-comment-backward ()
  (loop
    for count from 0
    for c1 = (following-char)
    for c2 = (preceding-char)
    do (if (and (syntax-start-block-comment-p c2 c1)
                (not (sexp-escape-p nil)))
           (progn
             (shift-position -1)
             (return))
           (unless (shift-position -1)
             (return)))))

(defun skip-space-backward ()
  (loop
    (if (bolp)
        (if (forward-line -1)
            (let ((pos (line-comment-charpos)))
              (if pos
                  (set-charpos pos)
                  (end-of-line)))
            (return nil))
        (let ((c1 (preceding-char))
              (c2 (char-before 2)))
          (cond ((or (syntax-space-char-p c1)
                     (and (syntax-line-comment-p c1 nil)
                          (not (sexp-escape-p nil))))
                 (unless (shift-position -1)
                   (return nil)))
                ((and (not (syntax-line-comment-p c2 nil))
                      (syntax-line-comment-p c2 c1)
                      (not (sexp-escape-p nil 1)))
                 (unless (shift-position -2)
                   (return nil)))
                ((or ;; (and (syntax-end-block-comment-p c1 nil)
                     ;;      (not (sexp-escape-p nil)))
                     (and (syntax-end-block-comment-p c2 c1)
                          (not (sexp-escape-p nil 1))))
                 (skip-block-comment-backward))
                (t
                 (return t)))))))
|#

(defparameter *converted-char-types*
  '(:space
    :symbol
    :open-paren
    :closed-paren
    :string-quote
    :expr-prefix
    :line-comment))

(defun sexp-get-char (dir)
  (if dir
      (following-char)
      (preceding-char)))

(defun sexp-escape-p (dir &optional (offset 0))
  (oddp (mod (loop for n from (- (if dir 1 2) offset)
               while (syntax-escape-char-p
                      (char-before n))
               count 1)
             2)))

(defun sexp-get-syntax-type (dir)
  (if (sexp-escape-p dir)
      :symbol
      (let ((c (sexp-get-char dir)))
        (cond ((syntax-space-char-p c) :space)
              ((syntax-symbol-char-p c) :symbol)
              ((syntax-open-paren-char-p c) :open-paren)
              ((syntax-closed-paren-char-p c) :closed-paren)
              ((syntax-string-quote-char-p c) :string-quote)
              ((syntax-escape-char-p c) :escape)
              ((syntax-expr-prefix-char-p c) :expr-prefix)
              ((syntax-line-comment-p c
                                      (if dir
                                          (char-after 1)
                                          (following-char)))
               :line-comment)
              (t :symbol)))))

(defun sexp-step-char (dir)
  (shift-position (if dir 1 -1)))

(defun skip-symbol (dir)
  (loop
    (if (and (null dir) (bobp))
        (return t)
        (case (sexp-get-syntax-type dir)
          ((:symbol :expr-prefix :escape)
           (unless (sexp-step-char dir)
             (return)))
          (t
           (return t))))))

(defun skip-symbol-forward ()
  (skip-symbol t))

(defun skip-symbol-backward ()
  (skip-symbol nil))

(defun skip-list (depth dir)
  (let ((paren-stack))
    (loop
      (unless (if dir
                  (skip-space-and-comment-forward)
                  (skip-space-and-comment-backward))
        (return nil))
      (when (if dir (eobp) (bobp))
        (return nil))
      (let ((type (sexp-get-syntax-type dir)))
        (case type
          ((:open-paren :closed-paren)
           (let ((c (sexp-get-char dir)))
             (cond ((or (and (eq type :open-paren) dir)
                        (and (eq type :closed-paren) (not dir)))
                    (push c paren-stack)
                    (incf depth))
                   ((or (null paren-stack)
                        (syntax-equal-paren-p c (car paren-stack)))
                    (pop paren-stack)
                    (decf depth))
                   (t
                    (return nil)))
             (sexp-step-char dir)
             (when (zerop depth)<commit_msg>skip-listを修正<commit_after>(in-package :lem)

(export '(raw-forward-sexp
          forward-sexp
          backward-sexp
          scan-lists
          forward-list
          backward-list
          down-list
          up-list
          top-of-defun
          mark-sexp
          kill-sexp
          transpose-sexps
          beginning-of-defun-abstract))

#|
(defun skip-block-comment-forward ()
  (loop
    for c1 = (following-char)
    for c2 = (char-after 1)
    do (if (and (syntax-end-block-comment-p c1 c2)
                (not (sexp-escape-p t)))
           (progn
             (shift-position 2)
             (return))
           (unless (shift-position 1)
             (return)))))

(defun skip-space-forward ()
  (loop
    for c1 = (following-char)
    for c2 = (char-after 1)
    do (cond ((syntax-space-char-p c1)
              (unless (shift-position 1)
                (return nil)))
             ((and (syntax-line-comment-p c1 c2)
                   (not (sexp-escape-p t)))
              (unless (forward-line 1)
                (return nil))
              (beginning-of-line))
             ((and (syntax-start-block-comment-p c1 c2)
                   (not (sexp-escape-p t)))
              (skip-block-comment-forward))
             (t
              (return t)))))

(defun line-comment-charpos ()
  (labels ((f (str i in-string-p)
              (if (>= i (length str))
                  nil
                  (let ((c (schar str i)))
                    (cond ((syntax-escape-char-p c)
                           (f str (+ i 2) in-string-p))
                          (in-string-p
                           (f str (1+ i) t))
                          ((syntax-string-quote-char-p c)
                           (f str (1+ i) (not in-string-p)))
                          ((syntax-line-comment-p c
                                                  (safe-aref str
                                                             (1+ i)
                                                             #\newline))
                           i)
                          (t
                           (f str (1+ i) in-string-p)))))))
    (f (current-line-string)
       0
       nil)))

(defun skip-block-comment-backward ()
  (loop
    for count from 0
    for c1 = (following-char)
    for c2 = (preceding-char)
    do (if (and (syntax-start-block-comment-p c2 c1)
                (not (sexp-escape-p nil)))
           (progn
             (shift-position -1)
             (return))
           (unless (shift-position -1)
             (return)))))

(defun skip-space-backward ()
  (loop
    (if (bolp)
        (if (forward-line -1)
            (let ((pos (line-comment-charpos)))
              (if pos
                  (set-charpos pos)
                  (end-of-line)))
            (return nil))
        (let ((c1 (preceding-char))
              (c2 (char-before 2)))
          (cond ((or (syntax-space-char-p c1)
                     (and (syntax-line-comment-p c1 nil)
                          (not (sexp-escape-p nil))))
                 (unless (shift-position -1)
                   (return nil)))
                ((and (not (syntax-line-comment-p c2 nil))
                      (syntax-line-comment-p c2 c1)
                      (not (sexp-escape-p nil 1)))
                 (unless (shift-position -2)
                   (return nil)))
                ((or ;; (and (syntax-end-block-comment-p c1 nil)
                     ;;      (not (sexp-escape-p nil)))
                     (and (syntax-end-block-comment-p c2 c1)
                          (not (sexp-escape-p nil 1))))
                 (skip-block-comment-backward))
                (t
                 (return t)))))))
|#

(defparameter *converted-char-types*
  '(:space
    :symbol
    :open-paren
    :closed-paren
    :string-quote
    :expr-prefix
    :line-comment))

(defun sexp-get-char (dir)
  (if dir
      (following-char)
      (preceding-char)))

(defun sexp-escape-p (dir &optional (offset 0))
  (oddp (mod (loop for n from (- (if dir 1 2) offset)
               while (syntax-escape-char-p
                      (char-before n))
               count 1)
             2)))

(defun sexp-get-syntax-type (dir)
  (if (sexp-escape-p dir)
      :symbol
      (let ((c (sexp-get-char dir)))
        (cond ((syntax-space-char-p c) :space)
              ((syntax-symbol-char-p c) :symbol)
              ((syntax-open-paren-char-p c) :open-paren)
              ((syntax-closed-paren-char-p c) :closed-paren)
              ((syntax-string-quote-char-p c) :string-quote)
              ((syntax-escape-char-p c) :escape)
              ((syntax-expr-prefix-char-p c) :expr-prefix)
              ((syntax-line-comment-p c
                                      (if dir
                                          (char-after 1)
                                          (following-char)))
               :line-comment)
              (t :symbol)))))

(defun sexp-step-char (dir)
  (shift-position (if dir 1 -1)))

(defun skip-symbol (dir)
  (loop
    (if (and (null dir) (bobp))
        (return t)
        (case (sexp-get-syntax-type dir)
          ((:symbol :expr-prefix :escape)
           (unless (sexp-step-char dir)
             (return)))
          (t
           (return t))))))

(defun skip-symbol-forward ()
  (skip-symbol t))

(defun skip-symbol-backward ()
  (skip-symbol nil))

(defun skip-list (depth dir)
  (let ((paren-stack))
    (loop
      (unless (if dir
                  (skip-space-and-comment-forward)
                  (skip-space-and-comment-backward))
        (return nil))
      (when (if dir (eobp) (bobp))
        (return nil))
      (let ((type (sexp-get-syntax-type dir)))
        (case type
          ((:open-paren :closed-paren)
           (let ((c (sexp-get-char dir)))
             (cond ((or (and (eq type :open-paren) dir)
                        (and (eq type :closed-paren) (not dir)))
                    (push c paren-stack)
                    (incf depth))
                   ((or (and (not (zerop depth)) (null paren-stack))
                        (syntax-equal-paren-p c (car paren-stack)))
                    (pop paren-stack)
                    (decf depth))
                   (t
                    (return nil)))
             (sexp-step-char dir)
             (when (zerop depth)
{'fraction_non_alphanumeric': 0.13497176489302473, 'fraction_numerical': 0.008589835361488905, 'mean_word_length': 1.1261413594859655, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1481520', 'n_tokens_mistral': 3897, 'n_tokens_neox': 3730, 'n_words': 889}