<commit_before>;; Ported from github.com/darius/dole

;; Glossary:
;;   p   coordinate (position between characters in text, or `nowhere`)
;;   dir direction
;;   cs  charset

;; A text is a sequence of characters. Coordinates denote the spaces
;; *between* character positions (or before or after them, at the
;; ends), in [0..size]. The coordinate before the first character is
;; 0, then after the first character and before the second is 1, and
;; so on, until the coordinate after the last character is `size`.
;; 
;; We store the characters at integer indices in a flexarray
;; (starting at index 0) in two chunks which we call the head and the
;; tail, separated by the gap. The fields `head` and `gap` (and `tail`
;; if it weren't implicit) denote the lengths of these spans. `size`
;; denotes head+tail, i.e. the total length of text. The whole array
;; is of size size+gap, i.e. head+gap+tail.
;; 
;; The gap lets us insert or delete text by moving the gap instead
;; of the whole tail; if there's locality, this will be cheaper.


;; A coordinate that's never an actual text position. We'll use
;; -nowhere for "off the left end" and +nowhere for the right.
(let nowhere (expt 2 40))               ;XXX a dangerous pretense

;; Directions from a coordinate.
(let backward -1)
(let forward   1)

(to (text<-)

  (let t    (flexarray<-))
  (let head (box<- 0))
  (let gap  (box<- 0))
  (let size (box<- 0))

  ;; Return coordinate p clipped to the text's actual range.
  (to (clip p)
    (max 0 (min p size.^)))

  (to (clip-range p span)
    (let q (clip p))
    (let h (clip (+ q (max 0 span))))
    `(,q ,(- h q)))

  ;; Pre: p is in [0..size).
  (to (get-char-after p)
    (t (if (< p head.^) p (+ p gap.^))))

  ;; Return the `span` characters after `p0` as a string.
  (to (get p0 span0)
    (let `(,p ,span) (clip-range p0 span0))
    (string<-list (each get-char-after (range<- p (+ p span)))))

  ;; Return the position after the character in the text that is in
  ;; char-set and that comes up first in searching from p in
  ;; direction dir. If none, return nowhere or -nowhere.
  ;; XXX untested, probably broken
  (to (find-char-set p dir char-set)
    ((if (= dir forward)
         find-char-set-forward
         find-char-set-backward)
     (clip p)
     char-set))

  (to (find-char-set-forward p char-set)
    (let H head.^)
    (begin searching ((p p))
      (if (< p H)
          (if (char-set .maps? (t p))
              p.+
              (searching p.+))
          (do
            (let G gap.^)
            (let S size.^)
            (begin persevering ((p p))
              (if (< p S)
                  (if (char-set .maps? (t (+ G p)))
                      p.+
                      (persevering p.+))
                  nowhere))))))

  (to (find-char-set-backward p char-set)
    (let H head.^)
    (let G gap.^)
    (begin searching ((p p))
      (if (< H p)
          (if (char-set .maps? (t (+ G p -1)))
              p
              (searching (- p 1)))
          (begin persevering ((p p))
            (if (< 0 p)
                (if (char-set .maps? (t (- p 1)))
                    p
                    (persevering (- p 1)))
                (- nowhere))))))

  (to (grow n)
    (+ n (n .quotient 2)))

  ;; Replace the `span` characters after `p` by `replacement`.<commit_msg>tweak<commit_after>;; Ported from github.com/darius/dole

;; Glossary:
;;   p   coordinate (position between characters in text, or `nowhere`)
;;   dir direction
;;   cs  charset

;; A text is a sequence of characters. Coordinates denote the spaces
;; *between* character positions (or before or after them, at the
;; ends), in [0..size]. The coordinate before the first character is
;; 0, then after the first character and before the second is 1, and
;; so on, until the coordinate after the last character is `size`.
;; 
;; We store the characters at integer indices in a flexarray
;; (starting at index 0) in two chunks which we call the head and the
;; tail, separated by the gap. The fields `head` and `gap` (and `tail`
;; if it weren't implicit) denote the lengths of these spans. `size`
;; denotes head+tail, i.e. the total length of text. The whole array
;; is of size size+gap, i.e. head+gap+tail.
;; 
;; The gap lets us insert or delete text by moving the gap instead
;; of the whole tail; if there's locality, this will be cheaper.


;; A coordinate that's never an actual text position. We'll use
;; -nowhere for "off the left end" and +nowhere for the right.
(let nowhere (expt 2 40))               ;XXX a dangerous pretense

;; Directions from a coordinate.
(let backward -1)
(let forward   1)

(to (text<-)

  (let t    (flexarray<-))
  (let head (box<- 0))
  (let gap  (box<- 0))
  (let size (box<- 0))

  ;; Return coordinate p clipped to the text's actual range.
  (to (clip p)
    (max 0 (min p size.^)))

  (to (clip-range p span)
    (let q (clip p))
    (let h (clip (+ q (max 0 span))))
    `(,q ,(- h q)))

  ;; Pre: p is in [0..size).
  (to (get-char-after p)
    (t (if (< p head.^) p (+ p gap.^))))

  ;; Return the `span` characters after `p0` as a string.
  (to (get p0 span0)
    (let `(,p ,span) (clip-range p0 span0))
    (string<-list (each get-char-after (p .span span))))

  ;; Return the position after the character in the text that is in
  ;; char-set and that comes up first in searching from p in
  ;; direction dir. If none, return nowhere or -nowhere.
  ;; XXX untested, probably broken
  (to (find-char-set p dir char-set)
    ((if (= dir forward)
         find-char-set-forward
         find-char-set-backward)
     (clip p)
     char-set))

  (to (find-char-set-forward p char-set)
    (let H head.^)
    (begin searching ((p p))
      (if (< p H)
          (if (char-set .maps? (t p))
              p.+
              (searching p.+))
          (do
            (let G gap.^)
            (let S size.^)
            (begin persevering ((p p))
              (if (< p S)
                  (if (char-set .maps? (t (+ G p)))
                      p.+
                      (persevering p.+))
                  nowhere))))))

  (to (find-char-set-backward p char-set)
    (let H head.^)
    (let G gap.^)
    (begin searching ((p p))
      (if (< H p)
          (if (char-set .maps? (t (+ G p.-)))
              p
              (searching p.-))
          (begin persevering ((p p))
            (if (< 0 p)
                (if (char-set .maps? (t p.-))
                    p
                    (persevering p.-))
                (- nowhere))))))

  (to (grow n)
    (+ n (n .quotient 2)))

  ;; Replace the `span` characters after `p` by `replacement`.
{'fraction_non_alphanumeric': 0.1409771592799879, 'fraction_numerical': 0.007260626229012252, 'mean_word_length': 2.0724907063197024, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7004104', 'n_tokens_mistral': 2233, 'n_tokens_neox': 2070, 'n_words': 892}