<commit_before>#lang racket

(require
  racket/path
  stiki/parameters
  sugar)

(provide
 (all-defined-out))

(define (without-trailing-slash path)
    (bytes->path (regexp-replace #rx#"/$" (path->bytes path) "")))

(define (with-trailing-slash path)
    (bytes->path (regexp-replace #rx#"/$" (path->bytes path) "")))

(define (change-ext path ext)
  (add-ext (remove-ext path) ext))

(define (exclude-path? path)
  (for/and ([pattern (exclude-patterns)])
    (not (regexp-match pattern (path->string path)))))


; Are two paths equal?
(define (path=? a b)
  (string=? (path->string a)
            (path->string b)))

(define (path<? a b)
  (string<? (path->string a)
            (path->string b)))

(define (symlink? path)
  (not (=
        (file-or-directory-identity path #t)
        (file-or-directory-identity path #f))))

(define (symlink-to? link dest)
  (and (symlink? link)
       (= (file-or-directory-identity link #f)
          (file-or-directory-identity dest #f))))

; Construct a relative path from a src and dst
(define (relativize src dst)
  (if (path=? src dst) (build-path 'same)
      (find-relative-path src dst)))

; Title from path
(define (path->title path)
  (remove-ext (last (explode-path path))))

(define (find path args)
 (map (compose without-trailing-slash string->path)
       (string-split	 
        (with-output-to-string
         (λ () (system (string-join
                        (list* "find \"" (path->string path) "\"" args) ""))))
        "\n")))

; Recursively list all links
(define (lslr path)
  (find path
        '(" -type l -not -path \"*/\\.*\"")))

(define (lsdr path)
  (find path
        '(" -type d -not -path \"*/\\.*\"")))
  

; Recursively list all markdown files
(define (lsfr-md path)
  (find path
        '(" -type f ! -type l -path \"*.md\" -not -path \"*/\\.*\"")))

; List Immediate Subfiles
(define (lsf path)
  (find path
        '(" -maxdepth 1 -type f -not -path \"*/\\.*\"")))

; List Immediate Sublinks
(define (lsl path)
  (find path
        '(" -maxdepth 1 -type l -not -path \"*/\\.*\"")))

; List Immediate Subdirectories
(define (lsd path)
  (find path
        '(" -maxdepth 1 -type d -not -path \"*/\\.*\"")))

; List the categories a given path belongs to
(define (categories-of path)
  (map identity
       (list* (get-enclosing-dir path)
              (map path-only
                   (filter ((curryr symlink-to?) path)
                           (lslr (source-directory)))))))

(define (subarticles-of path)
  (append (lsf path)
          (filter (compose file-exists? normalize-path)
                  (lsl path))))

; Subcategories Of
(define (subcategories-of path)
  (filter-not ((curry path=?) path)
              (append (lsd path)
                      (filter (compose directory-exists? resolve-path)
                              (lsl path)))))

(define base
  (without-trailing-slash
   (simple-form-path "../Encyclopedia Occulta")))

(define sampdir
  (without-trailing-slash
   (simple-form-path "../Encyclopedia Occulta/categories/Buildings.md")))

(define categories
  (first (subcategories-of base)))

(define periodicals
  (first (subcategories-of categories)))

(define sampfile
  (without-trailing-slash
   (simple-form-path "../Encyclopedia Occulta/articles/Adams House.md")))<commit_msg>cleaned up code<commit_after>#lang racket

(require
  racket/path
  stiki/parameters
  sugar)

(provide
 (all-defined-out))

(define (without-trailing-slash path)
    (bytes->path (regexp-replace #rx#"/$" (path->bytes path) "")))

(define (with-trailing-slash path)
    (bytes->path (regexp-replace #rx#"/$" (path->bytes path) "")))

(define (change-ext path ext)
  (add-ext (remove-ext path) ext))

(define (exclude-path? path)
  (for/and ([pattern (exclude-patterns)])
    (not (regexp-match pattern (path->string path)))))


; Are two paths equal?
(define (path=? a b)
  (string=? (path->string a)
            (path->string b)))

(define (path<? a b)
  (string<? (path->string a)
            (path->string b)))

(define (symlink? path)
  (not (=
        (file-or-directory-identity path #t)
        (file-or-directory-identity path #f))))

(define (symlink-to? link dest)
  (and (symlink? link)
       (= (file-or-directory-identity link #f)
          (file-or-directory-identity dest #f))))

; Construct a relative path from a src and dst
(define (relativize src dst)
  (if (path=? src dst) (build-path 'same)
      (find-relative-path src dst)))

; Title from path
(define (path->title path)
  (remove-ext (last (explode-path path))))

(define (find path args)
 (map (compose without-trailing-slash string->path)
       (string-split	 
        (with-output-to-string
         (λ () (system (string-join
                        (list* "find \"" (path->string path) "\"" args) ""))))
        "\n")))

; Recursively list all links
(define (lslr path)
  (find path
        '(" -type l -not -path \"*/\\.*\"")))

(define (lsdr path)
  (find path
        '(" -type d -not -path \"*/\\.*\"")))
  

; Recursively list all markdown files
(define (lsfr-md path)
  (find path
        '(" -type f ! -type l -path \"*.md\" -not -path \"*/\\.*\"")))

; List Immediate Subfiles
(define (lsf path)
  (find path
        '(" -maxdepth 1 -type f -not -path \"*/\\.*\"")))

; List Immediate Sublinks
(define (lsl path)
  (find path
        '(" -maxdepth 1 -type l -not -path \"*/\\.*\"")))

; List Immediate Subdirectories
(define (lsd path)
  (find path
        '(" -maxdepth 1 -type d -not -path \"*/\\.*\"")))

; List the categories a given path belongs to
(define (categories-of path)
  (map identity
       (list* (get-enclosing-dir path)
              (map path-only
                   (filter ((curryr symlink-to?) path)
                           (lslr (source-directory)))))))

(define (subarticles-of path)
  (append (lsf path)
          (filter (compose file-exists? normalize-path)
                  (lsl path))))

; Subcategories Of
(define (subcategories-of path)
  (filter-not ((curry path=?) path)
              (append (lsd path)
                      (filter (compose directory-exists? resolve-path)
                              (lsl path)))))
{'fraction_non_alphanumeric': 0.1762308444734268, 'fraction_numerical': 0.00097815454841865, 'mean_word_length': 2.436974789915966, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2587849', 'n_tokens_mistral': 2093, 'n_tokens_neox': 2049, 'n_words': 580}