<commit_before>#lang scheme/base
(require net/url
         mzlib/list
         mzlib/contract)
(require "../private/util.ss")
(define url-path?
  ((url?) . ->* . (path? (listof path-element?))))

(provide/contract
 [url-path? contract?]
 [make-url->path (path? . -> . url-path?)]
 [make-url->valid-path (url-path? . -> . url-path?)])

(define (build-path* . l)
  (if (empty? l)
      (build-path 'same)
      (apply build-path l)))

(define ((make-url->path base) u)
  (define nbase (path->complete-path base))
  (define the-path
    ; Complete it against the base
    (path->complete-path
     ; Build a path
     (apply build-path*
            ; Remove all ".."s
            (strip-prefix-ups
             (map (lambda (p)
                    (if (and (string? p) (string=? "" p))
                        'same
                        p))
                  ; Extract the paths from the url-path
                  (map path/param-path 
                       (url-path u)))))
     nbase))
  (define w/o-base (path-without-base nbase the-path))
  #;(printf "~S~n" `(url->path ,base ,nbase ,(url->string u) ,the-path ,w/o-base))
  (values the-path w/o-base))

(define ((make-url->valid-path url->path) u)
  (let loop ([up (url-path u)])
    #;(printf "~S~n" `(url->valid-path ,(url->string u) ,up))
    (with-handlers ([exn? (lambda (exn)
                            #;((error-display-handler) (exn-message exn) exn)
                            (if (empty? up)
                                (raise exn)
                                (loop (reverse (rest (reverse up))))))])
      (define-values (p w/o-base)
        (url->path (url-replace-path (lambda _ up) u)))
      (unless (or (file-exists? p) (link-exists? p))
        (raise (make-exn:fail:filesystem:exists (string->immutable-string "No valid path") (current-continuation-marks))))
      (values p w/o-base))))
<commit_msg>up<commit_after>#lang scheme/base
(require net/url
         mzlib/list
         mzlib/contract)
(require "../private/util.ss")
(define url-path?
  ((url?) . ->* . (path? (listof path-element?))))

(provide/contract
 [url-path? contract?]
 [make-url->path (path? . -> . url-path?)]
 [make-url->valid-path (url-path? . -> . url-path?)])

(define (build-path* . l)
  (if (empty? l)
      (build-path 'same)
      (apply build-path l)))

(define ((make-url->path base) u)
  (define nbase (path->complete-path base))
  (define the-path
    ; Complete it against the base
    (path->complete-path
     ; Build a path
     (apply build-path*
            ; Remove all ".."s
            (strip-prefix-ups
             (map (lambda (p)
                    (if (and (string? p) (string=? "" p))
                        'same
                        p))
                  ; Extract the paths from the url-path
                  (map path/param-path 
                       (url-path u)))))
     nbase))
  (define w/o-base (path-without-base nbase the-path))
  #;(printf "~S~n" `(url->path ,base ,nbase ,(url->string u) ,the-path ,w/o-base))
  (values the-path w/o-base))

(define ((make-url->valid-path url->path) u)
  (let loop ([up (url-path u)])
    #;(printf "~S~n" `(url->valid-path ,(url->string u) ,up))
    (with-handlers ([exn? (lambda (exn)
                            #;((error-display-handler) (exn-message exn) exn)
                            (if (empty? up)
                                (raise exn)
                                (loop (reverse (rest (reverse up))))))])
      (define-values (p w/o-base)
        (url->path (url-replace-path (lambda _ up) u)))
      (unless (or (file-exists? p) (link-exists? p))
        (raise (make-exn:fail:filesystem:exists (string->immutable-string (format "No valid path: ~a" p)) (current-continuation-marks))))
      (values p w/o-base))))

{'fraction_non_alphanumeric': 0.17868504772004243, 'fraction_numerical': 0.0, 'mean_word_length': 1.8325825825825826, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3394492', 'n_tokens_mistral': 1286, 'n_tokens_neox': 1253, 'n_words': 313}