<commit_before>#lang scheme
(require "svn.ss"
         "path-utils.ss"
         net/url
         scheme/system)
(provide
 (all-from-out "svn.ss"))

(define git-path (make-parameter "/opt/local/bin/git"))
(provide/contract
 [git-path (parameter/c string?)])

(define git-url-base "http://git.racket-lang.org/plt.git")

(define (newest-push)
  (string->number (port->string (get-pure-port (string->url (format "~a/push-counter" git-url-base))))))

(define (pad2zeros n)
  (format "~a~a"
          (if (n . < . 10)
              "0" "")
          (number->string n)))

(define-struct push-data (who end-commit branches) #:prefab)

(define (push-info push-n)
  (define push-n100s (quotient push-n 100))
  (define push-nrem (pad2zeros (modulo push-n 100)))
  (define ls 
    (port->lines 
     (get-pure-port 
      (string->url 
       (format "~a/pushes/~a/~a" git-url-base push-n100s push-nrem)))))
  (match ls
    [(list (regexp #rx"^([^ ]+) +([0-9abcdef]+)$" (list _ who end-commit))
           (regexp #rx"^([0-9abcdef]+) +([0-9abcdef]+) +(.+)$" (list _ bstart bend branch))
           ...)
     (make-push-data who end-commit 
                     (make-immutable-hash
                      (map (lambda (b bs be) (cons b (vector bs be)))
                           branch bstart bend)))]
    [_
     #f]))

(define (system/output-port #:k k #:stdout [init-stdout #f] . as)
  (define _ (printf "~S~n" as))
  (define-values (sp stdout stdin stderr)
    (apply subprocess init-stdout #f #f as))
  (begin0 (k stdout)
          (subprocess-wait sp)))

(define-struct git-push (num author commits) #:prefab)
(define-struct git-commit (hash author date msg) #:prefab)
(define-struct (git-diff git-commit) (mfiles) #:prefab)
(define-struct (git-merge git-commit) (from to) #:prefab)

(define (read-until-empty-line in-p)
  (let loop ()
    (let ([l (read-line in-p)])
      (cond
        [(eof-object? l)
         (close-input-port in-p)
         empty]
        [(string=? l "")
         empty]
        [else
         (list* (regexp-replace #rx"^ +" l "") (loop))]))))

(define (read-commit in-p)
  (match (read-line in-p)
    [(? eof-object?)
     #f]
    [(regexp #rx"^commit +(.+)$" (list _ hash))
     (match (read-line in-p)
       [(regexp #rx"^Merge: +(.+) +(.+)$" (list _ from to))
        (match-define (regexp #rx"^Author: +(.+)$" (list _ author)) (read-line in-p))
        (match-define (regexp #rx"^Date: +(.+)$" (list _ date)) (read-line in-p))
        (define _1 (read-line in-p))
        (define msg (read-until-empty-line in-p))
        (make-git-merge hash author date msg from to)]
       [(regexp #rx"^Author: +(.+)$" (list _ author))
        (match-define (regexp #rx"^Date: +(.+)$" (list _ date)) (read-line in-p))
        (define _1 (read-line in-p))
        (define msg (read-until-empty-line in-p))
        (define mfiles (read-until-empty-line in-p))
        (make-git-diff hash author date msg mfiles)])]))

(define port-empty? port-closed?)

(define (read-commits in-p)
  (cond
    [(port-empty? in-p)
     empty]
    [(read-commit in-p)
     => (lambda (c) 
          (printf "~S~n" c)
          (list* c (read-commits in-p)))]
    [else
     empty]))

(define (parse-push num author in-p)
  (make-git-push num author (read-commits in-p)))

(define (get-scm-commit-msg rev repo)
  (match-define (struct push-data (who _ branches)) (push-info rev))
  (match-define (vector start-commit end-commit) (hash-ref branches master-branch))
  (parameterize ([current-directory repo])
    (system/output-port 
     #:k (curry parse-push rev who)
     (git-path)
     "--no-pager" "log" "--date=iso" "--name-only"
     (format "~a..~a" start-commit end-commit))))
(provide/contract
 [struct git-push 
         ([num exact-nonnegative-integer?]
          [author string?]
          [commits (listof git-commit?)])]
 [struct git-commit 
         ([hash string?]
          [author string?]
          [date string?]
          [msg (listof string?)])]
 [struct git-diff 
         ([hash string?]
          [author string?]
          [date string?]
          [msg (listof string?)]
          [mfiles (listof string?)])]<commit_msg>Sam's advice<commit_after>#lang scheme
(require "svn.ss"
         "path-utils.ss"
         net/url
         scheme/system)
(provide
 (all-from-out "svn.ss"))

(define git-path (make-parameter "/opt/local/bin/git"))
(provide/contract
 [git-path (parameter/c string?)])

(define git-url-base "http://git.racket-lang.org/plt.git")

(define (newest-push)
  (string->number (port->string (get-pure-port (string->url (format "~a/push-counter" git-url-base))))))

(define (pad2zeros n)
  (format "~a~a"
          (if (n . < . 10)
              "0" "")
          (number->string n)))

(define-struct push-data (who end-commit branches) #:prefab)

(define (push-info push-n)
  (define push-n100s (quotient push-n 100))
  (define push-nrem (pad2zeros (modulo push-n 100)))
  (define ls 
    (port->lines 
     (get-pure-port 
      (string->url 
       (format "~a/pushes/~a/~a" git-url-base push-n100s push-nrem)))))
  (match ls
    [(list (regexp #rx"^([^ ]+) +([0-9abcdef]+)$" (list _ who end-commit))
           (regexp #rx"^([0-9abcdef]+) +([0-9abcdef]+) +(.+)$" (list _ bstart bend branch))
           ...)
     (make-push-data who end-commit 
                     (make-immutable-hash
                      (map (lambda (b bs be) (cons b (vector bs be)))
                           branch bstart bend)))]
    [_
     #f]))

(define (system/output-port #:k k #:stdout [init-stdout #f] . as)
  (define _ (printf "~S~n" as))
  (define-values (sp stdout stdin stderr)
    (apply subprocess init-stdout #f #f as))
  (begin0 (k stdout)
          (subprocess-wait sp)))

(define-struct git-push (num author commits) #:prefab)
(define-struct git-commit (hash author date msg) #:prefab)
(define-struct (git-diff git-commit) (mfiles) #:prefab)
(define-struct (git-merge git-commit) (from to) #:prefab)

(define (read-until-empty-line in-p)
  (let loop ()
    (let ([l (read-line in-p)])
      (cond
        [(eof-object? l)
         (close-input-port in-p)
         empty]
        [(string=? l "")
         empty]
        [else
         (list* (regexp-replace #rx"^ +" l "") (loop))]))))

(define (read-commit in-p)
  (match (read-line in-p)
    [(? eof-object?)
     #f]
    [(regexp #rx"^commit +(.+)$" (list _ hash))
     (match (read-line in-p)
       [(regexp #rx"^Merge: +(.+) +(.+)$" (list _ from to))
        (match-define (regexp #rx"^Author: +(.+)$" (list _ author)) (read-line in-p))
        (match-define (regexp #rx"^Date: +(.+)$" (list _ date)) (read-line in-p))
        (define _1 (read-line in-p))
        (define msg (read-until-empty-line in-p))
        (make-git-merge hash author date msg from to)]
       [(regexp #rx"^Author: +(.+)$" (list _ author))
        (match-define (regexp #rx"^Date: +(.+)$" (list _ date)) (read-line in-p))
        (define _1 (read-line in-p))
        (define msg (read-until-empty-line in-p))
        (define mfiles (read-until-empty-line in-p))
        (make-git-diff hash author date msg mfiles)])]))

(define port-empty? port-closed?)

(define (read-commits in-p)
  (cond
    [(port-empty? in-p)
     empty]
    [(read-commit in-p)
     => (lambda (c) 
          (printf "~S~n" c)
          (list* c (read-commits in-p)))]
    [else
     empty]))

(define (parse-push num author in-p)
  (make-git-push num author (read-commits in-p)))

(define (get-scm-commit-msg rev repo)
  (match-define (struct push-data (who _ branches)) (push-info rev))
  (match-define (vector start-commit end-commit) (hash-ref branches master-branch))
  (parameterize ([current-directory repo])
    (system/output-port 
     #:k (curry parse-push rev who)
     (git-path)
     "--no-pager" "log" "--date=iso" "--name-only" "--no-merges"
     (format "~a..~a" start-commit end-commit))))
(provide/contract
 [struct git-push 
         ([num exact-nonnegative-integer?]
          [author string?]
          [commits (listof git-commit?)])]
 [struct git-commit 
         ([hash string?]
          [author string?]
          [date string?]
          [msg (listof string?)])]
 [struct git-diff 
         ([hash string?]
          [author string?]
          [date string?]
          [msg (listof string?)]
          [mfiles (listof string?)])]
{'fraction_non_alphanumeric': 0.18969470254615664, 'fraction_numerical': 0.0062748883793894055, 'mean_word_length': 2.3718470301057772, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7360170', 'n_tokens_mistral': 3103, 'n_tokens_neox': 2961, 'n_words': 727}