<commit_before>(module read-lines mzscheme

  (require (lib "etc.ss")
	   (lib "pregexp.ss")
	   "util.ss")	

  (provide read-lines)

  (define read-lines
    (opt-lambda (file caption [offset #f])
      (template caption (get-the-lines file offset))))
  
  (define (semi-flatten lst)
    (if (null? lst) 
	'()
	(cons (caar lst)
	      (cons (cadar lst)
		    (semi-flatten (cdr lst))))))

  (define temp-anchor `(A ((NAME "temp")) ""))

  (define (spacify s)
    (if (and (string? s) (string=? s "")) 
	" " ; to appease IE
	s))

  (define (template caption lines) 
    `(TABLE ((CELLPADDING "0")
	     (CELLSPACING "0"))
	    (B ,(with-color "blue" caption))
	    (P)
	    (PRE ((STYLE "font-family:monospace"))
            ; use <BR>'s instead of newlines, for Opera
            ; don't put in a <BR> for the temp-anchor, which wasn't a line in the source
		 ,@(semi-flatten 
		    (map (lambda (ln) 
			   (if (eq? ln temp-anchor)
			       `(,ln "")
			       `(,(spacify ln) (BR)))) lines)))))
    
  (define eoregexp
    "($|\\s|(\\.(\\s|$))|>)")

  (define trailing-regexp
    (pregexp "[\\s>)(\"]"))

  (define url-regexp-base
    (string-append "://([^\\s]*)" eoregexp))

  (define (make-url-regexp ty)
    (pregexp 
     (string-append 
      ty
      url-regexp-base)))

  (define http-regexp (make-url-regexp "http"))
  (define cheap-http-regexp (regexp "http://"))
  (define (http-format url)
    `(A ((HREF ,url)) ,url))
  (define ftp-regexp (make-url-regexp "ftp"))
  (define cheap-ftp-regexp (regexp "ftp://"))
  (define ftp-format http-format) 

  (define email-regexp
    (let ([chars "[^\\s)(<>\"']"])
      (pregexp (string-append chars "+" "@" chars "{3,}"))))
  (define cheap-email-regexp (regexp "@"))
  (define (email-format addr)
    `(A ((HREF ,(string-append "mailto:" addr))) ,addr))

  (define (rtrim s)
    (let* ([presult (pregexp-replace* trailing-regexp s "")]
	   [plen (string-length presult)]
	   [qlen (sub1 plen)])
      (if (and (> qlen 0)
	       (char=? (string-ref presult qlen)
		       #\.))
	  (substring presult 0 qlen)
	  presult)))

  (define (process-for-urls line)
    (let loop ([built-line line])
      (let ([curr-len (string-length built-line)])
	(let-values 
	 ([(raw-indices formatter)
	   (let regexp-loop ([regexps (list http-regexp
                                            ftp-regexp
                                            email-regexp)]
                             [cheap-regexps 
                              (list cheap-http-regexp
                                    cheap-ftp-regexp
                                    cheap-email-regexp)]
                             [formats (list http-format
                                            ftp-format
                                            email-format)])
	     (if (null? regexps)
		 (values #f #f)
		 (let* ([curr-regexp (car regexps)]
			[curr-cheap-regexp (car cheap-regexps)]
			[curr-formatter (car formats)]
			[match-indices
			 (and (regexp-match-positions
			       curr-cheap-regexp built-line)
			      (pregexp-match-positions 
			       curr-regexp built-line))])
		   (if match-indices
		       (values match-indices curr-formatter)
		       (regexp-loop (cdr regexps) (cdr cheap-regexps)
				    (cdr formats))))))])
	 (if raw-indices
	     (let* ([indices (car raw-indices)]
		    [string-start (car indices)]
		    [string-end (cdr indices)]
		    [raw-item 
		     (substring built-line 
				string-start string-end)]
		    [raw-item-len (string-length raw-item)]
		    [item (rtrim raw-item)]
		    [item-len (string-length item)])
		`(TT
		  ,(substring built-line 0 string-start)
		  ,(formatter item)
		  ,(substring raw-item ; text removed by rtrim
			      item-len
			      raw-item-len)
		  ,(loop (substring built-line string-end 
				    curr-len))))
	     built-line)))))<commit_msg>removed use of regular regexps (they were just there as a 'performance improvement') and improved regexp for email addresses a little so it doesn't match ,@<commit_after>(module read-lines mzscheme

  (require (lib "etc.ss")
	   (lib "pregexp.ss")
	   "util.ss")	

  (provide read-lines)

  (define read-lines
    (opt-lambda (file caption [offset #f])
      (template caption (get-the-lines file offset))))
  
  (define (semi-flatten lst)
    (if (null? lst) 
	'()
	(cons (caar lst)
	      (cons (cadar lst)
		    (semi-flatten (cdr lst))))))

  (define temp-anchor `(A ((NAME "temp")) ""))

  (define (spacify s)
    (if (and (string? s) (string=? s "")) 
	" " ; to appease IE
	s))

  (define (template caption lines) 
    `(TABLE ((CELLPADDING "0")
	     (CELLSPACING "0"))
	    (B ,(with-color "blue" caption))
	    (P)
	    (PRE ((STYLE "font-family:monospace"))
            ; use <BR>'s instead of newlines, for Opera
            ; don't put in a <BR> for the temp-anchor, which wasn't a line in the source
		 ,@(semi-flatten 
		    (map (lambda (ln) 
			   (if (eq? ln temp-anchor)
			       `(,ln "")
			       `(,(spacify ln) (BR)))) lines)))))
    
  (define eoregexp-str "($|\\s|(\\.(\\s|$))|>)")
  (define url-regexp-base (string-append "://([^\\s]*)" eoregexp-str))

  (define trailing-regexp (pregexp "[\\s>)(\"]"))

  (define (make-url-regexp ty)
    (pregexp 
     (string-append 
      ty
      url-regexp-base)))

  (define http-regexp (make-url-regexp "http"))
  (define (http-format url) `(A ((HREF ,url)) ,url))
  (define ftp-regexp (make-url-regexp "ftp"))
  (define ftp-format http-format) 

  (define email-regexp
    (let ([chars "[^\\s)(<>\"']"]
          [no-comma-chars "[^\\s)(<>\"',]"])
      (pregexp (string-append no-comma-chars chars "*" "@" chars "{3,}"))))
  (define (email-format addr)
    `(A ((HREF ,(string-append "mailto:" addr))) ,addr))

  (define (rtrim s)
    (let* ([presult (pregexp-replace* trailing-regexp s "")]
	   [plen (string-length presult)]
	   [qlen (sub1 plen)])
      (if (and (> qlen 0)
	       (char=? (string-ref presult qlen)
		       #\.))
	  (substring presult 0 qlen)
	  presult)))

  (define (process-for-urls line)
    (let loop ([built-line line])
      (let ([curr-len (string-length built-line)])
	(let-values 
	 ([(raw-indices formatter)
	   (let regexp-loop ([regexps (list http-regexp
                                            ftp-regexp
                                            email-regexp)]
                             [formats (list http-format
                                            ftp-format
                                            email-format)])
	     (if (null? regexps)
		 (values #f #f)
		 (let* ([curr-regexp (car regexps)]
			[curr-formatter (car formats)]
			[match-indices
			 (pregexp-match-positions curr-regexp built-line)])
		   (if match-indices
		       (values match-indices curr-formatter)
		       (regexp-loop (cdr regexps) (cdr formats))))))])
	 (if raw-indices
	     (let* ([indices (car raw-indices)]
		    [string-start (car indices)]
		    [string-end (cdr indices)]
		    [raw-item 
		     (substring built-line 
				string-start string-end)]
		    [raw-item-len (string-length raw-item)]
		    [item (rtrim raw-item)]
		    [item-len (string-length item)])
		`(TT
		  ,(substring built-line 0 string-start)
		  ,(formatter item)
		  ,(substring raw-item ; text removed by rtrim
			      item-len
			      raw-item-len)
		  ,(loop (substring built-line string-end 
				    curr-len))))
	     built-line)))))
{'fraction_non_alphanumeric': 0.16757123473541383, 'fraction_numerical': 0.0018995929443690637, 'mean_word_length': 2.0408415841584158, 'pattern_counts': {'":': 2, '<': 10, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7579376', 'n_tokens_mistral': 2767, 'n_tokens_neox': 2461, 'n_words': 634}