<commit_before>;;;  --------------------------------------------------------------  ;;;
;;;                                                                  ;;;
;;;                 Namespace choosing functionality                 ;;;
;;;                                                                  ;;;
;;;  --------------------------------------------------------------  ;;;


(##define-syntax get-path
  (lambda (a)
    (vector-ref a 2)))

(define ns-file (path-expand "ns.dat"
                             (path-expand
                              (path-directory
                               (get-path)))))
(define ns-table #f)
(define ns-table-timestamp #f)
(define *ns-table-dont-read-file*
  (make-parameter #f))

(define (ns-file-timestamp)
  (time->seconds
   (if ns-file
       (file-info-last-modification-time
        (file-info ns-file))
       (current-time))))

(define (ns-table-up-to-date?)
  (or (not ns-file)
      (and ns-table-timestamp
           (>= ns-table-timestamp
               (ns-file-timestamp)))))

(define (read-ns-table)
  (if (and ns-file
           (file-exists? ns-file))
      (let* ((size (file-info-size
                    (file-info ns-file)))
             (vec (make-u8vector size)))
        (with-input-from-file
            ns-file
          (lambda ()
            (read-subu8vector vec 0 size)))

        ;; This is here to fix an odd bug that put table-set! into an
        ;; infinite loop sometimes.
        (list->table
         (table->list
          (u8vector->object vec))))
      (make-table)))

(define (save-ns-table tbl)
  (if (not tbl)
      (error "Cannot save non-existent ns-table"))
  (if ns-file
      (with-output-to-file
          ns-file
        (lambda ()
          (let ((vect (object->u8vector tbl)))
            (write-subu8vector vect
                               0
                               (u8vector-length vect))))))
  (set! ns-table-timestamp
        (ns-file-timestamp)))

(define (ns-table-cleanup tbl)
  (let ((updated #f))
    (table-for-each
     (lambda (k v)
       (if (not (file-exists? k))
           (begin
             (table-set! tbl k)
             (set! updated #t))))
     tbl)

    (if updated
        (save-ns-table tbl))))

(define (get-ns-table)
  (cond
   ((or (*ns-table-dont-read-file*)
        (and ns-table
             (ns-table-up-to-date?)))
    ns-table)

   (else
    (let ((tbl (read-ns-table)))
      ;; If this is the first time ns-table is loaded, do a cleanup
      (if (not ns-table)
          (ns-table-cleanup tbl))
      
      (set! ns-table tbl)<commit_msg>Changed the format of ns.dat to be an alist sexp instead of a serialized hashtable<commit_after>;;;  --------------------------------------------------------------  ;;;
;;;                                                                  ;;;
;;;                 Namespace choosing functionality                 ;;;
;;;                                                                  ;;;
;;;  --------------------------------------------------------------  ;;;


(##define-syntax get-path
  (lambda (a)
    (vector-ref a 2)))

(define ns-file (path-expand "ns.dat"
                             (path-expand
                              (path-directory
                               (get-path)))))
(define ns-table #f)
(define ns-table-timestamp #f)
(define *ns-table-dont-read-file*
  (make-parameter #f))

(define (ns-file-timestamp)
  (time->seconds
   (if ns-file
       (file-info-last-modification-time
        (file-info ns-file))
       (current-time))))

(define (ns-table-up-to-date?)
  (or (not ns-file)
      (and ns-table-timestamp
           (>= ns-table-timestamp
               (ns-file-timestamp)))))

(define (read-ns-table)
  (if (and ns-file
           (file-exists? ns-file))
      (with-input-from-file
          ns-file
        (lambda ()
          (list->table
           (read))))
      (make-table)))

(define (save-ns-table tbl)
  (if (not tbl)
      (error "Cannot save non-existent ns-table"))
  (if ns-file
      (with-output-to-file
          ns-file
        (lambda ()
          (write (table->list tbl)))))
  (set! ns-table-timestamp
        (ns-file-timestamp)))

(define (ns-table-cleanup tbl)
  (let ((updated #f))
    (table-for-each
     (lambda (k v)
       (if (not (file-exists? k))
           (begin
             (table-set! tbl k)
             (set! updated #t))))
     tbl)

    (if updated
        (save-ns-table tbl))))

(define (get-ns-table)
  (cond
   ((or (*ns-table-dont-read-file*)
        (and ns-table
             (ns-table-up-to-date?)))
    ns-table)

   (else
    (let ((tbl (read-ns-table)))
      ;; If this is the first time ns-table is loaded, do a cleanup
      (if (not ns-table)
          (ns-table-cleanup tbl))
      
      (set! ns-table tbl)
{'fraction_non_alphanumeric': 0.19441550322983955, 'fraction_numerical': 0.002083767451552407, 'mean_word_length': 1.2284122562674096, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2041026', 'n_tokens_mistral': 1448, 'n_tokens_neox': 1407, 'n_words': 331}