<commit_before>;;;  --------------------------------------------------------------  ;;;
;;;                                                                  ;;;
;;;                    Loaded module data structure                  ;;;
;;;                                                                  ;;;
;;;  --------------------------------------------------------------  ;;;

(define-type loaded-module
  id: 08E43248-AC39-4DA1-863E-49AFA9FFE84E
  constructor: make-loaded-module/internal

  (invoke-runtime read-only:)
  (invoke-compiletime read-only:)
  (visit read-only:)
  (info read-only:)
  (stamp read-only:)
  ;; An absolute module reference object
  (reference read-only:)
  
  (runtime-invoked init: #f)
  ;; A list of the currently loaded loaded-module objects that
  ;; directly depend on this loaded-module
  (dependent-modules init: '()))

(define (make-loaded-module #!key
                            invoke-runtime
                            invoke-compiletime
                            visit
                            info
                            stamp
                            reference)
  (if (not (and (procedure? invoke-runtime)
                (procedure? invoke-compiletime)
                (procedure? visit)
                (module-info? info)
                (module-reference? reference)))
      (error "Invalid parameters"))
  (make-loaded-module/internal invoke-runtime
                               invoke-compiletime
                               visit
                               info
                               stamp
                               reference))

(define (loaded-module-loader mod)
  (module-reference-loader (loaded-module-reference mod)))

(define (loaded-module-path mod)
  (module-reference-path (loaded-module-reference mod)))

(define (loaded-module-stamp-is-fresh? lm)
  (loader-compare-stamp (loaded-module-loader lm)
                        (module-reference-path
                         (loaded-module-reference lm))
                        (loaded-module-stamp lm)))

(define *loaded-module-registry* (make-table))

;; Loads a module, regardless of whether it's already loaded or not.
(define (module-reference-load! ref)
  (let ((loaded-module
         (loader-load-module (module-reference-loader ref)
                             (module-reference-path ref))))
    (table-set! *loaded-module-registry* ref loaded-module)
    loaded-module))

;; This function will break if invoked when macroexpanding a module
(define (loaded-module-unload! lm) ;; This function isn't used or tested
  ;; First, de-instantiate the modules that depend on this module.
  (let rec ((lm lm))
    (for-each rec
      (loaded-module-dependent-modules lm))

    (let ((ref (loaded-module-reference lm)))
      (vector-for-each
       (lambda (phase)
         (table-set! (expansion-phase-module-instances phase)
                     ref))
       (syntactic-tower-phases *repl-syntactic-tower*))))
  
  (table-set! *loaded-module-registry*
              (loaded-module-reference lm)))

(define (module-reference-ref ref #!key compare-stamps)
  (if (not (module-reference-absolute? ref))
      (error "Module reference must be absolute"))
  (or (let ((loaded-module (table-ref *loaded-module-registry* ref #f)))
        (if compare-stamps
            (and loaded-module
                 (loaded-module-stamp-is-fresh? loaded-module)
                 loaded-module)
            loaded-module))
      (module-reference-load! ref)))

;;;; Visiting and invoking functionality

;; Invoking a module means loading everything that needs to be loaded
;; to be able to use its exported variable bindings. Before invoking a
;; module, its runtime dependencies must be invoked.
;;
;; Visiting a module means loading everything that needs to be loaded
;; to be able to perform macro expansions on macros defined in that
;; module.  Before visiting a module, its compile time dependencies
;; must be invoked, and its runtime dependencies must be visited.
;;
;; When importing a module to the REPL the module should be invoked
;; and visited.

;; A module instance object implicitly belongs to an expansion phase
;; and to a module reference, because it's stored in the expansion
;; phase object's instance table, with a module reference as the key.
;;
;; Because of this, there's no need to store pointers back to these
;; objects.
(define-type module-instance
  id: F366DFDB-6F11-47EB-9558-04BF7B31225D

  (getter init: #f)
  (setter init: #f)
  
  (macros init: #f)
  (macro-env init: #f))

(define (module-instance-ref phase ref)
  (let ((table (expansion-phase-module-instances phase)))
    (and table
         (table-ref table ref #f))))

(define (module-instance-get! phase ref)
  (or (module-instance-ref phase ref)
      (let ((instance (make-module-instance)))
        (table-set! (expansion-phase-module-instances phase)
                    ref
                    instance)
        instance)))

(define (loaded-module-invoked? lm phase)
  (if (zero? (expansion-phase-number phase))
      (loaded-module-runtime-invoked lm)
      (let ((instance (module-instance-ref
                       phase
                       (loaded-module-reference lm))))
        (and instance
             (module-instance-getter instance)
             (module-instance-setter instance)
             #t))))

(define (loaded-module-visited? lm phase)
  (let ((instance (module-instance-ref
                   phase
                   (loaded-module-reference lm))))
    (and instance
         (module-instance-macros instance)
         (module-instance-macro-env instance)
         #t)))

;; This procedure doesn't make sure that the module's dependencies are
;; invoked first, nor that the modules that depend on this module
;; are reinvoked or that the dependent-modules field of the
;; relevant loaded-module objects are kept up-to-date.
;;
;; It is not intended to be used directly, it is just a utility
;; function for loaded-modules-invoke/deps and
;; loaded-modules-reinvoke.
(define (loaded-module-invoke! lm phase)
  (if (zero? (expansion-phase-number phase))
      ((loaded-module-invoke-runtime lm))
      (let ((getter
             setter
             ((loaded-module-invoke-compiletime lm)
              lm phase))
            (instance (module-instance-get!
                       phase
                       (loaded-module-reference lm))))
        (module-instance-getter-set! instance getter)
        (module-instance-setter-set! instance setter)))
  ;; It doesn't make sense to return anything, because
  ;; loaded-module-invoke-runtime doesn't return anything useful.
  (void))

;; Analogous to loaded-module-invoke!
(define (loaded-module-visit! lm phase)
  (let* ((next-phase (expansion-phase-next-phase phase))
         (phase-number (expansion-phase-number phase))
         (macro-procedures (list->table
                            ((loaded-module-visit lm) lm next-phase)))
         (macros (make-table))
         
         (module-ref (loaded-module-reference lm))
         (module-info (loaded-module-info lm))
         (namespace-string (module-info-namespace-string module-info))
         
         (env (make-top-environment module-ref))
         (memo-table (make-table
                      test: eq?
                      hash: eq?-hash)))
    (letrec
        ((make-env-from-letsyntax-environment<commit_msg>First stab at loading single instance modules<commit_after>;;;  --------------------------------------------------------------  ;;;
;;;                                                                  ;;;
;;;                    Loaded module data structure                  ;;;
;;;                                                                  ;;;
;;;  --------------------------------------------------------------  ;;;

(define-type loaded-module
  id: 08E43248-AC39-4DA1-863E-49AFA9FFE84E
  constructor: make-loaded-module/internal

  (invoke-runtime read-only:)
  (invoke-compiletime read-only:)
  (visit read-only:)
  (info read-only:)
  (stamp read-only:)
  ;; An absolute module reference object
  (reference read-only:)
  
  (runtime-invoked init: #f)
  ;; A list of the currently loaded loaded-module objects that
  ;; directly depend on this loaded-module
  (dependent-modules init: '()))

(define (make-loaded-module #!key
                            invoke-runtime
                            invoke-compiletime
                            visit
                            info
                            stamp
                            reference)
  (if (not (and (procedure? invoke-runtime)
                (or (procedure? invoke-compiletime)
                    (not invoke-compiletime))
                (procedure? visit)
                (module-info? info)
                (module-reference? reference)))
      (error "Invalid parameters"))
  (make-loaded-module/internal invoke-runtime
                               invoke-compiletime
                               visit
                               info
                               stamp
                               reference))

(define (loaded-module-loader mod)
  (module-reference-loader (loaded-module-reference mod)))

(define (loaded-module-path mod)
  (module-reference-path (loaded-module-reference mod)))

(define (loaded-module-stamp-is-fresh? lm)
  (loader-compare-stamp (loaded-module-loader lm)
                        (module-reference-path
                         (loaded-module-reference lm))
                        (loaded-module-stamp lm)))

(define *loaded-module-registry* (make-table))

;; Loads a module, regardless of whether it's already loaded or not.
(define (module-reference-load! ref)
  (let ((loaded-module
         (loader-load-module (module-reference-loader ref)
                             (module-reference-path ref))))
    (table-set! *loaded-module-registry* ref loaded-module)
    loaded-module))

;; This function will break if invoked when macroexpanding a module
(define (loaded-module-unload! lm) ;; This function isn't used or tested
  ;; First, de-instantiate the modules that depend on this module.
  (let rec ((lm lm))
    (for-each rec
      (loaded-module-dependent-modules lm))

    (let ((ref (loaded-module-reference lm)))
      (vector-for-each
       (lambda (phase)
         (table-set! (expansion-phase-module-instances phase)
                     ref))
       (syntactic-tower-phases *repl-syntactic-tower*))))
  
  (table-set! *loaded-module-registry*
              (loaded-module-reference lm)))

(define (module-reference-ref ref #!key compare-stamps)
  (if (not (module-reference-absolute? ref))
      (error "Module reference must be absolute"))
  (or (let ((loaded-module (table-ref *loaded-module-registry* ref #f)))
        (if compare-stamps
            (and loaded-module
                 (loaded-module-stamp-is-fresh? loaded-module)
                 loaded-module)
            loaded-module))
      (module-reference-load! ref)))

;;;; Visiting and invoking functionality

;; Invoking a module means loading everything that needs to be loaded
;; to be able to use its exported variable bindings. Before invoking a
;; module, its runtime dependencies must be invoked.
;;
;; Visiting a module means loading everything that needs to be loaded
;; to be able to perform macro expansions on macros defined in that
;; module.  Before visiting a module, its compile time dependencies
;; must be invoked, and its runtime dependencies must be visited.
;;
;; When importing a module to the REPL the module should be invoked
;; and visited.

;; A module instance object implicitly belongs to an expansion phase
;; and to a module reference, because it's stored in the expansion
;; phase object's instance table, with a module reference as the key.
;;
;; Because of this, there's no need to store pointers back to these
;; objects.
(define-type module-instance
  id: F366DFDB-6F11-47EB-9558-04BF7B31225D

  (getter init: #f)
  (setter init: #f)
  
  (macros init: #f)
  (macro-env init: #f))

(define (module-instance-ref phase ref)
  (let ((table (expansion-phase-module-instances phase)))
    (and table
         (table-ref table ref #f))))

(define (module-instance-get! phase ref)
  (or (module-instance-ref phase ref)
      (let ((instance (make-module-instance)))
        (table-set! (expansion-phase-module-instances phase)
                    ref
                    instance)
        instance)))

(define (loaded-module-invoked? lm phase)
  (if (zero? (expansion-phase-number phase))
      (loaded-module-runtime-invoked lm)
      (let ((instance (module-instance-ref
                       phase
                       (loaded-module-reference lm))))
        (and instance
             (module-instance-getter instance)
             (module-instance-setter instance)
             #t))))

(define (loaded-module-visited? lm phase)
  (let ((instance (module-instance-ref
                   phase
                   (loaded-module-reference lm))))
    (and instance
         (module-instance-macros instance)
         (module-instance-macro-env instance)
         #t)))

;; This procedure doesn't make sure that the module's dependencies are
;; invoked first, nor that the modules that depend on this module
;; are reinvoked or that the dependent-modules field of the
;; relevant loaded-module objects are kept up-to-date.
;;
;; It is not intended to be used directly, it is just a utility
;; function for loaded-modules-invoke/deps and
;; loaded-modules-reinvoke.
(define (loaded-module-invoke! lm phase)
  (if (zero? (expansion-phase-number phase))
      ((loaded-module-invoke-runtime lm))
      (let ((invoke-compiletime
             (loaded-module-invoke-compiletime lm)))
        (if invoke-compiletime
            (let ((getter
                   setter
                   ((loaded-module-invoke-compiletime lm)
                    lm phase))
                  (instance (module-instance-get!
                             phase
                             (loaded-module-reference lm))))
              (module-instance-getter-set! instance getter)
              (module-instance-setter-set! instance setter)))))
  ;; It doesn't make sense to return anything, because
  ;; loaded-module-invoke-runtime doesn't return anything useful.
  (void))

;; Analogous to loaded-module-invoke!
(define (loaded-module-visit! lm phase)
  (let* ((next-phase (expansion-phase-next-phase phase))
         (phase-number (expansion-phase-number phase))
         (macro-procedures (list->table
                            ((loaded-module-visit lm) lm next-phase)))
         (macros (make-table))
         
         (module-ref (loaded-module-reference lm))
         (module-info (loaded-module-info lm))
         (namespace-string (module-info-namespace-string module-info))
         
         (env (make-top-environment module-ref))
         (memo-table (make-table
                      test: eq?
                      hash: eq?-hash)))
    (letrec
        ((make-env-from-letsyntax-environment
{'fraction_non_alphanumeric': 0.12401758358865059, 'fraction_numerical': 0.005195151192220594, 'mean_word_length': 1.9366321142186582, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2923976', 'n_tokens_mistral': 4075, 'n_tokens_neox': 3907, 'n_words': 1166}