<commit_before>;; Copyright (c) 2013-2016 by <NAME>, All Rights Reserved.

;; A package system for Slogan.

(define (build-package pkg-path)
  (let* ((currdir (current-directory))
         (build-cmd (string-append "cd " pkg-path "; ./build; cd " currdir)))
    (let ((r (shell-command build-cmd)))
      (if (scm-not (zero? r))
          (error "build-package - build failed -" build-cmd ", " r)
          #t))))

(define (install-git-package pkg-name pkg-url pkg-path)
  (let ((cmd (string-append "git clone " pkg-url " " pkg-path)))
    (let ((r (shell-command cmd)))
      (if (scm-not (zero? r))
          (error "install-git-package - clone failed -" cmd ", " r)
          (build-package pkg-path))
      pkg-name)))

(define (get-file-name-from-pkg-url url)
  (let loop ((paths (string_split url #\/)))
    (if (null? (scm-cdr paths))
        (scm-car paths)
        (loop (scm-cdr paths)))))

(define *untar-cmds* '((".gz" . "tar xzf") (".bz2" . "tar xjf") (".zip" . "unzip")))

(define (untar-cmd file-name)
  (let ((extn (path-extension file-name)))
    (let ((cmd (scm-assoc extn *untar-cmds*)))
      (if cmd
          (let ((in-file (string-append *pkg-root* file-name)))
            (scm-cons (string-append (scm-cdr cmd) " " in-file
                                 (if (string=? extn ".zip")
                                     " -d " " -C ")
                                 *pkg-root*)
                  in-file))
          (error "cannot decompress package - " file-name)))))

(define (untar&build-package pkg-name pkg-url pkg-path)
  (let ((file-name (get-file-name-from-pkg-url pkg-url)))
    (let ((cmd&infile (untar-cmd file-name)))
      (let ((r (shell-command (scm-car cmd&infile))))
        (delete_file (scm-cdr cmd&infile))
        (if (zero? r)
            (build-package pkg-path)
            (error "failed to decompress package - " (scm-car cmd&infile) ", " r)))))
  pkg-name)
          
(define (install-remote-package pkg-name pkg-url pkg-path)
  (let ((cmd (string-append "curl " pkg-url " -o " (string-append *pkg-root* (get-file-name-from-pkg-url pkg-url)))))
    (let ((r (shell-command cmd)))
      (if (scm-not (zero? r))
          (error "install-remote-package failed -" cmd ", " r)
          (untar&build-package pkg-name pkg-url pkg-path))))
  pkg-name)

(define (copy-dir src dest)
  (let ((cmd (string-append "cp -R " src " " dest)))
    (let ((r (shell-command cmd)))
      (if (scm-not (zero? r))
          (error "copy-dir - failed - " cmd ", " r)
          #t))))

(define (install-local-package pkg-name pkg-url pkg-path)
  (if (file-exists? pkg-url)
      (if (string=? "" (path-extension pkg-url))
          (begin (copy-dir pkg-url pkg-path)
                 (build-package pkg-path))
          (begin (copy-file pkg-url (string-append *pkg-root* (get-file-name-from-pkg-url pkg-url)))
                 (untar&build-package pkg-name pkg-url pkg-path)))
      (error "install-local-package - file not found - " pkg-url))
  pkg-name)

(define (load_package pkg-name)
  (let ((pkg-init-path (string-append *pkg-root* pkg-name "/init")))
    (load pkg-init-path)
    pkg-name))

(define (force-rm-dir path)
  (let ((r (shell-command (string-append "rm -rf " path))))
    (if (zero? r)
        #t
        (error "failed to remove directory - " path ", " r))))

(define (install_package pkg-name pkg-type pkg-url #!optional force)
  (if (scm-not (file-exists? *pkg-root*))
      (create-directory *pkg-root*))
  (let* ((pkg-path (string-append *pkg-root* pkg-name))
         (pkg-path-old (string-append pkg-path ".bck")))
    (if (file-exists? pkg-path)
        (if (scm-not force)
            (error "package already installed - " pkg-path)
            (rename-file pkg-path pkg-path-old)))
    (case pkg-type
      ((git) (install-git-package pkg-name pkg-url pkg-path))
      ((remote) (install-remote-package pkg-name pkg-url pkg-path))
      ((local) (install-local-package pkg-name pkg-url pkg-path))
      (else (error "install_package - type not supported -" pkg-type)))
    (if (file-exists? pkg-path-old)
        (force-rm-dir pkg-path-old))
    (load_package pkg-name)))

(define (uninstall_package pkg-name)
  (let ((pkg-path (string-append *pkg-root* pkg-name)))
    (if (file-exists? pkg-path)
        (if (force-rm-dir pkg-path)
            pkg-name
            #f)
        #f)))
<commit_msg>init_package function added<commit_after>;; Copyright (c) 2013-2016 by <NAME>, All Rights Reserved.

;; A package system for Slogan.

(define (build-package pkg-path)
  (let* ((currdir (current-directory))
         (build-cmd (string-append "cd " pkg-path "; ./build; cd " currdir)))
    (let ((r (shell-command build-cmd)))
      (if (scm-not (zero? r))
          (error "build-package - build failed -" build-cmd ", " r)
          #t))))

(define (install-git-package pkg-name pkg-url pkg-path)
  (let ((cmd (string-append "git clone " pkg-url " " pkg-path)))
    (let ((r (shell-command cmd)))
      (if (scm-not (zero? r))
          (error "install-git-package - clone failed -" cmd ", " r)
          (build-package pkg-path))
      pkg-name)))

(define (get-file-name-from-pkg-url url)
  (let loop ((paths (string_split url #\/)))
    (if (null? (scm-cdr paths))
        (scm-car paths)
        (loop (scm-cdr paths)))))

(define *untar-cmds* '((".gz" . "tar xzf") (".bz2" . "tar xjf") (".zip" . "unzip")))

(define (untar-cmd file-name)
  (let ((extn (path-extension file-name)))
    (let ((cmd (scm-assoc extn *untar-cmds*)))
      (if cmd
          (let ((in-file (string-append *pkg-root* file-name)))
            (scm-cons (string-append (scm-cdr cmd) " " in-file
                                 (if (string=? extn ".zip")
                                     " -d " " -C ")
                                 *pkg-root*)
                  in-file))
          (error "cannot decompress package - " file-name)))))

(define (untar&build-package pkg-name pkg-url pkg-path)
  (let ((file-name (get-file-name-from-pkg-url pkg-url)))
    (let ((cmd&infile (untar-cmd file-name)))
      (let ((r (shell-command (scm-car cmd&infile))))
        (delete_file (scm-cdr cmd&infile))
        (if (zero? r)
            (build-package pkg-path)
            (error "failed to decompress package - " (scm-car cmd&infile) ", " r)))))
  pkg-name)
          
(define (install-remote-package pkg-name pkg-url pkg-path)
  (let ((cmd (string-append "curl " pkg-url " -o " (string-append *pkg-root* (get-file-name-from-pkg-url pkg-url)))))
    (let ((r (shell-command cmd)))
      (if (scm-not (zero? r))
          (error "install-remote-package failed -" cmd ", " r)
          (untar&build-package pkg-name pkg-url pkg-path))))
  pkg-name)

(define (copy-dir src dest)
  (let ((cmd (string-append "cp -R " src " " dest)))
    (let ((r (shell-command cmd)))
      (if (scm-not (zero? r))
          (error "copy-dir - failed - " cmd ", " r)
          #t))))

(define (install-local-package pkg-name pkg-url pkg-path)
  (if (file-exists? pkg-url)
      (if (string=? "" (path-extension pkg-url))
          (begin (copy-dir pkg-url pkg-path)
                 (build-package pkg-path))
          (begin (copy-file pkg-url (string-append *pkg-root* (get-file-name-from-pkg-url pkg-url)))
                 (untar&build-package pkg-name pkg-url pkg-path)))
      (error "install-local-package - file not found - " pkg-url))
  pkg-name)

(define (load_package pkg-name)
  (let ((pkg-init-path (string-append *pkg-root* pkg-name "/init")))
    (with-exception-catcher
     (lambda (e)
       (if (no-such-file-or-directory-exception? e)
           (raise e)
           (show_exception e)))
     (lambda () (load pkg-init-path)))
    pkg-name))

(define (init_package pkg-name)
  (let ((load-path (string-append (slogan_root) "/packages/" pkg-name)))
    (begin (if (not (file-exists? load-path))
               (set! load-path "."))
           (link (string-append load-path "/src/core")))))

(define (force-rm-dir path)
  (let ((r (shell-command (string-append "rm -rf " path))))
    (if (zero? r)
        #t
        (error "failed to remove directory - " path ", " r))))

(define (install_package pkg-name pkg-type pkg-url #!optional force)
  (if (scm-not (file-exists? *pkg-root*))
      (create-directory *pkg-root*))
  (let* ((pkg-path (string-append *pkg-root* pkg-name))
         (pkg-path-old (string-append pkg-path ".bck")))
    (if (file-exists? pkg-path)
        (if (scm-not force)
            (error "package already installed - " pkg-path)
            (rename-file pkg-path pkg-path-old)))
    (case pkg-type
      ((git) (install-git-package pkg-name pkg-url pkg-path))
      ((remote) (install-remote-package pkg-name pkg-url pkg-path))
      ((local) (install-local-package pkg-name pkg-url pkg-path))
      (else (error "install_package - type not supported -" pkg-type)))
    (if (file-exists? pkg-path-old)
        (force-rm-dir pkg-path-old))
    (load_package pkg-name)))

(define (uninstall_package pkg-name)
  (let ((pkg-path (string-append *pkg-root* pkg-name)))
    (if (file-exists? pkg-path)
        (if (force-rm-dir pkg-path)
            pkg-name
            #f)
        #f)))

{'fraction_non_alphanumeric': 0.18157952308028877, 'fraction_numerical': 0.001968934587617589, 'mean_word_length': 2.2595365418894833, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3594646', 'n_tokens_mistral': 3320, 'n_tokens_neox': 3158, 'n_words': 797}