<commit_before>;;;; tar.lisp -- reading and writing tar files from Common Lisp

;;; Yes, perfectly good implementations of tar already exist.  However,
;;; there are none for Common Lisp. :)  Actually, the main motivation
;;; behind writing this was to eventually provide a TAR-OP or something
;;; similar for ASDF.  This would make packaging up systems very easy.
;;; Furthermore, something like asdf-install could use this package as
;;; part of its installation routines.
;;;
;;; Implementation notes:
;;;
;;; Tar archives make ugly conflations between characters and bytes, as
;;; is typical of C programs.  This implementation deals strictly with
;;; bytes internally and will only convert to actual strings when the
;;; user requests it.  Hooks are provided for the user to customize the
;;; conversion of byte vectors to strings.  Bind
;;; *BYTEVEC-TO-STRING-CONVERSION-FUNCTION* or
;;; *STRING-TO-BYTEVEC-CONVERSION-FUNCTION* to use your own custom
;;; conversion functions.  The default functions will convert strings as
;;; ASCII (Latin-1, really), which may be good enough for you.
;;;
;;; Bugs almost certainly remain.  In particular, the implementation
;;; cannot yet handle archiving directories, so the equivalent of
;;;
;;;   tar cf foo.tar bar/
;;;
;;; will fail, whereas the equivalent of
;;;
;;;   tar cf foo.tar bar/*
;;;
;;; will work (so long as `bar' does not contain directories, of
;;; course).  Regular files are the only archivable entities at the
;;; moment; attempting to back up your /dev directory with the tools
;;; contained herein will be a futile exercise.
;;;
;;; The implementation only handles ustar archives (POSIX circa 1988 or
;;; so) and does not handle pax archives (POSIX circa 2001 or so).  This
;;; is a shortcoming that I would like to see fixed, but have not yet
;;; examined what would be necessary to do so.  Archive buffer handling
;;; would likely need to be altered somewhat, as the current version is
;;; biased towards handling tar files (fixed records of 512 bytes).

(in-package :archive)



;;; constants and class definitions
(defconstant +tar-n-block-bytes+ 512)
;;; GNU tar makes this configurable, actually.
(defconstant +tar-n-record-blocks+ 20)
(defconstant +tar-n-record-bytes+ (* +tar-n-block-bytes+ +tar-n-record-blocks+))

;;; values for tar's `typeflag' field
(defconstant +tar-regular-file+ #x30)
;;; backwards compatibility
(defconstant +tar-regular-alternate-file+ #x00)
(defconstant +tar-hard-link+ #x31)
(defconstant +tar-symbolic-link+ #x32)
(defconstant +tar-character-device+ #x33)
(defconstant +tar-block-device+ #x34)
(defconstant +tar-directory-file+ #x35)
(defconstant +tar-fifo-device+ #x36)
(defconstant +tar-implementation-specific-file+ #x37)

;;; non-standard typeflags<commit_msg>add POSIX extended and global header typeflags<commit_after>;;;; tar.lisp -- reading and writing tar files from Common Lisp

;;; Yes, perfectly good implementations of tar already exist.  However,
;;; there are none for Common Lisp. :)  Actually, the main motivation
;;; behind writing this was to eventually provide a TAR-OP or something
;;; similar for ASDF.  This would make packaging up systems very easy.
;;; Furthermore, something like asdf-install could use this package as
;;; part of its installation routines.
;;;
;;; Implementation notes:
;;;
;;; Tar archives make ugly conflations between characters and bytes, as
;;; is typical of C programs.  This implementation deals strictly with
;;; bytes internally and will only convert to actual strings when the
;;; user requests it.  Hooks are provided for the user to customize the
;;; conversion of byte vectors to strings.  Bind
;;; *BYTEVEC-TO-STRING-CONVERSION-FUNCTION* or
;;; *STRING-TO-BYTEVEC-CONVERSION-FUNCTION* to use your own custom
;;; conversion functions.  The default functions will convert strings as
;;; ASCII (Latin-1, really), which may be good enough for you.
;;;
;;; Bugs almost certainly remain.  In particular, the implementation
;;; cannot yet handle archiving directories, so the equivalent of
;;;
;;;   tar cf foo.tar bar/
;;;
;;; will fail, whereas the equivalent of
;;;
;;;   tar cf foo.tar bar/*
;;;
;;; will work (so long as `bar' does not contain directories, of
;;; course).  Regular files are the only archivable entities at the
;;; moment; attempting to back up your /dev directory with the tools
;;; contained herein will be a futile exercise.
;;;
;;; The implementation only handles ustar archives (POSIX circa 1988 or
;;; so) and does not handle pax archives (POSIX circa 2001 or so).  This
;;; is a shortcoming that I would like to see fixed, but have not yet
;;; examined what would be necessary to do so.  Archive buffer handling
;;; would likely need to be altered somewhat, as the current version is
;;; biased towards handling tar files (fixed records of 512 bytes).

(in-package :archive)



;;; constants and class definitions
(defconstant +tar-n-block-bytes+ 512)
;;; GNU tar makes this configurable, actually.
(defconstant +tar-n-record-blocks+ 20)
(defconstant +tar-n-record-bytes+ (* +tar-n-block-bytes+ +tar-n-record-blocks+))

;;; values for tar's `typeflag' field
(defconstant +tar-regular-file+ #x30)
;;; backwards compatibility
(defconstant +tar-regular-alternate-file+ #x00)
(defconstant +tar-hard-link+ #x31)
(defconstant +tar-symbolic-link+ #x32)
(defconstant +tar-character-device+ #x33)
(defconstant +tar-block-device+ #x34)
(defconstant +tar-directory-file+ #x35)
(defconstant +tar-fifo-device+ #x36)
(defconstant +tar-implementation-specific-file+ #x37)

(defconstant +posix-extended-header+ #x78)
(defconstant +posix-global-header+ #x67)

;;; non-standard typeflags
{'fraction_non_alphanumeric': 0.11464968152866242, 'fraction_numerical': 0.013092710544939844, 'mean_word_length': 4.744918699186992, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4831904', 'n_tokens_mistral': 1716, 'n_tokens_neox': 1613, 'n_words': 697}