<commit_before>(defpackage :shcl/core/expand
  (:use
   :common-lisp :shcl/core/utility :shcl/core/lexer :shcl/core/environment
   :shcl/core/working-directory)
  (:import-from :fset)
  (:import-from :cl-ppcre #:create-scanner #:scan)
  (:import-from :cffi #:foreign-string-to-lisp #:foreign-slot-pointer #:null-pointer-p)
  (:import-from :shcl/core/posix #:syscall-error)
  (:import-from :shcl/core/posix-types #:dirent #:d-name)
  (:import-from :shcl/core/fd-table #:with-dir-ptr-for-fd #:openat-retained #:fd-release)
  (:export
   #:expansion-for-words #:set-alias #:unalias #:expand #:make-string-fragment
   #:word-boundary #:*split-fields* #:split))
(in-package :shcl/core/expand)

(optimization-settings)

(defstruct (string-fragment
             (:constructor %make-string-fragment))
  "This struct represents a part of a word.

`string-fragment-string' stores the characters associated with this
fragment.

`string-fragment-quoted' is a boolean value indicating whether the
characters should be treated as though they were quoted.  This impacts
how the characters are interpreted in some expansion contexts.  For
example, a #\~ appearing in a quoted string fragment will not expand
to the user's home directory.

`string-fragment-literal' is a boolean value indicating whether the
characters should be treated as though they were typed by the user
directly.  Tokens like `shcl/core/lexer:variable-expansion-word'
should expand to non-literal string fragments.  Tokens like
`shcl/core/lexer:if-word' should expand to a literal string fragment."
  string
  quoted
  literal)

(defun make-string-fragment (string &key quoted literal)
  "Create a `string-fragment'."
  (%make-string-fragment :string string :quoted quoted :literal literal))

(defparameter *split-fields* t
  "Non-nil if field splitting should be performed after expansion
takes place.

Methods on the `expand' generic function should not emit word boundary
markers (see `+soft-word-boundary+' and `+hard-word-boundary+') in
their expansions when this variable is bound to nil.  When this
variable is non-nil, methods on `expand' are permitted to split their
expanded strings into multiple strings by inserting word boundary
markers into their return value.  If appropriate, they are encouraged
to use the `split' function to do this.")

(defconstant +soft-word-boundary+ '+soft-word-boundary+
  "A weaker version of `+hard-word-boundary+'.

Unlike `+hard-word-boundary+', this marker only introduces a new field
in specific circumstances.
- Soft boundaries at the beginning or end of a fragment sequence have
  no effect.
- Consecutive soft boundaries can be coalesced into a single soft
  boundary.
- If a soft word boundary is next to a hard word boundary, then the
  soft word boundary can be removed from the fragment sequence without
  changing the end result.")
(defconstant +hard-word-boundary+ '+hard-word-boundary+
  "A marker indicating that a new field has begun.

When fragments are being joined together, a hard word boundary will
always introduce a new field, even if there are no fragments on one
side of the boundary.  As a result,
- Hard boundaries at the beginning or end of a fragment sequence
  introduce an empty field.
- Consecutive hard boundaries introduce empty a sequence of empty
  fields.")

(defun soft-word-boundary-p (thing)
  "Returns non-nil iff the given object is a soft word boundary."
  (eq +soft-word-boundary+ thing))<commit_msg>Whoops.  word-boundary is needed for lisp interpolation<commit_after>(defpackage :shcl/core/expand
  (:use
   :common-lisp :shcl/core/utility :shcl/core/lexer :shcl/core/environment
   :shcl/core/working-directory)
  (:import-from :fset)
  (:import-from :cl-ppcre #:create-scanner #:scan)
  (:import-from :cffi #:foreign-string-to-lisp #:foreign-slot-pointer #:null-pointer-p)
  (:import-from :shcl/core/posix #:syscall-error)
  (:import-from :shcl/core/posix-types #:dirent #:d-name)
  (:import-from :shcl/core/fd-table #:with-dir-ptr-for-fd #:openat-retained #:fd-release)
  (:export
   #:expansion-for-words #:set-alias #:unalias #:expand #:make-string-fragment
   #:word-boundary #:*split-fields* #:split))
(in-package :shcl/core/expand)

(optimization-settings)

(defstruct (string-fragment
             (:constructor %make-string-fragment))
  "This struct represents a part of a word.

`string-fragment-string' stores the characters associated with this
fragment.

`string-fragment-quoted' is a boolean value indicating whether the
characters should be treated as though they were quoted.  This impacts
how the characters are interpreted in some expansion contexts.  For
example, a #\~ appearing in a quoted string fragment will not expand
to the user's home directory.

`string-fragment-literal' is a boolean value indicating whether the
characters should be treated as though they were typed by the user
directly.  Tokens like `shcl/core/lexer:variable-expansion-word'
should expand to non-literal string fragments.  Tokens like
`shcl/core/lexer:if-word' should expand to a literal string fragment."
  string
  quoted
  literal)

(defun make-string-fragment (string &key quoted literal)
  "Create a `string-fragment'."
  (%make-string-fragment :string string :quoted quoted :literal literal))

(defparameter *split-fields* t
  "Non-nil if field splitting should be performed after expansion
takes place.

Methods on the `expand' generic function should not emit word boundary
markers (see `+soft-word-boundary+' and `+hard-word-boundary+') in
their expansions when this variable is bound to nil.  When this
variable is non-nil, methods on `expand' are permitted to split their
expanded strings into multiple strings by inserting word boundary
markers into their return value.  If appropriate, they are encouraged
to use the `split' function to do this.")

(defconstant +soft-word-boundary+ '+soft-word-boundary+
  "A weaker version of `+hard-word-boundary+'.

Unlike `+hard-word-boundary+', this marker only introduces a new field
in specific circumstances.
- Soft boundaries at the beginning or end of a fragment sequence have
  no effect.
- Consecutive soft boundaries can be coalesced into a single soft
  boundary.
- If a soft word boundary is next to a hard word boundary, then the
  soft word boundary can be removed from the fragment sequence without
  changing the end result.")

(defconstant +hard-word-boundary+ '+hard-word-boundary+
  "A marker indicating that a new field has begun.

When fragments are being joined together, a hard word boundary will
always introduce a new field, even if there are no fragments on one
side of the boundary.  As a result,
- Hard boundaries at the beginning or end of a fragment sequence
  introduce an empty field.
- Consecutive hard boundaries introduce empty a sequence of empty
  fields.")

(defun word-boundary ()
  "Returns a marker which indicates that the previous string fragment
should be considered a seperate word from the next one."
  +soft-word-boundary+)

(defun soft-word-boundary-p (thing)
  "Returns non-nil iff the given object is a soft word boundary."
  (eq +soft-word-boundary+ thing))
{'fraction_non_alphanumeric': 0.09266300521053372, 'fraction_numerical': 0.0, 'mean_word_length': 4.404870624048706, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6564120', 'n_tokens_mistral': 2088, 'n_tokens_neox': 2007, 'n_words': 840}