<commit_before>## This module provides a Nim command line parser that is mostly API compatible
## with the Nim standard library parseopt (and the code derives from that).
## It supports one convenience iterator over all command line options and some
## lower-level features.
## Supported command syntax (here ``=``|``:`` may be any char in ``sepChars``):
##
## 1. short option bundles: ``-abx``  (where a, b, x *are in* `shortNoVal`)
##
## 1a. bundles with one final value: ``-abc:Bar``, ``-abc=Bar``, ``-c Bar``,
## ``-abcBar`` (where ``c`` is *not in* ``shortNoVal``)
##
## 2. long options with values: ``--foo:bar``, ``--foo=bar``, ``--foo bar``
## (where ``foo`` is *not in* ``longNoVal``)
##
## 2a. long options without vals: ``--baz`` (where ``baz`` is in ``longNoVal``)
##
## 3. command parameters: everything else | anything after "--" or a stop word.
##
## The above is a *superset* of usual POSIX command syntax - it should accept
## any POSIX-inspired input, but it also accepts more forms/styles. (Note that
## POSIX itself is not super strict about this part of the standard.  See:
## http://pubs.opengroup.org/onlinepubs/009604499/basedefs/xbd_chap12.html)
##
## When ``optionNormalize(key)`` is used, command authors provide command users
## additional flexibility to ``--spell_multi-word_options -aVarietyOfWays
## --as-Per_User-Preference``.  This is similar to Nim style-insensitive
## identifier syntax, but by default allows dash ('-') as well as underscore
## ('_') word separation.
##
## The "separator free" forms above require appropriate ``shortNoVal`` and
## ``longNoVal`` lists to designate option keys that take no value (as well
## as ``requireSeparator == false``).  If such lists are empty, the user must
## use separators when providing any value.
##
## A notable subtlety is when the first character of an option value is one of
## ``sepChars``.  Even if ``requireSeparator`` is ``false``, passing such option
## values requires either A) putting the value in the next command parameter,
## as in ``"-c :"`` or B) prefixing the value with an element of ``sepChars``,
## as in ``-c=:`` or ``-c::``.  Both choices fit into common quoting styles.
## It seems likely a POSIX-habituated end-user's second guess (after ``"-c:"``
## errored out with "argument expected") would just work as they expected.
## POSIX itself encourages authors & users to use the ``"-c :"`` form anyway.
## This small deviation lets this parser accept valid invocations with the
## original Nim option parser command syntax (with the same semantics), easing
## transition.
##
## To ease "nested" command-line parsing (such as with "git" where there may be
## early global options, a subcommand and later subcommand options), this parser
## also supports a set of "stop words" - special whole command parameters that
## prevent subsequent parameters being interpreted as options.  This feature
## makes it easy to fully process a command line and then re-process its tail
## rather than mandating breaking out at a stop word with a manual test.  Stop
## words are basically just like a POSIX "--" (which this parser also supports -
## even if "--" is not in ``stopWords``).  Such stop words (or "--") can still
## be the **values** of option keys with no effect.  Only usage as a non-option
## command parameter acts to stop possible option-treatment of later parameters.
##
## To facilitate syntax for operations beyond simple assignment, ``opChars`` is
## a set of chars that may prefix an element of ``sepChars``. The ``sep`` member
## of ``OptParser`` is the actual separator used for the current option, if any.
## E.g, a user entering "="  causes ``sep == "="`` while entering "+=" gets
## ``sep == "+="``, and "+/-+=" gets ``sep == "+/-+="``.

import os, strutils, critbits

proc optionNormalize*(s: string, wordSeparators="_-"): string {.noSideEffect.} =
  ## Normalizes option key `s` to allow command syntax to be style-insensitive
  ## in a similar way to Nim identifier syntax.
  ##
  ## Specifically this means to convert *all but the first* char to lower case
  ## and remove chars in ``wordSeparators`` ('_' and '-') by default.  This way
  ## users can type "command --my-opt-key" or "command --myOptKey" and so on.
  ##
  ## Example:
  ##
  ## .. code-block:: nim
  ##   for kind, key, val in p.getopt():
  ##     case kind
  ##     of cmdLongOption, cmdShortOption:
  ##       case optionNormalize(key)
  ##       of "myoptkey", "m": doSomething()
  result = newString(s.len)
  if s.len == 0: return
  var wordSeps: set[char]   # compile a set[char] from ``wordSeparators``
  for c in wordSeparators:
    wordSeps.incl(c)
  result[0] = s[0]
  var j = 1
  for i in 1..len(s) - 1:
    if s[i] in {'A'..'Z'}:
      result[j] = chr(ord(s[i]) + (ord('a') - ord('A')))
      inc j
    elif s[i] notin wordSeps:
      result[j] = s[i]
      inc j
  if j != s.len:
    setLen(result, j)

{.push warning[ProveField]: off.}
proc valsWithPfx*(cb: CritBitTree[string], key: string): seq[string] =
  for v in cb.valuesWithPrefix(optionNormalize(key)): result.add(v)

proc lengthen*(cb: CritBitTree[string], key: string): string =
  ## Use ``cb`` to find normalized long form of ``key``. Return empty string if
  ## ambiguous or unchanged string on no match.
  let n = optionNormalize(key)
  var ks: seq[string]
  for k in cb.keysWithPrefix(n): ks.add(k)<commit_msg>There's no reason these two procs should not track `CritBitTree` genericness.<commit_after>## This module provides a Nim command line parser that is mostly API compatible
## with the Nim standard library parseopt (and the code derives from that).
## It supports one convenience iterator over all command line options and some
## lower-level features.
## Supported command syntax (here ``=``|``:`` may be any char in ``sepChars``):
##
## 1. short option bundles: ``-abx``  (where a, b, x *are in* `shortNoVal`)
##
## 1a. bundles with one final value: ``-abc:Bar``, ``-abc=Bar``, ``-c Bar``,
## ``-abcBar`` (where ``c`` is *not in* ``shortNoVal``)
##
## 2. long options with values: ``--foo:bar``, ``--foo=bar``, ``--foo bar``
## (where ``foo`` is *not in* ``longNoVal``)
##
## 2a. long options without vals: ``--baz`` (where ``baz`` is in ``longNoVal``)
##
## 3. command parameters: everything else | anything after "--" or a stop word.
##
## The above is a *superset* of usual POSIX command syntax - it should accept
## any POSIX-inspired input, but it also accepts more forms/styles. (Note that
## POSIX itself is not super strict about this part of the standard.  See:
## http://pubs.opengroup.org/onlinepubs/009604499/basedefs/xbd_chap12.html)
##
## When ``optionNormalize(key)`` is used, command authors provide command users
## additional flexibility to ``--spell_multi-word_options -aVarietyOfWays
## --as-Per_User-Preference``.  This is similar to Nim style-insensitive
## identifier syntax, but by default allows dash ('-') as well as underscore
## ('_') word separation.
##
## The "separator free" forms above require appropriate ``shortNoVal`` and
## ``longNoVal`` lists to designate option keys that take no value (as well
## as ``requireSeparator == false``).  If such lists are empty, the user must
## use separators when providing any value.
##
## A notable subtlety is when the first character of an option value is one of
## ``sepChars``.  Even if ``requireSeparator`` is ``false``, passing such option
## values requires either A) putting the value in the next command parameter,
## as in ``"-c :"`` or B) prefixing the value with an element of ``sepChars``,
## as in ``-c=:`` or ``-c::``.  Both choices fit into common quoting styles.
## It seems likely a POSIX-habituated end-user's second guess (after ``"-c:"``
## errored out with "argument expected") would just work as they expected.
## POSIX itself encourages authors & users to use the ``"-c :"`` form anyway.
## This small deviation lets this parser accept valid invocations with the
## original Nim option parser command syntax (with the same semantics), easing
## transition.
##
## To ease "nested" command-line parsing (such as with "git" where there may be
## early global options, a subcommand and later subcommand options), this parser
## also supports a set of "stop words" - special whole command parameters that
## prevent subsequent parameters being interpreted as options.  This feature
## makes it easy to fully process a command line and then re-process its tail
## rather than mandating breaking out at a stop word with a manual test.  Stop
## words are basically just like a POSIX "--" (which this parser also supports -
## even if "--" is not in ``stopWords``).  Such stop words (or "--") can still
## be the **values** of option keys with no effect.  Only usage as a non-option
## command parameter acts to stop possible option-treatment of later parameters.
##
## To facilitate syntax for operations beyond simple assignment, ``opChars`` is
## a set of chars that may prefix an element of ``sepChars``. The ``sep`` member
## of ``OptParser`` is the actual separator used for the current option, if any.
## E.g, a user entering "="  causes ``sep == "="`` while entering "+=" gets
## ``sep == "+="``, and "+/-+=" gets ``sep == "+/-+="``.

import os, strutils, critbits

proc optionNormalize*(s: string, wordSeparators="_-"): string {.noSideEffect.} =
  ## Normalizes option key `s` to allow command syntax to be style-insensitive
  ## in a similar way to Nim identifier syntax.
  ##
  ## Specifically this means to convert *all but the first* char to lower case
  ## and remove chars in ``wordSeparators`` ('_' and '-') by default.  This way
  ## users can type "command --my-opt-key" or "command --myOptKey" and so on.
  ##
  ## Example:
  ##
  ## .. code-block:: nim
  ##   for kind, key, val in p.getopt():
  ##     case kind
  ##     of cmdLongOption, cmdShortOption:
  ##       case optionNormalize(key)
  ##       of "myoptkey", "m": doSomething()
  result = newString(s.len)
  if s.len == 0: return
  var wordSeps: set[char]   # compile a set[char] from ``wordSeparators``
  for c in wordSeparators:
    wordSeps.incl(c)
  result[0] = s[0]
  var j = 1
  for i in 1..len(s) - 1:
    if s[i] in {'A'..'Z'}:
      result[j] = chr(ord(s[i]) + (ord('a') - ord('A')))
      inc j
    elif s[i] notin wordSeps:
      result[j] = s[i]
      inc j
  if j != s.len:
    setLen(result, j)

{.push warning[ProveField]: off.}
proc valsWithPfx*[T](cb: CritBitTree[T], key: string): seq[T] =
  for v in cb.valuesWithPrefix(optionNormalize(key)): result.add(v)

proc lengthen*[T](cb: CritBitTree[T], key: string): string =
  ## Use ``cb`` to find normalized long form of ``key``. Return empty string if
  ## ambiguous or unchanged string on no match.
  let n = optionNormalize(key)
  var ks: seq[string]
  for k in cb.keysWithPrefix(n): ks.add(k)
{'fraction_non_alphanumeric': 0.14458946588910487, 'fraction_numerical': 0.004072942701101546, 'mean_word_length': 3.9650735294117645, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '72431', 'n_tokens_mistral': 3352, 'n_tokens_neox': 3218, 'n_words': 1407}