<commit_before>#!/usr/bin/env python

import re, os, copy
from plasTeX.ConfigManager import BUILTIN, CODE, ENVIRON, ENVIRONMENT
from plasTeX.ConfigManager import CODE, REGISTRY, COMMANDLINE, InvalidOptionError
from plasTeX.ConfigManager import ConfigManager, TooManyValues, TooFewValues

DEFAULTS = \
{
   'docstring': '',
   'options': None,
   'default': None,
   'optional': None,
   'values': None,
   'category': None,
   'name': None,
   'source': BUILTIN,
   'callback': None,
   'synopsis': None,
   'environ': None,
   'registry': None,
   'mandatory': None,
}

RegexType = type(re.compile(r''))


class GenericParser:

   def getOptionalArgument(self, args):
      return self.getArgument(args, range=[0,1])

   def _hasFollowingArgument(self, args, delim):
      """ Return boolean indicating the existence of another argument """
      if not ConfigManager.has_following_argument(args):
         return 0
      
      # No arguments left
      if not args:
         return 0

      # The current argument ends with the proper delimiter
      if args[0].strip()[-1] == delim:
         return 1

      # The next argument begins with the proper delimiter
      if len(args) > 1 and args[1].strip()[0] == delim:
         return 1

      return 0

   def getArgument(self, args, range=[1,1], delim=','):
      """ Parse argument """

      range = self.validateRange(range[:])

      # If we have an optional default, make sure that the range reflects that
      if range[0] < 2 and self.optional not in [None, [], ()]:
         range[0] = 0

      # No arguments left
      if not args:
         # Arguments that aren't mandatory can bail out now.
         if isinstance(self, GenericArgument) and not(self.mandatory):
            return None, args

         if range[0] > 0:
            name = self.name
            if self.actual: name = self.actual
            if range[0] != range[1]:
               raise TooFewValues, "Expecting at least %s value(s) for '%s'." % (range[0], name)
            elif range[0] == 1:
               raise TooFewValues, "Expecting a value for '%s'." % name
            else:
               raise TooFewValues, "Expecting %s values for '%s'." % (range[0], name)
         if self.optional:
            return self.optional, args
         else:
            return None, args

      # If delimiter is whitespace...
      if not delim or not str(delim).strip():
         new_args, args = self._getSpaceDelimitedArguments(args, range)
         delim = ' '

      else:
         new_args = []
         while self._hasFollowingArgument(args, delim):
   
            # If the current argument ends with a delimiter
            if args[0].strip()[-1] == delim:
               new_args.append(args.pop(0).strip()[:-1])
   
            # If next argument is an isolated delimiter character
            elif len(args) > 1 and args[1].strip() == delim:
               new_args.append(args.pop(0).strip())
               args.pop(0)
   
            # If next argument begins with a delimiter
            else:
               new_args.append(args.pop(0).strip()[1:])
   
               # If this argument ends with a delimiter, rip it off
               # and put it back into the stream.
               if new_args[-1].endswith(delim):
                  args.insert(0, delim)
                  new_args[-1] = new_args[-1][:-1]
                   
         # The following argument is always part of the list
         # unless the user accidentally put a trailing delimiter.
         if ConfigManager.has_following_argument(args):
            new_args.append(args.pop(0).strip())

         if new_args and new_args[-1].startswith(delim):
            new_args[-1] = new_args[-1][1:]

      _args = []
      for i in new_args:
         _args += i.split(delim)
      new_args = _args

      # Check number of values
      name = self.name
      if self.actual: name = self.actual
      if len(new_args) < range[0]:
         plural = 's'
         if range[0] == 1:
            plural = ''
         raise TooFewValues, "Expecting at least %s value%s for '%s'." % (range[0], plural, name)
      elif len(new_args) > range[1]:
         raise TooManyValues, "Expecting at most %s values for '%s'." % (range[1], name)

      # Collapse argument list to a value if possible
      if len(new_args) < 1:
         if self.optional is not None:
            new_args = self.optional
         else:
            new_args = None
      elif len(new_args) < 2:
         new_args = new_args[0]

      return new_args, args 

   def _getSpaceDelimitedArguments(self, args, range=[1,'*']):
      """ Slurp up any following arguments that don't begin with '-' """
      minimum, maximum = self.validateRange(range)
      arg_len = len(args)
      new_args = []
      while ConfigManager.has_following_argument(args):
          new_args.append(args.pop(0))<commit_msg>Fixed problem with commas in single valued options<commit_after>#!/usr/bin/env python

import re, os, copy
from plasTeX.ConfigManager import BUILTIN, CODE, ENVIRON, ENVIRONMENT
from plasTeX.ConfigManager import CODE, REGISTRY, COMMANDLINE, InvalidOptionError
from plasTeX.ConfigManager import ConfigManager, TooManyValues, TooFewValues

DEFAULTS = \
{
   'docstring': '',
   'options': None,
   'default': None,
   'optional': None,
   'values': None,
   'category': None,
   'name': None,
   'source': BUILTIN,
   'callback': None,
   'synopsis': None,
   'environ': None,
   'registry': None,
   'mandatory': None,
}

RegexType = type(re.compile(r''))


class GenericParser:

   def getOptionalArgument(self, args):
      return self.getArgument(args, range=[0,1])

   def _hasFollowingArgument(self, args, delim):
      """ Return boolean indicating the existence of another argument """
      if not ConfigManager.has_following_argument(args):
         return 0
      
      # No arguments left
      if not args:
         return 0

      # The current argument ends with the proper delimiter
      if args[0].strip()[-1] == delim:
         return 1

      # The next argument begins with the proper delimiter
      if len(args) > 1 and args[1].strip()[0] == delim:
         return 1

      return 0

   def getArgument(self, args, range=[1,1], delim=','):
      """ Parse argument """

      range = self.validateRange(range[:])

      # If we have an optional default, make sure that the range reflects that
      if range[0] < 2 and self.optional not in [None, [], ()]:
         range[0] = 0

      # No arguments left
      if not args:
         # Arguments that aren't mandatory can bail out now.
         if isinstance(self, GenericArgument) and not(self.mandatory):
            return None, args

         if range[0] > 0:
            name = self.name
            if self.actual: name = self.actual
            if range[0] != range[1]:
               raise TooFewValues, "Expecting at least %s value(s) for '%s'." % (range[0], name)
            elif range[0] == 1:
               raise TooFewValues, "Expecting a value for '%s'." % name
            else:
               raise TooFewValues, "Expecting %s values for '%s'." % (range[0], name)
         if self.optional:
            return self.optional, args
         else:
            return None, args

      # If delimiter is whitespace...
      if not delim or not str(delim).strip():
         new_args, args = self._getSpaceDelimitedArguments(args, range)
         delim = ' '

      else:
         new_args = []
         while self._hasFollowingArgument(args, delim):
   
            # If the current argument ends with a delimiter
            if args[0].strip()[-1] == delim:
               new_args.append(args.pop(0).strip()[:-1])
   
            # If next argument is an isolated delimiter character
            elif len(args) > 1 and args[1].strip() == delim:
               new_args.append(args.pop(0).strip())
               args.pop(0)
   
            # If next argument begins with a delimiter
            else:
               new_args.append(args.pop(0).strip()[1:])
   
               # If this argument ends with a delimiter, rip it off
               # and put it back into the stream.
               if new_args[-1].endswith(delim):
                  args.insert(0, delim)
                  new_args[-1] = new_args[-1][:-1]
                   
         # The following argument is always part of the list
         # unless the user accidentally put a trailing delimiter.
         if ConfigManager.has_following_argument(args):
            new_args.append(args.pop(0).strip())

         if new_args and new_args[-1].startswith(delim):
            new_args[-1] = new_args[-1][1:]

      if len(range) == 2 and range[0] == 1 and range[1] == 1:
          pass
      else:
          _args = []
          for i in new_args:
             _args += i.split(delim)
          new_args = _args

      # Check number of values
      name = self.name
      if self.actual: name = self.actual
      if len(new_args) < range[0]:
         plural = 's'
         if range[0] == 1:
            plural = ''
         raise TooFewValues, "Expecting at least %s value%s for '%s'." % (range[0], plural, name)
      elif len(new_args) > range[1]:
         raise TooManyValues, "Expecting at most %s values for '%s'." % (range[1], name)

      # Collapse argument list to a value if possible
      if len(new_args) < 1:
         if self.optional is not None:
            new_args = self.optional
         else:
            new_args = None
      elif len(new_args) < 2:
         new_args = new_args[0]

      return new_args, args 

   def _getSpaceDelimitedArguments(self, args, range=[1,'*']):
      """ Slurp up any following arguments that don't begin with '-' """
      minimum, maximum = self.validateRange(range)
      arg_len = len(args)
      new_args = []
      while ConfigManager.has_following_argument(args):
          new_args.append(args.pop(0))
{'fraction_non_alphanumeric': 0.11275303643724696, 'fraction_numerical': 0.012044534412955466, 'mean_word_length': 2.0686335403726708, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5373894', 'n_tokens_mistral': 2993, 'n_tokens_neox': 2837, 'n_words': 889}