the original test data set was from shellwords by hartmut goebel posixdata rxx foo barfoobar foo barfoobar foo bar foobar foo bar bla faselfoobarblafasel x y z xxxxxyzxxxx x barxbar x bar xbar bar bar foo x barfooxbar foo x barfoo xbar foo barfoo bar foo bar blafoobarbla foo bar blafoobarbla foo bar blafoobarbla foo bar blafoobarbla foo bar blafoobarbla foo bar bla foobarbla foo bar bla foobarbla foo bar blafoobarbla blurb foobarbarfasel bazblurbfoobarbarfaselbaz blurb foo bar bar fasel bazblurbfoobarbarfaselbaz foo barfoobar foo barfoobar foo barfoobar foo barfoobar foo barfoo bar foo barfoo bar foo barfoo bar foo barfoobar foo bar dfadffoo bar dfadf foo bar dfadffoo bar dfadf foox bar dfadffoox bar dfadf foox bar dfadffoox bar dfadf foo bar foo bar foo bar foo bar foox bar df a dffoox bar df a df foofoo fooxfoox fooxfoox foo foo foo xxfoo xx foo xxfoo xx foo xxfoo xx foo xxfoo xx foo xxfoo xx foo xxfoobarfoo xxfoobar foo xx foobarfoo xx foobar foo xx fo obarfoo xx fo obar foo xx fo obar don t foo xx fo obardon t foo xx fo obar don t foo xx fo obardon t foo bar foo bar foo bar foo bar foo barfoo bar foobarnbazfoobaz test data splitting with posix parser self splittestself posixdata commentstrue def testcompatself test handling of syntax splitting of could take these forms of course the same applies to and these should all parse to the same output for delimiter in src echo hi s echo bye delimiter echo hisecho bye delimiter ref echo hi delimiter echo bye for ss ws in itertools productsrc false true s shlex shlexss punctuationcharstrue s whitespacesplit ws result lists self assertequalref result while splitting s wss ss ws def testsyntaxsplitsemicolonself could take these forms these should all parse to the same output test handling of syntax splitting of of course the same applies to these should all parse to the same output for delimiter in src echo hi s out delimiter echo his out delimiter echo hisout delimiter ref echo hi delimiter out for ss ws in itertools productsrc false true s shlex shlexss punctuationcharstrue result lists self assertequalref result while splitting s wss ss ws def testsyntaxsplitparenself these should all parse to the same output test handling of syntax splitting with custom chars ss abc colorautod py ref a bc colorauto d py s shlex shlexss punctuationchars result lists self assertequalref result while splitting s wsfalse ss ref abc colorauto d py s shlex shlexss punctuationchars s whitespacesplit true result lists self assertequalref result while splitting s wstrue ss def testtokentypesself test that any punctuation chars are removed from wordchars s shlex shlex abc punctuationchars self assertnotin s wordchars self assertequallists a b c def testpunctuationwithwhitespacesplitself whitespacesplit is false so splitting will be based on punctuationchars whitespacesplit is true so splitting will be based on white space test that punctuationchars and posix behave correctly together see issue 29132 s shlex shlex f abc posixtrue punctuationcharstrue self assertequallists f abc s shlex shlex f abc posixtrue punctuationcharstrue self assertequallists f abc def testemptystringhandlingself see issue 21999 test punctuationchars and whitespacesplit handle unicode ss u2119u01b4u2602u210cu00f8u1f24 should be parsed as one complete token whitespacesplittrue ref u2119u01b4u2602u210cu00f8u1f24 s shlex shlexss punctuationcharstrue s whitespacesplit true self assertequallists ref without whitespacesplit uses wordchars and splits on all ref u2119 u01b4 u2602 u210c u00f8 u1f24 s shlex shlexss punctuationcharstrue self assertequallists ref def testquoteself safeunquoted string asciiletters string digits unicodesample xe9xe0xdf e acute accent a grave sharp s unsafe unicodesample self assertequalshlex quote self assertequalshlex quotesafeunquoted safeunquoted self assertequalshlex quote test file name test file name for u in unsafe self assertequalshlex quote testsname u testsname u for u in unsafe self assertequalshlex quotetests name u tests name u def testjoinself for splitcommand command in a b a b a b a b a b a b a b a b with self subtestcommandcommand joined shlex joinsplitcommand self assertequaljoined command def testjoinroundtripself alldata self data self posixdata for command splitcommand in alldata with self subtestcommandcommand joined shlex joinsplitcommand resplit shlex splitjoined self assertequalsplitcommand resplit def testpunctuationcharsreadonlyself punctuationchars shlexinstance shlex shlexpunctuationcharspunctuationchars self assertequalshlexinstance punctuationchars punctuationchars with self assertraisesattributeerror shlexinstance punctuationchars false allow this test to be used with old shlex py if not getattrshlex split none for methname in dirshlextest if methname startswithtest and methname testcompat delattrshlextest methname if name main unittest main the original test data set was from shellwords by hartmut goebel x x foo bar foo bar foo bar foo bar foo bar foo bar foo bar bla fasel foo bar bla fasel x y z xxxx x y z xxxx x bar x bar x bar x bar bar bar foo x bar foo x bar foo x bar foo x bar foo bar foo bar foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla blurb foo bar bar fasel baz blurb foo bar bar fasel baz blurb foo bar bar fasel baz blurb foo bar bar fasel baz foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar dfadf foo bar dfadf foo bar dfadf foo bar dfadf foo x bar dfadf foo x bar dfadf foo x bar dfadf foo x bar dfadf foo bar foo bar foo bar foo bar foo x bar df a df foo x bar df a df foo foo foo x foo x foo x foo x foo foo foo xx foo xx foo x x foo x x foo x x foo x x foo x x foo x x foo x x foo x x foo x x foobar foo x x foobar foo x x foobar foo x x foobar foo x x fo obar foo x x fo obar foo x x fo obar don t foo x x fo obar don t foo bar foo bar foo bar foo bar foo bar foo bar foo bar nbaz foobaz áéíóú á é í ó ú x x foo bar foo bar foo bar foo bar foo bar foo bar foo bar bla fasel foo bar bla fasel x y z xxxx x y z xxxx x bar x bar x bar x bar bar bar foo x bar foo x bar foo x bar foo x bar foo bar foo bar foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla foo bar bla blurb foo bar bar fasel baz blurb foobarbarfasel baz blurb foo bar bar fasel baz blurb foobarbarfasel baz foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar foo bar dfadf foo bar dfadf foo bar dfadf foo bar dfadf foo x bar dfadf foo x bar dfadf foo x bar dfadf foo x bar dfadf foo bar foo bar foo bar foo bar foo x bar df a df foo x bar df a df foo foo foo x foox foo x foo x foo foo foo xx foo xx foo x x foo xx foo x x foo xx foo x x foo x x foo x x foo x x foo x x foobar foo x x foobar foo x x foobar foo x x foobar foo x x fo obar foo x x fo obar foo x x fo obar don t foo x x fo obar don t foo x x fo obar don t foo x x fo obar don t foo bar foo bar foo bar foo bar foo bar foo bar foo bar nbaz foo baz áéíóú áéíóú test data splitting with posix parser test compatibility interface test handling of syntax splitting of could take these forms of course the same applies to and these should all parse to the same output test handling of syntax splitting of could take these forms these should all parse to the same output test handling of syntax splitting of of course the same applies to these should all parse to the same output test handling of syntax splitting of these should all parse to the same output test handling of syntax splitting with custom chars test that tokens are split with types as expected test that any punctuation chars are removed from wordchars test that with whitespace_split behaviour is as expected whitespace_split is false so splitting will be based on punctuation_chars whitespace_split is true so splitting will be based on white space test that punctuation_chars and posix behave correctly together see issue 29132 test that parsing of empty strings is correctly handled see issue 21999 test punctuation_chars and whitespace_split handle unicode should be parsed as one complete token whitespace_split true without whitespace_split uses wordchars and splits on all e acute accent a grave sharp s allow this test to be used with old shlex py
import io
import itertools
import shlex
import string
import unittest




data = r

posix_data = r

class ShlexTest(unittest.TestCase):
    def setUp(self):
        self.data = [x.split("|")[:-1]
                     for x in data.splitlines()]
        self.posix_data = [x.split("|")[:-1]
                           for x in posix_data.splitlines()]
        for item in self.data:
            item[0] = item[0].replace(r"\n", "\n")
        for item in self.posix_data:
            item[0] = item[0].replace(r"\n", "\n")

    def splitTest(self, data, comments):
        for i in range(len(data)):
            l = shlex.split(data[i][0], comments=comments)
            self.assertEqual(l, data[i][1:],
                             "%s: %s != %s" %
                             (data[i][0], l, data[i][1:]))

    def oldSplit(self, s):
        ret = []
        lex = shlex.shlex(io.StringIO(s))
        tok = lex.get_token()
        while tok:
            ret.append(tok)
            tok = lex.get_token()
        return ret

    def testSplitNone(self):
        with self.assertRaises(ValueError):
            shlex.split(None)

    def testSplitPosix(self):
        
        self.splitTest(self.posix_data, comments=True)

    def testCompat(self):
        
        for i in range(len(self.data)):
            l = self.oldSplit(self.data[i][0])
            self.assertEqual(l, self.data[i][1:],
                             "%s: %s != %s" %
                             (self.data[i][0], l, self.data[i][1:]))

    def testSyntaxSplitAmpersandAndPipe(self):
        
        
        
        
        for delimiter in ('&&', '&', '|&', ';&', ';;&',
                          '||', '|', '&|', ';|', ';;|'):
            src = ['echo hi %s echo bye' % delimiter,
                   'echo hi%secho bye' % delimiter]
            ref = ['echo', 'hi', delimiter, 'echo', 'bye']
            for ss, ws in itertools.product(src, (False, True)):
                s = shlex.shlex(ss, punctuation_chars=True)
                s.whitespace_split = ws
                result = list(s)
                self.assertEqual(ref, result,
                                 "While splitting '%s' [ws=%s]" % (ss, ws))

    def testSyntaxSplitSemicolon(self):
        
        
        
        for delimiter in (';', ';;', ';&', ';;&'):
            src = ['echo hi %s echo bye' % delimiter,
                   'echo hi%s echo bye' % delimiter,
                   'echo hi%secho bye' % delimiter]
            ref = ['echo', 'hi', delimiter, 'echo', 'bye']
            for ss, ws in itertools.product(src, (False, True)):
                s = shlex.shlex(ss, punctuation_chars=True)
                s.whitespace_split = ws
                result = list(s)
                self.assertEqual(ref, result,
                                 "While splitting '%s' [ws=%s]" % (ss, ws))

    def testSyntaxSplitRedirect(self):
        
        
        
        for delimiter in ('<', '|'):
            src = ['echo hi %s out' % delimiter,
                   'echo hi%s out' % delimiter,
                   'echo hi%sout' % delimiter]
            ref = ['echo', 'hi', delimiter, 'out']
            for ss, ws in itertools.product(src, (False, True)):
                s = shlex.shlex(ss, punctuation_chars=True)
                result = list(s)
                self.assertEqual(ref, result,
                                 "While splitting '%s' [ws=%s]" % (ss, ws))

    def testSyntaxSplitParen(self):
        
        
        src = ['( echo hi )',
               '(echo hi)']
        ref = ['(', 'echo', 'hi', ')']
        for ss, ws in itertools.product(src, (False, True)):
            s = shlex.shlex(ss, punctuation_chars=True)
            s.whitespace_split = ws
            result = list(s)
            self.assertEqual(ref, result,
                             "While splitting '%s' [ws=%s]" % (ss, ws))

    def testSyntaxSplitCustom(self):
        
        ss = "~/a&&b-c --color=auto||d *.py?"
        ref = ['~/a', '&', '&', 'b-c', '--color=auto', '||', 'd', '*.py?']
        s = shlex.shlex(ss, punctuation_chars="|")
        result = list(s)
        self.assertEqual(ref, result, "While splitting '%s' [ws=False]" % ss)
        ref = ['~/a&&b-c', '--color=auto', '||', 'd', '*.py?']
        s = shlex.shlex(ss, punctuation_chars="|")
        s.whitespace_split = True
        result = list(s)
        self.assertEqual(ref, result, "While splitting '%s' [ws=True]" % ss)

    def testTokenTypes(self):
        
        for source, expected in (
                                ('a && b || c',
                                 [('a', 'a'), ('&&', 'c'), ('b', 'a'),
                                  ('||', 'c'), ('c', 'a')]),
                              ):
            s = shlex.shlex(source, punctuation_chars=True)
            observed = []
            while True:
                t = s.get_token()
                if t == s.eof:
                    break
                if t[0] in s.punctuation_chars:
                    tt = 'c'
                else:
                    tt = 'a'
                observed.append((t, tt))
            self.assertEqual(observed, expected)

    def testPunctuationInWordChars(self):
        
        s = shlex.shlex('a_b__c', punctuation_chars='_')
        self.assertNotIn('_', s.wordchars)
        self.assertEqual(list(s), ['a', '_', 'b', '__', 'c'])

    def testPunctuationWithWhitespaceSplit(self):
        
        s = shlex.shlex('a  && b  ||  c', punctuation_chars='&')
        
        
        self.assertEqual(list(s), ['a', '&&', 'b', '|', '|', 'c'])
        s = shlex.shlex('a  && b  ||  c', punctuation_chars='&')
        s.whitespace_split = True
        
        
        self.assertEqual(list(s), ['a', '&&', 'b', '||', 'c'])

    def testPunctuationWithPosix(self):
        
        
        s = shlex.shlex('f >"abc"', posix=True, punctuation_chars=True)
        self.assertEqual(list(s), ['f', '>', 'abc'])
        s = shlex.shlex('f >\\"abc\\"', posix=True, punctuation_chars=True)
        self.assertEqual(list(s), ['f', '>', '"abc"'])

    def testEmptyStringHandling(self):
        
        
        expected = ['', ')', 'abc']
        for punct in (False, True):
            s = shlex.shlex("'')abc", posix=True, punctuation_chars=punct)
            slist = list(s)
            self.assertEqual(slist, expected)
        expected = ["''", ')', 'abc']
        s = shlex.shlex("'')abc", punctuation_chars=True)
        self.assertEqual(list(s), expected)

    def testUnicodeHandling(self):
        
        ss = "\u2119\u01b4\u2602\u210c\u00f8\u1f24"
        
        ref = ['\u2119\u01b4\u2602\u210c\u00f8\u1f24']
        s = shlex.shlex(ss, punctuation_chars=True)
        s.whitespace_split = True
        self.assertEqual(list(s), ref)
        
        ref = ['\u2119', '\u01b4', '\u2602', '\u210c', '\u00f8', '\u1f24']
        s = shlex.shlex(ss, punctuation_chars=True)
        self.assertEqual(list(s), ref)

    def testQuote(self):
        safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./'
        unicode_sample = '\xe9\xe0\xdf'  
        unsafe = '"`$\\!' + unicode_sample

        self.assertEqual(shlex.quote(''), "''")
        self.assertEqual(shlex.quote(safeunquoted), safeunquoted)
        self.assertEqual(shlex.quote('test file name'), "'test file name'")
        for u in unsafe:
            self.assertEqual(shlex.quote('test%sname' % u),
                             "'test%sname'" % u)
        for u in unsafe:
            self.assertEqual(shlex.quote("test%s'name'" % u),
                             "'test%s'\"'\"'name'\"'\"''" % u)

    def testJoin(self):
        for split_command, command in [
            (['a ', 'b'], "'a ' b"),
            (['a', ' b'], "a ' b'"),
            (['a', ' ', 'b'], "a ' ' b"),
            (['"a', 'b"'], '\'"a\' \'b"\''),
        ]:
            with self.subTest(command=command):
                joined = shlex.join(split_command)
                self.assertEqual(joined, command)

    def testJoinRoundtrip(self):
        all_data = self.data + self.posix_data
        for command, *split_command in all_data:
            with self.subTest(command=command):
                joined = shlex.join(split_command)
                resplit = shlex.split(joined)
                self.assertEqual(split_command, resplit)

    def testPunctuationCharsReadOnly(self):
        punctuation_chars = "/|$%^"
        shlex_instance = shlex.shlex(punctuation_chars=punctuation_chars)
        self.assertEqual(shlex_instance.punctuation_chars, punctuation_chars)
        with self.assertRaises(AttributeError):
            shlex_instance.punctuation_chars = False



if not getattr(shlex, "split", None):
    for methname in dir(ShlexTest):
        if methname.startswith("test") and methname != "testCompat":
            delattr(ShlexTest, methname)

if __name__ == "__main__":
    unittest.main()