import os import sys import shutil import tempfile import unittest import warnings template r coding s a x assert orda 120 b x01 assert ordb 1 c r x01 assert listmapord c 92 120 48 49 d x81 assert ordd 0x81 e r x81 assert listmapord e 92 120 56 49 f u1881 assert ordf 0x1881 g r u1881 assert listmapord g 92 117 49 56 56 49 h u0001d120 assert ordh 0x1d120 i r u0001d120 assert listmapord i 92 85 48 48 48 49 100 49 50 48 check that the template doesn t contain any nonprintables except for n check that the warning is raised ony once if there are syntax errors test correct treatment of various string literals by the parser there are four types of string literals abc normal str r abc raw str b xyz normal bytes br xyz rb xyz raw bytes the difference between normal and raw strings is of course that in a raw string escapes while still used to determine the end of the literal are not interpreted so that r x00 contains four characters a backslash an x and two zeros while x00 contains a single character code point zero the tricky thing is what should happen when non ascii bytes are used inside literals for bytes literals this is considered illegal but for str literals those bytes are supposed to be decoded using the encoding declared for the file utf 8 by default we have to test this with various file encodings we also test it with exec eval which uses a different code path this file is really about correct treatment of encodings and backslashes it doesn t concern itself with issues like single vs double quotes or singly vs triply quoted strings that s dealt with elsewhere i assume coding s a x assert ord a 120 b x01 assert ord b 1 c r x01 assert list map ord c 92 120 48 49 d x81 assert ord d 0x81 e r x81 assert list map ord e 92 120 56 49 f u1881 assert ord f 0x1881 g r u1881 assert list map ord g 92 117 49 56 56 49 h u0001d120 assert ord h 0x1d120 i r u0001d120 assert list map ord i 92 85 48 48 48 49 100 49 50 48 check that the template doesn t contain any non printables except for n x x01 x01 x81 x81 u1881 u1881 u0001d120 u0001d120 x x0 u u0 u00 u000 u u0 u00 u000 u0000 u00000 u000000 u0000000 n r 01234567nu abfnrtuvx n z n z check that the warning is raised ony once if there are syntax errors n 407 n 407 r x r x01 r x01 r x81 r x81 r u1881 r u1881 r u0001d120 r u0001d120 b x b x01 b x01 b x81 b x81 br u1881 b u1881 br u0001d120 b u0001d120 b x b x0 n r 01234567 abfnrtvx n z n z n 407 n 407 br x rb x br x01 rb x01 br x01 rb x01 br x81 rb x81 br x81 rb x81 br u1881 rb u1881 br u1881 rb u1881 br u0001d120 rb u0001d120 br u0001d120 rb u0001d120 bb rr brr bbr rrb rbb u x u u00e4 u n latin small letter a with diaeresis ur ru bu ub b x r x01 br x01 f 1 1 u u0001d120
r

import os
import sys
import shutil
import tempfile
import unittest
import warnings


TEMPLATE = r


def byte(i):
    return bytes([i])


class TestLiterals(unittest.TestCase):

    def setUp(self):
        self.save_path = sys.path[:]
        self.tmpdir = tempfile.mkdtemp()
        sys.path.insert(0, self.tmpdir)

    def tearDown(self):
        sys.path[:] = self.save_path
        shutil.rmtree(self.tmpdir, ignore_errors=True)

    def test_template(self):
        
        
        for c in TEMPLATE:
            assert c == '\n' or ' ' <= c <= '~', repr(c)

    def test_eval_str_normal(self):
        self.assertEqual(eval(), 'x')
        self.assertEqual(eval(r), chr(1))
        self.assertEqual(eval(), chr(1))
        self.assertEqual(eval(r), chr(0x81))
        self.assertEqual(eval(), chr(0x81))
        self.assertEqual(eval(r), chr(0x1881))
        self.assertEqual(eval(), chr(0x1881))
        self.assertEqual(eval(r), chr(0x1d120))
        self.assertEqual(eval(), chr(0x1d120))

    def test_eval_str_incomplete(self):
        self.assertRaises(SyntaxError, eval, r)
        self.assertRaises(SyntaxError, eval, r)
        self.assertRaises(SyntaxError, eval, r)
        self.assertRaises(SyntaxError, eval, r)
        self.assertRaises(SyntaxError, eval, r)
        self.assertRaises(SyntaxError, eval, r)
        self.assertRaises(SyntaxError, eval, r)
        self.assertRaises(SyntaxError, eval, r)
        self.assertRaises(SyntaxError, eval, r)
        self.assertRaises(SyntaxError, eval, r)
        self.assertRaises(SyntaxError, eval, r)
        self.assertRaises(SyntaxError, eval, r)
        self.assertRaises(SyntaxError, eval, r)
        self.assertRaises(SyntaxError, eval, r)

    def test_eval_str_invalid_escape(self):
        for b in range(1, 128):
            if b in b:
                continue
            with self.assertWarns(SyntaxWarning):
                self.assertEqual(eval(r"'\%c'" % b), '\\' + chr(b))

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always', category=SyntaxWarning)
            eval("")
        self.assertEqual(len(w), 1)
        self.assertEqual(str(w[0].message), r"invalid escape sequence '\z'")
        self.assertEqual(w[0].filename, '<string>')
        self.assertEqual(w[0].lineno, 1)

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('error', category=SyntaxWarning)
            with self.assertRaises(SyntaxError) as cm:
                eval("")
            exc = cm.exception
        self.assertEqual(w, [])
        self.assertEqual(exc.msg, r"invalid escape sequence '\z'")
        self.assertEqual(exc.filename, '<string>')
        self.assertEqual(exc.lineno, 1)
        self.assertEqual(exc.offset, 1)

        

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always', category=SyntaxWarning)
            with self.assertRaises(SyntaxError) as cm:
                eval("'\\e' $")
            exc = cm.exception
        self.assertEqual(len(w), 1)
        self.assertEqual(w[0].category, SyntaxWarning)
        self.assertRegex(str(w[0].message), 'invalid escape sequence')
        self.assertEqual(w[0].filename, '<string>')

    def test_eval_str_invalid_octal_escape(self):
        for i in range(0o400, 0o1000):
            with self.assertWarns(SyntaxWarning):
                self.assertEqual(eval(r"'\%o'" % i), chr(i))

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always', category=SyntaxWarning)
            eval("")
        self.assertEqual(len(w), 1)
        self.assertEqual(str(w[0].message),
                         r"invalid octal escape sequence '\407'")
        self.assertEqual(w[0].filename, '<string>')
        self.assertEqual(w[0].lineno, 1)

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('error', category=SyntaxWarning)
            with self.assertRaises(SyntaxError) as cm:
                eval("")
            exc = cm.exception
        self.assertEqual(w, [])
        self.assertEqual(exc.msg, r"invalid octal escape sequence '\407'")
        self.assertEqual(exc.filename, '<string>')
        self.assertEqual(exc.lineno, 1)
        self.assertEqual(exc.offset, 1)

    def test_eval_str_raw(self):
        self.assertEqual(eval(), 'x')
        self.assertEqual(eval(r), '\\' + 'x01')
        self.assertEqual(eval(), chr(1))
        self.assertEqual(eval(r), '\\' + 'x81')
        self.assertEqual(eval(), chr(0x81))
        self.assertEqual(eval(r), '\\' + 'u1881')
        self.assertEqual(eval(), chr(0x1881))
        self.assertEqual(eval(r), '\\' + 'U0001d120')
        self.assertEqual(eval(), chr(0x1d120))

    def test_eval_bytes_normal(self):
        self.assertEqual(eval(), b'x')
        self.assertEqual(eval(r), byte(1))
        self.assertEqual(eval(), byte(1))
        self.assertEqual(eval(r), byte(0x81))
        self.assertRaises(SyntaxError, eval, )
        self.assertEqual(eval(r), b'\\' + b'u1881')
        self.assertRaises(SyntaxError, eval, )
        self.assertEqual(eval(r), b'\\' + b'U0001d120')
        self.assertRaises(SyntaxError, eval, )

    def test_eval_bytes_incomplete(self):
        self.assertRaises(SyntaxError, eval, r)
        self.assertRaises(SyntaxError, eval, r)

    def test_eval_bytes_invalid_escape(self):
        for b in range(1, 128):
            if b in b:
                continue
            with self.assertWarns(SyntaxWarning):
                self.assertEqual(eval(r"b'\%c'" % b), b'\\' + bytes([b]))

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always', category=SyntaxWarning)
            eval("b")
        self.assertEqual(len(w), 1)
        self.assertEqual(str(w[0].message), r"invalid escape sequence '\z'")
        self.assertEqual(w[0].filename, '<string>')
        self.assertEqual(w[0].lineno, 1)

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('error', category=SyntaxWarning)
            with self.assertRaises(SyntaxError) as cm:
                eval("b")
            exc = cm.exception
        self.assertEqual(w, [])
        self.assertEqual(exc.msg, r"invalid escape sequence '\z'")
        self.assertEqual(exc.filename, '<string>')
        self.assertEqual(exc.lineno, 1)

    def test_eval_bytes_invalid_octal_escape(self):
        for i in range(0o400, 0o1000):
            with self.assertWarns(SyntaxWarning):
                self.assertEqual(eval(r"b'\%o'" % i), bytes([i & 0o377]))

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always', category=SyntaxWarning)
            eval("b")
        self.assertEqual(len(w), 1)
        self.assertEqual(str(w[0].message),
                         r"invalid octal escape sequence '\407'")
        self.assertEqual(w[0].filename, '<string>')
        self.assertEqual(w[0].lineno, 1)

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('error', category=SyntaxWarning)
            with self.assertRaises(SyntaxError) as cm:
                eval("b")
            exc = cm.exception
        self.assertEqual(w, [])
        self.assertEqual(exc.msg, r"invalid octal escape sequence '\407'")
        self.assertEqual(exc.filename, '<string>')
        self.assertEqual(exc.lineno, 1)

    def test_eval_bytes_raw(self):
        self.assertEqual(eval(), b'x')
        self.assertEqual(eval(), b'x')
        self.assertEqual(eval(r), b'\\' + b'x01')
        self.assertEqual(eval(r), b'\\' + b'x01')
        self.assertEqual(eval(), byte(1))
        self.assertEqual(eval(), byte(1))
        self.assertEqual(eval(r), b"\\" + b"x81")
        self.assertEqual(eval(r), b"\\" + b"x81")
        self.assertRaises(SyntaxError, eval, )
        self.assertRaises(SyntaxError, eval, )
        self.assertEqual(eval(r), b"\\" + b"u1881")
        self.assertEqual(eval(r), b"\\" + b"u1881")
        self.assertRaises(SyntaxError, eval, )
        self.assertRaises(SyntaxError, eval, )
        self.assertEqual(eval(r), b"\\" + b"U0001d120")
        self.assertEqual(eval(r), b"\\" + b"U0001d120")
        self.assertRaises(SyntaxError, eval, )
        self.assertRaises(SyntaxError, eval, )
        self.assertRaises(SyntaxError, eval, )
        self.assertRaises(SyntaxError, eval, )
        self.assertRaises(SyntaxError, eval, )
        self.assertRaises(SyntaxError, eval, )
        self.assertRaises(SyntaxError, eval, )
        self.assertRaises(SyntaxError, eval, )

    def test_eval_str_u(self):
        self.assertEqual(eval(), 'x')
        self.assertEqual(eval(), 'ä')
        self.assertEqual(eval(), 'ä')
        self.assertRaises(SyntaxError, eval, )
        self.assertRaises(SyntaxError, eval, )
        self.assertRaises(SyntaxError, eval, )
        self.assertRaises(SyntaxError, eval, )

    def test_uppercase_prefixes(self):
        self.assertEqual(eval(), b'x')
        self.assertEqual(eval(r), r'\x01')
        self.assertEqual(eval(r), br'\x01')
        self.assertEqual(eval(), f'{1+1}')
        self.assertEqual(eval(r), u'\U0001d120')

    def check_encoding(self, encoding, extra=""):
        modname = "xx_" + encoding.replace("-", "_")
        fn = os.path.join(self.tmpdir, modname + ".py")
        f = open(fn, "w", encoding=encoding)
        try:
            f.write(TEMPLATE % encoding)
            f.write(extra)
        finally:
            f.close()
        __import__(modname)
        del sys.modules[modname]

    def test_file_utf_8(self):
        extra = "z = '\u1234'; assert ord(z) == 0x1234\n"
        self.check_encoding("utf-8", extra)

    def test_file_utf_8_error(self):
        extra = "b'\x80'\n"
        self.assertRaises(SyntaxError, self.check_encoding, "utf-8", extra)

    def test_file_utf8(self):
        self.check_encoding("utf-8")

    def test_file_iso_8859_1(self):
        self.check_encoding("iso-8859-1")

    def test_file_latin_1(self):
        self.check_encoding("latin-1")

    def test_file_latin9(self):
        self.check_encoding("latin9")


if __name__ == "__main__":
    unittest.main()