todo test the following functions pyunicodebuildencodingmap pyunicodefsconverter pyunicodefsdecoder pyunicodedecodembcs pyunicodedecodembcsstateful pyunicodedecodecodepagestateful pyunicodeasmbcsstring pyunicodeencodecodepage pyunicodedecodelocaleandsize pyunicodedecodelocale pyunicodeencodelocale pyunicodedecodefsdefault pyunicodedecodefsdefaultandsize pyunicodeencodefsdefault test pyunicodefromencodedobject fromencodedobject testcapi unicodefromencodedobject self assertequalfromencodedobjectb abc null abc self assertequalfromencodedobjectb abc ascii abc b b axc2xa1xe4xbdxa0xf0x9fx98x80 s axa1u4f60u0001f600 self assertequalfromencodedobjectb null s self assertequalfromencodedobjectb utf8 s self assertequalfromencodedobjectb latin1 b decode latin1 self assertraisesunicodedecodeerror fromencodedobject b ascii self assertequalfromencodedobjectb ascii replace a ufffd 9 self assertequalfromencodedobjectbytearrayb null s self assertequalfromencodedobjectbytearrayb utf8 s self assertraiseslookuperror fromencodedobject b abc foo self assertraiseslookuperror fromencodedobject b ascii foo self assertraisestypeerror fromencodedobject abc null self assertraisestypeerror fromencodedobject abc ascii self assertraisestypeerror fromencodedobject null self assertraisestypeerror fromencodedobject ascii self assertraisessystemerror fromencodedobject null null self assertraisessystemerror fromencodedobject null ascii def testdecodeself todo test pyunicodedecode with null as data and negative size test pyunicodeasencodedstring asencodedstring testcapi unicodeasencodedstring self assertequalasencodedstring abc null b abc self assertequalasencodedstring abc ascii b abc s axa1u4f60u0001f600 b b axc2xa1xe4xbdxa0xf0x9fx98x80 self assertequalasencodedstrings null b self assertequalasencodedstrings utf8 b self assertequalasencodedstring xa1xa2 latin1 b xa1xa2 self assertraisesunicodeencodeerror asencodedstring xa1xa2 ascii self assertequalasencodedstrings ascii replace b a self assertraiseslookuperror asencodedstring abc foo self assertraiseslookuperror asencodedstring s ascii foo self assertraisestypeerror asencodedstring b abc null self assertraisestypeerror asencodedstring b abc ascii self assertraisestypeerror asencodedstring null self assertraisestypeerror asencodedstring ascii crashes asencodedstringnull null crashes asencodedstringnull ascii def testdecodeutf8self todo test pyunicodedecodeutf8 with null as data and negative size test pyunicodedecodeutf8stateful decodeutf8stateful testcapi unicodedecodeutf8stateful for s in abc xa1xa2 u4f60u597d au0001f600 b s encode utf8 self assertequaldecodeutf8statefulb s lenb self assertequaldecodeutf8statefulb strict s lenb self assertraisesunicodedecodeerror decodeutf8stateful b x80 self assertraisesunicodedecodeerror decodeutf8stateful b xc0 self assertraisesunicodedecodeerror decodeutf8stateful b xff self assertequaldecodeutf8statefulb axf0x9f a 1 self assertequaldecodeutf8statefulb axf0x9f replace a 1 self assertraisesunicodedecodeerror decodeutf8stateful b axf0x9fb self assertequaldecodeutf8statefulb axf0x9fb replace aufffdb 4 self assertraiseslookuperror decodeutf8stateful b ax80 foo todo test pyunicodedecodeutf8stateful with null as data and negative size todo test pyunicodedecodeutf8stateful with null as the address of consumed def testasutf8stringself crashes asutf8stringnull test pyunicodedecodeutf16 decodeutf16 testcapi unicodedecodeutf16 naturalbyteorder 1 if sys byteorder little else 1 for s in abc xa1xa2 u4f60u597d au0001f600 b s encode utf16 self assertequaldecodeutf160 b naturalbyteorder s b s encode utf16le self assertequaldecodeutf161 b 1 s self assertequaldecodeutf160 b xffxfe b 1 s b s encode utf16be self assertequaldecodeutf161 b 1 s self assertequaldecodeutf160 b xfexff b 1 s self assertraisesunicodedecodeerror decodeutf16 1 b a self assertraisesunicodedecodeerror decodeutf16 1 b a self assertraisesunicodedecodeerror decodeutf16 0 b xffxfea self assertraisesunicodedecodeerror decodeutf16 0 b xfexffa self assertraisesunicodedecodeerror decodeutf16 1 b x00xde self assertraisesunicodedecodeerror decodeutf16 1 b xdex00 self assertraisesunicodedecodeerror decodeutf16 0 b xdexde self assertequaldecodeutf161 b x00xde replace 1 ufffd self assertequaldecodeutf161 b xdex00 replace 1 ufffd self assertequaldecodeutf160 b xdexde replace 0 ufffd self assertequaldecodeutf160 b xffxfex00xde replace 1 ufffd self assertequaldecodeutf160 b xfexffxdex00 replace 1 ufffd self assertraisesunicodedecodeerror decodeutf16 1 b x3dxd8 self assertraisesunicodedecodeerror decodeutf16 1 b xd8x3d self assertraisesunicodedecodeerror decodeutf16 0 b xd8xd8 self assertequaldecodeutf161 b x3dxd8 replace 1 ufffd self assertequaldecodeutf161 b xd8x3d replace 1 ufffd self assertequaldecodeutf160 b xd8xd8 replace 0 ufffd self assertequaldecodeutf160 b xffxfex3dxd8 replace 1 ufffd self assertequaldecodeutf160 b xfexffxd8x3d replace 1 ufffd self assertraiseslookuperror decodeutf16 1 b x00xde foo self assertraiseslookuperror decodeutf16 1 b xdex00 foo self assertraiseslookuperror decodeutf16 0 b xdexde foo todo test pyunicodedecodeutf16 with null as data and negative size def testdecodeutf16statefulself todo test pyunicodedecodeutf16stateful with null as data and negative size todo test pyunicodedecodeutf16stateful with null as the address of consumed test pyunicodeasutf16string asutf16string testcapi unicodeasutf16string for s in abc xa1xa2 u4f60u597d au0001f600 self assertequalasutf16strings s encode utf16 self assertraisesunicodeencodeerror asutf16string ud8ff self assertraisestypeerror asutf16string b abc self assertraisestypeerror asutf16string crashes asutf16stringnull def testdecodeutf32self todo test pyunicodedecodeutf32 with null as data and negative size test pyunicodedecodeutf32stateful decodeutf32stateful testcapi unicodedecodeutf32stateful naturalbyteorder 1 if sys byteorder little else 1 for s in abc xa1xa2 u4f60u597d au0001f600 b s encode utf32 self assertequaldecodeutf32stateful0 b naturalbyteorder s lenb b s encode utf32le self assertequaldecodeutf32stateful1 b 1 s lenb self assertequaldecodeutf32stateful0 b xffxfex00x00 b 1 s lenb4 b s encode utf32be self assertequaldecodeutf32stateful1 b 1 s lenb self assertequaldecodeutf32stateful0 b x00x00xfexff b 1 s lenb4 self assertequaldecodeutf32stateful1 b x61x00x00x00x00 1 a 4 self assertequaldecodeutf32stateful1 b x61x00x00x00x00xf6 1 a 4 self assertequaldecodeutf32stateful1 b x61x00x00x00x00xf6x01 1 a 4 self assertequaldecodeutf32stateful1 b x00x00x00x61x00 1 a 4 self assertequaldecodeutf32stateful1 b x00x00x00x61x00x01 1 a 4 self assertequaldecodeutf32stateful1 b x00x00x00x61x00x01xf6 1 a 4 self assertequaldecodeutf32stateful0 b xffxfex00x00x61x00x00x00x00xf6x01 1 a 8 self assertequaldecodeutf32stateful0 b x00x00xfexffx00x00x00x61x00x01xf6 1 a 8 for b in b xff b xffxff b xffxffxff self assertequaldecodeutf32stateful1 b 1 0 self assertequaldecodeutf32stateful1 b 1 0 self assertequaldecodeutf32stateful0 b 0 0 self assertequaldecodeutf32stateful0 b xffxfex00x00 b 1 4 self assertequaldecodeutf32stateful0 b x00x00xfexff b 1 4 self assertraisesunicodedecodeerror decodeutf32stateful 1 b xffxffxffxff self assertraisesunicodedecodeerror decodeutf32stateful 1 b xffxffxffxff self assertraisesunicodedecodeerror decodeutf32stateful 0 b xffxffxffxff self assertequaldecodeutf32stateful1 b xffxffxffxff replace 1 ufffd 4 self assertequaldecodeutf32stateful1 b xffxffxffxff replace 1 ufffd 4 self assertequaldecodeutf32stateful0 b xffxffxffxff replace 0 ufffd 4 self assertequaldecodeutf32stateful0 b xffxfex00x00xffxffxffxff replace 1 ufffd 8 self assertequaldecodeutf32stateful0 b x00x00xfexffxffxffxffxff replace 1 ufffd 8 self assertraisesunicodedecodeerror decodeutf32stateful 1 b x3dxd8x00x00 self assertequaldecodeutf32stateful1 b x3dxd8x00x00 replace 1 ufffd 4 self assertraisesunicodedecodeerror decodeutf32stateful 1 b x00x00xd8x3d self assertequaldecodeutf32stateful1 b x00x00xd8x3d replace 1 ufffd 4 self assertraiseslookuperror decodeutf32stateful 1 b xffxffxffxff foo self assertraiseslookuperror decodeutf32stateful 1 b xffxffxffxff foo self assertraiseslookuperror decodeutf32stateful 0 b xffxffxffxff foo todo test pyunicodedecodeutf32stateful with null as data and negative size todo test pyunicodedecodeutf32stateful with null as the address of consumed def testasutf32stringself crashes asutf32stringnull test pyunicodedecodelatin1 decodelatin1 testcapi unicodedecodelatin1 self assertequaldecodelatin1b abc abc self assertequaldecodelatin1b abc strict abc self assertequaldecodelatin1b xa1xa2 xa1xa2 self assertequaldecodelatin1b xa1xa2 strict xa1xa2 todo test pyunicodedecodelatin1 with null as data and negative size def testaslatin1stringself crashes aslatin1stringnull test pyunicodedecodeascii decodeascii testcapi unicodedecodeascii self assertequaldecodeasciib abc abc self assertequaldecodeasciib abc strict abc self assertraisesunicodedecodeerror decodeascii b xff self assertequaldecodeasciib axff replace aufffd self assertequaldecodeasciib axffb replace aufffdb self assertraiseslookuperror decodeascii b axff foo todo test pyunicodedecodeascii with null as data and negative size def testasasciistringself crashes asasciistringnull test pyunicodedecodecharmap decodecharmap testcapi unicodedecodecharmap self assertequaldecodecharmapb 307 0 a 3 b 7 c bac self assertequaldecodecharmapb 102 a b c bac self assertequaldecodecharmapb 102 abc bac self assertequaldecodecharmapb 102 xa1 xa2 xa3 xa2xa1xa3 self assertequaldecodecharmapb 102 u4f60 u597d u4e16 u597du4f60u4e16 self assertequaldecodecharmapb 102 u0001f600 u0001f601 u0001f602 u0001f601u0001f600u0001f602 self assertequaldecodecharmapb 102 97 98 99 bac self assertequaldecodecharmapb 102 b cd bcd self assertraisesunicodedecodeerror decodecharmap b 0 self assertraisesunicodedecodeerror decodecharmap b 0 0 none self assertequaldecodecharmapb 102 none b c replace bufffdc self assertequaldecodecharmapb 102xff null 102xff self assertraisestypeerror decodecharmap b 0 42 todo test pyunicodedecodecharmap with null as data and negative size def testascharmapstringself crashes ascharmapstringnull test pyunicodedecodeunicodeescape decodeunicodeescape testcapi unicodedecodeunicodeescape self assertequaldecodeunicodeescapeb abc abc self assertequaldecodeunicodeescapebr tnrx0bx0cx00 tnrvf0 self assertequaldecodeunicodeescapeb tnrx0bx0cx00 tnrvf0 self assertequaldecodeunicodeescapebr xa1xa2 xa1xa2 self assertequaldecodeunicodeescapeb xa1xa2 xa1xa2 self assertequaldecodeunicodeescapebr u4f60u597d u4f60u597d self assertequaldecodeunicodeescapebr u0001f600 u0001f600 with self assertwarnsdeprecationwarning self assertequaldecodeunicodeescapebr z r z for b in b br xa br u4f6 br u0001f60 self assertraisesunicodedecodeerror decodeunicodeescape b self assertraisesunicodedecodeerror decodeunicodeescape b strict self assertequaldecodeunicodeescapebr xu0001f60 replace xufffd self assertequaldecodeunicodeescapebr xu0001f60y replace xufffdy self assertraiseslookuperror decodeunicodeescape b foo todo test pyunicodedecodeunicodeescape with null as data and negative size def testasunicodeescapestringself crashes asunicodeescapestringnull test pyunicodedecoderawunicodeescape decoderawunicodeescape testcapi unicodedecoderawunicodeescape self assertequaldecoderawunicodeescapeb abc abc self assertequaldecoderawunicodeescapeb tnrvf0 tnrvf0 self assertequaldecoderawunicodeescapeb xa1xa2 xa1xa2 self assertequaldecoderawunicodeescapebr u4f60u597d u4f60u597d self assertequaldecoderawunicodeescapebr u0001f600 u0001f600 self assertequaldecoderawunicodeescapebr xa1xa2 r xa1xa2 self assertequaldecoderawunicodeescapebr z r z for b in br u4f6 br u0001f60 self assertraisesunicodedecodeerror decoderawunicodeescape b self assertraisesunicodedecodeerror decoderawunicodeescape b strict self assertequaldecoderawunicodeescapebr xu0001f60 replace xufffd self assertequaldecoderawunicodeescapebr xu0001f60y replace xufffdy self assertraiseslookuperror decoderawunicodeescape br u0001f60 foo todo test pyunicodedecoderawunicodeescape with null as data and negative size def testasrawunicodeescapestringself crashes asrawunicodeescapestringnull todo test the following functions pyunicode_buildencodingmap pyunicode_fsconverter pyunicode_fsdecoder pyunicode_decodembcs pyunicode_decodembcsstateful pyunicode_decodecodepagestateful pyunicode_asmbcsstring pyunicode_encodecodepage pyunicode_decodelocaleandsize pyunicode_decodelocale pyunicode_encodelocale pyunicode_decodefsdefault pyunicode_decodefsdefaultandsize pyunicode_encodefsdefault test pyunicode_fromencodedobject test pyunicode_decode todo test pyunicode_decode with null as data and negative size test pyunicode_asencodedstring crashes asencodedstring null null crashes asencodedstring null ascii test pyunicode_decodeutf8 todo test pyunicode_decodeutf8 with null as data and negative size test pyunicode_decodeutf8stateful todo test pyunicode_decodeutf8stateful with null as data and negative size todo test pyunicode_decodeutf8stateful with null as the address of consumed test pyunicode_asutf8string crashes asutf8string null test pyunicode_decodeutf16 todo test pyunicode_decodeutf16 with null as data and negative size test pyunicode_decodeutf16stateful todo test pyunicode_decodeutf16stateful with null as data and negative size todo test pyunicode_decodeutf16stateful with null as the address of consumed test pyunicode_asutf16string crashes asutf16string null test pyunicode_decodeutf8 todo test pyunicode_decodeutf32 with null as data and negative size test pyunicode_decodeutf32stateful todo test pyunicode_decodeutf32stateful with null as data and negative size todo test pyunicode_decodeutf32stateful with null as the address of consumed test pyunicode_asutf32string crashes asutf32string null test pyunicode_decodelatin1 todo test pyunicode_decodelatin1 with null as data and negative size test pyunicode_aslatin1string crashes aslatin1string null test pyunicode_decodeascii todo test pyunicode_decodeascii with null as data and negative size test pyunicode_asasciistring crashes asasciistring null test pyunicode_decodecharmap todo test pyunicode_decodecharmap with null as data and negative size test pyunicode_ascharmapstring crashes ascharmapstring null test pyunicode_decodeunicodeescape todo test pyunicode_decodeunicodeescape with null as data and negative size test pyunicode_asunicodeescapestring crashes asunicodeescapestring null test pyunicode_decoderawunicodeescape todo test pyunicode_decoderawunicodeescape with null as data and negative size test pyunicode_asrawunicodeescapestring crashes asrawunicodeescapestring null
import unittest
import sys
from test.support import import_helper

_testcapi = import_helper.import_module('_testcapi')

NULL = None


class CAPITest(unittest.TestCase):
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    

    def test_fromencodedobject(self):
        
        fromencodedobject = _testcapi.unicode_fromencodedobject

        self.assertEqual(fromencodedobject(b'abc', NULL), 'abc')
        self.assertEqual(fromencodedobject(b'abc', 'ascii'), 'abc')
        b = b'a\xc2\xa1\xe4\xbd\xa0\xf0\x9f\x98\x80'
        s = 'a\xa1\u4f60\U0001f600'
        self.assertEqual(fromencodedobject(b, NULL), s)
        self.assertEqual(fromencodedobject(b, 'utf-8'), s)
        self.assertEqual(fromencodedobject(b, 'latin1'), b.decode('latin1'))
        self.assertRaises(UnicodeDecodeError, fromencodedobject, b, 'ascii')
        self.assertEqual(fromencodedobject(b, 'ascii', 'replace'),
                         'a' + '\ufffd'*9)
        self.assertEqual(fromencodedobject(bytearray(b), NULL), s)
        self.assertEqual(fromencodedobject(bytearray(b), 'utf-8'), s)
        self.assertRaises(LookupError, fromencodedobject, b'abc', 'foo')
        self.assertRaises(LookupError, fromencodedobject, b, 'ascii', 'foo')
        self.assertRaises(TypeError, fromencodedobject, 'abc', NULL)
        self.assertRaises(TypeError, fromencodedobject, 'abc', 'ascii')
        self.assertRaises(TypeError, fromencodedobject, [], NULL)
        self.assertRaises(TypeError, fromencodedobject, [], 'ascii')
        self.assertRaises(SystemError, fromencodedobject, NULL, NULL)
        self.assertRaises(SystemError, fromencodedobject, NULL, 'ascii')

    def test_decode(self):
        
        decode = _testcapi.unicode_decode

        self.assertEqual(decode(b'[\xe2\x82\xac]', 'utf-8'), '[\u20ac]')
        self.assertEqual(decode(b'[\xa4]', 'iso8859-15'), '[\u20ac]')
        self.assertEqual(decode(b'[\xa4]', 'iso8859-15', 'strict'), '[\u20ac]')
        self.assertRaises(UnicodeDecodeError, decode, b'[\xa4]', 'utf-8')
        self.assertEqual(decode(b'[\xa4]', 'utf-8', 'replace'), '[\ufffd]')

        self.assertEqual(decode(b'[\xe2\x82\xac]', NULL), '[\u20ac]')
        self.assertEqual(decode(b'[\xa4]', NULL, 'replace'), '[\ufffd]')

        self.assertRaises(LookupError, decode, b'\xa4', 'foo')
        self.assertRaises(LookupError, decode, b'\xa4', 'utf-8', 'foo')
        
        

    def test_asencodedstring(self):
        
        asencodedstring = _testcapi.unicode_asencodedstring

        self.assertEqual(asencodedstring('abc', NULL), b'abc')
        self.assertEqual(asencodedstring('abc', 'ascii'), b'abc')
        s = 'a\xa1\u4f60\U0001f600'
        b = b'a\xc2\xa1\xe4\xbd\xa0\xf0\x9f\x98\x80'
        self.assertEqual(asencodedstring(s, NULL), b)
        self.assertEqual(asencodedstring(s, 'utf-8'), b)
        self.assertEqual(asencodedstring('\xa1\xa2', 'latin1'), b'\xa1\xa2')
        self.assertRaises(UnicodeEncodeError, asencodedstring, '\xa1\xa2', 'ascii')
        self.assertEqual(asencodedstring(s, 'ascii', 'replace'), b'a???')

        self.assertRaises(LookupError, asencodedstring, 'abc', 'foo')
        self.assertRaises(LookupError, asencodedstring, s, 'ascii', 'foo')
        self.assertRaises(TypeError, asencodedstring, b'abc', NULL)
        self.assertRaises(TypeError, asencodedstring, b'abc', 'ascii')
        self.assertRaises(TypeError, asencodedstring, [], NULL)
        self.assertRaises(TypeError, asencodedstring, [], 'ascii')
        
        

    def test_decodeutf8(self):
        
        decodeutf8 = _testcapi.unicode_decodeutf8

        for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
            b = s.encode('utf-8')
            self.assertEqual(decodeutf8(b), s)
            self.assertEqual(decodeutf8(b, 'strict'), s)

        self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80')
        self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0')
        self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff')
        self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f')
        self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd')
        self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb')

        self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo')
        
        

    def test_decodeutf8stateful(self):
        
        decodeutf8stateful = _testcapi.unicode_decodeutf8stateful

        for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
            b = s.encode('utf-8')
            self.assertEqual(decodeutf8stateful(b), (s, len(b)))
            self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b)))

        self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80')
        self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0')
        self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff')
        self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1))
        self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1))
        self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb')
        self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4))

        self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo')
        
        
        
        

    def test_asutf8string(self):
        
        asutf8string = _testcapi.unicode_asutf8string

        for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
            self.assertEqual(asutf8string(s), s.encode('utf-8'))

        self.assertRaises(UnicodeEncodeError, asutf8string, '\ud8ff')
        self.assertRaises(TypeError, asutf8string, b'abc')
        self.assertRaises(TypeError, asutf8string, [])
        

    def test_decodeutf16(self):
        
        decodeutf16 = _testcapi.unicode_decodeutf16

        naturalbyteorder = -1 if sys.byteorder == 'little' else 1
        for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
            b = s.encode('utf-16')
            self.assertEqual(decodeutf16(0, b), (naturalbyteorder, s))
            b = s.encode('utf-16le')
            self.assertEqual(decodeutf16(-1, b), (-1, s))
            self.assertEqual(decodeutf16(0, b'\xff\xfe'+b), (-1, s))
            b = s.encode('utf-16be')
            self.assertEqual(decodeutf16(1, b), (1, s))
            self.assertEqual(decodeutf16(0, b'\xfe\xff'+b), (1, s))

        self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'a')
        self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'a')
        self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xff\xfea')
        self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xfe\xffa')

        self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'\x00\xde')
        self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'\xde\x00')
        self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xde\xde')
        self.assertEqual(decodeutf16(-1, b'\x00\xde', 'replace'), (-1, '\ufffd'))
        self.assertEqual(decodeutf16(1, b'\xde\x00', 'replace'), (1, '\ufffd'))
        self.assertEqual(decodeutf16(0, b'\xde\xde', 'replace'), (0, '\ufffd'))
        self.assertEqual(decodeutf16(0, b'\xff\xfe\x00\xde', 'replace'), (-1, '\ufffd'))
        self.assertEqual(decodeutf16(0, b'\xfe\xff\xde\x00', 'replace'), (1, '\ufffd'))

        self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'\x3d\xd8')
        self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'\xd8\x3d')
        self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xd8\xd8')
        self.assertEqual(decodeutf16(-1, b'\x3d\xd8', 'replace'), (-1, '\ufffd'))
        self.assertEqual(decodeutf16(1, b'\xd8\x3d', 'replace'), (1, '\ufffd'))
        self.assertEqual(decodeutf16(0, b'\xd8\xd8', 'replace'), (0, '\ufffd'))
        self.assertEqual(decodeutf16(0, b'\xff\xfe\x3d\xd8', 'replace'), (-1, '\ufffd'))
        self.assertEqual(decodeutf16(0, b'\xfe\xff\xd8\x3d', 'replace'), (1, '\ufffd'))

        self.assertRaises(LookupError, decodeutf16, -1, b'\x00\xde', 'foo')
        self.assertRaises(LookupError, decodeutf16, 1, b'\xde\x00', 'foo')
        self.assertRaises(LookupError, decodeutf16, 0, b'\xde\xde', 'foo')
        
        

    def test_decodeutf16stateful(self):
        
        decodeutf16stateful = _testcapi.unicode_decodeutf16stateful

        naturalbyteorder = -1 if sys.byteorder == 'little' else 1
        for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
            b = s.encode('utf-16')
            self.assertEqual(decodeutf16stateful(0, b), (naturalbyteorder, s, len(b)))
            b = s.encode('utf-16le')
            self.assertEqual(decodeutf16stateful(-1, b), (-1, s, len(b)))
            self.assertEqual(decodeutf16stateful(0, b'\xff\xfe'+b), (-1, s, len(b)+2))
            b = s.encode('utf-16be')
            self.assertEqual(decodeutf16stateful(1, b), (1, s, len(b)))
            self.assertEqual(decodeutf16stateful(0, b'\xfe\xff'+b), (1, s, len(b)+2))

        self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d'), (-1, 'a', 2))
        self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d\xd8'), (-1, 'a', 2))
        self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d\xd8\x00'), (-1, 'a', 2))
        self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8'), (1, 'a', 2))
        self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8\x3d'), (1, 'a', 2))
        self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8\x3d\xde'), (1, 'a', 2))
        self.assertEqual(decodeutf16stateful(0, b'\xff\xfe\x61\x00\x3d\xd8\x00'), (-1, 'a', 4))
        self.assertEqual(decodeutf16stateful(0, b'\xfe\xff\x00\x61\xd8\x3d\xde'), (1, 'a', 4))

        self.assertRaises(UnicodeDecodeError, decodeutf16stateful, -1, b'\x00\xde')
        self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 1, b'\xde\x00')
        self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 0, b'\xde\xde')
        self.assertEqual(decodeutf16stateful(-1, b'\x00\xde', 'replace'), (-1, '\ufffd', 2))
        self.assertEqual(decodeutf16stateful(1, b'\xde\x00', 'replace'), (1, '\ufffd', 2))
        self.assertEqual(decodeutf16stateful(0, b'\xde\xde', 'replace'), (0, '\ufffd', 2))
        self.assertEqual(decodeutf16stateful(0, b'\xff\xfe\x00\xde', 'replace'), (-1, '\ufffd', 4))
        self.assertEqual(decodeutf16stateful(0, b'\xfe\xff\xde\x00', 'replace'), (1, '\ufffd', 4))

        self.assertRaises(UnicodeDecodeError, decodeutf16stateful, -1, b'\x3d\xd8\x61\x00')
        self.assertEqual(decodeutf16stateful(-1, b'\x3d\xd8\x61\x00', 'replace'), (-1, '\ufffda', 4))
        self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 1, b'\xd8\x3d\x00\x61')
        self.assertEqual(decodeutf16stateful(1, b'\xd8\x3d\x00\x61', 'replace'), (1, '\ufffda', 4))

        self.assertRaises(LookupError, decodeutf16stateful, -1, b'\x00\xde', 'foo')
        self.assertRaises(LookupError, decodeutf16stateful, 1, b'\xde\x00', 'foo')
        self.assertRaises(LookupError, decodeutf16stateful, 0, b'\xde\xde', 'foo')
        
        
        
        

    def test_asutf16string(self):
        
        asutf16string = _testcapi.unicode_asutf16string

        for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
            self.assertEqual(asutf16string(s), s.encode('utf-16'))

        self.assertRaises(UnicodeEncodeError, asutf16string, '\ud8ff')
        self.assertRaises(TypeError, asutf16string, b'abc')
        self.assertRaises(TypeError, asutf16string, [])
        

    def test_decodeutf32(self):
        
        decodeutf32 = _testcapi.unicode_decodeutf32

        naturalbyteorder = -1 if sys.byteorder == 'little' else 1
        for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
            b = s.encode('utf-32')
            self.assertEqual(decodeutf32(0, b), (naturalbyteorder, s))
            b = s.encode('utf-32le')
            self.assertEqual(decodeutf32(-1, b), (-1, s))
            self.assertEqual(decodeutf32(0, b'\xff\xfe\x00\x00'+b), (-1, s))
            b = s.encode('utf-32be')
            self.assertEqual(decodeutf32(1, b), (1, s))
            self.assertEqual(decodeutf32(0, b'\x00\x00\xfe\xff'+b), (1, s))

        self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\x61\x00\x00\x00\x00')
        self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\x00\x00\x00\x61\x00')
        self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\xff\xfe\x00\x00\x61\x00\x00\x00\x00')
        self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\x00\x00\xfe\xff\x00\x00\x00\x61\x00')

        self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\xff\xff\xff\xff')
        self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\xff\xff\xff\xff')
        self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\xff\xff\xff\xff')
        self.assertEqual(decodeutf32(-1, b'\xff\xff\xff\xff', 'replace'), (-1, '\ufffd'))
        self.assertEqual(decodeutf32(1, b'\xff\xff\xff\xff', 'replace'), (1, '\ufffd'))
        self.assertEqual(decodeutf32(0, b'\xff\xff\xff\xff', 'replace'), (0, '\ufffd'))
        self.assertEqual(decodeutf32(0, b'\xff\xfe\x00\x00\xff\xff\xff\xff', 'replace'), (-1, '\ufffd'))
        self.assertEqual(decodeutf32(0, b'\x00\x00\xfe\xff\xff\xff\xff\xff', 'replace'), (1, '\ufffd'))

        self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\x3d\xd8\x00\x00')
        self.assertEqual(decodeutf32(-1, b'\x3d\xd8\x00\x00', 'replace'), (-1, '\ufffd'))
        self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\x00\x00\xd8\x3d')
        self.assertEqual(decodeutf32(1, b'\x00\x00\xd8\x3d', 'replace'), (1, '\ufffd'))

        self.assertRaises(LookupError, decodeutf32, -1, b'\xff\xff\xff\xff', 'foo')
        self.assertRaises(LookupError, decodeutf32, 1, b'\xff\xff\xff\xff', 'foo')
        self.assertRaises(LookupError, decodeutf32, 0, b'\xff\xff\xff\xff', 'foo')
        
        

    def test_decodeutf32stateful(self):
        
        decodeutf32stateful = _testcapi.unicode_decodeutf32stateful

        naturalbyteorder = -1 if sys.byteorder == 'little' else 1
        for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
            b = s.encode('utf-32')
            self.assertEqual(decodeutf32stateful(0, b), (naturalbyteorder, s, len(b)))
            b = s.encode('utf-32le')
            self.assertEqual(decodeutf32stateful(-1, b), (-1, s, len(b)))
            self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00'+b), (-1, s, len(b)+4))
            b = s.encode('utf-32be')
            self.assertEqual(decodeutf32stateful(1, b), (1, s, len(b)))
            self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff'+b), (1, s, len(b)+4))

        self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00'), (-1, 'a', 4))
        self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00\xf6'), (-1, 'a', 4))
        self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00\xf6\x01'), (-1, 'a', 4))
        self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00'), (1, 'a', 4))
        self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00\x01'), (1, 'a', 4))
        self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00\x01\xf6'), (1, 'a', 4))
        self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00\x61\x00\x00\x00\x00\xf6\x01'), (-1, 'a', 8))
        self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff\x00\x00\x00\x61\x00\x01\xf6'), (1, 'a', 8))

        for b in b'\xff', b'\xff\xff', b'\xff\xff\xff':
            self.assertEqual(decodeutf32stateful(-1, b), (-1, '', 0))
            self.assertEqual(decodeutf32stateful(1, b), (1, '', 0))
            self.assertEqual(decodeutf32stateful(0, b), (0, '', 0))
            self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00'+b), (-1, '', 4))
            self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff'+b), (1, '', 4))
        self.assertRaises(UnicodeDecodeError, decodeutf32stateful, -1, b'\xff\xff\xff\xff')
        self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 1, b'\xff\xff\xff\xff')
        self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 0, b'\xff\xff\xff\xff')
        self.assertEqual(decodeutf32stateful(-1, b'\xff\xff\xff\xff', 'replace'), (-1, '\ufffd', 4))
        self.assertEqual(decodeutf32stateful(1, b'\xff\xff\xff\xff', 'replace'), (1, '\ufffd', 4))
        self.assertEqual(decodeutf32stateful(0, b'\xff\xff\xff\xff', 'replace'), (0, '\ufffd', 4))
        self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00\xff\xff\xff\xff', 'replace'), (-1, '\ufffd', 8))
        self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff\xff\xff\xff\xff', 'replace'), (1, '\ufffd', 8))

        self.assertRaises(UnicodeDecodeError, decodeutf32stateful, -1, b'\x3d\xd8\x00\x00')
        self.assertEqual(decodeutf32stateful(-1, b'\x3d\xd8\x00\x00', 'replace'), (-1, '\ufffd', 4))
        self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 1, b'\x00\x00\xd8\x3d')
        self.assertEqual(decodeutf32stateful(1, b'\x00\x00\xd8\x3d', 'replace'), (1, '\ufffd', 4))

        self.assertRaises(LookupError, decodeutf32stateful, -1, b'\xff\xff\xff\xff', 'foo')
        self.assertRaises(LookupError, decodeutf32stateful, 1, b'\xff\xff\xff\xff', 'foo')
        self.assertRaises(LookupError, decodeutf32stateful, 0, b'\xff\xff\xff\xff', 'foo')
        
        
        
        

    def test_asutf32string(self):
        
        asutf32string = _testcapi.unicode_asutf32string

        for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
            self.assertEqual(asutf32string(s), s.encode('utf-32'))

        self.assertRaises(UnicodeEncodeError, asutf32string, '\ud8ff')
        self.assertRaises(TypeError, asutf32string, b'abc')
        self.assertRaises(TypeError, asutf32string, [])
        

    def test_decodelatin1(self):
        
        decodelatin1 = _testcapi.unicode_decodelatin1

        self.assertEqual(decodelatin1(b'abc'), 'abc')
        self.assertEqual(decodelatin1(b'abc', 'strict'), 'abc')
        self.assertEqual(decodelatin1(b'\xa1\xa2'), '\xa1\xa2')
        self.assertEqual(decodelatin1(b'\xa1\xa2', 'strict'), '\xa1\xa2')
        
        

    def test_aslatin1string(self):
        
        aslatin1string = _testcapi.unicode_aslatin1string

        self.assertEqual(aslatin1string('abc'), b'abc')
        self.assertEqual(aslatin1string('\xa1\xa2'), b'\xa1\xa2')

        self.assertRaises(UnicodeEncodeError, aslatin1string, '\u4f60')
        self.assertRaises(TypeError, aslatin1string, b'abc')
        self.assertRaises(TypeError, aslatin1string, [])
        

    def test_decodeascii(self):
        
        decodeascii = _testcapi.unicode_decodeascii

        self.assertEqual(decodeascii(b'abc'), 'abc')
        self.assertEqual(decodeascii(b'abc', 'strict'), 'abc')

        self.assertRaises(UnicodeDecodeError, decodeascii, b'\xff')
        self.assertEqual(decodeascii(b'a\xff', 'replace'), 'a\ufffd')
        self.assertEqual(decodeascii(b'a\xffb', 'replace'), 'a\ufffdb')

        self.assertRaises(LookupError, decodeascii, b'a\xff', 'foo')
        
        

    def test_asasciistring(self):
        
        asasciistring = _testcapi.unicode_asasciistring

        self.assertEqual(asasciistring('abc'), b'abc')

        self.assertRaises(UnicodeEncodeError, asasciistring, '\x80')
        self.assertRaises(TypeError, asasciistring, b'abc')
        self.assertRaises(TypeError, asasciistring, [])
        

    def test_decodecharmap(self):
        
        decodecharmap = _testcapi.unicode_decodecharmap

        self.assertEqual(decodecharmap(b'\3\0\7', {0: 'a', 3: 'b', 7: 'c'}), 'bac')
        self.assertEqual(decodecharmap(b'\1\0\2', ['a', 'b', 'c']), 'bac')
        self.assertEqual(decodecharmap(b'\1\0\2', 'abc'), 'bac')
        self.assertEqual(decodecharmap(b'\1\0\2', ['\xa1', '\xa2', '\xa3']), '\xa2\xa1\xa3')
        self.assertEqual(decodecharmap(b'\1\0\2', ['\u4f60', '\u597d', '\u4e16']), '\u597d\u4f60\u4e16')
        self.assertEqual(decodecharmap(b'\1\0\2', ['\U0001f600', '\U0001f601', '\U0001f602']), '\U0001f601\U0001f600\U0001f602')

        self.assertEqual(decodecharmap(b'\1\0\2', [97, 98, 99]), 'bac')
        self.assertEqual(decodecharmap(b'\1\0\2', ['', 'b', 'cd']), 'bcd')

        self.assertRaises(UnicodeDecodeError, decodecharmap, b'\0', {})
        self.assertRaises(UnicodeDecodeError, decodecharmap, b'\0', {0: None})
        self.assertEqual(decodecharmap(b'\1\0\2', [None, 'b', 'c'], 'replace'), 'b\ufffdc')
        self.assertEqual(decodecharmap(b'\1\0\2\xff', NULL), '\1\0\2\xff')
        self.assertRaises(TypeError, decodecharmap, b'\0', 42)

        
        

    def test_ascharmapstring(self):
        
        ascharmapstring = _testcapi.unicode_ascharmapstring

        self.assertEqual(ascharmapstring('abc', {97: 3, 98: 0, 99: 7}), b'\3\0\7')
        self.assertEqual(ascharmapstring('\xa1\xa2\xa3', {0xa1: 3, 0xa2: 0, 0xa3: 7}), b'\3\0\7')
        self.assertEqual(ascharmapstring('\u4f60\u597d\u4e16', {0x4f60: 3, 0x597d: 0, 0x4e16: 7}), b'\3\0\7')
        self.assertEqual(ascharmapstring('\U0001f600\U0001f601\U0001f602', {0x1f600: 3, 0x1f601: 0, 0x1f602: 7}), b'\3\0\7')
        self.assertEqual(ascharmapstring('abc', {97: 3, 98: b'', 99: b'spam'}), b'\3spam')

        self.assertRaises(UnicodeEncodeError, ascharmapstring, 'a', {})
        self.assertRaises(UnicodeEncodeError, ascharmapstring, 'a', {97: None})
        self.assertRaises(TypeError, ascharmapstring, b'a', {})
        self.assertRaises(TypeError, ascharmapstring, [], {})
        self.assertRaises(TypeError, ascharmapstring, 'a', NULL)
        

    def test_decodeunicodeescape(self):
        
        decodeunicodeescape = _testcapi.unicode_decodeunicodeescape

        self.assertEqual(decodeunicodeescape(b'abc'), 'abc')
        self.assertEqual(decodeunicodeescape(br'\t\n\r\x0b\x0c\x00\\'), '\t\n\r\v\f\0\\')
        self.assertEqual(decodeunicodeescape(b'\t\n\r\x0b\x0c\x00'), '\t\n\r\v\f\0')
        self.assertEqual(decodeunicodeescape(br'\xa1\xa2'), '\xa1\xa2')
        self.assertEqual(decodeunicodeescape(b'\xa1\xa2'), '\xa1\xa2')
        self.assertEqual(decodeunicodeescape(br'\u4f60\u597d'), '\u4f60\u597d')
        self.assertEqual(decodeunicodeescape(br'\U0001f600'), '\U0001f600')
        with self.assertWarns(DeprecationWarning):
            self.assertEqual(decodeunicodeescape(br'\z'), r'\z')

        for b in b'\\', br'\xa', br'\u4f6', br'\U0001f60':
            self.assertRaises(UnicodeDecodeError, decodeunicodeescape, b)
            self.assertRaises(UnicodeDecodeError, decodeunicodeescape, b, 'strict')
        self.assertEqual(decodeunicodeescape(br'x\U0001f60', 'replace'), 'x\ufffd')
        self.assertEqual(decodeunicodeescape(br'x\U0001f60y', 'replace'), 'x\ufffdy')

        self.assertRaises(LookupError, decodeunicodeescape, b'\\', 'foo')
        
        

    def test_asunicodeescapestring(self):
        
        asunicodeescapestring = _testcapi.unicode_asunicodeescapestring

        self.assertEqual(asunicodeescapestring('abc'), b'abc')
        self.assertEqual(asunicodeescapestring('\t\n\r\v\f\0\\'), br'\t\n\r\x0b\x0c\x00\\')
        self.assertEqual(asunicodeescapestring('\xa1\xa2'), br'\xa1\xa2')
        self.assertEqual(asunicodeescapestring('\u4f60\u597d'), br'\u4f60\u597d')
        self.assertEqual(asunicodeescapestring('\U0001f600'), br'\U0001f600')

        self.assertRaises(TypeError, asunicodeescapestring, b'abc')
        self.assertRaises(TypeError, asunicodeescapestring, [])
        

    def test_decoderawunicodeescape(self):
        
        decoderawunicodeescape = _testcapi.unicode_decoderawunicodeescape

        self.assertEqual(decoderawunicodeescape(b'abc'), 'abc')
        self.assertEqual(decoderawunicodeescape(b'\t\n\r\v\f\0\\'), '\t\n\r\v\f\0\\')
        self.assertEqual(decoderawunicodeescape(b'\xa1\xa2'), '\xa1\xa2')
        self.assertEqual(decoderawunicodeescape(br'\u4f60\u597d'), '\u4f60\u597d')
        self.assertEqual(decoderawunicodeescape(br'\U0001f600'), '\U0001f600')
        self.assertEqual(decoderawunicodeescape(br'\xa1\xa2'), r'\xa1\xa2')
        self.assertEqual(decoderawunicodeescape(br'\z'), r'\z')

        for b in br'\u4f6', br'\U0001f60':
            self.assertRaises(UnicodeDecodeError, decoderawunicodeescape, b)
            self.assertRaises(UnicodeDecodeError, decoderawunicodeescape, b, 'strict')
        self.assertEqual(decoderawunicodeescape(br'x\U0001f60', 'replace'), 'x\ufffd')
        self.assertEqual(decoderawunicodeescape(br'x\U0001f60y', 'replace'), 'x\ufffdy')

        self.assertRaises(LookupError, decoderawunicodeescape, br'\U0001f60', 'foo')
        
        

    def test_asrawunicodeescapestring(self):
        
        asrawunicodeescapestring = _testcapi.unicode_asrawunicodeescapestring

        self.assertEqual(asrawunicodeescapestring('abc'), b'abc')
        self.assertEqual(asrawunicodeescapestring('\t\n\r\v\f\0\\'), b'\t\n\r\v\f\0\\')
        self.assertEqual(asrawunicodeescapestring('\xa1\xa2'), b'\xa1\xa2')
        self.assertEqual(asrawunicodeescapestring('\u4f60\u597d'), br'\u4f60\u597d')
        self.assertEqual(asrawunicodeescapestring('\U0001f600'), br'\U0001f600')

        self.assertRaises(TypeError, asrawunicodeescapestring, b'abc')
        self.assertRaises(TypeError, asrawunicodeescapestring, [])
        


if __name__ == "__main__":
    unittest.main()