parameters: (cls, in_bytes, main_encoding="utf8",
                    embedded_encoding="windows-1252")
return_statement: return b''.join(byte_chunks)
docstring: Fix characters from one encoding embedded in some other encoding.
  
          Currently the only situation supported is Windows-1252 (or its
          subset ISO-8859-1), embedded in UTF-8.
  
          The input must be a bytestring. If you've already converted
          the document to Unicode, you're too late.
  
          The output is a bytestring in which `embedded_encoding`
          characters have been converted to their `main_encoding`
          equivalents.
docstring_summary: Fix characters from one encoding embedded in some other encoding.
docstring_tokens: Fix characters from one encoding embedded in some other encoding .
function: def detwingle(cls, in_bytes, main_encoding="utf8",
                    embedded_encoding="windows-1252"):
          """Fix characters from one encoding embedded in some other encoding.
  
          Currently the only situation supported is Windows-1252 (or its
          subset ISO-8859-1), embedded in UTF-8.
  
          The input must be a bytestring. If you've already converted
          the document to Unicode, you're too late.
  
          The output is a bytestring in which `embedded_encoding`
          characters have been converted to their `main_encoding`
          equivalents.
          """
          if embedded_encoding.replace('_', '-').lower() not in (
              'windows-1252', 'windows_1252'):
              raise NotImplementedError(
                  "Windows-1252 and ISO-8859-1 are the only currently supported "
                  "embedded encodings.")
  
          if main_encoding.lower() not in ('utf8', 'utf-8'):
              raise NotImplementedError(
                  "UTF-8 is the only currently supported main encoding.")
  
          byte_chunks = []
  
          chunk_start = 0
          pos = 0
          while pos < len(in_bytes):
              byte = in_bytes[pos]
              if not isinstance(byte, int):
                  # Python 2.x
                  byte = ord(byte)
              if (byte >= cls.FIRST_MULTIBYTE_MARKER
                  and byte <= cls.LAST_MULTIBYTE_MARKER):
                  # This is the start of a UTF-8 multibyte character. Skip
                  # to the end.
                  for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES:
                      if byte >= start and byte <= end:
                          pos += size
                          break
              elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8:
                  # We found a Windows-1252 character!
                  # Save the string up to this point as a chunk.
                  byte_chunks.append(in_bytes[chunk_start:pos])
  
                  # Now translate the Windows-1252 character into UTF-8
                  # and add it as another, one-byte chunk.
                  byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte])
                  pos += 1
                  chunk_start = pos
              else:
                  # Go on to the next character.
                  pos += 1
          if chunk_start == 0:
              # The string is unchanged.
              return in_bytes
          else:
              # Store the final chunk.
              byte_chunks.append(in_bytes[chunk_start:])
          return b''.join(byte_chunks)
function_tokens: def detwingle ( cls , in_bytes , main_encoding = "utf8" , embedded_encoding = "windows-1252" ) : if embedded_encoding . replace ( '_' , '-' ) . lower ( ) not in ( 'windows-1252' , 'windows_1252' ) : raise NotImplementedError ( "Windows-1252 and ISO-8859-1 are the only currently supported " "embedded encodings." ) if main_encoding . lower ( ) not in ( 'utf8' , 'utf-8' ) : raise NotImplementedError ( "UTF-8 is the only currently supported main encoding." ) byte_chunks = [ ] chunk_start = 0 pos = 0 while pos < len ( in_bytes ) : byte = in_bytes [ pos ] if not isinstance ( byte , int ) : # Python 2.x byte = ord ( byte ) if ( byte >= cls . FIRST_MULTIBYTE_MARKER and byte <= cls . LAST_MULTIBYTE_MARKER ) : # This is the start of a UTF-8 multibyte character. Skip # to the end. for start , end , size in cls . MULTIBYTE_MARKERS_AND_SIZES : if byte >= start and byte <= end : pos += size break elif byte >= 0x80 and byte in cls . WINDOWS_1252_TO_UTF8 : # We found a Windows-1252 character! # Save the string up to this point as a chunk. byte_chunks . append ( in_bytes [ chunk_start : pos ] ) # Now translate the Windows-1252 character into UTF-8 # and add it as another, one-byte chunk. byte_chunks . append ( cls . WINDOWS_1252_TO_UTF8 [ byte ] ) pos += 1 chunk_start = pos else : # Go on to the next character. pos += 1 if chunk_start == 0 : # The string is unchanged. return in_bytes else : # Store the final chunk. byte_chunks . append ( in_bytes [ chunk_start : ] ) return b'' . join ( byte_chunks )