r = requests.get(url, headers={'User-Agent': '...'})
is_html = content_type_header.split(';', 1)[0].lower().startswith('text/html')
declared_encoding = UnicodeDammit.find_declared_encoding(r.text, is_html=
    is_html)
encodings_to_try = [r.encoding]
Correctly detect encoding without any guessing when using Beautiful Soup