import hashlib
import re

from lxml import etree


def hash_document(document):
    """Returns hash of document"""
    return hashlib.sha1(document).hexdigest()


def remove_tags_from_string(string):
    text = etree.fromstring(string, etree.HTMLParser())
    parsed = ' '.join(text.xpath("//text()"))

    return parsed


def extract_document_text_for_hash(soup):
    body = soup.find('body')
    tags_for_remove = ['style',
                       'form',
                       'input',
                       'label',
                       'textarea',
                       'select',
                       'button',
                       'output',
                       'script']

    classes_for_remove = ['hidden',
                          'hide']

    if body is None:
        body = soup.find('html')

        if body is None:
            document_text = soup.text.encode()
            return document_text

    for tag in soup(tags_for_remove):
        tag.extract()

    for tag in soup.find_all(True, {'class': classes_for_remove}):
        tag.decompose()

    for hidden in soup.find_all(style=re.compile(r'display:\s*none')):
        hidden.decompose()

    document_text = remove_tags_from_string(body.prettify())

    return document_text

import hashlib
import re

from lxml import etree


def hash_document(document):
    """Returns hash of document"""
    return hashlib.sha1(document).hexdigest()


def remove_tags_from_string(string):
    text = etree.fromstring(string, etree.HTMLParser())
    parsed = ' '.join(text.xpath("//text()"))

    return parsed


def extract_document_text_for_hash(soup):
    body = soup.find('body')
    tags_for_remove = ['style',
                       'form',
                       'input',
                       'label',
                       'textarea',
                       'select',
                       'button',
                       'output',
                       'script']

    classes_for_remove = ['hidden',
                          'hide']

    if body is None:
        body = soup.find('html')

        if body is None:
            document_text = soup.text.encode()
            return document_text

    for tag in soup(tags_for_remove):
        tag.extract()

    for tag in soup.find_all(True, {'class': classes_for_remove}):
        tag.decompose()

    for hidden in soup.find_all(style=re.compile(r'display:\s*none')):
        hidden.decompose()

    document_text = remove_tags_from_string(body.text).encode()

    return document_text

Fix problem with encoding