# Path: databot/utils/html.py
# def get_content(data, errors='strict'):
#     headers = {k.lower(): v for k, v in data.get('headers', {}).items()}
#     content_type_header = headers.get('content-type', '')
#     content_type, params = cgi.parse_header(content_type_header)
#     if content_type.lower() in ('text/html', 'text/xml'):
#         soup = bs4.BeautifulSoup(data['content'], 'lxml', from_encoding=data['encoding'])
#         encoding = get_page_encoding(soup, soup.original_encoding)
#         return data['content'].decode(encoding, errors)
#     elif content_type.startswith('text/'):
#         return data['content'].decode(data['encoding'], errors)
#     else:
#         return data['content']
# 
# Path: databot/expressions/utils.py
# def handler(types=None, item=None, eval_args=True):
#     items = item if item is None or isinstance(item, tuple) else (item,)
# 
#     for item in items:
#         if item not in ('func', 'method', 'attr'):
#             raise ValueError('Unknonw handler item: %r' % item)
# 
#     def decorator(func):
#         if func.__name__ in HANDLERS:
#             raise RuntimeError("Handler %r is already registered." % func.__name__)
# 
#         HANDLERS[func.__name__].append(Handler(func, types, items, eval_args))
# 
#         return func
# 
#     return decorator
# 
# Path: databot/expressions/base.py
# def argrepr(value, nested=False):
#     def __init__(self, handler, args, kwargs):
#     def __str__(self):
#     def __init__(self, stack=(), func=None):
#     def __str__(self):
#     def __eq__(self, other):
#     def __getitem__(self, key):
#     def __getattr__(self, key):
#     def __call__(self, *args, **kwargs):
#     def _add(self, *items):
#     def _reset(self):
#     def _eval_args(self, value, item, eval_args=True):
#     def _eval(self, value, base=None):
# class _HandlerRepr:
# class ExpressionError(Exception):
# class Expression:
# 
# Path: databot/tasks.py
# class Task:
#     pass
# 
# Path: databot/db/utils.py
# class Row(dict):
#     def __init__(self, *args, **kwargs):
#         super().__init__(*args, **kwargs)
#         self.__dict__ = self


import re
import lxml.html
import lxml.etree
import itertools
from cssselect.parser import SelectorSyntaxError
from cssselect.xpath import ExpressionError
from bs4 import BeautifulSoup
from databot.utils.html import get_content
from databot.expressions.utils import handler
from databot.expressions.base import Expression, Func
from databot.tasks import Task
from databot.db.utils import Row
            If check is a string, then if selector did not returned anything, do another query, to double check if we
            are string on the right page and if second query returns nothing, then raise exception.
        """
        self.key = key
        self.value = value
        self.check = check

    def __call__(self, row):
        self.set_row(row)

        if isinstance(self.key, (list, Call, Expression)) and self.value is None:
            return self.check_render(row, self.html, self.key)
        else:
            return [(self.render(row, self.html, self.key), self.check_render(row, self.html, self.value))]

    def set_row(self, row):
        if isinstance(row.value, dict) and 'content' in row.value:
            self.html = create_html_parser(row)
        else:
            self.html = None

    def check_render(self, row, html, value, **kwargs):
        result = self.render(row, html, value, **kwargs)
        if result:
            return result
        elif isinstance(result, list):
            if self.check is False:
                return result
            elif isinstance(self.check, str) and self.select(html, self.check):
                return result

# Path: databot/utils/html.py
# def get_content(data, errors='strict'):
#     headers = {k.lower(): v for k, v in data.get('headers', {}).items()}
#     content_type_header = headers.get('content-type', '')
#     content_type, params = cgi.parse_header(content_type_header)
#     if content_type.lower() in ('text/html', 'text/xml'):
#         soup = bs4.BeautifulSoup(data['content'], 'lxml', from_encoding=data['encoding'])
#         encoding = get_page_encoding(soup, soup.original_encoding)
#         return data['content'].decode(encoding, errors)
#     elif content_type.startswith('text/'):
#         return data['content'].decode(data['encoding'], errors)
#     else:
#         return data['content']
# 
# Path: databot/expressions/utils.py
# def handler(types=None, item=None, eval_args=True):
#     items = item if item is None or isinstance(item, tuple) else (item,)
# 
#     for item in items:
#         if item not in ('func', 'method', 'attr'):
#             raise ValueError('Unknonw handler item: %r' % item)
# 
#     def decorator(func):
#         if func.__name__ in HANDLERS:
#             raise RuntimeError("Handler %r is already registered." % func.__name__)
# 
#         HANDLERS[func.__name__].append(Handler(func, types, items, eval_args))
# 
#         return func
# 
#     return decorator
# 
# Path: databot/expressions/base.py
# def argrepr(value, nested=False):
#     def __init__(self, handler, args, kwargs):
#     def __str__(self):
#     def __init__(self, stack=(), func=None):
#     def __str__(self):
#     def __eq__(self, other):
#     def __getitem__(self, key):
#     def __getattr__(self, key):
#     def __call__(self, *args, **kwargs):
#     def _add(self, *items):
#     def _reset(self):
#     def _eval_args(self, value, item, eval_args=True):
#     def _eval(self, value, base=None):
# class _HandlerRepr:
# class ExpressionError(Exception):
# class Expression:
# 
# Path: databot/tasks.py
# class Task:
#     pass
# 
# Path: databot/db/utils.py
# class Row(dict):
#     def __init__(self, *args, **kwargs):
#         super().__init__(*args, **kwargs)
#         self.__dict__ = self


# Path: databot/handlers/html.py
import re
import lxml.html
import lxml.etree
import itertools
from cssselect.parser import SelectorSyntaxError
from cssselect.xpath import ExpressionError
from bs4 import BeautifulSoup
from databot.utils.html import get_content
from databot.expressions.utils import handler
from databot.expressions.base import Expression, Func
from databot.tasks import Task
from databot.db.utils import Row

            If check is a string, then if selector did not returned anything, do another query, to double check if we
            are string on the right page and if second query returns nothing, then raise exception.
        """
        self.key = key
        self.value = value
        self.check = check

    def __call__(self, row):
        self.set_row(row)

        if isinstance(self.key, (list, Call, Expression)) and self.value is None:
            return self.check_render(row, self.html, self.key)
        else:
            return [(self.render(row, self.html, self.key), self.check_render(row, self.html, self.value))]

    def set_row(self, row):
        if isinstance(row.value, dict) and 'content' in row.value:
            self.html = create_html_parser(row)
        else:
            self.html = None

    def check_render(self, row, html, value, **kwargs):
        result = self.render(row, html, value, **kwargs)
        if result:
            return result
        elif isinstance(result, list):
            if self.check is False:
                return result
            elif isinstance(self.check, str) and self.select(html, self.check):
                return result

            elif isinstance(self.check, Expression) and self.check._eval(row):