<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
import os
import base64
import copy
import json
import pprint
import weakref
import functools
from collections import namedtuple
from PyQt4.QtWebKit import QWebPage, QWebSettings, QWebView
from PyQt4.QtCore import (Qt, QUrl, QBuffer, QSize, QTimer, QObject,
                          pyqtSlot, QByteArray)
from PyQt4.QtGui import QPainter, QImage
from PyQt4.QtNetwork import QNetworkRequest
from twisted.internet import defer
from twisted.python import log
from splash import defaults
from splash.qtutils import qurl2ascii, OPERATION_QT_CONSTANTS, qt2py
from splash.har.utils import without_private

from .qwebpage import SplashQWebPage


class BrowserTab(object):
    """
    An object for controlling a single browser tab (QWebView).

    It is created by splash.pool.Pool. Pool attaches to tab's deferred
    and waits until either a callback or an errback is called, then destroys
    a BrowserTab.
    """

    def __init__(self, network_manager, splash_proxy_factory, verbosity, render_options):
        """ Create a new browser tab. """
        self.deferred = defer.Deferred()
        self.network_manager = network_manager
        self.verbosity = verbosity
        self._uid = render_options.get_uid()
        self._closing = False
        self._default_headers = None
        self._active_timers = set()
        self._cancel_on_redirect_timers = weakref.WeakKeyDictionary()  # timer: callback
        self._js_console = None
        self._history = []

        self._init_webpage(verbosity, network_manager, splash_proxy_factory, render_options)
        self._setup_logging(verbosity)

    def _init_webpage(self, verbosity, network_manager, splash_proxy_factory, render_options):
        """ Create and initialize QWebPage and QWebView """
        self.web_page = SplashQWebPage(verbosity)
        self.web_page.setNetworkAccessManager(network_manager)
        self.web_page.splash_proxy_factory = splash_proxy_factory
        self.web_page.render_options = render_options

        self._set_default_webpage_options(self.web_page)
        self._listen_to_urlchanges()

        self.web_view = QWebView()
        self.web_view.setPage(self.web_page)
        self.web_view.setAttribute(Qt.WA_DeleteOnClose, True)

    def _set_default_webpage_options(self, web_page):
        """
        Set QWebPage options.
        TODO: allow to customize them.
        """
        settings = web_page.settings()
        settings.setAttribute(QWebSettings.JavascriptEnabled, True)
        settings.setAttribute(QWebSettings.PluginsEnabled, False)
        settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
        settings.setAttribute(QWebSettings.LocalStorageEnabled, True)
        settings.setAttribute(QWebSettings.LocalContentCanAccessRemoteUrls, True)
        web_page.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff)
        web_page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)

    def _setup_logging(self, verbosity):
        """ Setup logging of various events """
        self.logger = _BrowserTabLogger(
            uid=self._uid,
            web_page=self.web_page,
            verbosity=verbosity,
        )
        self.logger.enable()

    def _listen_to_urlchanges(self):
        self.web_page.mainFrame().urlChanged.connect(self._on_url_changed)

    def return_result(self, result):
        """ Return a result to the Pool. """
        if self.result_already_returned():
            self.logger.log("error: result is already returned", min_level=1)

        self.deferred.callback(result)
        # self.deferred = None

    def return_error(self, error=None):
        """ Return an error to the Pool. """
        if self.result_already_returned():
            self.logger.log("error: result is already returned", min_level=1)
        self.deferred.errback(error)
        # self.deferred = None

    def result_already_returned(self):
        """ Return True if an error or a result is already returned to Pool """
        return self.deferred.called

    def set_default_headers(self, headers):
        """ Set default HTTP headers """
        self._default_headers = headers

    def set_images_enabled(self, enabled):
        self.web_page.settings().setAttribute(QWebSettings.AutoLoadImages, enabled)

    def set_viewport(self, size):
        """
        Set viewport size.
        If size is "full" viewport size is detected automatically.
        If can also be "<width>x<height>".
        """
        if size == 'full':
            size = self.web_page.mainFrame().contentsSize()
            if size.isEmpty():
                self.logger.log("contentsSize method doesn't work %s", min_level=1)
                size = defaults.VIEWPORT_FALLBACK

        if not isinstance(size, QSize):
            w, h = map(int, size.split('x'))
            size = QSize(w, h)

        self.web_page.setViewportSize(size)
        w, h = int(size.width()), int(size.height())
        self.logger.log("viewport size is set to %sx%s" % (w, h), min_level=2)
        return w, h

    def go(self, url, callback, errback, baseurl=None, http_method='GET', body=None):
        """
        Go to an URL. This is similar to entering an URL in
        address tab and pressing Enter.
        """

        # TODO / FIXME: cancel previous goto request.
        # We must call errback from previous goto because multiple
        # webpages per browser tab are not supported.
        # Does it happen automatically?

        self.store_har_timing("_onStarted")

        if baseurl:
            # If baseurl is used, we download the page manually,
            # then set its contents to the QWebPage and let it
            # download related resources and render the result.
            if http_method != 'GET':
                raise NotImplementedError()

            request = self._create_request(url)
            request.setOriginatingObject(self.web_page.mainFrame())

            # TODO / FIXME: add support for multiple replies
            # or discard/cancel previous replies
            self._reply = self.network_manager.get(request)

            cb = functools.partial(
                self._on_baseurl_request_finished,
                callback=callback,
                errback=errback,
                baseurl=baseurl,
                url=url,
            )
            self._reply.finished.connect(cb)
        else:
            cb = functools.partial(
                self._on_goto_load_finished,
                callback=callback,
                errback=errback,
            )
            self.web_page.loadFinished.connect(cb)
            self._load_url_to_mainframe(url, http_method, body)

    def stop_loading(self):
        """
        Stop loading of the current page and all pending page
        refresh/redirect requests.
        """
        self.web_view.pageAction(QWebPage.StopScheduledPageRefresh)<commit_msg>log stop_loading<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
import os
import base64
import copy
import json
import pprint
import weakref
import functools
from collections import namedtuple
from PyQt4.QtWebKit import QWebPage, QWebSettings, QWebView
from PyQt4.QtCore import (Qt, QUrl, QBuffer, QSize, QTimer, QObject,
                          pyqtSlot, QByteArray)
from PyQt4.QtGui import QPainter, QImage
from PyQt4.QtNetwork import QNetworkRequest
from twisted.internet import defer
from twisted.python import log
from splash import defaults
from splash.qtutils import qurl2ascii, OPERATION_QT_CONSTANTS, qt2py
from splash.har.utils import without_private

from .qwebpage import SplashQWebPage


class BrowserTab(object):
    """
    An object for controlling a single browser tab (QWebView).

    It is created by splash.pool.Pool. Pool attaches to tab's deferred
    and waits until either a callback or an errback is called, then destroys
    a BrowserTab.
    """

    def __init__(self, network_manager, splash_proxy_factory, verbosity, render_options):
        """ Create a new browser tab. """
        self.deferred = defer.Deferred()
        self.network_manager = network_manager
        self.verbosity = verbosity
        self._uid = render_options.get_uid()
        self._closing = False
        self._default_headers = None
        self._active_timers = set()
        self._cancel_on_redirect_timers = weakref.WeakKeyDictionary()  # timer: callback
        self._js_console = None
        self._history = []

        self._init_webpage(verbosity, network_manager, splash_proxy_factory, render_options)
        self._setup_logging(verbosity)

    def _init_webpage(self, verbosity, network_manager, splash_proxy_factory, render_options):
        """ Create and initialize QWebPage and QWebView """
        self.web_page = SplashQWebPage(verbosity)
        self.web_page.setNetworkAccessManager(network_manager)
        self.web_page.splash_proxy_factory = splash_proxy_factory
        self.web_page.render_options = render_options

        self._set_default_webpage_options(self.web_page)
        self._listen_to_urlchanges()

        self.web_view = QWebView()
        self.web_view.setPage(self.web_page)
        self.web_view.setAttribute(Qt.WA_DeleteOnClose, True)

    def _set_default_webpage_options(self, web_page):
        """
        Set QWebPage options.
        TODO: allow to customize them.
        """
        settings = web_page.settings()
        settings.setAttribute(QWebSettings.JavascriptEnabled, True)
        settings.setAttribute(QWebSettings.PluginsEnabled, False)
        settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
        settings.setAttribute(QWebSettings.LocalStorageEnabled, True)
        settings.setAttribute(QWebSettings.LocalContentCanAccessRemoteUrls, True)
        web_page.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff)
        web_page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)

    def _setup_logging(self, verbosity):
        """ Setup logging of various events """
        self.logger = _BrowserTabLogger(
            uid=self._uid,
            web_page=self.web_page,
            verbosity=verbosity,
        )
        self.logger.enable()

    def _listen_to_urlchanges(self):
        self.web_page.mainFrame().urlChanged.connect(self._on_url_changed)

    def return_result(self, result):
        """ Return a result to the Pool. """
        if self.result_already_returned():
            self.logger.log("error: result is already returned", min_level=1)

        self.deferred.callback(result)
        # self.deferred = None

    def return_error(self, error=None):
        """ Return an error to the Pool. """
        if self.result_already_returned():
            self.logger.log("error: result is already returned", min_level=1)
        self.deferred.errback(error)
        # self.deferred = None

    def result_already_returned(self):
        """ Return True if an error or a result is already returned to Pool """
        return self.deferred.called

    def set_default_headers(self, headers):
        """ Set default HTTP headers """
        self._default_headers = headers

    def set_images_enabled(self, enabled):
        self.web_page.settings().setAttribute(QWebSettings.AutoLoadImages, enabled)

    def set_viewport(self, size):
        """
        Set viewport size.
        If size is "full" viewport size is detected automatically.
        If can also be "<width>x<height>".
        """
        if size == 'full':
            size = self.web_page.mainFrame().contentsSize()
            if size.isEmpty():
                self.logger.log("contentsSize method doesn't work %s", min_level=1)
                size = defaults.VIEWPORT_FALLBACK

        if not isinstance(size, QSize):
            w, h = map(int, size.split('x'))
            size = QSize(w, h)

        self.web_page.setViewportSize(size)
        w, h = int(size.width()), int(size.height())
        self.logger.log("viewport size is set to %sx%s" % (w, h), min_level=2)
        return w, h

    def go(self, url, callback, errback, baseurl=None, http_method='GET', body=None):
        """
        Go to an URL. This is similar to entering an URL in
        address tab and pressing Enter.
        """

        # TODO / FIXME: cancel previous goto request.
        # We must call errback from previous goto because multiple
        # webpages per browser tab are not supported.
        # Does it happen automatically?

        self.store_har_timing("_onStarted")

        if baseurl:
            # If baseurl is used, we download the page manually,
            # then set its contents to the QWebPage and let it
            # download related resources and render the result.
            if http_method != 'GET':
                raise NotImplementedError()

            request = self._create_request(url)
            request.setOriginatingObject(self.web_page.mainFrame())

            # TODO / FIXME: add support for multiple replies
            # or discard/cancel previous replies
            self._reply = self.network_manager.get(request)

            cb = functools.partial(
                self._on_baseurl_request_finished,
                callback=callback,
                errback=errback,
                baseurl=baseurl,
                url=url,
            )
            self._reply.finished.connect(cb)
        else:
            cb = functools.partial(
                self._on_goto_load_finished,
                callback=callback,
                errback=errback,
            )
            self.web_page.loadFinished.connect(cb)
            self._load_url_to_mainframe(url, http_method, body)

    def stop_loading(self):
        """
        Stop loading of the current page and all pending page
        refresh/redirect requests.
        """
        self.logger.log("stop_loading", min_level=2)
        self.web_view.pageAction(QWebPage.StopScheduledPageRefresh)
{'fraction_non_alphanumeric': 0.08435802116099514, 'fraction_numerical': 0.0019302259079210752, 'mean_word_length': 2.5487062404870624, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7111205', 'n_tokens_mistral': 3954, 'n_tokens_neox': 3828, 'n_words': 978}