<commit_before>from __future__ import absolute_import
import json
import os

from six.moves.urllib_parse import urlparse

from autobahn.twisted.resource import WebSocketResource
from autobahn.twisted.websocket import (WebSocketServerFactory,
                                        WebSocketServerProtocol)
from weakref import WeakKeyDictionary, WeakValueDictionary
from monotonic import monotonic
from twisted.python import log
from twisted.python.failure import Failure

from scrapy.utils.serialize import ScrapyJSONEncoder
from splash import defaults
from splash import network_manager
from splash.browser_tab import BrowserTab
from splash.render_options import RenderOptions

from PyQt4.QtCore import QObject
from PyQt4.QtCore import pyqtSlot
from PyQt4.QtWebKit import QWebElement

from slybot.spider import IblSpider
from slyd.errors import BaseHTTPError

from .cookies import PortiaCookieJar
from .commands import (load_page, interact_page, close_tab, metadata, resize,
                       resolve, update_project_data, rename_project_data,
                       delete_project_data, pause, resume)
from .css_utils import process_css, wrap_url
import six
text = six.text_type  # unicode in py2, str in py3

_DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'
_DEFAULT_VIEWPORT = '1240x680'


def create_ferry_resource(spec_manager, factory):
    return FerryWebSocketResource(spec_manager, factory)


class PortiaBrowserTab(BrowserTab):
    def set_content(self, data, callback, errback, mime_type=None,
                    baseurl=None):
        self._raw_html = str(data)
        super(PortiaBrowserTab, self).set_content(data, callback, errback,
                                                  mime_type, baseurl)


class FerryWebSocketResource(WebSocketResource):
    def __init__(self, spec_manager, factory):
        self.spec_manager = spec_manager
        settings = spec_manager.settings
        self.settings = spec_manager.settings
        FerryServerProtocol.spec_manager = spec_manager
        FerryServerProtocol.settings = settings
        FerryServerProtocol.assets = factory.assets
        WebSocketResource.__init__(self, factory)

    def render(self, request):
        request.requestHeaders.setRawHeaders('X-Auth-Info',
                                             [json.dumps(request.auth_info)])
        return WebSocketResource.render(self, request)


class User(object):
    _by_id = WeakValueDictionary()

    def __init__(self, auth, tab=None, spider=None, spiderspec=None):
        self.auth = auth
        self.authorized_projects = auth.get('authorized_projects', None)
        self.tab = tab
        self.spider = spider
        self.spiderspec = spiderspec
        self.tabid = id(self)
        User._by_id[self.tabid] = self

    @classmethod
    def findById(cls, tabid):
        return cls._by_id.get(tabid, None)

    @property
    def name(self):
        return self.auth.get('username', '')

    def __getattr__(self, key):
        try:
            return self.auth[key]
        except KeyError:
            name = self.__class__.__name__
            raise AttributeError('"%s" has no attribute "%s"' % (name, key))


class SpiderSpec(object):
    def __init__(self, name, spider, items, extractors):
        self.name = name
        self.spider = spider
        self.items = items
        self.extractors = extractors

    @property
    def templates(self):
        return self.spider['templates']


class PortiaJSApi(QObject):
    def __init__(self, protocol):
        super(PortiaJSApi, self).__init__()
        self.protocol = protocol

    @pyqtSlot(QWebElement)
    def returnElement(self, element):
        """Hack to return an DOM node as a QWebElement instead of
        QVariant(QVariantMap) """
        self.element = element

    def getReturnedElement(self):
        element = self.element
        self.element = None
        return element

    @pyqtSlot('QString', 'QString', result='QString')
    def processCss(self, css, baseuri):
        return process_css(text(css), self.protocol.user.tabid, text(baseuri))

    @pyqtSlot('QString', 'QString', result='QString')
    def wrapUrl(self, url, baseuri):
        return wrap_url(text(url), self.protocol.user.tabid, text(baseuri))

    @pyqtSlot('QString')
    def log(self, s):
        print(s)

    @pyqtSlot('QString')
    def sendMessage(self, message):
        message = text(message)
        try:
            command, data = json.loads(message)
        except ValueError:  # XXX: Possibly null terminated string
            command, data = json.loads(message[:-1])
        self.protocol.sendMessage({
            '_command': command,
            '_data': data
        })
        if command == 'mutation':
            self.protocol.sendMessage(metadata(self.protocol))


class FerryServerProtocol(WebSocketServerProtocol):

    _handlers = {
        'load': load_page,
        'interact': interact_page,
        'close_tab': close_tab,
        'heartbeat': lambda d, s: None,
        'resize': resize,
        'saveChanges': update_project_data,
        'delete': delete_project_data,
        'rename': rename_project_data,
        'resolve': resolve,
        'resume': resume,
        'pause': pause
    }
    spec_manager = None
    settings = None
    assets = './'
    storage = None
<commit_msg>Strip null characters before decoding JSON<commit_after>from __future__ import absolute_import
import json
import os

from six.moves.urllib_parse import urlparse

from autobahn.twisted.resource import WebSocketResource
from autobahn.twisted.websocket import (WebSocketServerFactory,
                                        WebSocketServerProtocol)
from weakref import WeakKeyDictionary, WeakValueDictionary
from monotonic import monotonic
from twisted.python import log
from twisted.python.failure import Failure

from scrapy.utils.serialize import ScrapyJSONEncoder
from splash import defaults
from splash import network_manager
from splash.browser_tab import BrowserTab
from splash.render_options import RenderOptions

from PyQt4.QtCore import QObject
from PyQt4.QtCore import pyqtSlot
from PyQt4.QtWebKit import QWebElement

from slybot.spider import IblSpider
from slyd.errors import BaseHTTPError

from .cookies import PortiaCookieJar
from .commands import (load_page, interact_page, close_tab, metadata, resize,
                       resolve, update_project_data, rename_project_data,
                       delete_project_data, pause, resume)
from .css_utils import process_css, wrap_url
import six
text = six.text_type  # unicode in py2, str in py3

_DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'
_DEFAULT_VIEWPORT = '1240x680'


def create_ferry_resource(spec_manager, factory):
    return FerryWebSocketResource(spec_manager, factory)


class PortiaBrowserTab(BrowserTab):
    def set_content(self, data, callback, errback, mime_type=None,
                    baseurl=None):
        self._raw_html = str(data)
        super(PortiaBrowserTab, self).set_content(data, callback, errback,
                                                  mime_type, baseurl)


class FerryWebSocketResource(WebSocketResource):
    def __init__(self, spec_manager, factory):
        self.spec_manager = spec_manager
        settings = spec_manager.settings
        self.settings = spec_manager.settings
        FerryServerProtocol.spec_manager = spec_manager
        FerryServerProtocol.settings = settings
        FerryServerProtocol.assets = factory.assets
        WebSocketResource.__init__(self, factory)

    def render(self, request):
        request.requestHeaders.setRawHeaders('X-Auth-Info',
                                             [json.dumps(request.auth_info)])
        return WebSocketResource.render(self, request)


class User(object):
    _by_id = WeakValueDictionary()

    def __init__(self, auth, tab=None, spider=None, spiderspec=None):
        self.auth = auth
        self.authorized_projects = auth.get('authorized_projects', None)
        self.tab = tab
        self.spider = spider
        self.spiderspec = spiderspec
        self.tabid = id(self)
        User._by_id[self.tabid] = self

    @classmethod
    def findById(cls, tabid):
        return cls._by_id.get(tabid, None)

    @property
    def name(self):
        return self.auth.get('username', '')

    def __getattr__(self, key):
        try:
            return self.auth[key]
        except KeyError:
            name = self.__class__.__name__
            raise AttributeError('"%s" has no attribute "%s"' % (name, key))


class SpiderSpec(object):
    def __init__(self, name, spider, items, extractors):
        self.name = name
        self.spider = spider
        self.items = items
        self.extractors = extractors

    @property
    def templates(self):
        return self.spider['templates']


class PortiaJSApi(QObject):
    def __init__(self, protocol):
        super(PortiaJSApi, self).__init__()
        self.protocol = protocol

    @pyqtSlot(QWebElement)
    def returnElement(self, element):
        """Hack to return an DOM node as a QWebElement instead of
        QVariant(QVariantMap) """
        self.element = element

    def getReturnedElement(self):
        element = self.element
        self.element = None
        return element

    @pyqtSlot('QString', 'QString', result='QString')
    def processCss(self, css, baseuri):
        return process_css(text(css), self.protocol.user.tabid, text(baseuri))

    @pyqtSlot('QString', 'QString', result='QString')
    def wrapUrl(self, url, baseuri):
        return wrap_url(text(url), self.protocol.user.tabid, text(baseuri))

    @pyqtSlot('QString')
    def log(self, s):
        print(s)

    @pyqtSlot('QString')
    def sendMessage(self, message):
        message = text(message)
        message = message.strip('\x00') # Allocation bug somewhere leaves null characters at the end.
        try:
            command, data = json.loads(message)
        except ValueError as e:
            return log.err(ValueError(
                "%s JSON head: %r tail: %r" % (e.message, message[:100], message[-100:])
            ))
        self.protocol.sendMessage({
            '_command': command,
            '_data': data
        })
        if command == 'mutation':
            self.protocol.sendMessage(metadata(self.protocol))


class FerryServerProtocol(WebSocketServerProtocol):

    _handlers = {
        'load': load_page,
        'interact': interact_page,
        'close_tab': close_tab,
        'heartbeat': lambda d, s: None,
        'resize': resize,
        'saveChanges': update_project_data,
        'delete': delete_project_data,
        'rename': rename_project_data,
        'resolve': resolve,
        'resume': resume,
        'pause': pause
    }
    spec_manager = None
    settings = None
    assets = './'
    storage = None

{'fraction_non_alphanumeric': 0.09085110260616001, 'fraction_numerical': 0.007016584654638236, 'mean_word_length': 2.791018998272884, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5363802', 'n_tokens_mistral': 3213, 'n_tokens_neox': 3094, 'n_words': 775}