<commit_before># -*- coding: utf-8 -*-
from base64 import b64encode
from hashlib import sha1
import os
import socket
import ssl

from ws4py import WS_KEY, WS_VERSION
from ws4py.exc import HandshakeError
from ws4py.websocket import WebSocket
from ws4py.compat import urlsplit, enc, dec

__all__ = ['WebSocketBaseClient']

class WebSocketBaseClient(WebSocket):
    def __init__(self, url, protocols, extensions, heartbeat_freq=None):
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        WebSocket.__init__(self, sock, protocols=protocols, extensions=extensions,
                           heartbeat_freq=heartbeat_freq)
        self.stream.always_mask = True
        self.stream.expect_masking = False
        self.key = b64encode(os.urandom(16))
        self.url = url

        self.host = None
        self.scheme = None
        self.port = None
        self.resource = None

        self._parse_url()

    # Adpated from: https://github.com/liris/websocket-client/blob/master/websocket.py#L105
    def _parse_url(self):
        if ":" not in self.url:
            raise ValueError("Invalid URL: %s" % self.url)

        # Python 2.6.1 and below don't parse ws or wss urls properly. netloc is empty.
        # See: https://github.com/Lawouach/WebSocket-for-Python/issues/59
        scheme, url = self.url.split(":", 1)

        parsed = urlsplit(url, scheme="http")
        if parsed.hostname:
            self.host = parsed.hostname
        else:
            raise ValueError("Invalid hostname from: %s", self.url)

        if parsed.port:
            self.port = parsed.port

        if scheme == "ws":
            if not self.port:
                self.port = 80
        elif scheme == "wss":
            if not self.port:
                self.port = 443
        else:
            raise ValueError("Invalid scheme: %s" % scheme)

        if parsed.path:
            resource = parsed.path
        else:
            resource = "/"

        if parsed.query:
            resource += "?" + parsed.query

        self.scheme = scheme
        self.resource = resource

    def close(self, code=1000, reason=''):
        if not self.client_terminated:
            self.client_terminated = True
            self.sender(self.stream.close(code=code, reason=reason).single(mask=True))

    def connect(self):
        #self.sock.settimeout(3)
        if self.scheme == "wss":
            # default port is now 443; upgrade self.sender to send ssl
            self.sock = ssl.wrap_socket(self.sock)
            self.sender = self.sock.sendall

        self.sock.connect((self.host, int(self.port)))

        self.sender(self.handshake_request)

        response = enc('')
        doubleCLRF = enc('\r\n\r\n')
        while True:
            bytes = self.sock.recv(128)
            if not bytes:
                break
            response += bytes
            if doubleCLRF in response:
                break

        if not response:
            self.close_connection()
            raise HandshakeError("Invalid response")

        headers, _, body = response.partition(doubleCLRF)
        response_line, _, headers = headers.partition(enc('\r\n'))

        try:
            self.process_response_line(response_line)
            self.protocols, self.extensions = self.process_handshake_header(headers)
        except HandshakeError:
            self.close_connection()
            raise

        self.handshake_ok()
        if body:<commit_msg>never liked the pseudo shortcut this was. Also deactivating nagle's algorithm as I believe it doesn't help most of the time with short websocket messages<commit_after># -*- coding: utf-8 -*-
from base64 import b64encode
from hashlib import sha1
import os
import socket
import ssl

from ws4py import WS_KEY, WS_VERSION
from ws4py.exc import HandshakeError
from ws4py.websocket import WebSocket
from ws4py.compat import urlsplit, enc, dec

__all__ = ['WebSocketBaseClient']

class WebSocketBaseClient(WebSocket):
    def __init__(self, url, protocols=None, extensions=None, heartbeat_freq=None):
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        
        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

        WebSocket.__init__(self, sock, protocols=protocols,
                           extensions=extensions,
                           heartbeat_freq=heartbeat_freq)
        self.stream.always_mask = True
        self.stream.expect_masking = False
        self.key = b64encode(os.urandom(16))
        self.url = url

        self.host = None
        self.scheme = None
        self.port = None
        self.resource = None

        self._parse_url()

    # Adpated from: https://github.com/liris/websocket-client/blob/master/websocket.py#L105
    def _parse_url(self):
        if ":" not in self.url:
            raise ValueError("Invalid URL: %s" % self.url)

        # Python 2.6.1 and below don't parse ws or wss urls properly. netloc is empty.
        # See: https://github.com/Lawouach/WebSocket-for-Python/issues/59
        scheme, url = self.url.split(":", 1)

        parsed = urlsplit(url, scheme="http")
        if parsed.hostname:
            self.host = parsed.hostname
        else:
            raise ValueError("Invalid hostname from: %s", self.url)

        if parsed.port:
            self.port = parsed.port

        if scheme == "ws":
            if not self.port:
                self.port = 80
        elif scheme == "wss":
            if not self.port:
                self.port = 443
        else:
            raise ValueError("Invalid scheme: %s" % scheme)

        if parsed.path:
            resource = parsed.path
        else:
            resource = "/"

        if parsed.query:
            resource += "?" + parsed.query

        self.scheme = scheme
        self.resource = resource

    def close(self, code=1000, reason=''):
        if not self.client_terminated:
            self.client_terminated = True
            self.sock.sendall(self.stream.close(code=code, reason=reason).single(mask=True))

    def connect(self):
        #self.sock.settimeout(3)
        if self.scheme == "wss":
            # default port is now 443; upgrade self.sender to send ssl
            self.sock = ssl.wrap_socket(self.sock)
            self.sender = self.sock.sendall

        self.sock.connect((self.host, int(self.port)))

        self.sock.sendall(self.handshake_request)

        response = enc('')
        doubleCLRF = enc('\r\n\r\n')
        while True:
            bytes = self.sock.recv(128)
            if not bytes:
                break
            response += bytes
            if doubleCLRF in response:
                break

        if not response:
            self.close_connection()
            raise HandshakeError("Invalid response")

        headers, _, body = response.partition(doubleCLRF)
        response_line, _, headers = headers.partition(enc('\r\n'))

        try:
            self.process_response_line(response_line)
            self.protocols, self.extensions = self.process_handshake_header(headers)
        except HandshakeError:
            self.close_connection()
            raise

        self.handshake_ok()
        if body:
{'fraction_non_alphanumeric': 0.09893749137574169, 'fraction_numerical': 0.01131502690768594, 'mean_word_length': 1.9987587918907737, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4376131', 'n_tokens_mistral': 2151, 'n_tokens_neox': 2038, 'n_words': 505}