<commit_before>#!/usr/bin/env python
#
# Copyright 2004-2015, Martian Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import ctypes
import platform
import optparse
import os
import os.path
import Queue
import select
import socket
import struct
import sys
from threading import Condition, Event, Thread

# @author <a href="http://www.martiansoftware.com/contact.html"><NAME></a>
# @author <NAME> (Win32 port)
# @author <NAME> (Python port)
#
# Please try to keep this working on Python 2.6.

NAILGUN_VERSION = '0.9.0'
BUFSIZE = 2048
NAILGUN_PORT_DEFAULT = 2113
CHUNK_HEADER_LEN = 5

CHUNKTYPE_STDIN = '0'
CHUNKTYPE_STDOUT = '1'
CHUNKTYPE_STDERR = '2'
CHUNKTYPE_STDIN_EOF = '.'
CHUNKTYPE_ARG = 'A'
CHUNKTYPE_LONGARG = 'L'
CHUNKTYPE_ENV = 'E'
CHUNKTYPE_DIR = 'D'
CHUNKTYPE_CMD = 'C'
CHUNKTYPE_EXIT = 'X'
CHUNKTYPE_SENDINPUT = 'S'
CHUNKTYPE_HEARTBEAT = 'H'

NSEC_PER_SEC = 1000000000

# 500 ms heartbeat timeout
HEARTBEAT_TIMEOUT_NANOS = NSEC_PER_SEC / 2
HEARTBEAT_TIMEOUT_SECS = HEARTBEAT_TIMEOUT_NANOS / (NSEC_PER_SEC * 1.0)

# We need to support Python 2.6 hosts which lack memoryview().
import __builtin__
HAS_MEMORYVIEW = 'memoryview' in dir(__builtin__)

EVENT_STDIN_CHUNK = 0
EVENT_STDIN_CLOSED = 1
EVENT_STDIN_EXCEPTION = 2

class NailgunException(Exception):
    SOCKET_FAILED = 231
    CONNECT_FAILED = 230
    UNEXPECTED_CHUNKTYPE = 229
    CONNECTION_BROKEN = 227

    def __init__(self, message, code):
        self.message = message
        self.code = code

    def __str__(self):
        return self.message


class Transport(object):
    def close(self):
        raise NotImplementedError()

    def sendall(self, data):
        raise NotImplementedError()

    def recv_into(self, buffer, size=None):
        raise NotImplementedError()

    def select(self, timeout_secs):
        raise NotImplementedError()


class UnixTransport(Transport):
    def __init__(self, __socket):
        self.__socket = __socket
        self.recv_flags = 0
        self.send_flags = 0
        if hasattr(socket, 'MSG_WAITALL'):
            self.recv_flags |= socket.MSG_WAITALL
        if hasattr(socket, 'MSG_NOSIGNAL'):
            self.send_flags |= socket.MSG_NOSIGNAL

    def close(self):
        return self.__socket.close()

    def sendall(self, data):
        result = self.__socket.sendall(data, self.send_flags)
        return result

    def recv_into(self, buffer, nbytes=None):
        return self.__socket.recv_into(buffer, nbytes, self.send_flags)

    def select(self, timeout_secs):
        select_list = {self.__socket}
        readable, _, exceptional = select.select(
            select_list, [], select_list, timeout_secs)
        return (self.__socket in readable), (self.__socket in exceptional)


class NailgunConnection(object):<commit_msg>restoring ng.py to work with python2.6<commit_after>#!/usr/bin/env python
#
# Copyright 2004-2015, Martian Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import ctypes
import platform
import optparse
import os
import os.path
import Queue
import select
import socket
import struct
import sys
from threading import Condition, Event, Thread

# @author <a href="http://www.martiansoftware.com/contact.html"><NAME></a>
# @author <NAME> (Win32 port)
# @author <NAME> (Python port)
#
# Please try to keep this working on Python 2.6.

NAILGUN_VERSION = '0.9.0'
BUFSIZE = 2048
NAILGUN_PORT_DEFAULT = 2113
CHUNK_HEADER_LEN = 5

CHUNKTYPE_STDIN = '0'
CHUNKTYPE_STDOUT = '1'
CHUNKTYPE_STDERR = '2'
CHUNKTYPE_STDIN_EOF = '.'
CHUNKTYPE_ARG = 'A'
CHUNKTYPE_LONGARG = 'L'
CHUNKTYPE_ENV = 'E'
CHUNKTYPE_DIR = 'D'
CHUNKTYPE_CMD = 'C'
CHUNKTYPE_EXIT = 'X'
CHUNKTYPE_SENDINPUT = 'S'
CHUNKTYPE_HEARTBEAT = 'H'

NSEC_PER_SEC = 1000000000

# 500 ms heartbeat timeout
HEARTBEAT_TIMEOUT_NANOS = NSEC_PER_SEC / 2
HEARTBEAT_TIMEOUT_SECS = HEARTBEAT_TIMEOUT_NANOS / (NSEC_PER_SEC * 1.0)

# We need to support Python 2.6 hosts which lack memoryview().
import __builtin__
HAS_MEMORYVIEW = 'memoryview' in dir(__builtin__)

EVENT_STDIN_CHUNK = 0
EVENT_STDIN_CLOSED = 1
EVENT_STDIN_EXCEPTION = 2

class NailgunException(Exception):
    SOCKET_FAILED = 231
    CONNECT_FAILED = 230
    UNEXPECTED_CHUNKTYPE = 229
    CONNECTION_BROKEN = 227

    def __init__(self, message, code):
        self.message = message
        self.code = code

    def __str__(self):
        return self.message


class Transport(object):
    def close(self):
        raise NotImplementedError()

    def sendall(self, data):
        raise NotImplementedError()

    def recv(self, size):
        raise NotImplementedError()

    def recv_into(self, buffer, size=None):
        raise NotImplementedError()

    def select(self, timeout_secs):
        raise NotImplementedError()


class UnixTransport(Transport):
    def __init__(self, __socket):
        self.__socket = __socket
        self.recv_flags = 0
        self.send_flags = 0
        if hasattr(socket, 'MSG_WAITALL'):
            self.recv_flags |= socket.MSG_WAITALL
        if hasattr(socket, 'MSG_NOSIGNAL'):
            self.send_flags |= socket.MSG_NOSIGNAL

    def close(self):
        return self.__socket.close()

    def sendall(self, data):
        result = self.__socket.sendall(data, self.send_flags)
        return result

    def recv(self, nbytes):
        return self.__socket.recv(nbytes, self.recv_flags)

    def recv_into(self, buffer, nbytes=None):
        return self.__socket.recv_into(buffer, nbytes, self.recv_flags)

    def select(self, timeout_secs):
        select_list = [self.__socket]
        readable, _, exceptional = select.select(
            select_list, [], select_list, timeout_secs)
        return (self.__socket in readable), (self.__socket in exceptional)


class NailgunConnection(object):
{'fraction_non_alphanumeric': 0.09104433204403452, 'fraction_numerical': 0.019934543290687296, 'mean_word_length': 3.6817548746518107, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4105993', 'n_tokens_mistral': 2277, 'n_tokens_neox': 2162, 'n_words': 619}