<commit_before>"""
Twisted protocol for <NAME>'s exiftool.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import re
from twisted.internet import defer, error, protocol

try:
    from os import fsencode
except ImportError:
    from twisted_exiftool.compat import fsencode


class ExiftoolProtocol(protocol.Protocol):
    """
    A protocol that talks to <NAME>'s exiftool in batch mode.

    @cvar MAX_LENGTH: The maximum length of a response to allow (If an incoming
                      message is longer than this, the connection is dropped).
                      Default is 65536.
    """

    MAX_LENGTH = 2**16
    _buffer = None
    _pattern = re.compile(b'^{ready([0-9]+)}$', re.MULTILINE)
    _queue = None
    _stopped = None
    _tag = None

    def connectionMade(self):
        """
        Initializes the protocol.
        """
        self._buffer = b''
        self._queue = {}
        self._stopped = None
        self._tag = 0

    def dataReceived(self, data):
        """
        Parses chunks of bytes into responses.

        Whenever a complete response is received, this method extracts its
        payload and calls L{responseReceived} to process it.

        @param data: A chunk of data representing a (possibly partial) response
        @type data: C{bytes}
        """
        l = len(self._buffer) + len(data)
        if (l > self.MAX_LENGTH):
            self.lengthLimitExceeded(l)
        self._buffer += data

        start = 0
        for match in self._pattern.finditer(self._buffer):
            # The start of the sentinel marks the end of the response.
            end = match.start()
            tag = int(match.group(1))
            self.responseReceived(self._buffer[start:end], tag)

            # Advance start position to the beginning of the next line
            start = match.end() + 1

        if start:
            self._buffer = self._buffer[start:]

    def responseReceived(self, response, tag):
        """
        Receives some characters of a netstring.

        Whenever a complete response is received, this method calls the
        deferred associated with it.

        @param response: A complete response generated by exiftool.
        @type response: C{bytes}
        @param tag: The tag associated with the response
        @type tag: C{int}
        """
        self._queue.pop(tag).callback(response)

    def lengthLimitExceeded(self, length):
        """
        Callback invoked when the incomming data would exceed the length limit
        appended to the buffer. The default implementation disconnects the
        transport.

        @param length: The total number of bytes
        @type length: C{int}
        """
        self.transport.loseConnection()

    def execute(self, *args):
        """
        Pass one command to exiftool and return a deferred which is fired as
        soon as the command completes.

        @param *args: Command line arguments passed to exiftool
        @type *args: C{unicode}

        @rtype: C{Deferred}
        @return: A deferred whose callback will be invoked when the command
        completed.
        """
        self._tag += 1

        args = tuple(args) + ('-execute{:d}'.format(self._tag), '')
        safe_args = [fsencode(arg) for arg in args]
        self.transport.write(b'\n'.join(safe_args))

        d = defer.Deferred()
        self._queue[self._tag] = d

        return d


    def loseConnection(self):
        """
        Close the connection and terminate the exiftool process.

        @rtype: C{Deferred}
        @return: A deferred whose callback will be invoked when the connection
        was closed.
        """
        if self._stopped:
            d = self._stopped
        elif self.connected:
            d = defer.Deferred()
            self._stopped = d
            self.transport.write(b'\n'.join((b'-stay_open', b'False', b'')))
        else:
            # Already disconnected.
            d = defer.success(self)

        return d

    def connectionLost(self, reason=protocol.connectionDone):
        """
        Check whether termination was intended and invoke the deferred.

        If the connection terminated unexpectedly, reraise the failure.

        @type reason: L{twisted.python.failure.Failure}
        """
        self.connected = 0
        if self._stopped:
            self._stopped.callback(self if reason.check(error.ConnectionDone) else reason)
            self._stopped = None
        else:
            reason.raiseException()
<commit_msg>Rename variables to make pylint happy<commit_after>"""
Twisted protocol for <NAME>'s exiftool.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import re
from twisted.internet import defer, error, protocol

try:
    from os import fsencode
except ImportError:
    from twisted_exiftool.compat import fsencode


class ExiftoolProtocol(protocol.Protocol):
    """
    A protocol that talks to <NAME>'s exiftool in batch mode.

    @cvar MAX_LENGTH: The maximum length of a response to allow (If an incoming
                      message is longer than this, the connection is dropped).
                      Default is 65536.
    """

    MAX_LENGTH = 2**16
    _buffer = None
    _pattern = re.compile(b'^{ready([0-9]+)}$', re.MULTILINE)
    _queue = None
    _stopped = None
    _tag = None

    def connectionMade(self):
        """
        Initializes the protocol.
        """
        self._buffer = b''
        self._queue = {}
        self._stopped = None
        self._tag = 0

    def dataReceived(self, data):
        """
        Parses chunks of bytes into responses.

        Whenever a complete response is received, this method extracts its
        payload and calls L{responseReceived} to process it.

        @param data: A chunk of data representing a (possibly partial) response
        @type data: C{bytes}
        """
        size = len(self._buffer) + len(data)
        if size > self.MAX_LENGTH:
            self.lengthLimitExceeded(size)
        self._buffer += data

        start = 0
        for match in self._pattern.finditer(self._buffer):
            # The start of the sentinel marks the end of the response.
            end = match.start()
            tag = int(match.group(1))
            self.responseReceived(self._buffer[start:end], tag)

            # Advance start position to the beginning of the next line
            start = match.end() + 1

        if start:
            self._buffer = self._buffer[start:]

    def responseReceived(self, response, tag):
        """
        Receives some characters of a netstring.

        Whenever a complete response is received, this method calls the
        deferred associated with it.

        @param response: A complete response generated by exiftool.
        @type response: C{bytes}
        @param tag: The tag associated with the response
        @type tag: C{int}
        """
        self._queue.pop(tag).callback(response)

    def lengthLimitExceeded(self, length):
        """
        Callback invoked when the incomming data would exceed the length limit
        appended to the buffer. The default implementation disconnects the
        transport.

        @param length: The total number of bytes
        @type length: C{int}
        """
        self.transport.loseConnection()

    def execute(self, *args):
        """
        Pass one command to exiftool and return a deferred which is fired as
        soon as the command completes.

        @param *args: Command line arguments passed to exiftool
        @type *args: C{unicode}

        @rtype: C{Deferred}
        @return: A deferred whose callback will be invoked when the command
        completed.
        """
        self._tag += 1

        args = tuple(args) + ('-execute{:d}'.format(self._tag), '')
        safe_args = [fsencode(arg) for arg in args]
        self.transport.write(b'\n'.join(safe_args))

        result = defer.Deferred()
        self._queue[self._tag] = result

        return result


    def loseConnection(self):
        """
        Close the connection and terminate the exiftool process.

        @rtype: C{Deferred}
        @return: A deferred whose callback will be invoked when the connection
        was closed.
        """
        if self._stopped:
            result = self._stopped
        elif self.connected:
            result = defer.Deferred()
            self._stopped = result
            self.transport.write(b'\n'.join((b'-stay_open', b'False', b'')))
        else:
            # Already disconnected.
            result = defer.success(self)

        return result

    def connectionLost(self, reason=protocol.connectionDone):
        """
        Check whether termination was intended and invoke the deferred.

        If the connection terminated unexpectedly, reraise the failure.

        @type reason: L{twisted.python.failure.Failure}
        """
        self.connected = 0
        if self._stopped:
            result = self if reason.check(error.ConnectionDone) else reason
            self._stopped.callback(result)
            self._stopped = None
        else:
            reason.raiseException()

{'fraction_non_alphanumeric': 0.08516394329618007, 'fraction_numerical': 0.0034628286981928363, 'mean_word_length': 2.0940743220622697, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '492459', 'n_tokens_mistral': 2507, 'n_tokens_neox': 2359, 'n_words': 837}