<commit_before># TODO: test this on windows, this may turn out to be pretty broken :(
# even input() fails under wine

import io
import platform
import queue
import signal
import subprocess
import sys
import threading
import tkinter as tk

from porcupine import tabs, textwidget, utils


if platform.system() == 'Windows':
    _WINDOWS = True
    _INTERRUPT = signal.CTRL_C_EVENT
else:
    _WINDOWS = False
    _INTERRUPT = signal.SIGINT


def _tupleindex(index):
    """Convert 'line.column' to (line, column)."""
    line, column = index.split('.')
    return (int(line), int(column))


class PythonPrompt:

    def __init__(self, textwidget, close_callback):
        self.widget = textwidget
        self.close_callback = close_callback
        self.widget.bind('<Return>', self._on_return)
        self.widget.bind('<Control-c>', self._keyboard_interrupt)
        self.widget.bind('<Control-C>', self._copy)
        self.widget.bind('<Control-l>', self._clear)
        self.widget.bind('<Control-L>', self._clear)
        self.widget.bind('<Control-d>', self._send_eof)
        self.widget.bind('<Control-D>', self._send_eof)

        # without -u python buffers stdout and everything is one enter
        # press late :( see python --help
        self.process = subprocess.Popen(
            [sys.executable, '-i', '-u'], stdin=subprocess.PIPE,
            stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=0)

        # the queuer thread is a daemon thread because it makes exiting
        # porcupine easier and interrupting it isn't a problem
        self._queue = queue.Queue()
        threading.Thread(target=self._queuer, daemon=True).start()
        self.widget.after_idle(self._queue_clearer)

    def _keyboard_interrupt(self, event):
        try:
            self.process.send_signal(_INTERRUPT)
        except ProcessLookupError:
            # the subprocess has terminated, _queue_clearer should have
            # taken care of it already
            assert self.widget['state'] == 'disabled'

    def _copy(self, event):
        # i didn't find a way to do this like tkinter does it by default
        try:
            start, end = self.widget.tag_ranges('sel')
        except ValueError:
            return

        text = self.widget.get(start, end)
        if text:
            self.widget.clipboard_clear()
            self.widget.clipboard_append(text)

    def _clear(self, event):
        self.widget.delete('1.0', 'end-1l')

    def _send_eof(self, event):
        self.process.stdin.close()

    def _on_return(self, event):
        end_of_output = _tupleindex(str(self.widget.tag_ranges('output')[-1]))
        cursor = _tupleindex(self.widget.index('insert'))
        end = _tupleindex(self.widget.index('end-1c'))

        # (line, column) tuples compare nicely
        if not (end_of_output <= cursor <= end):
            return 'break'

        # this needs to return 'break' to allow pressing enter with the
        # cursor anywhere on the line
        text = self.widget.get('%d.%d' % end_of_output, 'end')   # ends with \n
        self.widget.insert('end', '\n')
        self.widget.mark_set('insert', 'end')
        self.process.stdin.write(text.encode('utf-8'))
        self.process.stdin.flush()
        return 'break'

    def _queuer(self):
        while True:
            output = self.process.stdout.read(io.DEFAULT_BUFFER_SIZE)
            if not output:
                # the process terminated, wait() will return the exit
                # code immediately instead of actually waiting
                self._queue.put(('exit', self.process.wait()))
                break
            self._queue.put(('output', output))

    def _queue_clearer(self):
        try:
            state, value = self._queue.get(block=False)<commit_msg>it wasn't wine/windows specific :(<commit_after># TODO: test this on windows, this may turn out to be pretty broken :(
# even input() fails under wine

import io
import platform
import queue
import signal
import subprocess
import sys
import threading
import tkinter as tk

from porcupine import tabs, textwidget, utils


if platform.system() == 'Windows':
    _WINDOWS = True
    _INTERRUPT = signal.CTRL_C_EVENT
else:
    _WINDOWS = False
    _INTERRUPT = signal.SIGINT


def _tupleindex(index):
    """Convert 'line.column' to (line, column)."""
    line, column = index.split('.')
    return (int(line), int(column))


class PythonPrompt:

    def __init__(self, textwidget, close_callback):
        self.widget = textwidget
        self.close_callback = close_callback
        self.widget.bind('<Return>', self._on_return)
        self.widget.bind('<Control-c>', self._keyboard_interrupt)
        self.widget.bind('<Control-C>', self._copy)
        self.widget.bind('<Control-l>', self._clear)
        self.widget.bind('<Control-L>', self._clear)
        self.widget.bind('<Control-d>', self._send_eof)
        self.widget.bind('<Control-D>', self._send_eof)

        # without -u python buffers stdout and everything is one enter
        # press late :( see python --help
        self.process = subprocess.Popen(
            [sys.executable, '-i', '-u'], stdin=subprocess.PIPE,
            stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=0)

        # the queuer thread is a daemon thread because it makes exiting
        # porcupine easier and interrupting it isn't a problem
        self._queue = queue.Queue()
        threading.Thread(target=self._queuer, daemon=True).start()
        self.widget.after_idle(self._queue_clearer)

    def _keyboard_interrupt(self, event):
        try:
            self.process.send_signal(_INTERRUPT)
        except ProcessLookupError:
            # the subprocess has terminated, _queue_clearer should have
            # taken care of it already
            assert self.widget['state'] == 'disabled'

    def _copy(self, event):
        # i didn't find a way to do this like tkinter does it by default
        try:
            start, end = self.widget.tag_ranges('sel')
        except ValueError:
            return

        text = self.widget.get(start, end)
        if text:
            self.widget.clipboard_clear()
            self.widget.clipboard_append(text)

    def _clear(self, event):
        self.widget.delete('1.0', 'end-1l')

    def _send_eof(self, event):
        self.process.stdin.close()

    def _on_return(self, event):
        end_of_output = _tupleindex(str(self.widget.tag_ranges('output')[-1]))
        cursor = _tupleindex(self.widget.index('insert'))
        end = _tupleindex(self.widget.index('end-1c'))

        # (line, column) tuples compare nicely
        if not (end_of_output <= cursor <= end):
            return 'break'

        # this happens when inputting multiple lines at once
        if end_of_output[0] < cursor[0]:
            end_of_output = (cursor[0], 0)

        # this needs to return 'break' to allow pressing enter with the
        # cursor anywhere on the line
        text = self.widget.get('%d.%d' % end_of_output, 'end')   # ends with \n
        self.widget.insert('end', '\n')
        self.widget.mark_set('insert', 'end')
        self.process.stdin.write(text.encode('utf-8'))
        self.process.stdin.flush()
        return 'break'

    def _queuer(self):
        while True:
            output = self.process.stdout.read(io.DEFAULT_BUFFER_SIZE)
            if not output:
                # the process terminated, wait() will return the exit
                # code immediately instead of actually waiting
                self._queue.put(('exit', self.process.wait()))
                break
            self._queue.put(('output', output))

    def _queue_clearer(self):
        try:
            state, value = self._queue.get(block=False)
{'fraction_non_alphanumeric': 0.11490241695747706, 'fraction_numerical': 0.0023264831329972858, 'mean_word_length': 2.466845878136201, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3197372', 'n_tokens_mistral': 2295, 'n_tokens_neox': 2237, 'n_words': 593}