<commit_before>#!/usr/bin/python
#
# Urwid raw display module
#    Copyright (C) 2004-2009  <NAME>
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# Urwid web site: http://excess.org/urwid/

"""
Direct terminal UI implementation
"""

import fcntl
import termios
import os
import select
import struct
import sys
import tty
import signal

from urwid import util
from urwid import escape
from urwid.display_common import BaseScreen, RealTerminal, \
    UPDATE_PALETTE_ENTRY, AttrSpec, UNPRINTABLE_TRANS_TABLE, \
    INPUT_DESCRIPTORS_CHANGED
from urwid import signals
from urwid.compat import PYTHON3, bytes, B

from subprocess import Popen, PIPE


class Screen(BaseScreen, RealTerminal):
    def __init__(self):
        """Initialize a screen that directly prints escape codes to an output
        terminal.
        """
        super(Screen, self).__init__()
        self._pal_escape = {}
        signals.connect_signal(self, UPDATE_PALETTE_ENTRY, 
            self._on_update_palette_entry)
        self.colors = 16 # FIXME: detect this
        self.has_underline = True # FIXME: detect this
        self.register_palette_entry( None, 'default','default')
        self._keyqueue = []
        self.prev_input_resize = 0
        self.set_input_timeouts()
        self.screen_buf = None
        self._screen_buf_canvas = None
        self._resized = False
        self.maxrow = None
        self.gpm_mev = None
        self.gpm_event_pending = False
        self.last_bstate = 0
        self._setup_G1_done = False
        self._rows_used = None
        self._cy = 0
        self.bright_is_bold = os.environ.get('TERM',None) != "xterm"
        self._next_timeout = None
        self._term_output_file = sys.stdout
        self._term_input_file = sys.stdin
        # pipe for signalling external event loops about resize events
        self._resize_pipe_rd, self._resize_pipe_wr = os.pipe()
        fcntl.fcntl(self._resize_pipe_rd, fcntl.F_SETFL, os.O_NONBLOCK)

    def _on_update_palette_entry(self, name, *attrspecs):
        # copy the attribute to a dictionary containing the escape seqences
        self._pal_escape[name] = self._attrspec_to_escape(
            attrspecs[{16:0,1:1,88:2,256:3}[self.colors]])

    def set_input_timeouts(self, max_wait=None, complete_wait=0.125, 
        resize_wait=0.125):
        """
        Set the get_input timeout values.  All values are in floating
        point numbers of seconds.
        
        max_wait -- amount of time in seconds to wait for input when
            there is no input pending, wait forever if None
        complete_wait -- amount of time in seconds to wait when
            get_input detects an incomplete escape sequence at the
            end of the available input
        resize_wait -- amount of time in seconds to wait for more input
            after receiving two screen resize requests in a row to
            stop Urwid from consuming 100% cpu during a gradual
            window resize operation
        """
        self.max_wait = max_wait
        if max_wait is not None:
            if self._next_timeout is None:
                self._next_timeout = max_wait
            else:
                self._next_timeout = min(self._next_timeout, self.max_wait)
        self.complete_wait = complete_wait
        self.resize_wait = resize_wait

    def _sigwinch_handler(self, signum, frame):
        if not self._resized:
            os.write(self._resize_pipe_wr, B('R'))
        self._resized = True
        self.screen_buf = None
      
    def signal_init(self):
        """
        Called in the startup of run wrapper to set the SIGWINCH 
        signal handler to self._sigwinch_handler.

        Override this function to call from main thread in threaded
        applications.
        """
        signal.signal(signal.SIGWINCH, self._sigwinch_handler)
    
    def signal_restore(self):
        """
        Called in the finally block of run wrapper to restore the
        SIGWINCH handler to the default handler.

        Override this function to call from main thread in threaded
        applications.
        """
        signal.signal(signal.SIGWINCH, signal.SIG_DFL)
      
    def set_mouse_tracking(self):
        """
        Enable mouse tracking.  
        
        After calling this function get_input will include mouse
        click events along with keystrokes.
        """
        self._term_output_file.write(escape.MOUSE_TRACKING_ON)

        self._start_gpm_tracking()
    
    def _start_gpm_tracking(self):
        if not os.path.isfile("/usr/bin/mev"):
            return
        if not os.environ.get('TERM',"").lower().startswith("linux"):
            return
        if not Popen:
            return
        m = Popen(["/usr/bin/mev","-e","158"], stdin=PIPE, stdout=PIPE,
            close_fds=True)
        fcntl.fcntl(m.stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
        self.gpm_mev = m
    
    def _stop_gpm_tracking(self):
        os.kill(self.gpm_mev.pid, signal.SIGINT)
        os.waitpid(self.gpm_mev.pid, 0)
        self.gpm_mev = None
    
    def start(self, alternate_buffer=True):
        """
        Initialize the screen and input mode.

        alternate_buffer -- use alternate screen buffer
        """
        assert not self._started
        if alternate_buffer:
            self._term_output_file.write(escape.SWITCH_TO_ALTERNATE_BUFFER)
            self._rows_used = None
        else:
            self._rows_used = 0

        fd = self._term_input_file.fileno()
        if os.isatty(fd):
            self._old_termios_settings = termios.tcgetattr(fd)
            tty.setcbreak(fd)

        self.signal_init()
        self._alternate_buffer = alternate_buffer
        self._input_iter = self._run_input_iter()
        self._next_timeout = self.max_wait

        if not self._signal_keys_set:
            self._old_signal_keys = self.tty_signal_keys(fileno=fd)

        super(Screen, self).start()

        signals.emit_signal(self, INPUT_DESCRIPTORS_CHANGED)


    def stop(self):
        """
        Restore the screen.
        """
        self.clear()
        if not self._started:
            return

        signals.emit_signal(self, INPUT_DESCRIPTORS_CHANGED)

        self.signal_restore()

        fd = self._term_input_file.fileno()
        if os.isatty(fd):
            termios.tcsetattr(fd, termios.TCSADRAIN,
        self._old_termios_settings)

        move_cursor = ""
        if self.gpm_mev:
            self._stop_gpm_tracking()
        if self._alternate_buffer:
            move_cursor = escape.RESTORE_NORMAL_BUFFER
        elif self.maxrow is not None:
            move_cursor = escape.set_cursor_position( 
                0, self.maxrow)
        self._term_output_file.write(self._attrspec_to_escape(AttrSpec('','')) 
            + escape.SI
            + escape.MOUSE_TRACKING_OFF
            + escape.SHOW_CURSOR
            + move_cursor + "\n" + escape.SHOW_CURSOR )
        self._input_iter = self._fake_input_iter()

        if self._old_signal_keys:
            self.tty_signal_keys(*(self._old_signal_keys + (fd,)))

        super(Screen, self).stop()


    def run_wrapper(self, fn, alternate_buffer=True):
        """
        Call start to initialize screen, then call fn.  
        When fn exits call stop to restore the screen to normal.

        alternate_buffer -- use alternate screen buffer and restore
            normal screen buffer on exit
        """
        try:
            self.start(alternate_buffer)
            return fn()
        finally:
            self.stop()

    def get_input(self, raw_keys=False):
        """Return pending input as a list.

        raw_keys -- return raw keycodes as well as translated versions

        This function will immediately return all the input since the
        last time it was called.  If there is no input pending it will
        wait before returning an empty list.  The wait time may be
        configured with the set_input_timeouts function.

        If raw_keys is False (default) this function will return a list
        of keys pressed.  If raw_keys is True this function will return
        a ( keys pressed, raw keycodes ) tuple instead.

        Examples of keys returned:

        * ASCII printable characters:  " ", "a", "0", "A", "-", "/" 
        * ASCII control characters:  "tab", "enter"
        * Escape sequences:  "up", "page up", "home", "insert", "f1"
        * Key combinations:  "shift f1", "meta a", "ctrl b"
        * Window events:  "window resize"

        When a narrow encoding is not enabled:

        * "Extended ASCII" characters:  "\\xa1", "\\xb2", "\\xfe"

        When a wide encoding is enabled:

        * Double-byte characters:  "\\xa1\\xea", "\\xb2\\xd4"

        When utf8 encoding is enabled:

        * Unicode characters: u"\\u00a5", u'\\u253c"

        Examples of mouse events returned:

        * Mouse button press: ('mouse press', 1, 15, 13), 
                              ('meta mouse press', 2, 17, 23)
        * Mouse drag: ('mouse drag', 1, 16, 13),
                      ('mouse drag', 1, 17, 13),
                      ('ctrl mouse drag', 1, 18, 13)
        * Mouse button release: ('mouse release', 0, 18, 13),
                                ('ctrl mouse release', 0, 17, 23)
        """
        assert self._started

        self._wait_for_input_ready(self._next_timeout)
        self._next_timeout, keys, raw = self._input_iter.next()

        # Avoid pegging CPU at 100% when slowly resizing
        if keys==['window resize'] and self.prev_input_resize:
            while True:
                self._wait_for_input_ready(self.resize_wait)
                self._next_timeout, keys, raw2 = \
                    self._input_iter.next()
                raw += raw2
                #if not keys:
                #    keys, raw2 = self._get_input( 
                #        self.resize_wait)
                #    raw += raw2
                if keys!=['window resize']:
                    break
            if keys[-1:]!=['window resize']:
                keys.append('window resize')

        if keys==['window resize']:
            self.prev_input_resize = 2
        elif self.prev_input_resize == 2 and not keys:
            self.prev_input_resize = 1
        else:
            self.prev_input_resize = 0

        if raw_keys:
            return keys, raw
        return keys

    def get_input_descriptors(self):
        """
        Return a list of integer file descriptors that should be
        polled in external event loops to check for user input.

        Use this method if you are implementing yout own event loop.
        """
        if not self._started:
            return []

        fd_list = [self._term_input_file.fileno(), self._resize_pipe_rd]
        if self.gpm_mev is not None:
            fd_list.append(self.gpm_mev.stdout.fileno())
        return fd_list

    def get_input_nonblocking(self):
        """
        Return a (next_input_timeout, keys_pressed, raw_keycodes)
        tuple.

        Use this method if you are implementing your own event loop.

        When there is input waiting on one of the descriptors returned
        by get_input_descriptors() this method should be called to
        read and process the input.

        This method expects to be called in next_input_timeout seconds
        (a floating point number) if there is no input waiting.
        """
        return self._input_iter.next()

    def _run_input_iter(self):
        def empty_resize_pipe():
            # clean out the pipe used to signal external event loops
            # that a resize has occured
            try:
                while True: os.read(self._resize_pipe_rd, 1)
            except OSError:
                pass

        while True:
            processed = []
            codes = self._get_gpm_codes() + \
                self._get_keyboard_codes()

            original_codes = codes
            try:
                while codes:
                    run, codes = escape.process_keyqueue(
                        codes, True)
                    processed.extend(run)
            except escape.MoreInputRequired:
                k = len(original_codes) - len(codes)
                yield (self.complete_wait, processed,
                    original_codes[:k])
                empty_resize_pipe()
                original_codes = codes
                processed = []

                codes += self._get_keyboard_codes() + \
                    self._get_gpm_codes()
                while codes:
                    run, codes = escape.process_keyqueue(
                        codes, False)
                    processed.extend(run)
            
            if self._resized:
                processed.append('window resize')
                self._resized = False

            yield (self.max_wait, processed, original_codes)
            empty_resize_pipe()

    def _fake_input_iter(self):
        """
        This generator is a placeholder for when the screen is stopped
        to always return that no input is available.
        """
        while True:
            yield (self.max_wait, [], [])

    def _get_keyboard_codes(self):
        codes = []
        while True:
            code = self._getch_nodelay()
            if code < 0:
                break
            codes.append(code)
        return codes

    def _get_gpm_codes(self):
        codes = []
        try:
            while self.gpm_mev is not None and self.gpm_event_pending:
                codes.extend(self._encode_gpm_event())
        except IOError, e:
            if e.args[0] != 11:
                raise
        return codes

    def _wait_for_input_ready(self, timeout):
        ready = None
        fd_list = [self._term_input_file.fileno()]
        if self.gpm_mev is not None:
            fd_list.append(self.gpm_mev.stdout.fileno())
        while True:
            try:
                if timeout is None:
                    ready,w,err = select.select(
                        fd_list, [], fd_list)
                else:
                    ready,w,err = select.select(
                        fd_list,[],fd_list, timeout)
                break
            except select.error, e:
                if e.args[0] != 4: 
                    raise
                if self._resized:
                    ready = []
                    break
        return ready    
        
    def _getch(self, timeout):
        ready = self._wait_for_input_ready(timeout)
        if self.gpm_mev is not None:
            if self.gpm_mev.stdout.fileno() in ready:
                self.gpm_event_pending = True
        if self._term_input_file.fileno() in ready:
            return ord(os.read(self._term_input_file.fileno(), 1))
        return -1
    
    def _encode_gpm_event( self ):
        self.gpm_event_pending = False
        s = self.gpm_mev.stdout.readline().decode('ascii')
        l = s.split(",")
        if len(l) != 6:
            # unexpected output, stop tracking
            self._stop_gpm_tracking()
            signals.emit_signal(self, INPUT_DESCRIPTORS_CHANGED)
            return []
        ev, x, y, ign, b, m = s.split(",")
        ev = int( ev.split("x")[-1], 16)
        x = int( x.split(" ")[-1] )
        y = int( y.lstrip().split(" ")[0] )
        b = int( b.split(" ")[-1] )
        m = int( m.split("x")[-1].rstrip(), 16 )

        # convert to xterm-like escape sequence

        last = next = self.last_bstate
        l = []
        
        mod = 0
        if m & 1:    mod |= 4 # shift
        if m & 10:    mod |= 8 # alt
        if m & 4:    mod |= 16 # ctrl

        def append_button( b ):
            b |= mod
            l.extend([ 27, ord('['), ord('M'), b+32, x+32, y+32 ])

        if ev == 20: # press
            if b & 4 and last & 1 == 0:
                append_button( 0 )
                next |= 1
            if b & 2 and last & 2 == 0:
                append_button( 1 )
                next |= 2
            if b & 1 and last & 4 == 0:
                append_button( 2 )
                next |= 4
        elif ev == 146: # drag
            if b & 4:
                append_button( 0 + escape.MOUSE_DRAG_FLAG )
            elif b & 2:
                append_button( 1 + escape.MOUSE_DRAG_FLAG )
            elif b & 1:
                append_button( 2 + escape.MOUSE_DRAG_FLAG )
        else: # release
            if b & 4 and last & 1:
                append_button( 0 + escape.MOUSE_RELEASE_FLAG )
                next &= ~ 1
            if b & 2 and last & 2:
                append_button( 1 + escape.MOUSE_RELEASE_FLAG )
                next &= ~ 2
            if b & 1 and last & 4:
                append_button( 2 + escape.MOUSE_RELEASE_FLAG )
                next &= ~ 4
            
        self.last_bstate = next
        return l
    
    def _getch_nodelay(self):
        return self._getch(0)
    
    
    def get_cols_rows(self):
        """Return the terminal dimensions (num columns, num rows)."""
        buf = fcntl.ioctl(0, termios.TIOCGWINSZ, ' '*4)
        y, x = struct.unpack('hh', buf)
        self.maxrow = y
        return x, y

    def _setup_G1(self):
        """
        Initialize the G1 character set to graphics mode if required.
        """
        if self._setup_G1_done:
            return
        
        while True:
            try:
                self._term_output_file.write(escape.DESIGNATE_G1_SPECIAL)
                self._term_output_file.flush()
                break
            except IOError:
                pass
        self._setup_G1_done = True

    
    def draw_screen(self, (maxcol, maxrow), r ):
        """Paint screen with rendered canvas."""
        assert self._started

        assert maxrow == r.rows()

        # quick return if nothing has changed
        if self.screen_buf and r is self._screen_buf_canvas:
            return

        self._setup_G1()
        
        if self._resized: 
            # handle resize before trying to draw screen
            return
        
        o = [escape.HIDE_CURSOR, self._attrspec_to_escape(AttrSpec('',''))]
        
        def partial_display():
            # returns True if the screen is in partial display mode
            # ie. only some rows belong to the display
            return self._rows_used is not None

        if not partial_display():
            o.append(escape.CURSOR_HOME)

        if self.screen_buf:
            osb = self.screen_buf
        else:
            osb = []
        sb = []
        cy = self._cy
        y = -1

        def set_cursor_home():
            if not partial_display():
                return escape.set_cursor_position(0, 0)
            return (escape.CURSOR_HOME_COL + 
                escape.move_cursor_up(cy))
        
        def set_cursor_row(y):
            if not partial_display():
                return escape.set_cursor_position(0, y)
            return escape.move_cursor_down(y - cy)

        def set_cursor_position(x, y):
            if not partial_display():
                return escape.set_cursor_position(x, y)
            if cy > y:
                return ('\b' + escape.CURSOR_HOME_COL +
                    escape.move_cursor_up(cy - y) +
                    escape.move_cursor_right(x))
            return ('\b' + escape.CURSOR_HOME_COL +
                escape.move_cursor_down(y - cy) +
                escape.move_cursor_right(x))
        
        def is_blank_row(row):
            if len(row) > 1:
                return False
            if row[0][2].strip():
                return False
            return True

        def attr_to_escape(a):
            if a in self._pal_escape:
                return self._pal_escape[a]
            elif isinstance(a, AttrSpec):
                return self._attrspec_to_escape(a)
            # undefined attributes use default/default
            # TODO: track and report these
            return self._attrspec_to_escape(
                AttrSpec('default','default'))

        ins = None
        o.append(set_cursor_home())
        cy = 0
        for row in r.content():
            y += 1
            if False and osb and osb[y] == row:
                # this row of the screen buffer matches what is
                # currently displayed, so we can skip this line
                sb.append( osb[y] )
                continue

            sb.append(row)

            # leave blank lines off display when we are using
            # the default screen buffer (allows partial screen)
            if partial_display() and y > self._rows_used:
                if is_blank_row(row):
                    continue
                self._rows_used = y

            if y or partial_display():
                o.append(set_cursor_position(0, y))
            # after updating the line we will be just over the
            # edge, but terminals still treat this as being
            # on the same line
            cy = y

            whitespace_at_end = False
            if row and row[-1][2][-1] == ' ':
                whitespace_at_end = True
                a, cs, run = row[-1]
                row = row[:-1] + [(a, cs, run.rstrip(' '))]
            elif y == maxrow-1:
                row, back, ins = self._last_row(row)

            first = True
            lasta = lastcs = None
            for (a,cs, run) in row:
                assert isinstance(run, bytes) # canvases should render with bytes
                if cs != 'U':
                    run = run.translate(UNPRINTABLE_TRANS_TABLE)
                if first or lasta != a:
                    o.append(attr_to_escape(a))
                    lasta = a
                if first or lastcs != cs:
                    assert cs in [None, "0", "U"], repr(cs)
                    if lastcs == "U":
                        o.append( escape.IBMPC_OFF )

                    if cs is None:
                        o.append( escape.SI )
                    elif cs == "U":
                        o.append( escape.IBMPC_ON )
                    else:
                        o.append( escape.SO )
                    lastcs = cs
                o.append( run )
                first = False
            if ins:<commit_msg>fix a raw_display regression related to clear-to-eol change<commit_after>#!/usr/bin/python
#
# Urwid raw display module
#    Copyright (C) 2004-2009  <NAME>
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# Urwid web site: http://excess.org/urwid/

"""
Direct terminal UI implementation
"""

import fcntl
import termios
import os
import select
import struct
import sys
import tty
import signal

from urwid import util
from urwid import escape
from urwid.display_common import BaseScreen, RealTerminal, \
    UPDATE_PALETTE_ENTRY, AttrSpec, UNPRINTABLE_TRANS_TABLE, \
    INPUT_DESCRIPTORS_CHANGED
from urwid import signals
from urwid.compat import PYTHON3, bytes, B

from subprocess import Popen, PIPE


class Screen(BaseScreen, RealTerminal):
    def __init__(self):
        """Initialize a screen that directly prints escape codes to an output
        terminal.
        """
        super(Screen, self).__init__()
        self._pal_escape = {}
        signals.connect_signal(self, UPDATE_PALETTE_ENTRY, 
            self._on_update_palette_entry)
        self.colors = 16 # FIXME: detect this
        self.has_underline = True # FIXME: detect this
        self.register_palette_entry( None, 'default','default')
        self._keyqueue = []
        self.prev_input_resize = 0
        self.set_input_timeouts()
        self.screen_buf = None
        self._screen_buf_canvas = None
        self._resized = False
        self.maxrow = None
        self.gpm_mev = None
        self.gpm_event_pending = False
        self.last_bstate = 0
        self._setup_G1_done = False
        self._rows_used = None
        self._cy = 0
        self.bright_is_bold = os.environ.get('TERM',None) != "xterm"
        self._next_timeout = None
        self._term_output_file = sys.stdout
        self._term_input_file = sys.stdin
        # pipe for signalling external event loops about resize events
        self._resize_pipe_rd, self._resize_pipe_wr = os.pipe()
        fcntl.fcntl(self._resize_pipe_rd, fcntl.F_SETFL, os.O_NONBLOCK)

    def _on_update_palette_entry(self, name, *attrspecs):
        # copy the attribute to a dictionary containing the escape seqences
        self._pal_escape[name] = self._attrspec_to_escape(
            attrspecs[{16:0,1:1,88:2,256:3}[self.colors]])

    def set_input_timeouts(self, max_wait=None, complete_wait=0.125, 
        resize_wait=0.125):
        """
        Set the get_input timeout values.  All values are in floating
        point numbers of seconds.
        
        max_wait -- amount of time in seconds to wait for input when
            there is no input pending, wait forever if None
        complete_wait -- amount of time in seconds to wait when
            get_input detects an incomplete escape sequence at the
            end of the available input
        resize_wait -- amount of time in seconds to wait for more input
            after receiving two screen resize requests in a row to
            stop Urwid from consuming 100% cpu during a gradual
            window resize operation
        """
        self.max_wait = max_wait
        if max_wait is not None:
            if self._next_timeout is None:
                self._next_timeout = max_wait
            else:
                self._next_timeout = min(self._next_timeout, self.max_wait)
        self.complete_wait = complete_wait
        self.resize_wait = resize_wait

    def _sigwinch_handler(self, signum, frame):
        if not self._resized:
            os.write(self._resize_pipe_wr, B('R'))
        self._resized = True
        self.screen_buf = None
      
    def signal_init(self):
        """
        Called in the startup of run wrapper to set the SIGWINCH 
        signal handler to self._sigwinch_handler.

        Override this function to call from main thread in threaded
        applications.
        """
        signal.signal(signal.SIGWINCH, self._sigwinch_handler)
    
    def signal_restore(self):
        """
        Called in the finally block of run wrapper to restore the
        SIGWINCH handler to the default handler.

        Override this function to call from main thread in threaded
        applications.
        """
        signal.signal(signal.SIGWINCH, signal.SIG_DFL)
      
    def set_mouse_tracking(self):
        """
        Enable mouse tracking.  
        
        After calling this function get_input will include mouse
        click events along with keystrokes.
        """
        self._term_output_file.write(escape.MOUSE_TRACKING_ON)

        self._start_gpm_tracking()
    
    def _start_gpm_tracking(self):
        if not os.path.isfile("/usr/bin/mev"):
            return
        if not os.environ.get('TERM',"").lower().startswith("linux"):
            return
        if not Popen:
            return
        m = Popen(["/usr/bin/mev","-e","158"], stdin=PIPE, stdout=PIPE,
            close_fds=True)
        fcntl.fcntl(m.stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
        self.gpm_mev = m
    
    def _stop_gpm_tracking(self):
        os.kill(self.gpm_mev.pid, signal.SIGINT)
        os.waitpid(self.gpm_mev.pid, 0)
        self.gpm_mev = None
    
    def start(self, alternate_buffer=True):
        """
        Initialize the screen and input mode.

        alternate_buffer -- use alternate screen buffer
        """
        assert not self._started
        if alternate_buffer:
            self._term_output_file.write(escape.SWITCH_TO_ALTERNATE_BUFFER)
            self._rows_used = None
        else:
            self._rows_used = 0

        fd = self._term_input_file.fileno()
        if os.isatty(fd):
            self._old_termios_settings = termios.tcgetattr(fd)
            tty.setcbreak(fd)

        self.signal_init()
        self._alternate_buffer = alternate_buffer
        self._input_iter = self._run_input_iter()
        self._next_timeout = self.max_wait

        if not self._signal_keys_set:
            self._old_signal_keys = self.tty_signal_keys(fileno=fd)

        super(Screen, self).start()

        signals.emit_signal(self, INPUT_DESCRIPTORS_CHANGED)


    def stop(self):
        """
        Restore the screen.
        """
        self.clear()
        if not self._started:
            return

        signals.emit_signal(self, INPUT_DESCRIPTORS_CHANGED)

        self.signal_restore()

        fd = self._term_input_file.fileno()
        if os.isatty(fd):
            termios.tcsetattr(fd, termios.TCSADRAIN,
        self._old_termios_settings)

        move_cursor = ""
        if self.gpm_mev:
            self._stop_gpm_tracking()
        if self._alternate_buffer:
            move_cursor = escape.RESTORE_NORMAL_BUFFER
        elif self.maxrow is not None:
            move_cursor = escape.set_cursor_position( 
                0, self.maxrow)
        self._term_output_file.write(self._attrspec_to_escape(AttrSpec('','')) 
            + escape.SI
            + escape.MOUSE_TRACKING_OFF
            + escape.SHOW_CURSOR
            + move_cursor + "\n" + escape.SHOW_CURSOR )
        self._input_iter = self._fake_input_iter()

        if self._old_signal_keys:
            self.tty_signal_keys(*(self._old_signal_keys + (fd,)))

        super(Screen, self).stop()


    def run_wrapper(self, fn, alternate_buffer=True):
        """
        Call start to initialize screen, then call fn.  
        When fn exits call stop to restore the screen to normal.

        alternate_buffer -- use alternate screen buffer and restore
            normal screen buffer on exit
        """
        try:
            self.start(alternate_buffer)
            return fn()
        finally:
            self.stop()

    def get_input(self, raw_keys=False):
        """Return pending input as a list.

        raw_keys -- return raw keycodes as well as translated versions

        This function will immediately return all the input since the
        last time it was called.  If there is no input pending it will
        wait before returning an empty list.  The wait time may be
        configured with the set_input_timeouts function.

        If raw_keys is False (default) this function will return a list
        of keys pressed.  If raw_keys is True this function will return
        a ( keys pressed, raw keycodes ) tuple instead.

        Examples of keys returned:

        * ASCII printable characters:  " ", "a", "0", "A", "-", "/" 
        * ASCII control characters:  "tab", "enter"
        * Escape sequences:  "up", "page up", "home", "insert", "f1"
        * Key combinations:  "shift f1", "meta a", "ctrl b"
        * Window events:  "window resize"

        When a narrow encoding is not enabled:

        * "Extended ASCII" characters:  "\\xa1", "\\xb2", "\\xfe"

        When a wide encoding is enabled:

        * Double-byte characters:  "\\xa1\\xea", "\\xb2\\xd4"

        When utf8 encoding is enabled:

        * Unicode characters: u"\\u00a5", u'\\u253c"

        Examples of mouse events returned:

        * Mouse button press: ('mouse press', 1, 15, 13), 
                              ('meta mouse press', 2, 17, 23)
        * Mouse drag: ('mouse drag', 1, 16, 13),
                      ('mouse drag', 1, 17, 13),
                      ('ctrl mouse drag', 1, 18, 13)
        * Mouse button release: ('mouse release', 0, 18, 13),
                                ('ctrl mouse release', 0, 17, 23)
        """
        assert self._started

        self._wait_for_input_ready(self._next_timeout)
        self._next_timeout, keys, raw = self._input_iter.next()

        # Avoid pegging CPU at 100% when slowly resizing
        if keys==['window resize'] and self.prev_input_resize:
            while True:
                self._wait_for_input_ready(self.resize_wait)
                self._next_timeout, keys, raw2 = \
                    self._input_iter.next()
                raw += raw2
                #if not keys:
                #    keys, raw2 = self._get_input( 
                #        self.resize_wait)
                #    raw += raw2
                if keys!=['window resize']:
                    break
            if keys[-1:]!=['window resize']:
                keys.append('window resize')

        if keys==['window resize']:
            self.prev_input_resize = 2
        elif self.prev_input_resize == 2 and not keys:
            self.prev_input_resize = 1
        else:
            self.prev_input_resize = 0

        if raw_keys:
            return keys, raw
        return keys

    def get_input_descriptors(self):
        """
        Return a list of integer file descriptors that should be
        polled in external event loops to check for user input.

        Use this method if you are implementing yout own event loop.
        """
        if not self._started:
            return []

        fd_list = [self._term_input_file.fileno(), self._resize_pipe_rd]
        if self.gpm_mev is not None:
            fd_list.append(self.gpm_mev.stdout.fileno())
        return fd_list

    def get_input_nonblocking(self):
        """
        Return a (next_input_timeout, keys_pressed, raw_keycodes)
        tuple.

        Use this method if you are implementing your own event loop.

        When there is input waiting on one of the descriptors returned
        by get_input_descriptors() this method should be called to
        read and process the input.

        This method expects to be called in next_input_timeout seconds
        (a floating point number) if there is no input waiting.
        """
        return self._input_iter.next()

    def _run_input_iter(self):
        def empty_resize_pipe():
            # clean out the pipe used to signal external event loops
            # that a resize has occured
            try:
                while True: os.read(self._resize_pipe_rd, 1)
            except OSError:
                pass

        while True:
            processed = []
            codes = self._get_gpm_codes() + \
                self._get_keyboard_codes()

            original_codes = codes
            try:
                while codes:
                    run, codes = escape.process_keyqueue(
                        codes, True)
                    processed.extend(run)
            except escape.MoreInputRequired:
                k = len(original_codes) - len(codes)
                yield (self.complete_wait, processed,
                    original_codes[:k])
                empty_resize_pipe()
                original_codes = codes
                processed = []

                codes += self._get_keyboard_codes() + \
                    self._get_gpm_codes()
                while codes:
                    run, codes = escape.process_keyqueue(
                        codes, False)
                    processed.extend(run)
            
            if self._resized:
                processed.append('window resize')
                self._resized = False

            yield (self.max_wait, processed, original_codes)
            empty_resize_pipe()

    def _fake_input_iter(self):
        """
        This generator is a placeholder for when the screen is stopped
        to always return that no input is available.
        """
        while True:
            yield (self.max_wait, [], [])

    def _get_keyboard_codes(self):
        codes = []
        while True:
            code = self._getch_nodelay()
            if code < 0:
                break
            codes.append(code)
        return codes

    def _get_gpm_codes(self):
        codes = []
        try:
            while self.gpm_mev is not None and self.gpm_event_pending:
                codes.extend(self._encode_gpm_event())
        except IOError, e:
            if e.args[0] != 11:
                raise
        return codes

    def _wait_for_input_ready(self, timeout):
        ready = None
        fd_list = [self._term_input_file.fileno()]
        if self.gpm_mev is not None:
            fd_list.append(self.gpm_mev.stdout.fileno())
        while True:
            try:
                if timeout is None:
                    ready,w,err = select.select(
                        fd_list, [], fd_list)
                else:
                    ready,w,err = select.select(
                        fd_list,[],fd_list, timeout)
                break
            except select.error, e:
                if e.args[0] != 4: 
                    raise
                if self._resized:
                    ready = []
                    break
        return ready    
        
    def _getch(self, timeout):
        ready = self._wait_for_input_ready(timeout)
        if self.gpm_mev is not None:
            if self.gpm_mev.stdout.fileno() in ready:
                self.gpm_event_pending = True
        if self._term_input_file.fileno() in ready:
            return ord(os.read(self._term_input_file.fileno(), 1))
        return -1
    
    def _encode_gpm_event( self ):
        self.gpm_event_pending = False
        s = self.gpm_mev.stdout.readline().decode('ascii')
        l = s.split(",")
        if len(l) != 6:
            # unexpected output, stop tracking
            self._stop_gpm_tracking()
            signals.emit_signal(self, INPUT_DESCRIPTORS_CHANGED)
            return []
        ev, x, y, ign, b, m = s.split(",")
        ev = int( ev.split("x")[-1], 16)
        x = int( x.split(" ")[-1] )
        y = int( y.lstrip().split(" ")[0] )
        b = int( b.split(" ")[-1] )
        m = int( m.split("x")[-1].rstrip(), 16 )

        # convert to xterm-like escape sequence

        last = next = self.last_bstate
        l = []
        
        mod = 0
        if m & 1:    mod |= 4 # shift
        if m & 10:    mod |= 8 # alt
        if m & 4:    mod |= 16 # ctrl

        def append_button( b ):
            b |= mod
            l.extend([ 27, ord('['), ord('M'), b+32, x+32, y+32 ])

        if ev == 20: # press
            if b & 4 and last & 1 == 0:
                append_button( 0 )
                next |= 1
            if b & 2 and last & 2 == 0:
                append_button( 1 )
                next |= 2
            if b & 1 and last & 4 == 0:
                append_button( 2 )
                next |= 4
        elif ev == 146: # drag
            if b & 4:
                append_button( 0 + escape.MOUSE_DRAG_FLAG )
            elif b & 2:
                append_button( 1 + escape.MOUSE_DRAG_FLAG )
            elif b & 1:
                append_button( 2 + escape.MOUSE_DRAG_FLAG )
        else: # release
            if b & 4 and last & 1:
                append_button( 0 + escape.MOUSE_RELEASE_FLAG )
                next &= ~ 1
            if b & 2 and last & 2:
                append_button( 1 + escape.MOUSE_RELEASE_FLAG )
                next &= ~ 2
            if b & 1 and last & 4:
                append_button( 2 + escape.MOUSE_RELEASE_FLAG )
                next &= ~ 4
            
        self.last_bstate = next
        return l
    
    def _getch_nodelay(self):
        return self._getch(0)
    
    
    def get_cols_rows(self):
        """Return the terminal dimensions (num columns, num rows)."""
        buf = fcntl.ioctl(0, termios.TIOCGWINSZ, ' '*4)
        y, x = struct.unpack('hh', buf)
        self.maxrow = y
        return x, y

    def _setup_G1(self):
        """
        Initialize the G1 character set to graphics mode if required.
        """
        if self._setup_G1_done:
            return
        
        while True:
            try:
                self._term_output_file.write(escape.DESIGNATE_G1_SPECIAL)
                self._term_output_file.flush()
                break
            except IOError:
                pass
        self._setup_G1_done = True

    
    def draw_screen(self, (maxcol, maxrow), r ):
        """Paint screen with rendered canvas."""
        assert self._started

        assert maxrow == r.rows()

        # quick return if nothing has changed
        if self.screen_buf and r is self._screen_buf_canvas:
            return

        self._setup_G1()
        
        if self._resized: 
            # handle resize before trying to draw screen
            return
        
        o = [escape.HIDE_CURSOR, self._attrspec_to_escape(AttrSpec('',''))]
        
        def partial_display():
            # returns True if the screen is in partial display mode
            # ie. only some rows belong to the display
            return self._rows_used is not None

        if not partial_display():
            o.append(escape.CURSOR_HOME)

        if self.screen_buf:
            osb = self.screen_buf
        else:
            osb = []
        sb = []
        cy = self._cy
        y = -1

        def set_cursor_home():
            if not partial_display():
                return escape.set_cursor_position(0, 0)
            return (escape.CURSOR_HOME_COL + 
                escape.move_cursor_up(cy))
        
        def set_cursor_row(y):
            if not partial_display():
                return escape.set_cursor_position(0, y)
            return escape.move_cursor_down(y - cy)

        def set_cursor_position(x, y):
            if not partial_display():
                return escape.set_cursor_position(x, y)
            if cy > y:
                return ('\b' + escape.CURSOR_HOME_COL +
                    escape.move_cursor_up(cy - y) +
                    escape.move_cursor_right(x))
            return ('\b' + escape.CURSOR_HOME_COL +
                escape.move_cursor_down(y - cy) +
                escape.move_cursor_right(x))
        
        def is_blank_row(row):
            if len(row) > 1:
                return False
            if row[0][2].strip():
                return False
            return True

        def attr_to_escape(a):
            if a in self._pal_escape:
                return self._pal_escape[a]
            elif isinstance(a, AttrSpec):
                return self._attrspec_to_escape(a)
            # undefined attributes use default/default
            # TODO: track and report these
            return self._attrspec_to_escape(
                AttrSpec('default','default'))

        ins = None
        o.append(set_cursor_home())
        cy = 0
        for row in r.content():
            y += 1
            if False and osb and osb[y] == row:
                # this row of the screen buffer matches what is
                # currently displayed, so we can skip this line
                sb.append( osb[y] )
                continue

            sb.append(row)

            # leave blank lines off display when we are using
            # the default screen buffer (allows partial screen)
            if partial_display() and y > self._rows_used:
                if is_blank_row(row):
                    continue
                self._rows_used = y

            if y or partial_display():
                o.append(set_cursor_position(0, y))
            # after updating the line we will be just over the
            # edge, but terminals still treat this as being
            # on the same line
            cy = y

            whitespace_at_end = False
            if row and row[-1][2][-1:] == ' ':
                whitespace_at_end = True
                a, cs, run = row[-1]
                row = row[:-1] + [(a, cs, run.rstrip(' '))]
            elif y == maxrow-1:
                row, back, ins = self._last_row(row)

            first = True
            lasta = lastcs = None
            for (a,cs, run) in row:
                assert isinstance(run, bytes) # canvases should render with bytes
                if cs != 'U':
                    run = run.translate(UNPRINTABLE_TRANS_TABLE)
                if first or lasta != a:
                    o.append(attr_to_escape(a))
                    lasta = a
                if first or lastcs != cs:
                    assert cs in [None, "0", "U"], repr(cs)
                    if lastcs == "U":
                        o.append( escape.IBMPC_OFF )

                    if cs is None:
                        o.append( escape.SI )
                    elif cs == "U":
                        o.append( escape.IBMPC_ON )
                    else:
                        o.append( escape.SO )
                    lastcs = cs
                o.append( run )
                first = False
            if ins:
{'fraction_non_alphanumeric': 0.08442481359193581, 'fraction_numerical': 0.009708525572343822, 'mean_word_length': 1.6154637996111174, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7351398', 'n_tokens_mistral': 13401, 'n_tokens_neox': 12612, 'n_words': 3710}