<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import sys
from os import path
import json
import asyncio
import webbrowser
from functools import partial

from tornado import web
from tornado import websocket
from tornado.platform.asyncio import AsyncIOMainLoop

from pygments.styles import STYLE_MAP
from pygments.formatters import HtmlFormatter

current_dir = path.dirname(__file__)
sys.path.insert(0, path.join(current_dir, 'wdom'))

from wdom import options
from wdom.tag import Div, Style, H2, Script, WebElement
from wdom.document import get_document
from wdom.server import get_app, start_server
from wdom.parser import parse_html

from converter import convert


connections = []
CURSOR_TAG = '<span id="vimcursor"></span>' 
cursor_move_js = '''
        function moveToElement(id) {
            var elm = document.getElementById(id)
            if (elm) {
                var x = window.scrollX
                var rect = elm.getBoundingClientRect()
                window.scrollTo(x, rect.top + window.scrollY)
            }
        }
    '''

options.parser.define('browser', default='google-chrome', type=str)
options.parser.define('browser-port', default=8089, type=int)
options.parser.define('vim-port', default=8090, type=int)
options.parser.define('js-files', default=[], nargs='+')
options.parser.define('css-files', default=[], nargs='+')
options.parser.define('highlight-theme', default='default', type=str)



class MainHandler(web.RequestHandler):
    def get(self):
        self.render('main.html', css=css, port=options.config.browser_port)


class WSHandler(websocket.WebSocketHandler):
    def open(self):
        connections.append(self)

    def on_close(self):
        connections.remove(self)


class SocketListener(asyncio.Protocol):
    pass


def _is_same_node(node1, node2):
    if node1.nodeType == node2.nodeType:
        if node1.nodeType == node1.TEXT_NODE:
            return node1.textContent == node2.textContent
        else:
            return node1.html_noid == node2.html_noid
    else:
        return False


def _next_noempty(node):
    new_node = node.nextSibling
    while new_node is not None:
        if isinstance(new_node, WebElement):
            return new_node
        else:
            text = new_node.textContent
            if text and not text.isspace():
                return new_node
            new_node = new_node.nextSibling
    return None


class Preview(Div):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.tlist = []
        self.dom_tree = None
        self._tasks = []

    def _canncel_tasks(self):
        for task in self._tasks:
            if not task.done() and not task.cancelled():
                task.cancel()
            self._tasks.remove(task)

    def data_received(self, data):
        self._canncel_tasks()
        line = data['line']
        event = data['event']
        if event == 'update':
            self.tlist = data['text']
            _task = self.update_preview
        elif event == 'move':
            _task = self.move_cursor
        else:
            raise ValueError('Get unknown event: {}'.format(event))
        try:
            self._tasks.append(asyncio.ensure_future(_task(line)))
        except asyncio.CancelledError:
            pass

    @asyncio.coroutine
    def update_preview(self, line):
        html = yield from self.convert_to_html(self.tlist)
        yield from self.mount_html(html)
        yield from self.move_cursor(line)

    @asyncio.coroutine
    def convert_to_html(self, tlist):
        md = '\n'.join(tlist)
        yield from asyncio.sleep(0.0)
        return convert(md)

    @asyncio.coroutine
    def mount_html(self, html):
        fragment = parse_html(html)
        self.dom_tree = fragment.cloneNode(True)
        if self.length < 1:
            self.appendChild(fragment)
        else:
            diff = yield from self.find_diff_node(fragment)
            for _i in diff['inserted']:
                self.insertBefore(_i[1], _i[0])
            for _d in diff['deleted']:
                self.removeChild(_d)
            for _a in diff['appended']:
                self.appendChild(_a)

    @asyncio.coroutine
    def move_cursor(self, line):
        blank_lines = 0
        i = 1
        while i < line:
            if self.tlist[i] == '' and self.tlist[i - 1] == '':
                blank_lines += 1
            i += 1
        cur_line = line - blank_lines

        node_n = 0
        if self.dom_tree is not None and self.ownerDocument:
            _l = 0
            elm = self.dom_tree.firstChild
            while elm is not None:
                if elm.nodeName != '#text':
                    node_n += 1
                _l += elm.textContent.count('\n')
                if _l >= cur_line:
                    break
                elm = elm.nextSibling

            if node_n >= len(self):
                top_node = self.lastChild
            else:<commit_msg>Not move if line number is negative<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import sys
from os import path
import json
import asyncio
import webbrowser
from functools import partial

from tornado import web
from tornado import websocket
from tornado.platform.asyncio import AsyncIOMainLoop

from pygments.styles import STYLE_MAP
from pygments.formatters import HtmlFormatter

current_dir = path.dirname(__file__)
sys.path.insert(0, path.join(current_dir, 'wdom'))

from wdom import options
from wdom.tag import Div, Style, H2, Script, WebElement
from wdom.document import get_document
from wdom.server import get_app, start_server
from wdom.parser import parse_html

from converter import convert


connections = []
CURSOR_TAG = '<span id="vimcursor"></span>' 
cursor_move_js = '''
        function moveToElement(id) {
            var elm = document.getElementById(id)
            if (elm) {
                var x = window.scrollX
                var rect = elm.getBoundingClientRect()
                window.scrollTo(x, rect.top + window.scrollY)
            }
        }
    '''

options.parser.define('browser', default='google-chrome', type=str)
options.parser.define('browser-port', default=8089, type=int)
options.parser.define('vim-port', default=8090, type=int)
options.parser.define('js-files', default=[], nargs='+')
options.parser.define('css-files', default=[], nargs='+')
options.parser.define('highlight-theme', default='default', type=str)



class MainHandler(web.RequestHandler):
    def get(self):
        self.render('main.html', css=css, port=options.config.browser_port)


class WSHandler(websocket.WebSocketHandler):
    def open(self):
        connections.append(self)

    def on_close(self):
        connections.remove(self)


class SocketListener(asyncio.Protocol):
    pass


def _is_same_node(node1, node2):
    if node1.nodeType == node2.nodeType:
        if node1.nodeType == node1.TEXT_NODE:
            return node1.textContent == node2.textContent
        else:
            return node1.html_noid == node2.html_noid
    else:
        return False


def _next_noempty(node):
    new_node = node.nextSibling
    while new_node is not None:
        if isinstance(new_node, WebElement):
            return new_node
        else:
            text = new_node.textContent
            if text and not text.isspace():
                return new_node
            new_node = new_node.nextSibling
    return None


class Preview(Div):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.tlist = []
        self.dom_tree = None
        self._tasks = []

    def _canncel_tasks(self):
        for task in self._tasks:
            if not task.done() and not task.cancelled():
                task.cancel()
            self._tasks.remove(task)

    def data_received(self, data):
        self._canncel_tasks()
        line = data['line']
        event = data['event']
        if event == 'update':
            self.tlist = data['text']
            _task = self.update_preview
        elif event == 'move':
            _task = self.move_cursor
        else:
            raise ValueError('Get unknown event: {}'.format(event))
        try:
            self._tasks.append(asyncio.ensure_future(_task(line)))
        except asyncio.CancelledError:
            pass

    @asyncio.coroutine
    def update_preview(self, line):
        html = yield from self.convert_to_html(self.tlist)
        yield from self.mount_html(html)
        yield from self.move_cursor(line)

    @asyncio.coroutine
    def convert_to_html(self, tlist):
        md = '\n'.join(tlist)
        yield from asyncio.sleep(0.0)
        return convert(md)

    @asyncio.coroutine
    def mount_html(self, html):
        fragment = parse_html(html)
        self.dom_tree = fragment.cloneNode(True)
        if self.length < 1:
            self.appendChild(fragment)
        else:
            diff = yield from self.find_diff_node(fragment)
            for _i in diff['inserted']:
                self.insertBefore(_i[1], _i[0])
            for _d in diff['deleted']:
                self.removeChild(_d)
            for _a in diff['appended']:
                self.appendChild(_a)

    @asyncio.coroutine
    def move_cursor(self, line):
        if line < 0:
            return

        blank_lines = 0
        i = 1
        while i < line:
            if self.tlist[i] == '' and self.tlist[i - 1] == '':
                blank_lines += 1
            i += 1
        cur_line = line - blank_lines

        node_n = 0
        if self.dom_tree is not None and self.ownerDocument:
            _l = 0
            elm = self.dom_tree.firstChild
            while elm is not None:
                if elm.nodeName != '#text':
                    node_n += 1
                _l += elm.textContent.count('\n')
                if _l >= cur_line:
                    break
                elm = elm.nextSibling

            if node_n >= len(self):
                top_node = self.lastChild
            else:
{'fraction_non_alphanumeric': 0.10488462691039856, 'fraction_numerical': 0.0070921985815602835, 'mean_word_length': 2.143485086342229, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '703541', 'n_tokens_mistral': 3053, 'n_tokens_neox': 2946, 'n_words': 715}