<commit_before>import os
import Queue
import json
import hashlib
from datetime import datetime

import sublime
import sublime_plugin
import dmp_monkey
dmp_monkey.monkey_patch()
from lib import diff_match_patch as dmp

import msg
import shared as G
import utils

MODIFIED_EVENTS = Queue.Queue()
BUFS = {}

settings = sublime.load_settings('Floobits.sublime-settings')


def get_text(view):
    return view.substr(sublime.Region(0, view.size()))


def get_view(buf_id):
    buf = BUFS[buf_id]
    for view in G.ROOM_WINDOW.views():
        if not view.file_name():
            continue
        if buf['path'] == utils.to_rel_path(view.file_name()):
            return view
    return None


def create_view(buf):
    path = utils.get_full_path(buf['path'])
    view = G.ROOM_WINDOW.open_file(path)
    if view:
        msg.debug('Created view', view.name() or view.file_name())
    return view


def get_buf(view):
    if view.is_scratch():
        return None
    if not view.file_name():
        return None
    rel_path = utils.to_rel_path(view.file_name())
    for buf_id, buf in BUFS.iteritems():
        if rel_path == buf['path']:
            return buf
    return None


def save_buf(buf):
    path = utils.get_full_path(buf['path'])
    utils.mkdir(os.path.split(path)[0])
    with open(path, 'wb') as fd:
        fd.write(buf['buf'])


def delete_buf(buf_id):
    # TODO: somehow tell the user about this. maybe delete on disk too?
    del BUFS[buf_id]


class FlooPatch(object):

    def __init__(self, view, buf):
        self.buf = buf
        self.view = view
        self.current = get_text(view)
        self.previous = buf['buf']
        self.md5_before = hashlib.md5(self.previous).hexdigest()

    def __str__(self):
        return '%s - %s - %s' % (self.buf['id'], self.buf['path'], self.view.buffer_id())

    def patches(self):
        return dmp.diff_match_patch().patch_make(self.previous, self.current)

    def to_json(self):
        patches = self.patches()
        if len(patches) == 0:
            return None
        msg.debug('sending %s patches' % len(patches))
        patch_str = ''
        for patch in patches:
            patch_str += str(patch)
        return json.dumps({
            'id': self.buf['id'],
            'md5_after': hashlib.md5(self.current).hexdigest(),
            'md5_before': self.md5_before,
            'path': self.buf['path'],
            'patch': patch_str,
            'name': 'patch'
        })


class Listener(sublime_plugin.EventListener):
    views_changed = []
    selection_changed = []
    agent = None

    def __init__(self, *args, **kwargs):
        sublime_plugin.EventListener.__init__(self, *args, **kwargs)
        self.between_save_events = {}

    @staticmethod
    def set_agent(agent):
        Listener.agent = agent

    @staticmethod
    def push():
        reported = set()
        while Listener.views_changed:
            view, buf = Listener.views_changed.pop()

            vb_id = view.buffer_id()
            if vb_id in reported:
                continue
            if 'buf' not in buf:
                msg.debug('No data for buf %s %s yet. Skipping sending patch' % (buf['id'], buf['path']))
                continue

            reported.add(vb_id)
            patch = FlooPatch(view, buf)
            # update the current copy of the buffer
            buf['buf'] = patch.current
            buf['md5'] = hashlib.md5(patch.current).hexdigest()
            if Listener.agent:
                Listener.agent.put(patch.to_json())
            else:
                msg.debug('Not connected. Discarding view change.')

        while Listener.selection_changed:
            view, buf = Listener.selection_changed.pop()
            vb_id = view.buffer_id()
            if vb_id in reported:
                continue

            reported.add(vb_id)
            sel = view.sel()
            highlight_json = json.dumps({
                'id': buf['id'],
                'name': 'highlight',
                'ranges': [[x.a, x.b] for x in sel]
            })
            if Listener.agent:
                Listener.agent.put(highlight_json)
            else:
                msg.debug('Not connected. Discarding selection change.')

        sublime.set_timeout(Listener.push, 100)

    @staticmethod
    def apply_patch(patch_data):
        buf_id = patch_data['id']
        buf = BUFS[buf_id]
        view = get_view(buf_id)
        DMP = dmp.diff_match_patch()
        if len(patch_data['patch']) == 0:
            msg.error('wtf? no patches to apply. server is being stupid')
            return
        msg.debug('patch is', patch_data['patch'])
        dmp_patches = DMP.patch_fromText(patch_data['patch'])
        # TODO: run this in a separate thread
        if view:
            old_text = get_text(view)
        else:
            old_text = buf['buf']
        md5_before = hashlib.md5(old_text).hexdigest()
        if md5_before != patch_data['md5_before']:
            msg.warn('starting md5s don\'t match. this is dangerous!')

        t = DMP.patch_apply(dmp_patches, old_text)

        clean_patch = True
        for applied_patch in t[1]:
            if not applied_patch:
                clean_patch = False
                break

        if G.DEBUG:
            if '\x01' in t[0]:
                msg.debug('FOUND CRAZY BYTE IN BUFFER')

        if not clean_patch:
            msg.error('failed to patch')
            return Listener.get_buf(buf_id)

        cur_hash = hashlib.md5(t[0]).hexdigest()
        if cur_hash != patch_data['md5_after']:
            msg.warn('new hash %s != expected %s' % (cur_hash, patch_data['md5_after']))
            # TODO: do something better than erasing local changes
            return Listener.get_buf(buf_id)

        buf['buf'] = str(t[0]).decode('utf-8')
        buf['md5'] = cur_hash

        if not view:
            save_buf(buf)
            return

        selections = [x for x in view.sel()]  # deep copy<commit_msg>Another debug thing<commit_after>import os
import Queue
import json
import hashlib
from datetime import datetime

import sublime
import sublime_plugin
import dmp_monkey
dmp_monkey.monkey_patch()
from lib import diff_match_patch as dmp

import msg
import shared as G
import utils

MODIFIED_EVENTS = Queue.Queue()
BUFS = {}

settings = sublime.load_settings('Floobits.sublime-settings')


def get_text(view):
    return view.substr(sublime.Region(0, view.size()))


def get_view(buf_id):
    buf = BUFS[buf_id]
    for view in G.ROOM_WINDOW.views():
        if not view.file_name():
            continue
        if buf['path'] == utils.to_rel_path(view.file_name()):
            return view
    return None


def create_view(buf):
    path = utils.get_full_path(buf['path'])
    view = G.ROOM_WINDOW.open_file(path)
    if view:
        msg.debug('Created view', view.name() or view.file_name())
    return view


def get_buf(view):
    if view.is_scratch():
        return None
    if not view.file_name():
        return None
    rel_path = utils.to_rel_path(view.file_name())
    for buf_id, buf in BUFS.iteritems():
        if rel_path == buf['path']:
            return buf
    return None


def save_buf(buf):
    path = utils.get_full_path(buf['path'])
    utils.mkdir(os.path.split(path)[0])
    with open(path, 'wb') as fd:
        fd.write(buf['buf'])


def delete_buf(buf_id):
    # TODO: somehow tell the user about this. maybe delete on disk too?
    del BUFS[buf_id]


class FlooPatch(object):

    def __init__(self, view, buf):
        self.buf = buf
        self.view = view
        self.current = get_text(view)
        self.previous = buf['buf']
        self.md5_before = hashlib.md5(self.previous).hexdigest()

    def __str__(self):
        return '%s - %s - %s' % (self.buf['id'], self.buf['path'], self.view.buffer_id())

    def patches(self):
        return dmp.diff_match_patch().patch_make(self.previous, self.current)

    def to_json(self):
        patches = self.patches()
        if len(patches) == 0:
            return None
        msg.debug('sending %s patches' % len(patches))
        patch_str = ''
        for patch in patches:
            patch_str += str(patch)
        return json.dumps({
            'id': self.buf['id'],
            'md5_after': hashlib.md5(self.current).hexdigest(),
            'md5_before': self.md5_before,
            'path': self.buf['path'],
            'patch': patch_str,
            'name': 'patch'
        })


class Listener(sublime_plugin.EventListener):
    views_changed = []
    selection_changed = []
    agent = None

    def __init__(self, *args, **kwargs):
        sublime_plugin.EventListener.__init__(self, *args, **kwargs)
        self.between_save_events = {}

    @staticmethod
    def set_agent(agent):
        Listener.agent = agent

    @staticmethod
    def push():
        reported = set()
        while Listener.views_changed:
            view, buf = Listener.views_changed.pop()

            vb_id = view.buffer_id()
            if vb_id in reported:
                continue
            if 'buf' not in buf:
                msg.debug('No data for buf %s %s yet. Skipping sending patch' % (buf['id'], buf['path']))
                continue

            reported.add(vb_id)
            patch = FlooPatch(view, buf)
            # update the current copy of the buffer
            buf['buf'] = patch.current
            buf['md5'] = hashlib.md5(patch.current).hexdigest()
            if Listener.agent:
                Listener.agent.put(patch.to_json())
            else:
                msg.debug('Not connected. Discarding view change.')

        while Listener.selection_changed:
            view, buf = Listener.selection_changed.pop()
            vb_id = view.buffer_id()
            if vb_id in reported:
                continue

            reported.add(vb_id)
            sel = view.sel()
            highlight_json = json.dumps({
                'id': buf['id'],
                'name': 'highlight',
                'ranges': [[x.a, x.b] for x in sel]
            })
            if Listener.agent:
                Listener.agent.put(highlight_json)
            else:
                msg.debug('Not connected. Discarding selection change.')

        sublime.set_timeout(Listener.push, 100)

    @staticmethod
    def apply_patch(patch_data):
        buf_id = patch_data['id']
        buf = BUFS[buf_id]
        view = get_view(buf_id)
        DMP = dmp.diff_match_patch()
        if len(patch_data['patch']) == 0:
            msg.error('wtf? no patches to apply. server is being stupid')
            return
        msg.debug('patch is', patch_data['patch'])
        dmp_patches = DMP.patch_fromText(patch_data['patch'])
        # TODO: run this in a separate thread
        if view:
            old_text = get_text(view)
        else:
            old_text = buf['buf']
        md5_before = hashlib.md5(old_text).hexdigest()
        if md5_before != patch_data['md5_before']:
            msg.warn('starting md5s don\'t match. this is dangerous!')

        t = DMP.patch_apply(dmp_patches, old_text)

        clean_patch = True
        for applied_patch in t[1]:
            if not applied_patch:
                clean_patch = False
                break

        if G.DEBUG:
            # TODO: log tons of stuff if these things happen
            if len(t[0]):
                msg.debug('OMG EMPTY!')
            if '\x01' in t[0]:
                msg.debug('FOUND CRAZY BYTE IN BUFFER')

        if not clean_patch:
            msg.error('failed to patch')
            return Listener.get_buf(buf_id)

        cur_hash = hashlib.md5(t[0]).hexdigest()
        if cur_hash != patch_data['md5_after']:
            msg.warn('new hash %s != expected %s' % (cur_hash, patch_data['md5_after']))
            # TODO: do something better than erasing local changes
            return Listener.get_buf(buf_id)

        buf['buf'] = str(t[0]).decode('utf-8')
        buf['md5'] = cur_hash

        if not view:
            save_buf(buf)
            return

        selections = [x for x in view.sel()]  # deep copy
{'fraction_non_alphanumeric': 0.1161210111893908, 'fraction_numerical': 0.005221715706589308, 'mean_word_length': 2.018009004502251, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3899367', 'n_tokens_mistral': 3815, 'n_tokens_neox': 3689, 'n_words': 890}