<commit_before>import os
import Queue
import json
import hashlib
from datetime import datetime

import sublime
import sublime_plugin
import dmp_monkey
dmp_monkey.monkey_patch()
from lib import diff_match_patch as dmp

import msg
import shared as G
import utils

MODIFIED_EVENTS = Queue.Queue()
BUFS = {}

settings = sublime.load_settings('Floobits.sublime-settings')


def get_text(view):
    return view.substr(sublime.Region(0, view.size()))


def get_view(buf_id):
    buf = BUFS[buf_id]
    for view in G.ROOM_WINDOW.views():
        if not view.file_name():
            continue
        if buf['path'] == utils.to_rel_path(view.file_name()):
            return view
    return None


def create_view(buf):
    path = utils.get_full_path(buf['path'])
    view = G.ROOM_WINDOW.open_file(path)
    if view:
        msg.debug('Created view', view.name() or view.file_name())
    return view


def get_buf(view):
    if view.is_scratch():
        return None
    if not view.file_name():
        return None
    rel_path = utils.to_rel_path(view.file_name())
    for buf_id, buf in BUFS.iteritems():
        if rel_path == buf['path']:
            return buf
    return None


def save_buf(buf):
    path = utils.get_full_path(buf['path'])
    utils.mkdir(os.path.split(path)[0])
    with open(path, 'wb') as fd:
        fd.write(buf['buf'])


def delete_buf(buf_id):
    # TODO: somehow tell the user about this. maybe delete on disk too?
    del BUFS[buf_id]


class FlooPatch(object):

    def __init__(self, view, buf):
        self.buf = buf
        self.view = view
        self.current = get_text(view)
        self.previous = buf['buf']
        self.md5_before = hashlib.md5(self.previous).hexdigest()

    def __str__(self):
        return '%s - %s - %s' % (self.buf['id'], self.buf['path'], self.view.buffer_id())

    def patches(self):
        return dmp.diff_match_patch().patch_make(self.previous, self.current)

    def to_json(self):
        patches = self.patches()
        if len(patches) == 0:
            return None
        msg.debug('sending %s patches' % len(patches))
        patch_str = ''
        for patch in patches:
            patch_str += str(patch)
        return json.dumps({
            'id': self.buf['id'],
            'md5_after': hashlib.md5(self.current).hexdigest(),
            'md5_before': self.md5_before,
            'path': self.buf['path'],
            'patch': patch_str,
            'name': 'patch'
        })


class Listener(sublime_plugin.EventListener):
    views_changed = []
    selection_changed = []
    agent = None

    def __init__(self, *args, **kwargs):
        sublime_plugin.EventListener.__init__(self, *args, **kwargs)
        self.between_save_events = {}
<commit_msg>OK this should fix it<commit_after>import os
import Queue
import json
import hashlib
from datetime import datetime

import sublime
import sublime_plugin
import dmp_monkey
dmp_monkey.monkey_patch()
from lib import diff_match_patch as dmp

import msg
import shared as G
import utils

MODIFIED_EVENTS = Queue.Queue()
BUFS = {}

settings = sublime.load_settings('Floobits.sublime-settings')


def get_text(view):
    return view.substr(sublime.Region(0, view.size()))


def get_view(buf_id):
    buf = BUFS[buf_id]
    for view in G.ROOM_WINDOW.views():
        if not view.file_name():
            continue
        if buf['path'] == utils.to_rel_path(view.file_name()):
            return view
    return None


def create_view(buf):
    path = utils.get_full_path(buf['path'])
    view = G.ROOM_WINDOW.open_file(path)
    if view:
        msg.debug('Created view', view.name() or view.file_name())
    return view


def get_buf(view):
    if view.is_scratch():
        return None
    if not view.file_name():
        return None
    rel_path = utils.to_rel_path(view.file_name())
    for buf_id, buf in BUFS.iteritems():
        if rel_path == buf['path']:
            return buf
    return None


def save_buf(buf):
    path = utils.get_full_path(buf['path'])
    utils.mkdir(os.path.split(path)[0])
    with open(path, 'wb') as fd:
        fd.write(buf['buf'])


def delete_buf(buf_id):
    # TODO: somehow tell the user about this. maybe delete on disk too?
    del BUFS[buf_id]


class FlooPatch(object):

    def __init__(self, view, buf):
        self.buf = buf
        self.view = view
        self.current = get_text(view)
        self.previous = buf['buf']
        self.md5_before = hashlib.md5(self.previous).hexdigest()

    def __str__(self):
        return '%s - %s - %s' % (self.buf['id'], self.buf['path'], self.view.buffer_id())

    def patches(self):
        return dmp.diff_match_patch().patch_make(self.previous, self.current)

    def to_json(self):
        patches = self.patches()
        if len(patches) == 0:
            return None
        msg.debug('sending %s patches' % len(patches))
        patch_str = ''
        for patch in patches:
            patch_str += str(patch)
        return json.dumps({
            'id': self.buf['id'],
            'md5_after': hashlib.md5(self.current.encode('utf-8')).hexdigest(),
            'md5_before': self.md5_before,
            'path': self.buf['path'],
            'patch': patch_str,
            'name': 'patch'
        })


class Listener(sublime_plugin.EventListener):
    views_changed = []
    selection_changed = []
    agent = None

    def __init__(self, *args, **kwargs):
        sublime_plugin.EventListener.__init__(self, *args, **kwargs)
        self.between_save_events = {}

{'fraction_non_alphanumeric': 0.12631962140516928, 'fraction_numerical': 0.003458318165271205, 'mean_word_length': 2.6608927381745504, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '60503', 'n_tokens_mistral': 1829, 'n_tokens_neox': 1801, 'n_words': 418}