<commit_before>﻿# -*- coding: utf-8 -*-

import win32file
import win32con

from os.path import realpath, abspath, sep as path_separator, join as path_join, isdir as path_isdir
from threading import Thread, Event as ThreadedEvent
from Queue import Queue
from polling_observer import PollingObserver, _Rule
from events import DirMovedEvent, DirDeletedEvent, DirCreatedEvent, DirModifiedEvent, \
    FileMovedEvent, FileDeletedEvent, FileCreatedEvent, FileModifiedEvent

from win32con import FILE_SHARE_READ, FILE_SHARE_WRITE, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, \
    FILE_NOTIFY_CHANGE_FILE_NAME, FILE_NOTIFY_CHANGE_DIR_NAME, FILE_NOTIFY_CHANGE_ATTRIBUTES, \
    FILE_NOTIFY_CHANGE_SIZE, FILE_NOTIFY_CHANGE_LAST_WRITE, FILE_NOTIFY_CHANGE_SECURITY
from win32file import ReadDirectoryChangesW


FILE_LIST_DIRECTORY = 0x0001
BUFFER_SIZE = 1024
FILE_ACTION_CREATED = 1
FILE_ACTION_DELETED = 2
FILE_ACTION_MODIFIED = 3
FILE_ACTION_RENAMED_OLD_NAME = 4
FILE_ACTION_RENAMED_NEW_NAME = 5

DIR_ACTION_EVENT_MAP = {
    FILE_ACTION_CREATED: DirCreatedEvent,
    FILE_ACTION_DELETED: DirDeletedEvent,
    FILE_ACTION_MODIFIED: DirModifiedEvent,
}
FILE_ACTION_EVENT_MAP = {
    FILE_ACTION_CREATED: FileCreatedEvent,
    FILE_ACTION_DELETED: FileDeletedEvent,
    FILE_ACTION_MODIFIED: FileModifiedEvent,
}


class _Win32EventEmitter(Thread):
    def __init__(self, path, out_event_queue, *args, **kwargs):
        Thread.__init__(self)
        self.stopped = ThreadedEvent()
        self.setDaemon(True)
        self.path = path
        self.out_event_queue = out_event_queue

    def stop(self):
        self.stopped.set()

    def run(self):
        handle_directory = win32file.CreateFile (
            self.path,
            FILE_LIST_DIRECTORY,
            FILE_SHARE_READ | FILE_SHARE_WRITE,
            None,
            OPEN_EXISTING,
            FILE_FLAG_BACKUP_SEMANTICS,
            None
        )
        while not self.stopped.is_set():
            results = ReadDirectoryChangesW (
                handle_directory,
                BUFFER_SIZE,
                True,
                FILE_NOTIFY_CHANGE_FILE_NAME |
                FILE_NOTIFY_CHANGE_DIR_NAME |
                FILE_NOTIFY_CHANGE_ATTRIBUTES |
                FILE_NOTIFY_CHANGE_SIZE |
                FILE_NOTIFY_CHANGE_LAST_WRITE |
                FILE_NOTIFY_CHANGE_SECURITY,
                None,
                None
                )
            last_renamed_from_filename = ""
            q = self.out_event_queue
            for action, file in results:
                filename = path_join(self.path, file)
                if action == FILE_ACTION_RENAMED_OLD_NAME:
                    last_renamed_from_filename = filename
                elif action == FILE_ACTION_RENAMED_NEW_NAME:
                    if path_isdir(filename):
                        q.put((self.path, DirMovedEvent(last_renamed_from_filename, filename)))
                    else:
                        q.put((self.path, FileMovedEvent(last_renamed_from_filename, filename)))
                else:
                    if path_isdir(filename):
                        action_event_map = DIR_ACTION_EVENT_MAP
                    else:
                        action_event_map = FILE_ACTION_EVENT_MAP
                    q.put((self.path, action_event_map[action](filename)))


class Win32Observer(PollingObserver):
    def _create_event_emitter(self, path):
        return _Win32EventEmitter(path=path, out_event_queue=self.event_queue)


if __name__ == '__main__':<commit_msg>Avoid recalculating flags per call.<commit_after>﻿# -*- coding: utf-8 -*-

import win32file
import win32con

from os.path import realpath, abspath, sep as path_separator, join as path_join, isdir as path_isdir
from threading import Thread, Event as ThreadedEvent
from Queue import Queue
from polling_observer import PollingObserver, _Rule
from events import DirMovedEvent, DirDeletedEvent, DirCreatedEvent, DirModifiedEvent, \
    FileMovedEvent, FileDeletedEvent, FileCreatedEvent, FileModifiedEvent

from win32con import FILE_SHARE_READ, FILE_SHARE_WRITE, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, \
    FILE_NOTIFY_CHANGE_FILE_NAME, FILE_NOTIFY_CHANGE_DIR_NAME, FILE_NOTIFY_CHANGE_ATTRIBUTES, \
    FILE_NOTIFY_CHANGE_SIZE, FILE_NOTIFY_CHANGE_LAST_WRITE, FILE_NOTIFY_CHANGE_SECURITY
from win32file import ReadDirectoryChangesW



FILE_LIST_DIRECTORY = 0x0001
BUFFER_SIZE = 1024
FILE_NOTIFY_FLAGS = FILE_NOTIFY_CHANGE_FILE_NAME | FILE_NOTIFY_CHANGE_DIR_NAME | \
    FILE_NOTIFY_CHANGE_ATTRIBUTES | FILE_NOTIFY_CHANGE_SIZE | \
    FILE_NOTIFY_CHANGE_LAST_WRITE | FILE_NOTIFY_CHANGE_SECURITY

FILE_SHARE_FLAGS = FILE_SHARE_READ | FILE_SHARE_WRITE
    
FILE_ACTION_CREATED = 1
FILE_ACTION_DELETED = 2
FILE_ACTION_MODIFIED = 3
FILE_ACTION_RENAMED_OLD_NAME = 4
FILE_ACTION_RENAMED_NEW_NAME = 5

DIR_ACTION_EVENT_MAP = {
    FILE_ACTION_CREATED: DirCreatedEvent,
    FILE_ACTION_DELETED: DirDeletedEvent,
    FILE_ACTION_MODIFIED: DirModifiedEvent,
}
FILE_ACTION_EVENT_MAP = {
    FILE_ACTION_CREATED: FileCreatedEvent,
    FILE_ACTION_DELETED: FileDeletedEvent,
    FILE_ACTION_MODIFIED: FileModifiedEvent,
}


class _Win32EventEmitter(Thread):
    def __init__(self, path, out_event_queue, *args, **kwargs):
        Thread.__init__(self)
        self.stopped = ThreadedEvent()
        self.setDaemon(True)
        self.path = path
        self.out_event_queue = out_event_queue

    def stop(self):
        self.stopped.set()

    def run(self):
        handle_directory = win32file.CreateFile (
            self.path,
            FILE_LIST_DIRECTORY,
            FILE_SHARE_FLAGS,
            None,
            OPEN_EXISTING,
            FILE_FLAG_BACKUP_SEMANTICS,
            None
        )
        while not self.stopped.is_set():
            results = ReadDirectoryChangesW (
                handle_directory,
                BUFFER_SIZE,
                True,
                FILE_NOTIFY_FLAGS,
                None,
                None
                )
            last_renamed_from_filename = ""
            q = self.out_event_queue
            for action, file in results:
                filename = path_join(self.path, file)
                if action == FILE_ACTION_RENAMED_OLD_NAME:
                    last_renamed_from_filename = filename
                elif action == FILE_ACTION_RENAMED_NEW_NAME:
                    if path_isdir(filename):
                        q.put((self.path, DirMovedEvent(last_renamed_from_filename, filename)))
                    else:
                        q.put((self.path, FileMovedEvent(last_renamed_from_filename, filename)))
                else:
                    if path_isdir(filename):
                        action_event_map = DIR_ACTION_EVENT_MAP
                    else:
                        action_event_map = FILE_ACTION_EVENT_MAP
                    q.put((self.path, action_event_map[action](filename)))


class Win32Observer(PollingObserver):
    def _create_event_emitter(self, path):
        return _Win32EventEmitter(path=path, out_event_queue=self.event_queue)


if __name__ == '__main__':
{'fraction_non_alphanumeric': 0.05988445822178385, 'fraction_numerical': 0.00873608567000141, 'mean_word_length': 2.6012176560121767, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7527817', 'n_tokens_mistral': 2210, 'n_tokens_neox': 2121, 'n_words': 385}