<commit_before>
'''
@author: <NAME>
'''
from nxdrive.logging_config import get_logger
from watchdog.events import FileSystemEventHandler
from nxdrive.engine.workers import EngineWorker, ThreadInterrupt
from nxdrive.utils import current_milli_time
from nxdrive.engine.activity import Action
from Queue import Queue
import sys
import os
from time import sleep, time
from threading import Lock
from PyQt4.QtCore import pyqtSignal, pyqtSlot
log = get_logger(__name__)

# Windows 2s between resolution of delete event
WIN_MOVE_RESOLUTION_PERIOD = 2000


class LocalWatcher(EngineWorker):
    localScanFinished = pyqtSignal()
    rootMoved = pyqtSignal(str)
    rootDeleted = pyqtSignal()
    '''
    classdocs
    '''
    def __init__(self, engine, dao):
        '''
        Constructor
        '''
        super(LocalWatcher, self).__init__(engine, dao)
        self.unhandle_fs_event = False
        self._event_handler = None
        self._windows = (sys.platform == 'win32')
        if self._windows:
            log.debug('Windows detected so delete event will be delayed by %dms', WIN_MOVE_RESOLUTION_PERIOD)
        # TODO Review to delete
        self._init()

    def _init(self):
        self.local_full_scan = dict()
        self._local_scan_finished = False
        self.client = self._engine.get_local_client()
        self._metrics = dict()
        self._metrics['last_local_scan_time'] = -1
        self._metrics['new_files'] = 0
        self._metrics['update_files'] = 0
        self._metrics['delete_files'] = 0
        self._metrics['last_event'] = 0
        self._observer = None
        self._root_observer = None
        self._win_lock = Lock()
        self._delete_events = dict()

    def _execute(self):
        try:
            trigger_local_scan = False
            self._init()
            if not self.client.exists('/'):
                self.rootDeleted.emit()
                return
            self._action = Action("Setup watchdog")
            self._watchdog_queue = Queue()
            self._setup_watchdog()
            log.debug("Watchdog setup finished")
            self._action = Action("Full local scan")
            self._scan()
            self._end_action()
            # Check windows dequeue only every 100 loops ( every 1s )
            self._win_delete_interval = int(round(time() * 1000))
            while (1):
                self._interact()
                sleep(0.01)
                if trigger_local_scan:
                    self._action = Action("Full local scan")
                    self._scan()
                    trigger_local_scan = False
                    self._end_action()
                while (not self._watchdog_queue.empty()):
                    # Dont retest if already local scan
                    if not trigger_local_scan and self._watchdog_queue.qsize() > 50:
                        trigger_local_scan = True
                    evt = self._watchdog_queue.get()
                    self.handle_watchdog_event(evt)
                    self._win_delete_check()
                self._win_delete_check()

        except ThreadInterrupt:
            raise
        finally:
            self._stop_watchdog()

    def win_queue_empty(self):
        return len(self._delete_events) == 0

    def _win_delete_check(self):
        if self._windows and self._win_delete_interval < int(round(time() * 1000)) - WIN_MOVE_RESOLUTION_PERIOD:
            self._action = Action("Dequeue delete")
            self._win_dequeue_delete()
            self._end_action()
            self._win_delete_interval = int(round(time() * 1000))

    def _win_dequeue_delete(self):
        self._win_lock.acquire()
        try:
            delete_events = self._delete_events
            for evt in delete_events.values():
                evt_time = evt[0]
                evt_pair = evt[1]
                if current_milli_time() - evt_time < WIN_MOVE_RESOLUTION_PERIOD:
                    log.debug("Win: ignoring delete event as waiting for move resolution period expiration: %r", evt)
                    continue
                if not self.client.exists(evt_pair.local_path):
                    log.debug("Win: handling watchdog delete for event: %r", evt)
                    self._handle_watchdog_delete(evt_pair)
                else:
                    log.debug("Win: ignoring delete event as file still exists: %r", evt)
                log.debug("Win: dequeuing delete event: %r", evt)
                del self._delete_events[evt_pair.remote_ref]
        except ThreadInterrupt:
            raise
        except Exception as e:
            log.exception(e)
        finally:
            self._win_lock.release()
<commit_msg>NXDRIVE-221: Add same remote_id verification on delete for Windows<commit_after>
'''
@author: <NAME>
'''
from nxdrive.logging_config import get_logger
from watchdog.events import FileSystemEventHandler
from nxdrive.engine.workers import EngineWorker, ThreadInterrupt
from nxdrive.utils import current_milli_time
from nxdrive.engine.activity import Action
from Queue import Queue
import sys
import os
from time import sleep, time
from threading import Lock
from PyQt4.QtCore import pyqtSignal, pyqtSlot
log = get_logger(__name__)

# Windows 2s between resolution of delete event
WIN_MOVE_RESOLUTION_PERIOD = 2000


class LocalWatcher(EngineWorker):
    localScanFinished = pyqtSignal()
    rootMoved = pyqtSignal(str)
    rootDeleted = pyqtSignal()
    '''
    classdocs
    '''
    def __init__(self, engine, dao):
        '''
        Constructor
        '''
        super(LocalWatcher, self).__init__(engine, dao)
        self.unhandle_fs_event = False
        self._event_handler = None
        self._windows = (sys.platform == 'win32')
        if self._windows:
            log.debug('Windows detected so delete event will be delayed by %dms', WIN_MOVE_RESOLUTION_PERIOD)
        # TODO Review to delete
        self._init()

    def _init(self):
        self.local_full_scan = dict()
        self._local_scan_finished = False
        self.client = self._engine.get_local_client()
        self._metrics = dict()
        self._metrics['last_local_scan_time'] = -1
        self._metrics['new_files'] = 0
        self._metrics['update_files'] = 0
        self._metrics['delete_files'] = 0
        self._metrics['last_event'] = 0
        self._observer = None
        self._root_observer = None
        self._win_lock = Lock()
        self._delete_events = dict()

    def _execute(self):
        try:
            trigger_local_scan = False
            self._init()
            if not self.client.exists('/'):
                self.rootDeleted.emit()
                return
            self._action = Action("Setup watchdog")
            self._watchdog_queue = Queue()
            self._setup_watchdog()
            log.debug("Watchdog setup finished")
            self._action = Action("Full local scan")
            self._scan()
            self._end_action()
            # Check windows dequeue only every 100 loops ( every 1s )
            self._win_delete_interval = int(round(time() * 1000))
            while (1):
                self._interact()
                sleep(0.01)
                if trigger_local_scan:
                    self._action = Action("Full local scan")
                    self._scan()
                    trigger_local_scan = False
                    self._end_action()
                while (not self._watchdog_queue.empty()):
                    # Dont retest if already local scan
                    if not trigger_local_scan and self._watchdog_queue.qsize() > 50:
                        trigger_local_scan = True
                    evt = self._watchdog_queue.get()
                    self.handle_watchdog_event(evt)
                    self._win_delete_check()
                self._win_delete_check()

        except ThreadInterrupt:
            raise
        finally:
            self._stop_watchdog()

    def win_queue_empty(self):
        return len(self._delete_events) == 0

    def _win_delete_check(self):
        if self._windows and self._win_delete_interval < int(round(time() * 1000)) - WIN_MOVE_RESOLUTION_PERIOD:
            self._action = Action("Dequeue delete")
            self._win_dequeue_delete()
            self._end_action()
            self._win_delete_interval = int(round(time() * 1000))

    def _win_dequeue_delete(self):
        self._win_lock.acquire()
        try:
            delete_events = self._delete_events
            for evt in delete_events.values():
                evt_time = evt[0]
                evt_pair = evt[1]
                if current_milli_time() - evt_time < WIN_MOVE_RESOLUTION_PERIOD:
                    log.debug("Win: ignoring delete event as waiting for move resolution period expiration: %r", evt)
                    continue
                if not self.client.exists(evt_pair.local_path):
                    log.debug("Win: handling watchdog delete for event: %r", evt)
                    self._handle_watchdog_delete(evt_pair)
                else:
                    remote_id = self.client.get_remote_id(evt_pair.local_path)
                    if remote_id == evt_pair.remote_ref or remote_id is None:
                        log.debug("Win: ignoring delete event as file still exists: %r", evt)
                    else:
                        log.debug("Win: handling watchdog delete for event: %r", evt)
                        self._handle_watchdog_delete(evt_pair)
                log.debug("Win: dequeuing delete event: %r", evt)
                del self._delete_events[evt_pair.remote_ref]
        except ThreadInterrupt:
            raise
        except Exception as e:
            log.exception(e)
        finally:
            self._win_lock.release()

{'fraction_non_alphanumeric': 0.08481181417290534, 'fraction_numerical': 0.008101733155573788, 'mean_word_length': 1.922385376086305, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6958244', 'n_tokens_mistral': 2744, 'n_tokens_neox': 2636, 'n_words': 629}