<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2011 <NAME> <<EMAIL>>
# Copyright 2012 Google, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


from watchdog.utils import platform

if platform.is_windows():
  import ctypes

  from watchdog.observers.winapi import\
    FILE_FLAG_BACKUP_SEMANTICS,\
    FILE_FLAG_OVERLAPPED,\
    FILE_SHARE_READ,\
    FILE_SHARE_WRITE,\
    FILE_SHARE_DELETE,\
    FILE_NOTIFY_CHANGE_FILE_NAME,\
    FILE_NOTIFY_CHANGE_DIR_NAME,\
    FILE_NOTIFY_CHANGE_ATTRIBUTES,\
    FILE_NOTIFY_CHANGE_SIZE,\
    FILE_NOTIFY_CHANGE_LAST_WRITE,\
    FILE_NOTIFY_CHANGE_SECURITY,\
    FILE_NOTIFY_CHANGE_LAST_ACCESS,\
    FILE_NOTIFY_CHANGE_CREATION,\
    FILE_ACTION_CREATED,\
    FILE_ACTION_DELETED,\
    FILE_ACTION_MODIFIED,\
    FILE_LIST_DIRECTORY,\
    OPEN_EXISTING,\
    INVALID_HANDLE_VALUE,\
    CreateFileW,\
    ReadDirectoryChangesW,\
    CreateIoCompletionPort,\
    CancelIoEx
  from watchdog.events import\
    DirDeletedEvent,\
    DirCreatedEvent,\
    DirModifiedEvent,\
    FileDeletedEvent,\
    FileCreatedEvent,\
    FileModifiedEvent

  # We don't need to recalculate these flags every time a call is made to
  # the win32 API functions.
  WATCHDOG_FILE_FLAGS = FILE_FLAG_BACKUP_SEMANTICS
  WATCHDOG_FILE_FLAGS_ASYNC = FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED
  WATCHDOG_FILE_SHARE_FLAGS = reduce(lambda x, y: x | y, [
    FILE_SHARE_READ,
    FILE_SHARE_WRITE,
    FILE_SHARE_DELETE,
    ])
  WATCHDOG_FILE_NOTIFY_FLAGS = reduce(lambda x, y: x | y, [
    FILE_NOTIFY_CHANGE_FILE_NAME,
    FILE_NOTIFY_CHANGE_DIR_NAME,
    FILE_NOTIFY_CHANGE_ATTRIBUTES,
    FILE_NOTIFY_CHANGE_SIZE,
    FILE_NOTIFY_CHANGE_LAST_WRITE,
    FILE_NOTIFY_CHANGE_SECURITY,
    FILE_NOTIFY_CHANGE_LAST_ACCESS,
    FILE_NOTIFY_CHANGE_CREATION,
    ])


  # HACK:
  WATCHDOG_TRAVERSE_MOVED_DIR_DELAY = 1   # seconds

  BUFFER_SIZE = 2048

  # Moved event is handled explicitly in the emitter thread.
  DIR_ACTION_EVENT_MAP = {
    FILE_ACTION_CREATED: DirCreatedEvent,
    FILE_ACTION_DELETED: DirDeletedEvent,
    FILE_ACTION_MODIFIED: DirModifiedEvent,
    }
  FILE_ACTION_EVENT_MAP = {
    FILE_ACTION_CREATED: FileCreatedEvent,
    FILE_ACTION_DELETED: FileDeletedEvent,
    FILE_ACTION_MODIFIED: FileModifiedEvent,
    }


  def get_directory_handle(path, file_flags):
    """Returns a Windows handle to the specified directory path."""
    handle = CreateFileW(path,
                         FILE_LIST_DIRECTORY,
                         WATCHDOG_FILE_SHARE_FLAGS,
                         None,
                         OPEN_EXISTING,
                         file_flags,
                         None)
    return handle


  def close_directory_handle(handle):
    CancelIoEx(handle, None) #force ReadDirectoryChangesW to return


  def read_directory_changes(handle, event_buffer, recursive):
    """Read changes to the directory using the specified directory handle.

    http://timgolden.me.uk/pywin32-docs/win32file__ReadDirectoryChangesW_meth.html
    """
    nbytes = ctypes.wintypes.DWORD()
    ReadDirectoryChangesW(handle,
                          ctypes.byref(event_buffer),
                          len(event_buffer),
                          recursive,
                          WATCHDOG_FILE_NOTIFY_FLAGS,
                          ctypes.byref(nbytes),
                          None,
                          None)<commit_msg>handle exceptions that occur during io canceling<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2011 <NAME> <<EMAIL>>
# Copyright 2012 Google, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


from watchdog.utils import platform

if platform.is_windows():
  import ctypes

  from watchdog.observers.winapi import\
    FILE_FLAG_BACKUP_SEMANTICS,\
    FILE_FLAG_OVERLAPPED,\
    FILE_SHARE_READ,\
    FILE_SHARE_WRITE,\
    FILE_SHARE_DELETE,\
    FILE_NOTIFY_CHANGE_FILE_NAME,\
    FILE_NOTIFY_CHANGE_DIR_NAME,\
    FILE_NOTIFY_CHANGE_ATTRIBUTES,\
    FILE_NOTIFY_CHANGE_SIZE,\
    FILE_NOTIFY_CHANGE_LAST_WRITE,\
    FILE_NOTIFY_CHANGE_SECURITY,\
    FILE_NOTIFY_CHANGE_LAST_ACCESS,\
    FILE_NOTIFY_CHANGE_CREATION,\
    FILE_ACTION_CREATED,\
    FILE_ACTION_DELETED,\
    FILE_ACTION_MODIFIED,\
    FILE_LIST_DIRECTORY,\
    OPEN_EXISTING,\
    INVALID_HANDLE_VALUE,\
    CreateFileW,\
    ReadDirectoryChangesW,\
    CreateIoCompletionPort,\
    CancelIoEx
  from watchdog.events import\
    DirDeletedEvent,\
    DirCreatedEvent,\
    DirModifiedEvent,\
    FileDeletedEvent,\
    FileCreatedEvent,\
    FileModifiedEvent

  # We don't need to recalculate these flags every time a call is made to
  # the win32 API functions.
  WATCHDOG_FILE_FLAGS = FILE_FLAG_BACKUP_SEMANTICS
  WATCHDOG_FILE_FLAGS_ASYNC = FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED
  WATCHDOG_FILE_SHARE_FLAGS = reduce(lambda x, y: x | y, [
    FILE_SHARE_READ,
    FILE_SHARE_WRITE,
    FILE_SHARE_DELETE,
    ])
  WATCHDOG_FILE_NOTIFY_FLAGS = reduce(lambda x, y: x | y, [
    FILE_NOTIFY_CHANGE_FILE_NAME,
    FILE_NOTIFY_CHANGE_DIR_NAME,
    FILE_NOTIFY_CHANGE_ATTRIBUTES,
    FILE_NOTIFY_CHANGE_SIZE,
    FILE_NOTIFY_CHANGE_LAST_WRITE,
    FILE_NOTIFY_CHANGE_SECURITY,
    FILE_NOTIFY_CHANGE_LAST_ACCESS,
    FILE_NOTIFY_CHANGE_CREATION,
    ])


  # HACK:
  WATCHDOG_TRAVERSE_MOVED_DIR_DELAY = 1   # seconds

  BUFFER_SIZE = 2048

  # Moved event is handled explicitly in the emitter thread.
  DIR_ACTION_EVENT_MAP = {
    FILE_ACTION_CREATED: DirCreatedEvent,
    FILE_ACTION_DELETED: DirDeletedEvent,
    FILE_ACTION_MODIFIED: DirModifiedEvent,
    }
  FILE_ACTION_EVENT_MAP = {
    FILE_ACTION_CREATED: FileCreatedEvent,
    FILE_ACTION_DELETED: FileDeletedEvent,
    FILE_ACTION_MODIFIED: FileModifiedEvent,
    }


  def get_directory_handle(path, file_flags):
    """Returns a Windows handle to the specified directory path."""
    handle = CreateFileW(path,
                         FILE_LIST_DIRECTORY,
                         WATCHDOG_FILE_SHARE_FLAGS,
                         None,
                         OPEN_EXISTING,
                         file_flags,
                         None)
    return handle


  def close_directory_handle(handle):
    try:
      CancelIoEx(handle, None) #force ReadDirectoryChangesW to return
    except WindowsError:
      return


  def read_directory_changes(handle, event_buffer, recursive):
    """Read changes to the directory using the specified directory handle.

    http://timgolden.me.uk/pywin32-docs/win32file__ReadDirectoryChangesW_meth.html
    """
    nbytes = ctypes.wintypes.DWORD()
    try:
      ReadDirectoryChangesW(handle,
                            ctypes.byref(event_buffer),
                            len(event_buffer),
                            recursive,
                            WATCHDOG_FILE_NOTIFY_FLAGS,
                            ctypes.byref(nbytes),
                            None,
                            None)
    except WindowsError:
      return [], 0
{'fraction_non_alphanumeric': 0.06583072100313479, 'fraction_numerical': 0.006144200626959247, 'mean_word_length': 2.8512795750845004, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4541004', 'n_tokens_mistral': 2525, 'n_tokens_neox': 2456, 'n_words': 540}