<commit_before>import ctypes
from ltk.actions import Action
from logger import logger
from utils import map_locale
import time
import requests
import os
from watchdog.observers import Observer
from ltk.watchhandler import WatchHandler

# import Queue

# import threading

# class WatchThread:
#     def __init__(self, interval=5):
#         self.interval = interval
#
#         self.thread = threading.Thread(target=self.run, args=())
#         self.thread.daemon = True
#         self.thread.start()_on_create
#
#     def run(self):
#         while True:
#             time.sleep(self.interval)

# retry decorator to retry connections
def retry(logger, timeout=5, exec_type=None):
    if not exec_type:
        exec_type = [requests.exceptions.ConnectionError]

    def decorator(function):
        def wrapper(*args, **kwargs):
            while True:
                try:
                    return function(*args, **kwargs)
                except Exception as e:
                    if e.__class__ in exec_type:
                        # some logging error here
                        logger.error("Connection has timed out. Retrying..")
                        # sleep for some time then retry
                        time.sleep(timeout)
                    else:
                        raise e
        return wrapper
    return decorator

def is_hidden_file(file_path):
    # todo more robust checking for OSX files that doesn't start with '.'
    name = os.path.basename(os.path.abspath(file_path))
    return name.startswith('.') or has_hidden_attribute(file_path)

def has_hidden_attribute(file_path):
    """ Detects if a file has hidden attributes """
    try:
        attrs = ctypes.windll.kernel32.GetFileAttributesW(unicode(file_path))
        assert attrs != -1
        result = bool(attrs & 2)
    except (AttributeError, AssertionError):
        result = False
    return result

class WatchAction(Action):
    # def __init__(self, path, remote=False):
    def __init__(self, path):
        Action.__init__(self, path)
        self.observer = Observer()  # watchdog observer that will watch the files
        self.handler = WatchHandler()
        self.handler.on_modified = self._on_modified
        self.handler.on_created = self._on_created
        self.watch_queue = []  # not much slower than deque unless expecting 100+ items
        self.locale_delimiter = None
        self.ignore_ext = []  # file types to ignore as specified by the user
        # if remote:  # poll lingotek cloud periodically if this option enabled
        # self.remote_thread = threading.Thread(target=self.poll_remote(), args=())
        # self.remote_thread.daemon = True
        # self.remote_thread.start()

    def check_remote_doc_exist(self, fn, document_id=None):
        """ check if a document exists remotely """
        if not document_id:
            entry = self.doc_manager.get_doc_by_prop('file_name', fn)
            document_id = entry['id']
        response = self.api.get_document(document_id)
        if response.status_code != 200:
            return False
        return True

    def _on_modified(self, event):
        """ Notify Lingotek cloud when a previously added file is modified """
        db_entries = self.doc_manager.get_all_entries()
        in_db = False
        fn = ''
        # print event.src_path
        for entry in db_entries:
            # print entry['file_name']
            if event.src_path.endswith(entry['file_name']):
                fn = entry['file_name']
                in_db = True
        if not event.is_directory and in_db:
            # check that document is added in TMS before updating
            if self.check_remote_doc_exist(fn):
                # logger.info('Detected local content modified: {0}'.format(fn))
                # self.update_document_action(os.path.join(self.path, fn))
                # logger.info('Updating remote content: {0}'.format(fn))
                self.update_content(fn)

    def _on_created(self, event):
        # get path
        # add action
        file_path = event.src_path
        relative_path = file_path.replace(self.path, '')
        title = os.path.basename(os.path.normpath(file_path))
        curr_ext = os.path.splitext(file_path)[1]
        if curr_ext in self.ignore_ext:
            logger.info("Detected a file with an extension in the ignore list, ignoring..")
            return
        if self.locale_delimiter:
            try:
                curr_locale = title.split(self.locale_delimiter)[1]
                fixed_locale = map_locale(curr_locale)
                if fixed_locale:
                    self.watch_locales.add(fixed_locale)
                else:
                    logger.warning('This document\'s detected locale: {0} is not supported.'.format(curr_locale))
            except IndexError:
                logger.warning('Cannot detect locales from file: {0}, not adding any locales'.format(title))
        # only add the document if it's not a hidden document and it's a new file
        if not is_hidden_file(file_path) and self.doc_manager.is_doc_new(relative_path):
            self.add_document(self.locale, file_path, title)
        elif self.doc_manager.is_doc_modified(relative_path):
            self.update_content(relative_path)
        document_id = self.doc_manager.get_doc_by_prop('name', title)['id']
        self.watch_add_target(title, document_id)
        # logger.info('Added new document {0}'.format(title))

    def watch_add_target(self, title, document_id):
        if self.check_remote_doc_exist(title, document_id):
            self.target_action(title, self.watch_locales, None, None, None, document_id)<commit_msg>fix adding folder while watch is running<commit_after>import ctypes
from ltk.actions import Action
from logger import logger
from utils import map_locale
import time
import requests
import os
from watchdog.observers import Observer
from ltk.watchhandler import WatchHandler

# import Queue

# import threading

# class WatchThread:
#     def __init__(self, interval=5):
#         self.interval = interval
#
#         self.thread = threading.Thread(target=self.run, args=())
#         self.thread.daemon = True
#         self.thread.start()_on_create
#
#     def run(self):
#         while True:
#             time.sleep(self.interval)

# retry decorator to retry connections
def retry(logger, timeout=5, exec_type=None):
    if not exec_type:
        exec_type = [requests.exceptions.ConnectionError]

    def decorator(function):
        def wrapper(*args, **kwargs):
            while True:
                try:
                    return function(*args, **kwargs)
                except Exception as e:
                    if e.__class__ in exec_type:
                        # some logging error here
                        logger.error("Connection has timed out. Retrying..")
                        # sleep for some time then retry
                        time.sleep(timeout)
                    else:
                        raise e
        return wrapper
    return decorator

def is_hidden_file(file_path):
    # todo more robust checking for OSX files that doesn't start with '.'
    name = os.path.basename(os.path.abspath(file_path))
    return name.startswith('.') or has_hidden_attribute(file_path)

def has_hidden_attribute(file_path):
    """ Detects if a file has hidden attributes """
    try:
        attrs = ctypes.windll.kernel32.GetFileAttributesW(unicode(file_path))
        assert attrs != -1
        result = bool(attrs & 2)
    except (AttributeError, AssertionError):
        result = False
    return result

class WatchAction(Action):
    # def __init__(self, path, remote=False):
    def __init__(self, path):
        Action.__init__(self, path)
        self.observer = Observer()  # watchdog observer that will watch the files
        self.handler = WatchHandler()
        self.handler.on_modified = self._on_modified
        self.handler.on_created = self._on_created
        self.watch_queue = []  # not much slower than deque unless expecting 100+ items
        self.locale_delimiter = None
        self.ignore_ext = []  # file types to ignore as specified by the user
        # if remote:  # poll lingotek cloud periodically if this option enabled
        # self.remote_thread = threading.Thread(target=self.poll_remote(), args=())
        # self.remote_thread.daemon = True
        # self.remote_thread.start()

    def check_remote_doc_exist(self, fn, document_id=None):
        """ check if a document exists remotely """
        if not document_id:
            entry = self.doc_manager.get_doc_by_prop('file_name', fn)
            document_id = entry['id']
        response = self.api.get_document(document_id)
        if response.status_code != 200:
            return False
        return True

    def _on_modified(self, event):
        """ Notify Lingotek cloud when a previously added file is modified """
        db_entries = self.doc_manager.get_all_entries()
        in_db = False
        fn = ''
        # print event.src_path
        for entry in db_entries:
            # print entry['file_name']
            if event.src_path.endswith(entry['file_name']):
                fn = entry['file_name']
                in_db = True
        if not event.is_directory and in_db:
            # check that document is added in TMS before updating
            if self.check_remote_doc_exist(fn):
                # logger.info('Detected local content modified: {0}'.format(fn))
                # self.update_document_action(os.path.join(self.path, fn))
                # logger.info('Updating remote content: {0}'.format(fn))
                self.update_content(fn)

    def _on_created(self, event):
        # get path
        # add action
        file_path = event.src_path
        relative_path = file_path.replace(self.path, '')
        title = os.path.basename(os.path.normpath(file_path))
        curr_ext = os.path.splitext(file_path)[1]
        # return if the extension should be ignored or if the path is not a file
        if curr_ext in self.ignore_ext or not os.path.isfile(file_path):
            # logger.info("Detected a file with an extension in the ignore list, ignoring..")
            return
        if self.locale_delimiter:
            try:
                curr_locale = title.split(self.locale_delimiter)[1]
                fixed_locale = map_locale(curr_locale)
                if fixed_locale:
                    self.watch_locales.add(fixed_locale)
                else:
                    logger.warning('This document\'s detected locale: {0} is not supported.'.format(curr_locale))
            except IndexError:
                logger.warning('Cannot detect locales from file: {0}, not adding any locales'.format(title))
        # only add the document if it's not a hidden document and it's a new file
        if not is_hidden_file(file_path) and self.doc_manager.is_doc_new(relative_path):
            self.add_document(self.locale, file_path, title)
        elif self.doc_manager.is_doc_modified(relative_path):
            self.update_content(relative_path)
        document_id = self.doc_manager.get_doc_by_prop('name', title)['id']
        self.watch_add_target(title, document_id)
        # logger.info('Added new document {0}'.format(title))

    def watch_add_target(self, title, document_id):
        if self.check_remote_doc_exist(title, document_id):
            self.target_action(title, self.watch_locales, None, None, None, document_id)
{'fraction_non_alphanumeric': 0.08922515001304461, 'fraction_numerical': 0.003304635185668319, 'mean_word_length': 2.3093525179856114, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1717920', 'n_tokens_mistral': 3236, 'n_tokens_neox': 3146, 'n_words': 845}