<commit_before>import imp
from hoverboard import config
from hoverboard import plugin
from hoverboard import exceptions
from hoverboard import clipboard
from hoverboard import actions
import threading
import collections
import logging
import traceback
import time
import datetime

plugins = {}
backends = {}
backend = None
last_modified = datetime.datetime.min
last_modified_device = datetime.datetime.min
last_checkin = datetime.datetime.min
last_pulled = None
access_revoked = False
pull_threads = []
devices = []

class UploadThread(threading.Thread):
    def __init__(self,group,queue_lock,queue):
        threading.Thread.__init__(self,group=group, target=self.upload_thread_func, args=(queue_lock,queue))
        self._stop = threading.Event()

    def stop(self):
        self._stop.set()

    def stopped(self):
        return self._stop.isSet()

    def upload_thread_func(self,lock, queue):
        import hoverboard
        while True:
            if self.stopped():
                break
            if len(queue):
                try:
                    if hoverboard.backend is not None and hoverboard.backend.check_validity():
                        data, format, device = queue.popleft()
                        if len(data) < hoverboard.config.max_size:
                            if device is None:
                                backend.push_clip(data,format)
                            else:
                                backend.push_clip(data,format,device)
                except exceptions.AccessRevokedException:
                    hoverboard.access_revoked = True
                except Exception as e:
                    logging.error(traceback.format_exc())
            time.sleep(.5)

class CleanupThread(threading.Thread):
    def __init__(self,group,backend_lock,device_name):
        threading.Thread.__init__(self,group=group, target=self.cleanup_thread_func, args=(backend_lock,))
        self._stop = threading.Event()
        self.device_name = device_name

    def stop(self):
        self._stop.set()

    def stopped(self):
        return self._stop.isSet()

    def cleanup_thread_func(self,lock):
        import hoverboard
        while True:
            if self.stopped():
                break
            try:
                if hoverboard.backend is not None:
                    # Not strictly cleanup, but I don't feel like starting up another thread.
                    hoverboard.devices = hoverboard.backend.get_devices(self.device_name)
                    timedelta = datetime.datetime.now() - hoverboard.last_checkin
                    # Check in every tenish minutes
                    if (timedelta.days * 86400 + timedelta.seconds)/60 > 10:
                        hoverboard.backend.checkin(hoverboard.settings.device_name)
                        hoverboard.last_checkin = datetime.datetime.now()
                    for device in (None,self.device_name):
                        if hoverboard.backend.check_validity():
                            with lock:
                                files = sorted(hoverboard.backend.list_clips(device), key=lambda x: x.modified)
                            if files and not hoverboard.access_revoked:
                                totalsize = sum(x.size for x in files)
                                while totalsize > config.max_size:<commit_msg>Increased time for device fetching to reduce server load<commit_after>import imp
from hoverboard import config
from hoverboard import plugin
from hoverboard import exceptions
from hoverboard import clipboard
from hoverboard import actions
import threading
import collections
import logging
import traceback
import time
import datetime

plugins = {}
backends = {}
backend = None
last_modified = datetime.datetime.min
last_modified_device = datetime.datetime.min
last_checkin = datetime.datetime.min
last_pulled = None
access_revoked = False
pull_threads = []
devices = []

class UploadThread(threading.Thread):
    def __init__(self,group,queue_lock,queue):
        threading.Thread.__init__(self,group=group, target=self.upload_thread_func, args=(queue_lock,queue))
        self._stop = threading.Event()

    def stop(self):
        self._stop.set()

    def stopped(self):
        return self._stop.isSet()

    def upload_thread_func(self,lock, queue):
        import hoverboard
        while True:
            if self.stopped():
                break
            if len(queue):
                try:
                    if hoverboard.backend is not None and hoverboard.backend.check_validity():
                        data, format, device = queue.popleft()
                        if len(data) < hoverboard.config.max_size:
                            if device is None:
                                backend.push_clip(data,format)
                            else:
                                backend.push_clip(data,format,device)
                except exceptions.AccessRevokedException:
                    hoverboard.access_revoked = True
                except Exception as e:
                    logging.error(traceback.format_exc())
            time.sleep(.5)

class CleanupThread(threading.Thread):
    def __init__(self,group,backend_lock,device_name):
        threading.Thread.__init__(self,group=group, target=self.cleanup_thread_func, args=(backend_lock,))
        self._stop = threading.Event()
        self.device_name = device_name

    def stop(self):
        self._stop.set()

    def stopped(self):
        return self._stop.isSet()

    def cleanup_thread_func(self,lock):
        import hoverboard
        while True:
            if self.stopped():
                break
            try:
                if hoverboard.backend is not None:
                    timedelta = datetime.datetime.now() - hoverboard.last_checkin
                    # Not strictly cleanup, but I don't feel like starting up another thread.
                    # Check for devices every 2 minutes
                    if (timedelta.days * 86400 + timedelta.seconds)/60 > 2:
                        hoverboard.devices = hoverboard.backend.get_devices(self.device_name)
                    
                    # Check in every tenish minutes
                    if (timedelta.days * 86400 + timedelta.seconds)/60 > 10:
                        hoverboard.backend.checkin(hoverboard.settings.device_name)
                        hoverboard.last_checkin = datetime.datetime.now()
                    for device in (None,self.device_name):
                        if hoverboard.backend.check_validity():
                            with lock:
                                files = sorted(hoverboard.backend.list_clips(device), key=lambda x: x.modified)
                            if files and not hoverboard.access_revoked:
                                totalsize = sum(x.size for x in files)
                                while totalsize > config.max_size:
{'fraction_non_alphanumeric': 0.07538994800693241, 'fraction_numerical': 0.004188330444829578, 'mean_word_length': 1.8334697217675942, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4855136', 'n_tokens_mistral': 1784, 'n_tokens_neox': 1692, 'n_words': 418}