<commit_before>
import time
import threading
import xlog
import sys
from config import config

# change to False when exit: system tray exit menu, or Ctrl+C in console
# then GoAgent will quit
# Every long running thread should check it and exit when False
keep_running = True

#=============================================
# concurrent connect control
# Windows10 will block sound when too many concurrent connect out in the same time.
# so when user request web, scan thread will stop to reduce concurrent action.
def check_win10():
    if sys.platform != "win32":
        return False

    import ctypes
    class OSVERSIONINFOEXW(ctypes.Structure):
        _fields_ = [('dwOSVersionInfoSize', ctypes.c_ulong),
                    ('dwMajorVersion', ctypes.c_ulong),
                    ('dwMinorVersion', ctypes.c_ulong),
                    ('dwBuildNumber', ctypes.c_ulong),
                    ('dwPlatformId', ctypes.c_ulong),
                    ('szCSDVersion', ctypes.c_wchar*128),
                    ('wServicePackMajor', ctypes.c_ushort),
                    ('wServicePackMinor', ctypes.c_ushort),
                    ('wSuiteMask', ctypes.c_ushort),
                    ('wProductType', ctypes.c_byte),
                    ('wReserved', ctypes.c_byte)]

    os_version = OSVERSIONINFOEXW()
    os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version)
    retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version))
    if retcode != 0:
        xlog.warn("Failed to get win32 OS version")
        return False

    if os_version.dwMajorVersion == 10:
        xlog.info("detect Win10, enable connect concurent control.")
        return True

    return False

is_win10 = check_win10()

ccc_lock = threading.Lock()
high_prior_lock = []
low_prior_lock = []
high_prior_connecting_num = 0
low_prior_connecting_num = 0

def start_connect_register(high_prior=False):
    global high_prior_connecting_num, low_prior_connecting_num
    if not is_win10:
        return

    ccc_lock.acquire()
    try:
        if high_prior_connecting_num + low_prior_connecting_num > config.https_max_connect_thread:
            atom_lock = threading.Lock()
            atom_lock.acquire()
            if high_prior:
                high_prior_lock.append(atom_lock)
            else:
                low_prior_lock.append(atom_lock)
            ccc_lock.release()
            atom_lock.acquire()

            ccc_lock.acquire()

        if high_prior:
            high_prior_connecting_num += 1
        else:
            low_prior_connecting_num += 1
    finally:
        ccc_lock.release()


def end_connect_register(high_prior=False):
    global high_prior_connecting_num, low_prior_connecting_num
    if not is_win10:
        return

    ccc_lock.acquire()
    try:
        if high_prior:
            high_prior_connecting_num -= 1
        else:
            low_prior_connecting_num -= 1

        if high_prior_connecting_num + low_prior_connecting_num < config.https_max_connect_thread:
            if len(high_prior_lock):
                atom_lock = high_prior_lock.pop()
                atom_lock.release()
                return

            if len(low_prior_lock):
                atom_lock = low_prior_lock.pop()
                atom_lock.release()<commit_msg>add connect interval control for win10<commit_after>
import time
import threading
import xlog
import sys
from config import config

# change to False when exit: system tray exit menu, or Ctrl+C in console
# then GoAgent will quit
# Every long running thread should check it and exit when False
keep_running = True

#=============================================
# concurrent connect control
# Windows10 will block sound when too many concurrent connect out in the same time.
# so when user request web, scan thread will stop to reduce concurrent action.
def check_win10():
    if sys.platform != "win32":
        return False

    import ctypes
    class OSVERSIONINFOEXW(ctypes.Structure):
        _fields_ = [('dwOSVersionInfoSize', ctypes.c_ulong),
                    ('dwMajorVersion', ctypes.c_ulong),
                    ('dwMinorVersion', ctypes.c_ulong),
                    ('dwBuildNumber', ctypes.c_ulong),
                    ('dwPlatformId', ctypes.c_ulong),
                    ('szCSDVersion', ctypes.c_wchar*128),
                    ('wServicePackMajor', ctypes.c_ushort),
                    ('wServicePackMinor', ctypes.c_ushort),
                    ('wSuiteMask', ctypes.c_ushort),
                    ('wProductType', ctypes.c_byte),
                    ('wReserved', ctypes.c_byte)]

    os_version = OSVERSIONINFOEXW()
    os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version)
    retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version))
    if retcode != 0:
        xlog.warn("Failed to get win32 OS version")
        return False

    if os_version.dwMajorVersion == 10:
        xlog.info("detect Win10, enable connect concurent control.")
        return True

    return False

is_win10 = check_win10()

ccc_lock = threading.Lock()
high_prior_lock = []
low_prior_lock = []
high_prior_connecting_num = 0
low_prior_connecting_num = 0
last_connect_time = 0

min_connect_interval = 0.03

def start_connect_register(high_prior=False):
    global high_prior_connecting_num, low_prior_connecting_num, last_connect_time
    if not is_win10:
        return

    ccc_lock.acquire()
    try:
        if high_prior_connecting_num + low_prior_connecting_num > config.https_max_connect_thread:
            atom_lock = threading.Lock()
            atom_lock.acquire()
            if high_prior:
                high_prior_lock.append(atom_lock)
            else:
                low_prior_lock.append(atom_lock)
            ccc_lock.release()
            atom_lock.acquire()

            ccc_lock.acquire()

        last_connect_interval = time.time() - last_connect_time
        if last_connect_interval < 0:
            xlog.error("last_connect_interval:%f", last_connect_interval)
            return

        if last_connect_interval < min_connect_interval:
            wait_time = min_connect_interval - last_connect_interval
            time.sleep(wait_time)

        if high_prior:
            high_prior_connecting_num += 1
        else:
            low_prior_connecting_num += 1
    finally:
        last_connect_time = time.time()
        ccc_lock.release()


def end_connect_register(high_prior=False):
    global high_prior_connecting_num, low_prior_connecting_num
    if not is_win10:
        return

    ccc_lock.acquire()
    try:
        if high_prior:
            high_prior_connecting_num -= 1
        else:
            low_prior_connecting_num -= 1

        if high_prior_connecting_num + low_prior_connecting_num < config.https_max_connect_thread:
            if len(high_prior_lock):
                atom_lock = high_prior_lock.pop()
                atom_lock.release()
                return

            if len(low_prior_lock):
                atom_lock = low_prior_lock.pop()
                atom_lock.release()
{'fraction_non_alphanumeric': 0.08780903665814152, 'fraction_numerical': 0.00951974992895709, 'mean_word_length': 2.4070667957405614, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6325233', 'n_tokens_mistral': 2212, 'n_tokens_neox': 2032, 'n_words': 457}