<commit_before>import sys
import subprocess
import os
import signal
import select
import stat
import xbmc
import xbmcaddon
import urllib2
from pulsar.config import PULSARD_HOST
from pulsar.platform import PLATFORM
from pulsar.util import notify


def ensure_exec_perms(file_):
    st = os.stat(file_)
    os.chmod(file_, st.st_mode | stat.S_IEXEC)
    return file_


def android_get_current_appid():
    uid = os.getuid()
    with open("/proc/%d/cmdline", uid) as fp:
        return fp.read().rstrip()


def get_pulsar_binary():
    binary = "pulsar" + (PLATFORM["os"] == "windows" and ".exe" or "")

    platform = PLATFORM.copy()
    if platform["os"] == "darwin": # 64 bits anyway on Darwin
        platform["arch"] = "x64"
    elif platform["os"] == "windows": # 32 bits anyway on Windows
        platform["arch"] = "x86"

    binary_dir = os.path.join(os.path.dirname(__file__), "..", "..", "bin", "%(os)s_%(arch)s" % platform)
    binary_path = os.path.join(binary_dir, binary)

    # On Android, we need to copy pulsard to ext4, since the sdcard is noexec
    if platform["os"] == "android":
        app_id = android_get_current_appid()
        xbmc_data_path = os.path.join("/data", "data", app_id)
        if os.path.exists(xbmc_data_path) and uid == os.stat(xbmc_data_path).st_uid:
            android_binary_dir = os.path.join(xbmc_data_path, "files", ADDON.getAddonInfo("id"))

        if not os.path.exists(android_binary_dir):
            os.makedirs(android_binary_dir)
        android_binary_path = os.path.join(android_binary_dir, binary)
        # Testing for size to see if update is needed. This is a poor test indeed, but it's sufficient.
        if not os.path.exists(android_binary_path) or os.path.getsize(android_binary_path) != os.path.getsize(binary_path):
            import shutil
            shutil.copy2(binary_path, android_binary_path)
        binary_path = android_binary_path
        binary_dir = android_binary_dir

    return binary_dir, ensure_exec_perms(binary_path)


def start_pulsard(**kwargs):
    pulsar_dir, pulsar_binary = get_pulsar_binary()

    bind_port = 8000
    args = [pulsar_binary]
    kwargs["cwd"] = pulsar_dir
    if sys.platform == "win32":
        si = subprocess.STARTUPINFO()
        si.dwFlags |= 1
        si.wShowWindow = 01
        kwargs["startupinfo"] = si
    return subprocess.Popen(args, **kwargs)


def pulsard_thread():
    while not xbmc.abortRequested:
        xbmc.log("pulsard: starting pulsard")
        proc = start_pulsard(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        while proc.poll() is None:
            to_read, _, _ = select.select([proc.stdout], [], [], 1)
            if xbmc.abortRequested:
                break
            for fd in to_read:
                xbmc.log(fd.readline())
        if proc.returncode == 0 or xbmc.abortRequested:
            break
        notify("Pulsar has crashed, restarting...", time=1000)
    xbmc.log("pulsard: exiting pulsard daemon")
    if PLATFORM["os"] == "windows":
        proc.send_signal(signal.CTRL_C_EVENT)
    else:
        proc.terminate()
    xbmc.log("pulsard: pulsard daemon exited")<commit_msg>Properly handle Windows' lack of select.select via a thread to read from pulsard stdout.<commit_after>import sys
import subprocess
import os
import signal
import stat
import xbmc
import xbmcaddon
import urllib2
import threading
from pulsar.config import PULSARD_HOST
from pulsar.platform import PLATFORM
from pulsar.util import notify


def ensure_exec_perms(file_):
    st = os.stat(file_)
    os.chmod(file_, st.st_mode | stat.S_IEXEC)
    return file_


def android_get_current_appid():
    uid = os.getuid()
    with open("/proc/%d/cmdline", uid) as fp:
        return fp.read().rstrip()


def get_pulsar_binary():
    binary = "pulsar" + (PLATFORM["os"] == "windows" and ".exe" or "")

    platform = PLATFORM.copy()
    if platform["os"] == "darwin": # 64 bits anyway on Darwin
        platform["arch"] = "x64"
    elif platform["os"] == "windows": # 32 bits anyway on Windows
        platform["arch"] = "x86"

    binary_dir = os.path.join(os.path.dirname(__file__), "..", "..", "bin", "%(os)s_%(arch)s" % platform)
    binary_path = os.path.join(binary_dir, binary)

    # On Android, we need to copy pulsard to ext4, since the sdcard is noexec
    if platform["os"] == "android":
        app_id = android_get_current_appid()
        xbmc_data_path = os.path.join("/data", "data", app_id)
        if os.path.exists(xbmc_data_path) and uid == os.stat(xbmc_data_path).st_uid:
            android_binary_dir = os.path.join(xbmc_data_path, "files", ADDON.getAddonInfo("id"))

        if not os.path.exists(android_binary_dir):
            os.makedirs(android_binary_dir)
        android_binary_path = os.path.join(android_binary_dir, binary)
        # Testing for size to see if update is needed. This is a poor test indeed, but it's sufficient.
        if not os.path.exists(android_binary_path) or os.path.getsize(android_binary_path) != os.path.getsize(binary_path):
            import shutil
            shutil.copy2(binary_path, android_binary_path)
        binary_path = android_binary_path
        binary_dir = android_binary_dir

    return binary_dir, ensure_exec_perms(binary_path)


def start_pulsard(**kwargs):
    pulsar_dir, pulsar_binary = get_pulsar_binary()

    bind_port = 8000
    args = [pulsar_binary]
    kwargs["cwd"] = pulsar_dir
    if sys.platform == "win32":
        si = subprocess.STARTUPINFO()
        si.dwFlags |= 1
        si.wShowWindow = 01
        kwargs["startupinfo"] = si
    return subprocess.Popen(args, **kwargs)


def wait_for_abortRequested(proc):
    while not xbmc.abortRequested:
        xbmc.sleep(1000)
    xbmc.log("pulsard: exiting pulsard daemon")
    proc.terminate()
    xbmc.log("pulsard: pulsard daemon exited")


def pulsard_thread():
    while not xbmc.abortRequested:
        xbmc.log("pulsard: starting pulsard")
        proc = start_pulsard(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        threading.Thread(target=wait_for_abortRequested, args=[proc]).start()
        while proc.poll() is None:
            xbmc.log(proc.stdout.readline())
        if proc.returncode == 0 or xbmc.abortRequested:
            break
        notify("Pulsar has crashed, restarting...", time=1000)
{'fraction_non_alphanumeric': 0.1165634429093219, 'fraction_numerical': 0.008734317929172623, 'mean_word_length': 3.135259356533158, 'pattern_counts': {'":': 9, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6633500', 'n_tokens_mistral': 2158, 'n_tokens_neox': 2082, 'n_words': 483}