<commit_before>import sys
import subprocess
import os
import signal
import stat
import xbmc
import xbmcaddon
import urllib2
import threading
import time
from pulsar.logger import log
from pulsar.config import PULSARD_HOST
from pulsar.platform import PLATFORM
from pulsar.addon import ADDON, ADDON_ID
from pulsar.util import notify


SW_HIDE = 0
STARTF_USESHOWWINDOW = 1


def ensure_exec_perms(file_):
    st = os.stat(file_)
    os.chmod(file_, st.st_mode | stat.S_IEXEC)
    return file_


def android_get_current_appid():
    with open("/proc/%d/cmdline" % os.getpid()) as fp:
        return fp.read().rstrip("\0")


def get_pulsard_checksum(path):
    with open(path) as fp:
        fp.seek(-40, os.SEEK_END) # we put a sha1 there
        return fp.read()


def get_pulsar_binary():
    binary = "pulsar" + (PLATFORM["os"] == "windows" and ".exe" or "")

    platform = PLATFORM.copy()
    if platform["os"] == "darwin": # 64 bits anyway on Darwin
        platform["arch"] = "x64"
    elif platform["os"] == "windows": # 32 bits anyway on Windows
        platform["arch"] = "x86"

    binary_dir = os.path.join(ADDON.getAddonInfo("path"), "resources", "bin", "%(os)s_%(arch)s" % platform)
    if platform["os"] == "android":
        binary_dir = binary_dir.replace("/storage/emulated/0", "/storage/emulated/legacy")
        app_id = android_get_current_appid()
        xbmc_data_path = os.path.join("/data", "data", app_id)
        dest_binary_dir = os.path.join(xbmc_data_path, "files", ADDON_ID, "bin", "%(os)s_%(arch)s" % platform)
    else:
        dest_binary_dir = os.path.join(xbmc.translatePath(ADDON.getAddonInfo("profile")), "bin", "%(os)s_%(arch)s" % platform)

    try:
        binary_dir = binary_dir.decode("latin1")
        dest_binary_dir = dest_binary_dir.decode("latin1")
    except UnicodeEncodeError:
        log.info("Unable to decode: binary_dir=%s dest_binary_dir=%s" % (repr(binary_dir), repr(dest_binary_dir)))

    binary_path = os.path.join(binary_dir, binary)
    dest_binary_path = os.path.join(dest_binary_dir, binary)

    if not os.path.exists(dest_binary_path) or get_pulsard_checksum(dest_binary_path) != get_pulsard_checksum(binary_path):
        log.info("Updating pulsar daemon...")
        import shutil
        try:
            os.makedirs(dest_binary_dir)
        except OSError:
            pass
        try:
            shutil.rmtree(dest_binary_dir)
        except:
            pass
        shutil.copytree(binary_dir, dest_binary_dir)

    return dest_binary_dir, ensure_exec_perms(dest_binary_path)


def start_pulsard(**kwargs):
    # Make sure all other pulsard instances are closed
    try:
        urllib2.urlopen(PULSARD_HOST + "/shutdown")
    except urllib2.URLError:
        pass
    pulsar_dir, pulsar_binary = get_pulsar_binary()
    args = [pulsar_binary]

    env = os.environ.copy()
    env["LD_LIBRARY_PATH"] = pulsar_dir

    kwargs = {
        "cwd": pulsar_dir,
        "env": env,
    }

    if PLATFORM["os"] == "windows":
        si = subprocess.STARTUPINFO()
        si.dwFlags = STARTF_USESHOWWINDOW
        si.wShowWindow = SW_HIDE
        kwargs["startupinfo"] = si

    return subprocess.Popen(args, **kwargs)


def wait_for_abortRequested(proc, monitor):
    monitor.closing.wait()
    log.info("pulsard: exiting pulsard daemon")
    proc.terminate()
    log.info("pulsard: pulsard daemon exited")


def pulsard_thread(monitor):
    try:
        import xbmc
        while not xbmc.abortRequested:
            log.info("pulsard: starting pulsard")
            proc = start_pulsard(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            threading.Thread(target=wait_for_abortRequested, args=[proc, monitor]).start()
            while proc.poll() is None:
                log.info(proc.stdout.readline())
            if proc.returncode == 0 or xbmc.abortRequested:
                break
            notify("Pulsar has crashed, restarting...", time=1000)
    except Exception, e:
        import xbmc
        import traceback
        map(xbmc.log, traceback.format_exc().split("\n"))
        raise
<commit_msg>Remove stalled files when starting up in the Pulsar executable folder<commit_after>import sys
import subprocess
import os
import signal
import stat
import xbmc
import xbmcaddon
import urllib2
import threading
import time
import shutil
from pulsar.logger import log
from pulsar.config import PULSARD_HOST
from pulsar.platform import PLATFORM
from pulsar.addon import ADDON, ADDON_ID
from pulsar.util import notify


SW_HIDE = 0
STARTF_USESHOWWINDOW = 1


def ensure_exec_perms(file_):
    st = os.stat(file_)
    os.chmod(file_, st.st_mode | stat.S_IEXEC)
    return file_


def android_get_current_appid():
    with open("/proc/%d/cmdline" % os.getpid()) as fp:
        return fp.read().rstrip("\0")


def get_pulsard_checksum(path):
    with open(path) as fp:
        fp.seek(-40, os.SEEK_END) # we put a sha1 there
        return fp.read()


def get_pulsar_binary():
    binary = "pulsar" + (PLATFORM["os"] == "windows" and ".exe" or "")

    platform = PLATFORM.copy()
    if platform["os"] == "darwin": # 64 bits anyway on Darwin
        platform["arch"] = "x64"
    elif platform["os"] == "windows": # 32 bits anyway on Windows
        platform["arch"] = "x86"

    binary_dir = os.path.join(ADDON.getAddonInfo("path"), "resources", "bin", "%(os)s_%(arch)s" % platform)
    if platform["os"] == "android":
        binary_dir = binary_dir.replace("/storage/emulated/0", "/storage/emulated/legacy")
        app_id = android_get_current_appid()
        xbmc_data_path = os.path.join("/data", "data", app_id)
        dest_binary_dir = os.path.join(xbmc_data_path, "files", ADDON_ID, "bin", "%(os)s_%(arch)s" % platform)
    else:
        dest_binary_dir = os.path.join(xbmc.translatePath(ADDON.getAddonInfo("profile")), "bin", "%(os)s_%(arch)s" % platform)

    try:
        binary_dir = binary_dir.decode("latin1")
        dest_binary_dir = dest_binary_dir.decode("latin1")
    except UnicodeEncodeError:
        log.info("Unable to decode: binary_dir=%s dest_binary_dir=%s" % (repr(binary_dir), repr(dest_binary_dir)))

    binary_path = os.path.join(binary_dir, binary)
    dest_binary_path = os.path.join(dest_binary_dir, binary)

    if not os.path.exists(dest_binary_path) or get_pulsard_checksum(dest_binary_path) != get_pulsard_checksum(binary_path):
        log.info("Updating pulsar daemon...")
        import shutil
        try:
            os.makedirs(dest_binary_dir)
        except OSError:
            pass
        try:
            shutil.rmtree(dest_binary_dir)
        except:
            pass
        shutil.copytree(binary_dir, dest_binary_dir)

    # Clean stale files in the directory, as this can cause headaches on
    # Android when they are unreachable
    dest_files = set(os.listdir(dest_binary_dir))
    orig_files = set(os.listdir(binary_dir))
    log.info("Deleting stale files %s" % (dest_files - orig_files))
    for file_ in (dest_files - orig_files):
        path = os.path.join(dest_binary_dir, file_)
        if os.path.isdir(path):
            shutil.rmtree(path)
        else:
            os.remove(path)

    return dest_binary_dir, ensure_exec_perms(dest_binary_path)


def start_pulsard(**kwargs):
    # Make sure all other pulsard instances are closed
    try:
        urllib2.urlopen(PULSARD_HOST + "/shutdown")
    except urllib2.URLError:
        pass
    pulsar_dir, pulsar_binary = get_pulsar_binary()
    args = [pulsar_binary]

    env = os.environ.copy()
    env["LD_LIBRARY_PATH"] = pulsar_dir

    kwargs = {
        "cwd": pulsar_dir,
        "env": env,
    }

    if PLATFORM["os"] == "windows":
        si = subprocess.STARTUPINFO()
        si.dwFlags = STARTF_USESHOWWINDOW
        si.wShowWindow = SW_HIDE
        kwargs["startupinfo"] = si

    return subprocess.Popen(args, **kwargs)


def wait_for_abortRequested(proc, monitor):
    monitor.closing.wait()
    log.info("pulsard: exiting pulsard daemon")
    proc.terminate()
    log.info("pulsard: pulsard daemon exited")


def pulsard_thread(monitor):
    try:
        import xbmc
        while not xbmc.abortRequested:
            log.info("pulsard: starting pulsard")
            proc = start_pulsard(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            threading.Thread(target=wait_for_abortRequested, args=[proc, monitor]).start()
            while proc.poll() is None:
                log.info(proc.stdout.readline())
            if proc.returncode == 0 or xbmc.abortRequested:
                break
            notify("Pulsar has crashed, restarting...", time=1000)
    except Exception, e:
        import xbmc
        import traceback
        map(xbmc.log, traceback.format_exc().split("\n"))
        raise

{'fraction_non_alphanumeric': 0.11727616645649433, 'fraction_numerical': 0.005731972945087699, 'mean_word_length': 3.014726184997699, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3617489', 'n_tokens_mistral': 3004, 'n_tokens_neox': 2915, 'n_words': 622}