<commit_before>#!/usr/bin/env python3

# Copyright (C) 2015-2016 <NAME>. All rights reserved.
# Licensed under the Artistic License 2.0.

import asyncio
import os
import signal

import psutil

import tools


def _make_process_nicest(pid):
    process = psutil.Process(pid)
    process.nice(19)
    process.ionice(psutil.IOPRIO_CLASS_IDLE)


class Worker:

    def __init__(self, is_sandboxed, is_already_paused, is_being_tested):
        self.is_sandboxed = is_sandboxed
        self.is_already_paused = is_already_paused
        self.is_being_tested = is_being_tested
        self.result = None
        self.process = None
        self.child_pid = None

    async def create_process(self):
        if self.is_sandboxed:
            sandbox_fs_path = os.path.join(os.path.dirname(__file__),
                                           "sandbox_fs")
            cache_path = os.path.join(os.getcwd(), tools.CACHE_PATH)
            command = [sandbox_fs_path, cache_path, "--", __file__]
        else:
            command = [__file__]
        create = asyncio.create_subprocess_exec(
            *command, stdin=asyncio.subprocess.PIPE,
            stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
        self.process = await create
        pid_line = await self.process.stdout.readline()
        self.child_pid = int(pid_line.strip())

    async def run_tool(self, path, tool):
        self.process.stdin.write(("%s\n%s\n" %
                                  (tool.__qualname__, path)).encode("utf-8"))
        data = await self.process.stdout.readline()
        return tools.Status(int(data))

    async def job_runner(self, summary, log, jobs_added_event,
                         appearance_changed_event):
        await self.create_process()
        _make_process_nicest(self.child_pid)
        while True:
            await jobs_added_event.wait()
            while True:
                try:
                    self.result = summary.get_closest_placeholder()
                except StopIteration:
                    self.result = None
                    if summary.result_total == summary.completed_total:
                        log.log_message("All results are up to date.")
                        if self.is_being_tested:
                            os.kill(os.getpid(), signal.SIGINT)
                    break
                await self.result.run(log, appearance_changed_event,
                                      self)
                summary.completed_total += 1
            jobs_added_event.clear()

    def pause(self):
        if self.result is not None and \
           self.result.status == tools.Status.running:
            os.kill(self.child_pid, signal.SIGSTOP)
            self.result.set_status(tools.Status.paused)

    def continue_(self):
        if self.result is not None and \
           self.result.status == tools.Status.paused:
            self.result.set_status(tools.Status.running)
            os.kill(self.child_pid, signal.SIGCONT)


def main():
    print(os.getpid(), flush=True)<commit_msg>Pause workers and their subprocesses together.<commit_after>#!/usr/bin/env python3

# Copyright (C) 2015-2016 <NAME>. All rights reserved.
# Licensed under the Artistic License 2.0.

import asyncio
import os
import signal

import psutil

import tools


def _make_process_nicest(pid):
    process = psutil.Process(pid)
    process.nice(19)
    process.ionice(psutil.IOPRIO_CLASS_IDLE)


class Worker:

    def __init__(self, is_sandboxed, is_already_paused, is_being_tested):
        self.is_sandboxed = is_sandboxed
        self.is_already_paused = is_already_paused
        self.is_being_tested = is_being_tested
        self.result = None
        self.process = None
        self.child_pgid = None

    async def create_process(self):
        if self.is_sandboxed:
            sandbox_fs_path = os.path.join(os.path.dirname(__file__),
                                           "sandbox_fs")
            cache_path = os.path.join(os.getcwd(), tools.CACHE_PATH)
            command = [sandbox_fs_path, cache_path, "--", __file__]
        else:
            command = [__file__]
        create = asyncio.create_subprocess_exec(
            *command, stdin=asyncio.subprocess.PIPE,
            stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
            preexec_fn=os.setsid)
        self.process = await create
        pid_line = await self.process.stdout.readline()
        self.child_pgid = int(pid_line.strip())

    async def run_tool(self, path, tool):
        self.process.stdin.write(("%s\n%s\n" %
                                  (tool.__qualname__, path)).encode("utf-8"))
        data = await self.process.stdout.readline()
        return tools.Status(int(data))

    async def job_runner(self, summary, log, jobs_added_event,
                         appearance_changed_event):
        await self.create_process()
        _make_process_nicest(self.child_pgid)
        while True:
            await jobs_added_event.wait()
            while True:
                try:
                    self.result = summary.get_closest_placeholder()
                except StopIteration:
                    self.result = None
                    if summary.result_total == summary.completed_total:
                        log.log_message("All results are up to date.")
                        if self.is_being_tested:
                            os.kill(os.getpid(), signal.SIGINT)
                    break
                await self.result.run(log, appearance_changed_event,
                                      self)
                summary.completed_total += 1
            jobs_added_event.clear()

    def pause(self):
        if self.result is not None and \
           self.result.status == tools.Status.running:
            os.killpg(self.child_pgid, signal.SIGSTOP)
            self.result.set_status(tools.Status.paused)

    def continue_(self):
        if self.result is not None and \
           self.result.status == tools.Status.paused:
            self.result.set_status(tools.Status.running)
            os.killpg(self.child_pgid, signal.SIGCONT)


def main():
    print(os.getpgid(os.getpid()), flush=True)
{'fraction_non_alphanumeric': 0.09259857789269554, 'fraction_numerical': 0.004848093083387201, 'mean_word_length': 1.9193396226415094, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7262385', 'n_tokens_mistral': 1872, 'n_tokens_neox': 1768, 'n_words': 343}