<commit_before>#!/usr/bin/env python3.6

# Copyright (C) 2015-2018 <NAME>. All rights reserved.
# Licensed under the Artistic License 2.0.

import asyncio
import os
import signal

import vigil.tools as tools


class Worker:

    AUTOSAVE_MESSAGE = "Auto-saving..."
    unsaved_jobs_total = 0

    def __init__(self, is_already_paused, is_being_tested):
        self.is_already_paused = is_already_paused
        self.is_being_tested = is_being_tested
        self.result = None
        self.process = None
        self.child_pgid = None

    async def create_process(self):
        create = asyncio.create_subprocess_exec(
            "vigil-worker", stdin=asyncio.subprocess.PIPE,
            stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
            preexec_fn=os.setsid)
        self.process = await create
        pid_line = await self.process.stdout.readline()
        self.child_pgid = int(pid_line.strip())
        os.setpriority(os.PRIO_PGRP, self.child_pgid, 19)

    async def run_tool(self, path, tool):
        self.process.stdin.write(f"{tool.__qualname__}\n{path}\n".encode("utf-8"))
        data = await self.process.stdout.readline()
        return tools.Status(int(data))

    async def job_runner(self, screen, summary, log, jobs_added_event,
                   appearance_changed_event):
        await self.create_process()
        while True:
            await jobs_added_event.wait()
            while True:
                try:
                    self.result = await summary.get_closest_placeholder()
                except StopAsyncIteration:
                    self.result = None
                    if summary.result_total == summary.completed_total:
                        log.log_message("All results are up to date.")
                        log.log_message(Worker.AUTOSAVE_MESSAGE)
                        screen.save()
                        if self.is_being_tested:
                            os.kill(os.getpid(), signal.SIGINT)
                    break
                await self.result.run(log, appearance_changed_event, self)
                Worker.unsaved_jobs_total += 1
                if Worker.unsaved_jobs_total == 100:
                    log.log_message(Worker.AUTOSAVE_MESSAGE)
                    screen.save()
                summary.completed_total += 1
            jobs_added_event.clear()

    def pause(self):
        if self.result is not None and \
           self.result.status == tools.Status.running:
            os.killpg(self.child_pgid, signal.SIGSTOP)
            self.result.set_status(tools.Status.paused)

    def continue_(self):
        if self.result is not None and \
           self.result.status == tools.Status.paused:
            self.result.set_status(tools.Status.running)<commit_msg>Fixed too many auto-saves occuring after restart.<commit_after>#!/usr/bin/env python3.6

# Copyright (C) 2015-2018 <NAME>. All rights reserved.
# Licensed under the Artistic License 2.0.

import asyncio
import os
import signal

import vigil.tools as tools


class Worker:

    AUTOSAVE_MESSAGE = "Auto-saving..."
    unsaved_jobs_total = 0

    def __init__(self, is_already_paused, is_being_tested):
        self.is_already_paused = is_already_paused
        self.is_being_tested = is_being_tested
        self.result = None
        self.process = None
        self.child_pgid = None

    async def create_process(self):
        create = asyncio.create_subprocess_exec(
            "vigil-worker", stdin=asyncio.subprocess.PIPE,
            stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
            preexec_fn=os.setsid)
        self.process = await create
        pid_line = await self.process.stdout.readline()
        self.child_pgid = int(pid_line.strip())
        os.setpriority(os.PRIO_PGRP, self.child_pgid, 19)

    async def run_tool(self, path, tool):
        self.process.stdin.write(f"{tool.__qualname__}\n{path}\n".encode("utf-8"))
        data = await self.process.stdout.readline()
        return tools.Status(int(data))

    async def job_runner(self, screen, summary, log, jobs_added_event,
                   appearance_changed_event):
        await self.create_process()
        while True:
            await jobs_added_event.wait()
            while True:
                try:
                    self.result = await summary.get_closest_placeholder()
                except StopAsyncIteration:
                    self.result = None
                    break
                await self.result.run(log, appearance_changed_event, self)
                Worker.unsaved_jobs_total += 1
                if Worker.unsaved_jobs_total == 100:
                    log.log_message(Worker.AUTOSAVE_MESSAGE)
                    screen.save()
                summary.completed_total += 1
                if summary.result_total == summary.completed_total:
                    log.log_message("All results are up to date.")
                    log.log_message(Worker.AUTOSAVE_MESSAGE)
                    screen.save()
                    if self.is_being_tested:
                        os.kill(os.getpid(), signal.SIGINT)
            jobs_added_event.clear()

    def pause(self):
        if self.result is not None and \
           self.result.status == tools.Status.running:
            os.killpg(self.child_pgid, signal.SIGSTOP)
            self.result.set_status(tools.Status.paused)

    def continue_(self):
        if self.result is not None and \
           self.result.status == tools.Status.paused:
            self.result.set_status(tools.Status.running)
{'fraction_non_alphanumeric': 0.08976660682226212, 'fraction_numerical': 0.0075403949730700175, 'mean_word_length': 1.97437266417512, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7509059', 'n_tokens_mistral': 1695, 'n_tokens_neox': 1566, 'n_words': 315}