<commit_before>import collections
from contextlib import closing
import io
import logging
import os
import subprocess
import time
from subprocess import Popen, PIPE
import threading
import signal
try:
    from urllib.request import urlopen, Request
    from urllib.error import URLError, HTTPError
except ImportError:
    from urllib2 import urlopen, Request, URLError, HTTPError
from pyxform.errors import PyXFormError


HERE = os.path.abspath(os.path.dirname(__file__))
XFORM_SPEC_PATH = os.path.join(os.path.dirname(HERE), "tests",
                               "test_expected_output", "xlsform_spec_test.xml")


# Adapted from:
# http://betabug.ch/blogs/ch-athens/1093
def run_popen_with_timeout(command, timeout):
    """
    Run a sub-program in subprocess.Popen, pass it the input_data,
    kill it if the specified timeout has passed.
    returns a tuple of resultcode, timeout, stdout, stderr
    """
    kill_check = threading.Event()

    def _kill_process_after_a_timeout(pid):
        os.kill(pid, signal.SIGTERM)
        kill_check.set()  # tell the main routine that we had to kill
        # use SIGKILL if hard to kill...
        return

    # Workarounds for pyinstaller
    # https://github.com/pyinstaller/pyinstaller/wiki/Recipe-subprocess
    startup_info = None
    if os.name == 'nt':
        # disable command window when run from pyinstaller
        startup = subprocess.STARTUPINFO()
        # Less fancy version of bitwise-or-assignment (x |= y) shown in ref url.
        if startup.dwFlags == 1 or subprocess.STARTF_USESHOWWINDOW == 1:
            startup.dwFlags = 1
        else:
            startup.dwFlags = 0

    p = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE,
              startupinfo=startup_info)
    watchdog = threading.Timer(
        timeout, _kill_process_after_a_timeout, args=(p.pid,))
    watchdog.start()
    (stdout, stderr) = p.communicate()
    watchdog.cancel()  # if it's still waiting to run
    timeout = kill_check.isSet()
    kill_check.clear()
    return p.returncode, timeout, stdout, stderr<commit_msg>Flags are not set due to misnamed variable<commit_after>import collections
from contextlib import closing
import io
import logging
import os
import subprocess
import time
from subprocess import Popen, PIPE
import threading
import signal
try:
    from urllib.request import urlopen, Request
    from urllib.error import URLError, HTTPError
except ImportError:
    from urllib2 import urlopen, Request, URLError, HTTPError
from pyxform.errors import PyXFormError


HERE = os.path.abspath(os.path.dirname(__file__))
XFORM_SPEC_PATH = os.path.join(os.path.dirname(HERE), "tests",
                               "test_expected_output", "xlsform_spec_test.xml")


# Adapted from:
# http://betabug.ch/blogs/ch-athens/1093
def run_popen_with_timeout(command, timeout):
    """
    Run a sub-program in subprocess.Popen, pass it the input_data,
    kill it if the specified timeout has passed.
    returns a tuple of resultcode, timeout, stdout, stderr
    """
    kill_check = threading.Event()

    def _kill_process_after_a_timeout(pid):
        os.kill(pid, signal.SIGTERM)
        kill_check.set()  # tell the main routine that we had to kill
        # use SIGKILL if hard to kill...
        return

    # Workarounds for pyinstaller
    # https://github.com/pyinstaller/pyinstaller/wiki/Recipe-subprocess
    startup_info = None
    if os.name == 'nt':
        # disable command window when run from pyinstaller
        startup_info = subprocess.STARTUPINFO()
        # Less fancy version of bitwise-or-assignment (x |= y) shown in ref url.
        if startup_info.dwFlags == 1 or subprocess.STARTF_USESHOWWINDOW == 1:
            startup_info.dwFlags = 1
        else:
            startup_info.dwFlags = 0

    p = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE,
              startupinfo=startup_info)
    watchdog = threading.Timer(
        timeout, _kill_process_after_a_timeout, args=(p.pid,))
    watchdog.start()
    (stdout, stderr) = p.communicate()
    watchdog.cancel()  # if it's still waiting to run
    timeout = kill_check.isSet()
    kill_check.clear()
    return p.returncode, timeout, stdout, stderr
{'fraction_non_alphanumeric': 0.08689424683695393, 'fraction_numerical': 0.004296968250179041, 'mean_word_length': 3.1159135559921416, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1522652', 'n_tokens_mistral': 1325, 'n_tokens_neox': 1308, 'n_words': 366}