<commit_before>"""
Helper script to combine coverage (as codecov seems to have problems...).
Written in python to be cross-platform
"""

import argparse
from os import chdir, listdir, environ
from pathlib import Path
import platform
from pprint import pprint
import signal
from subprocess import PIPE, Popen, CalledProcessError, TimeoutExpired
import sys

THIS_FILE = Path(__file__)
GIT_MAIN_DIR = THIS_FILE.parent.parent.resolve()
COVERAGE_DIR = GIT_MAIN_DIR.joinpath('.coverage_dir')
PYVERSION = '{}.{}'.format(sys.version_info[0], sys.version_info[1])


def msg(*args):
    print('ERR:', *args, file=sys.stderr, flush=True)


def pmsg(*args):
    pprint(*args, stream=sys.stderr)
    sys.stderr.flush()


def _run(proc: Popen, timeout=None):
    try:
        return proc.wait(timeout=timeout)
    except TimeoutExpired:
        pass
    if sys.platform != 'win32':
        proc.send_signal(signal.SIGINT)
        try:
            return proc.wait(timeout=5)
        except TimeoutExpired:
            pass

    proc.terminate()
    try:
        return proc.wait(timeout=5)
    except TimeoutExpired:
        pass

    proc.kill()
    return proc.wait(timeout=5)

def run_with_python(args, timeout=None, **kwargs):
    if platform.system() == 'Windows':
        exe = ['py', '-' + PYVERSION, '-m']
    else:
        exe = []
    cmd = exe + args
    msg("Running:", *cmd)

    proc = Popen(cmd, stdin=PIPE, **kwargs)
    proc.stdin.close()
    retcode = _run(proc, timeout=timeout)

    if retcode != 0:
        raise CalledProcessError(retcode, cmd)


def send_coverage(*, workdir, coverage_files, codecov_token):
    chdir(workdir)
    run_with_python(['coverage', 'combine'] + coverage_files)
    msg(f"Combined coverage, listing {GIT_MAIN_DIR}")
    pmsg(sorted(listdir(GIT_MAIN_DIR)))
    run_with_python(['coverage', 'xml', '--ignore-errors'])
    msg(f"Created coverage xml, listing {GIT_MAIN_DIR}")
    pmsg(sorted(listdir(GIT_MAIN_DIR)))
    codecov_args = []
    if codecov_token is not None:
        codecov_args.extend(['-t', codecov_token])
    codecov_args.extend(['--file', 'coverage.xml'])
    run_with_python(['codecov'] + codecov_args, timeout=60)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--codecov-token", default=None)
    args = parser.parse_args()
    msg(f"Working in {GIT_MAIN_DIR}, listing coverage dir {COVERAGE_DIR}")
    pmsg(sorted(listdir(COVERAGE_DIR)))
    coverage_files = [str(f) for f in COVERAGE_DIR.glob('coverage-*')]
    if coverage_files:
        send_coverage(
            workdir=GIT_MAIN_DIR,
            coverage_files=coverage_files,
            codecov_token=args.codecov_token,
        )
    else:
        msg("No coverage files found")
        if environ.get("H5PY_ENFORCE_COVERAGE") is not None:
            raise RuntimeError(
                "Coverage required, no coverage found, failing..."
            )


if __name__ == '__main__':
    main()<commit_msg>Some cleanup in upload_coverage.py script<commit_after>"""
Helper script to combine coverage (as codecov seems to have problems...).
Written in python to be cross-platform
"""

import argparse
from os import chdir, environ
from pathlib import Path
import platform
from pprint import pprint
import signal
from subprocess import PIPE, Popen, CalledProcessError, TimeoutExpired
import sys

THIS_FILE = Path(__file__)
GIT_MAIN_DIR = THIS_FILE.parent.parent.resolve()
COVERAGE_DIR = GIT_MAIN_DIR.joinpath('.coverage_dir')
PYVERSION = '{}.{}'.format(sys.version_info[0], sys.version_info[1])


def msg(*args):
    print('ERR:', *args, file=sys.stderr, flush=True)


def pmsg(*args):
    pprint(*args, stream=sys.stderr)
    sys.stderr.flush()


def _run(proc: Popen, timeout):
    """Run process, with several steps of signalling if timeout is reached"""
    try:
        return proc.wait(timeout=timeout)
    except TimeoutExpired:
        pass
    if sys.platform != 'win32':
        proc.send_signal(signal.SIGINT)
        try:
            return proc.wait(timeout=5)
        except TimeoutExpired:
            pass

    proc.terminate()  # SIGTERM
    try:
        return proc.wait(timeout=5)
    except TimeoutExpired:
        pass

    proc.kill()  # SIGKILL
    return proc.wait(timeout=5)

def run_with_python(args, timeout=30, **kwargs):
    if platform.system() == 'Windows':
        exe = ['py', '-' + PYVERSION, '-m']
    else:
        exe = []
    cmd = exe + args
    msg("Running:", *cmd)

    proc = Popen(cmd, stdin=PIPE, **kwargs)
    proc.stdin.close()
    retcode = _run(proc, timeout)

    if retcode != 0:
        raise CalledProcessError(retcode, cmd)


def send_coverage(*, workdir, coverage_files, codecov_token):
    chdir(workdir)
    run_with_python(['coverage', 'combine'] + coverage_files)
    msg(f"Combined coverage")
    run_with_python(['coverage', 'xml', '--ignore-errors'])
    msg(f"Created coverage xml")

    # Upload coverage.xml to codecov
    codecov_args = []
    if codecov_token is not None:
        codecov_args.extend(['-t', codecov_token])
    codecov_args.extend(['--file', 'coverage.xml'])
    run_with_python(['codecov'] + codecov_args)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--codecov-token", default=None)
    args = parser.parse_args()
    msg(f"Working in {GIT_MAIN_DIR}, looking for coverage files...")
    coverage_files = [str(f) for f in COVERAGE_DIR.glob('coverage-*')]
    pmsg(sorted(coverage_files))
    if coverage_files:
        send_coverage(
            workdir=GIT_MAIN_DIR,
            coverage_files=coverage_files,
            codecov_token=args.codecov_token,
        )
    else:
        msg("No coverage files found")
        if environ.get("H5PY_ENFORCE_COVERAGE") is not None:
            raise RuntimeError(
                "Coverage required, no coverage found, failing..."
            )


if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.1254035683942226, 'fraction_numerical': 0.003738317757009346, 'mean_word_length': 3.18932384341637, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '314008', 'n_tokens_mistral': 1938, 'n_tokens_neox': 1902, 'n_words': 442}