<commit_before># coding=utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

"""Allows the funfuzz harness to run continuously.
"""

import argparse
import io
import json
import os
from pathlib import Path
import platform
import subprocess
import sys
from textwrap import dedent
import time
import zipfile

from . import compare_jit
from . import js_interesting
from . import link_fuzzer
from . import shell_flags
from . import with_binaryen
from ..util import create_collector
from ..util import file_manipulation
from ..util import file_system_helpers
from ..util import lithium_helpers
from ..util import os_ops


def parseOpts(args):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc
    parser = argparse.ArgumentParser()
    parser.add_argument("--compare-jit",
                        action="store_true", dest="use_compare_jit",
                        help="After running the fuzzer, run the FCM lines against the engine "
                             "in two configurations and compare the output.")
    parser.add_argument("--random-flags",
                        action="store_true", dest="randomFlags",
                        help="Pass a random set of flags (e.g. --ion-eager) to the js engine")
    parser.add_argument("--repo", type=Path, default=Path.home() / "trees" / "mozilla-central",
                        help="The hg repository (e.g. ~/trees/mozilla-central/), for bisection")
    # if you run loop directly w/o --build, lithium_helpers.pinpoint will try to guess
    parser.add_argument("--build",
                        dest="build_options_str",
                        help="The build options, for bisection")
    parser.add_argument("--valgrind",
                        action="store_true",
                        help="use valgrind with a reasonable set of options")
    parser.add_argument("--no-fuzz-wasm", action="store_false", dest="fuzz_wasm",
                        help="Don't run the fuzzer against wasm")
    # jsfunfuzz will quit after half this time if it's not looping.
    # higher = more complex mixing, especially with regression tests.
    # lower = less time wasted in timeouts and in compare_jit testcases that are thrown away due to OOMs.
    parser.add_argument("timeout", type=int, help="kill js shell if it runs this long")
    # FIXME: We can probably remove args[1]  # pylint: disable=fixme
    parser.add_argument("unused", help="Legacy option (value ignored)")
    parser.add_argument("jsEngine", type=Path)
    parser.add_argument("engineFlags", nargs=argparse.REMAINDER)
    parser.set_defaults(knownPath="mozilla-central")
    options = parser.parse_args(args)

    if options.valgrind and options.use_compare_jit:
        print("Note: When running compare_jit, the --valgrind option will be ignored", file=sys.stderr)

    return options


def showtail(filename):  # pylint: disable=missing-docstring
    # pylint: disable=fixme
    # FIXME: Get jsfunfuzz to output start & end of interesting result boundaries instead of this.
    cmd = []
    cmd.extend(["tail", "-n", "20"])
    cmd.append(str(filename))
    print(" ".join(cmd))
    print()
    subprocess.run(cmd, check=True)
    print()
    print()


def makeRegressionTestPrologue(repo):  # pylint: disable=invalid-name,missing-param-doc
    # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc
    """Generate a JS string to tell jsfunfuzz where to find SpiderMonkey's regression tests."""
    libdir = Path("js") / "src" / "jit-test" / "lib"
    js_src_tests_dir = Path("js") / "src" / "tests"
    non262_tests_dir = js_src_tests_dir / "non262"
    test262_tests_dir = js_src_tests_dir / "test262"
    w_pltfrm_res_dir = Path("testing") / "web-platform" / "tests" / "resources"
    return dedent(f"""
        const regressionTestsRoot = {json.dumps(str(repo) + os.sep)};
        const libdir = regressionTestsRoot + {json.dumps(str(libdir) + os.sep)}; // needed by jit-tests
        const js_src_tests_dir = regressionTestsRoot + {json.dumps(str(js_src_tests_dir) + os.sep)}; // streams tests
        const non262_tests_dir = regressionTestsRoot + {json.dumps(str(non262_tests_dir) + os.sep)}; // non262 tests
        const test262_tests_dir = regressionTestsRoot + {json.dumps(str(test262_tests_dir) + os.sep)}; // test262 tests
        const w_pltfrm_res_dir = regressionTestsRoot + {json.dumps(str(w_pltfrm_res_dir) + os.sep)}; // streams tests
        const regressionTestList = {json.dumps(inTreeRegressionTests(repo))};
    """)


def inTreeRegressionTests(repo):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc
    # pylint: disable=missing-return-type-doc
    jit_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "jit-test" / "tests")
    js_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "tests")
    non262_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "tests" / "non262")
    test262_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "tests" / "test262")
    streams_tests = jsFilesIn(len(str(repo)), repo / "testing" / "web-platform" / "tests" / "streams")
    return jit_tests + js_tests + non262_tests + test262_tests + streams_tests


def jsFilesIn(repoPathLength, root):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc
    # pylint: disable=missing-return-type-doc
    return [os.path.join(path, filename)[repoPathLength + 1:]
            for path, _dirs, files in os.walk(str(root))
            for filename in files
            if filename.endswith(".js")]


def many_timed_runs(target_time, wtmp_dir, args, collector, ccoverage):
    """As long as the run length duration is less than target_time, the harness will run the fuzzers in a loop.

    Args:
        target_time (int): Target time the harness runs before restarting
        wtmp_dir (Path): Path to the wtmp directory
        args (list): Extra arguments
        collector (object): Collector object for FuzzManager submission
        ccoverage (bool): Whether we are running in coverage gathering mode
    """
    # pylint: disable=too-complex,too-many-branches,too-many-locals,too-many-statements
    options = parseOpts([str(arg) for arg in args])
    startTime = time.time()  # pylint: disable=invalid-name

    if options.repo.is_dir():
        regressionTestPrologue = makeRegressionTestPrologue(options.repo)  # pylint: disable=invalid-name
    else:
        regressionTestPrologue = ""  # pylint: disable=invalid-name

    fuzzjs = wtmp_dir / "jsfunfuzz.js"

    link_fuzzer.link_fuzzer(fuzzjs, regressionTestPrologue)
    assert fuzzjs.is_file()

    iteration = 0
    while True:
        if target_time and time.time() > startTime + target_time:
            print("Out of time!")
            fuzzjs.unlink()
            if not os.listdir(str(wtmp_dir)):
                wtmp_dir.rmdir()
            break

        # Construct command needed to loop jsfunfuzz fuzzing.
        js_interesting_args = []
        js_interesting_args.append(f"--timeout={options.timeout}")
        if options.valgrind:
            js_interesting_args.append("--valgrind")
        js_interesting_args.append(str(options.knownPath))
        js_interesting_args.append(str(options.jsEngine))
        if options.randomFlags:
            options.engineFlags = shell_flags.random_flag_set(options.jsEngine)
            js_interesting_args.extend(options.engineFlags)
        js_interesting_args.extend(["-e", f"maxRunTime={options.timeout * (1000 // 2)}"])
        js_interesting_args.extend(["-f", fuzzjs])
        js_interesting_opts = js_interesting.parseOptions(js_interesting_args)

        iteration += 1
        log_prefix = wtmp_dir / f"w{iteration}"

        env = {}  # default environment will be used
        if ccoverage:
            env["GCOV_PREFIX_STRIP"] = "6"  # Assumes ccoverage build from Taskcluster via fuzzfetch
            cov_build_path = Path(args[-2]).parent.parent.parent
            assert "cov-build" in str(cov_build_path)
            env["GCOV_PREFIX"] = str(cov_build_path)

        res, out_log = run_to_report(options, js_interesting_opts, env, log_prefix,
                                     fuzzjs, ccoverage, collector, target_time)

        # funbind - integrate with binaryen wasm project but only on Linux x86_64
        # (aarch64 got activated as we now first compile our own binaryen aarch64 Linux builds)
        # For binaryen Linux x86, first wait for https://github.com/WebAssembly/binaryen/issues/1615 to be fixed
        # I do not believe binaryen x86 builds are needed since all our host OS'es are 64-bit
        if options.fuzz_wasm and platform.system() == "Linux" and "64" in platform.machine() \
                and out_log.is_file() and not options.valgrind:
            run_to_report_wasm(options, js_interesting_opts, env, log_prefix,
                               out_log, ccoverage, collector, target_time)

        # pylint: disable=no-member<commit_msg>Fix GCOV_PREFIX_STRIP value<commit_after># coding=utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

"""Allows the funfuzz harness to run continuously.
"""

import argparse
import io
import json
import os
from pathlib import Path
import platform
import subprocess
import sys
from textwrap import dedent
import time
import zipfile

from . import compare_jit
from . import js_interesting
from . import link_fuzzer
from . import shell_flags
from . import with_binaryen
from ..util import create_collector
from ..util import file_manipulation
from ..util import file_system_helpers
from ..util import lithium_helpers
from ..util import os_ops


def parseOpts(args):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc
    parser = argparse.ArgumentParser()
    parser.add_argument("--compare-jit",
                        action="store_true", dest="use_compare_jit",
                        help="After running the fuzzer, run the FCM lines against the engine "
                             "in two configurations and compare the output.")
    parser.add_argument("--random-flags",
                        action="store_true", dest="randomFlags",
                        help="Pass a random set of flags (e.g. --ion-eager) to the js engine")
    parser.add_argument("--repo", type=Path, default=Path.home() / "trees" / "mozilla-central",
                        help="The hg repository (e.g. ~/trees/mozilla-central/), for bisection")
    # if you run loop directly w/o --build, lithium_helpers.pinpoint will try to guess
    parser.add_argument("--build",
                        dest="build_options_str",
                        help="The build options, for bisection")
    parser.add_argument("--valgrind",
                        action="store_true",
                        help="use valgrind with a reasonable set of options")
    parser.add_argument("--no-fuzz-wasm", action="store_false", dest="fuzz_wasm",
                        help="Don't run the fuzzer against wasm")
    # jsfunfuzz will quit after half this time if it's not looping.
    # higher = more complex mixing, especially with regression tests.
    # lower = less time wasted in timeouts and in compare_jit testcases that are thrown away due to OOMs.
    parser.add_argument("timeout", type=int, help="kill js shell if it runs this long")
    # FIXME: We can probably remove args[1]  # pylint: disable=fixme
    parser.add_argument("unused", help="Legacy option (value ignored)")
    parser.add_argument("jsEngine", type=Path)
    parser.add_argument("engineFlags", nargs=argparse.REMAINDER)
    parser.set_defaults(knownPath="mozilla-central")
    options = parser.parse_args(args)

    if options.valgrind and options.use_compare_jit:
        print("Note: When running compare_jit, the --valgrind option will be ignored", file=sys.stderr)

    return options


def showtail(filename):  # pylint: disable=missing-docstring
    # pylint: disable=fixme
    # FIXME: Get jsfunfuzz to output start & end of interesting result boundaries instead of this.
    cmd = []
    cmd.extend(["tail", "-n", "20"])
    cmd.append(str(filename))
    print(" ".join(cmd))
    print()
    subprocess.run(cmd, check=True)
    print()
    print()


def makeRegressionTestPrologue(repo):  # pylint: disable=invalid-name,missing-param-doc
    # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc
    """Generate a JS string to tell jsfunfuzz where to find SpiderMonkey's regression tests."""
    libdir = Path("js") / "src" / "jit-test" / "lib"
    js_src_tests_dir = Path("js") / "src" / "tests"
    non262_tests_dir = js_src_tests_dir / "non262"
    test262_tests_dir = js_src_tests_dir / "test262"
    w_pltfrm_res_dir = Path("testing") / "web-platform" / "tests" / "resources"
    return dedent(f"""
        const regressionTestsRoot = {json.dumps(str(repo) + os.sep)};
        const libdir = regressionTestsRoot + {json.dumps(str(libdir) + os.sep)}; // needed by jit-tests
        const js_src_tests_dir = regressionTestsRoot + {json.dumps(str(js_src_tests_dir) + os.sep)}; // streams tests
        const non262_tests_dir = regressionTestsRoot + {json.dumps(str(non262_tests_dir) + os.sep)}; // non262 tests
        const test262_tests_dir = regressionTestsRoot + {json.dumps(str(test262_tests_dir) + os.sep)}; // test262 tests
        const w_pltfrm_res_dir = regressionTestsRoot + {json.dumps(str(w_pltfrm_res_dir) + os.sep)}; // streams tests
        const regressionTestList = {json.dumps(inTreeRegressionTests(repo))};
    """)


def inTreeRegressionTests(repo):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc
    # pylint: disable=missing-return-type-doc
    jit_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "jit-test" / "tests")
    js_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "tests")
    non262_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "tests" / "non262")
    test262_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "tests" / "test262")
    streams_tests = jsFilesIn(len(str(repo)), repo / "testing" / "web-platform" / "tests" / "streams")
    return jit_tests + js_tests + non262_tests + test262_tests + streams_tests


def jsFilesIn(repoPathLength, root):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc
    # pylint: disable=missing-return-type-doc
    return [os.path.join(path, filename)[repoPathLength + 1:]
            for path, _dirs, files in os.walk(str(root))
            for filename in files
            if filename.endswith(".js")]


def many_timed_runs(target_time, wtmp_dir, args, collector, ccoverage):
    """As long as the run length duration is less than target_time, the harness will run the fuzzers in a loop.

    Args:
        target_time (int): Target time the harness runs before restarting
        wtmp_dir (Path): Path to the wtmp directory
        args (list): Extra arguments
        collector (object): Collector object for FuzzManager submission
        ccoverage (bool): Whether we are running in coverage gathering mode
    """
    # pylint: disable=too-complex,too-many-branches,too-many-locals,too-many-statements
    options = parseOpts([str(arg) for arg in args])
    startTime = time.time()  # pylint: disable=invalid-name

    if options.repo.is_dir():
        regressionTestPrologue = makeRegressionTestPrologue(options.repo)  # pylint: disable=invalid-name
    else:
        regressionTestPrologue = ""  # pylint: disable=invalid-name

    fuzzjs = wtmp_dir / "jsfunfuzz.js"

    link_fuzzer.link_fuzzer(fuzzjs, regressionTestPrologue)
    assert fuzzjs.is_file()

    iteration = 0
    while True:
        if target_time and time.time() > startTime + target_time:
            print("Out of time!")
            fuzzjs.unlink()
            if not os.listdir(str(wtmp_dir)):
                wtmp_dir.rmdir()
            break

        # Construct command needed to loop jsfunfuzz fuzzing.
        js_interesting_args = []
        js_interesting_args.append(f"--timeout={options.timeout}")
        if options.valgrind:
            js_interesting_args.append("--valgrind")
        js_interesting_args.append(str(options.knownPath))
        js_interesting_args.append(str(options.jsEngine))
        if options.randomFlags:
            options.engineFlags = shell_flags.random_flag_set(options.jsEngine)
            js_interesting_args.extend(options.engineFlags)
        js_interesting_args.extend(["-e", f"maxRunTime={options.timeout * (1000 // 2)}"])
        js_interesting_args.extend(["-f", fuzzjs])
        js_interesting_opts = js_interesting.parseOptions(js_interesting_args)

        iteration += 1
        log_prefix = wtmp_dir / f"w{iteration}"

        env = {}  # default environment will be used
        if ccoverage:
            env["GCOV_PREFIX_STRIP"] = "4"  # Assumes ccoverage build from Taskcluster via fuzzfetch
            cov_build_path = Path(args[-2]).parent.parent.parent
            assert "cov-build" in str(cov_build_path)
            env["GCOV_PREFIX"] = str(cov_build_path)

        res, out_log = run_to_report(options, js_interesting_opts, env, log_prefix,
                                     fuzzjs, ccoverage, collector, target_time)

        # funbind - integrate with binaryen wasm project but only on Linux x86_64
        # (aarch64 got activated as we now first compile our own binaryen aarch64 Linux builds)
        # For binaryen Linux x86, first wait for https://github.com/WebAssembly/binaryen/issues/1615 to be fixed
        # I do not believe binaryen x86 builds are needed since all our host OS'es are 64-bit
        if options.fuzz_wasm and platform.system() == "Linux" and "64" in platform.machine() \
                and out_log.is_file() and not options.valgrind:
            run_to_report_wasm(options, js_interesting_opts, env, log_prefix,
                               out_log, ccoverage, collector, target_time)

        # pylint: disable=no-member
{'fraction_non_alphanumeric': 0.10842844600526778, 'fraction_numerical': 0.009438103599648815, 'mean_word_length': 3.0455049944506105, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2606047', 'n_tokens_mistral': 5774, 'n_tokens_neox': 5476, 'n_words': 1439}