<commit_before>#!/usr/bin/env python
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import with_statement

import os
import platform
import random
import shutil
import subprocess
import sys

from copy import deepcopy
from optparse import OptionParser
from tempfile import mkdtemp

from compileShell import CompiledShell, cfgJsCompileCopy, copyJsSrcDirs
from inspectShell import archOfBinary, testDbgOrOpt, verifyBinary
import buildOptions

path0 = os.path.dirname(os.path.abspath(__file__))
path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util'))
sys.path.append(path1)
from countCpus import cpuCount
from downloadBuild import downloadBuild, downloadLatestBuild, mozPlatform
from hgCmds import getRepoHashAndId, getRepoNameFromHgrc, patchHgRepoUsingMq
from lithOps import knownBugsDir
from subprocesses import captureStdout, dateStr, isLinux, isMac, isWin, normExpUserPath, shellify, \
    vdump

def machineTimeoutDefaults(options):
    '''Sets different defaults depending on the machine type or debugger used.'''
    # FIXME: Set defaults for Pandaboard ES w/ & w/o Valgrind.
    if options.buildOptions.runWithVg:
        return 300
    elif platform.uname()[1] == 'tegra-ubuntu':
        return 180
    elif platform.uname()[4] == 'armv7l':
        return 600
    else:
        return 10  # If no timeout preference is specified, use 10 seconds.

def parseOptions():
    usage = 'Usage: %prog [options]'
    parser = OptionParser(usage)

    parser.set_defaults(
        disableCompareJit = False,
        disableRndFlags = False,
        noStart = False,
        timeout = 0,
        buildOptions = ""
    )

    parser.add_option('--disable-comparejit', dest='disableCompareJit', action='store_true',
                      help='Disable comparejit fuzzing.')
    parser.add_option('--disable-random-flags', dest='disableRndFlags', action='store_true',
                      help='Disable random flag fuzzing.')
    parser.add_option('--nostart', dest='noStart', action='store_true',
                      help='Compile shells only, do not start fuzzing.')

    # Specify how the shell will be built.
    # See buildOptions.py for details.
    parser.add_option('-b', '--build',
                      dest='buildOptions',
                      help='Specify build options, e.g. -b "-c opt --arch=32" (python buildOptions.py --help)')

    parser.add_option('-t', '--timeout', type='int', dest='timeout',
                      help='Sets the timeout for loopjsfunfuzz.py. ' + \
                           'Defaults to taking into account the speed of the computer and ' + \
                           'debugger (if any).')

    parser.add_option('-p', '--set-patchDir', dest='patchDir',
                      #help='Define the path to a single patch or to a directory containing mq ' + \
                      #     'patches. Must have a "series" file present, containing the names ' + \
                      #     'of the patches, the first patch required at the bottom of the list.')
                      help='Define the path to a single patch. Multiple patches are not yet ' + \
                           'supported.')

    options, args = parser.parse_args()

    if options.patchDir:
        options.patchDir = normExpUserPath(options.patchDir)

    if not options.disableCompareJit:
        options.buildOptions += " --enable-more-deterministic"

    options.buildOptions = buildOptions.parseShellOptions(options.buildOptions)

    options.timeout = options.timeout or machineTimeoutDefaults(options)

    return options

def envDump(shell, log):
    '''Dumps environment to file.'''
    with open(log, 'ab') as f:
        f.write('Information about shell:\n\n')

        f.write('Create another shell in autobisect-cache like this one:\n')
        f.write(shellify(["python", "-u", os.path.join(path0, "compileShell.py"),
            "-R", shell.getRepoDir(), "-b", shell.buildOptions.inputArgs]) + "\n\n")

        jobs = ((cpuCount() * 5) // 4) if cpuCount() > 2 else 3 # From compileShell.py
        rndNum = str(random.random()).split('.')[1][-5:]
        newTmpDir = os.path.join(shell.getCompilePathJsSrc(), 'objdir-' + rndNum)
        newTmpDir = newTmpDir.replace('\\', '\\\\') if isWin else newTmpDir
        finalBinName = os.path.join(shell.getBaseTempDir(), 'otherJs-' + rndNum)
        finalBinName = (finalBinName + '.exe').replace('\\', '\\\\') if isWin else finalBinName
        f.write('Create another shell from compilePath sources like this one, and copy here:\n')
        f.write('mkdir ' + newTmpDir + ';\n')
        f.write('cd ' + newTmpDir + ' && ' + \
            shellify(shell.getEnvAdded()) + ' ' + shellify(shell.getCfgCmdExclEnv()) + ' && ' + \
            ' '.join(['make', '-j' + str(jobs), '-s']) + ' && ' + \
            'cp js' + ('.exe ' if isWin else ' ') + finalBinName + ' && ' + \
            'cd ' + shell.getBaseTempDir() + ' && ' + 'echo' + ' && ' + \
            'echo "Finished compiling the required shell from the compilePath source."' + \
            '\n\n')

        f.write('Full environment is: ' + str(shell.getEnvFull()) + '\n')
        f.write('Environment variables added are:\n')
        f.write(shellify(shell.getEnvAdded()) + '\n\n')

        f.write('Configuration command was:\n')
        f.write(shellify(shell.getCfgCmdExclEnv()) + '\n\n')

        f.write('Full configuration command with needed environment variables is:\n')
        f.write(shellify(shell.getEnvAdded()) + ' ' + shellify(shell.getCfgCmdExclEnv()) + '\n\n')

def cmdDump(shell, cmdList, log):
    '''Dump commands to file.'''
    with open(log, 'ab') as f:
        f.write('Command to be run is:\n')
        f.write(shellify(cmdList) + '\n')
        f.write('========================================================\n')
        f.write('|  Fuzzing %s js shell builds\n' %
                     (shell.getRepoName() ))
        f.write('|  DATE: %s\n' % dateStr())
        f.write('========================================================\n\n')<commit_msg>Retweak envDump for Windows. Take 2.<commit_after>#!/usr/bin/env python
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import with_statement

import os
import platform
import random
import shutil
import subprocess
import sys

from copy import deepcopy
from optparse import OptionParser
from tempfile import mkdtemp

from compileShell import CompiledShell, cfgJsCompileCopy, copyJsSrcDirs
from inspectShell import archOfBinary, testDbgOrOpt, verifyBinary
import buildOptions

path0 = os.path.dirname(os.path.abspath(__file__))
path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util'))
sys.path.append(path1)
from countCpus import cpuCount
from downloadBuild import downloadBuild, downloadLatestBuild, mozPlatform
from hgCmds import getRepoHashAndId, getRepoNameFromHgrc, patchHgRepoUsingMq
from lithOps import knownBugsDir
from subprocesses import captureStdout, dateStr, isLinux, isMac, isWin, normExpUserPath, shellify, \
    vdump

def machineTimeoutDefaults(options):
    '''Sets different defaults depending on the machine type or debugger used.'''
    # FIXME: Set defaults for Pandaboard ES w/ & w/o Valgrind.
    if options.buildOptions.runWithVg:
        return 300
    elif platform.uname()[1] == 'tegra-ubuntu':
        return 180
    elif platform.uname()[4] == 'armv7l':
        return 600
    else:
        return 10  # If no timeout preference is specified, use 10 seconds.

def parseOptions():
    usage = 'Usage: %prog [options]'
    parser = OptionParser(usage)

    parser.set_defaults(
        disableCompareJit = False,
        disableRndFlags = False,
        noStart = False,
        timeout = 0,
        buildOptions = ""
    )

    parser.add_option('--disable-comparejit', dest='disableCompareJit', action='store_true',
                      help='Disable comparejit fuzzing.')
    parser.add_option('--disable-random-flags', dest='disableRndFlags', action='store_true',
                      help='Disable random flag fuzzing.')
    parser.add_option('--nostart', dest='noStart', action='store_true',
                      help='Compile shells only, do not start fuzzing.')

    # Specify how the shell will be built.
    # See buildOptions.py for details.
    parser.add_option('-b', '--build',
                      dest='buildOptions',
                      help='Specify build options, e.g. -b "-c opt --arch=32" (python buildOptions.py --help)')

    parser.add_option('-t', '--timeout', type='int', dest='timeout',
                      help='Sets the timeout for loopjsfunfuzz.py. ' + \
                           'Defaults to taking into account the speed of the computer and ' + \
                           'debugger (if any).')

    parser.add_option('-p', '--set-patchDir', dest='patchDir',
                      #help='Define the path to a single patch or to a directory containing mq ' + \
                      #     'patches. Must have a "series" file present, containing the names ' + \
                      #     'of the patches, the first patch required at the bottom of the list.')
                      help='Define the path to a single patch. Multiple patches are not yet ' + \
                           'supported.')

    options, args = parser.parse_args()

    if options.patchDir:
        options.patchDir = normExpUserPath(options.patchDir)

    if not options.disableCompareJit:
        options.buildOptions += " --enable-more-deterministic"

    options.buildOptions = buildOptions.parseShellOptions(options.buildOptions)

    options.timeout = options.timeout or machineTimeoutDefaults(options)

    return options

def envDump(shell, log):
    '''Dumps environment to file.'''
    with open(log, 'ab') as f:
        f.write('Information about shell:\n\n')

        f.write('Create another shell in autobisect-cache like this one:\n')
        f.write(shellify(["python", "-u", os.path.join(path0, "compileShell.py"),
            "-R", shell.getRepoDir(), "-b", shell.buildOptions.inputArgs]) + "\n\n")

        jobs = ((cpuCount() * 5) // 4) if cpuCount() > 2 else 3 # From compileShell.py
        rndNum = str(random.random()).split('.')[1][-5:]
        newTmpDir = os.path.join(shell.getCompilePathJsSrc(), 'objdir-' + rndNum)
        newTmpDir = newTmpDir.replace('\\', '\\\\') if isWin else newTmpDir
        finalBinName = os.path.join(shell.getBaseTempDir(), 'otherJs-' + rndNum)
        finalBinName = (finalBinName + '.exe').replace('\\', '\\\\') if isWin else finalBinName
        f.write('Create another shell from compilePath sources like this one, and copy here:\n')
        f.write('mkdir ' + newTmpDir + ';\n')
        f.write('cd ' + newTmpDir + ' && ' + \
            shellify(shell.getEnvAdded()) + ' ' + shellify(shell.getCfgCmdExclEnv()) + ' && ' + \
            ' '.join(['make', '-j' + str(jobs), '-s']) + ' && ' + \
            'cp js' + ('.exe ' if isWin else ' ') + finalBinName + ' && ' + 'cd ' + \
            (shell.getBaseTempDir().replace('\\', '\\\\') if isWin else shell.getBaseTempDir()) + \
            ' && ' + 'echo' + ' && ' + \
            'echo "Finished compiling the required shell from the compilePath source."' + \
            '\n\n')

        f.write('Full environment is: ' + str(shell.getEnvFull()) + '\n')
        f.write('Environment variables added are:\n')
        f.write(shellify(shell.getEnvAdded()) + '\n\n')

        f.write('Configuration command was:\n')
        f.write(shellify(shell.getCfgCmdExclEnv()) + '\n\n')

        f.write('Full configuration command with needed environment variables is:\n')
        f.write(shellify(shell.getEnvAdded()) + ' ' + shellify(shell.getCfgCmdExclEnv()) + '\n\n')

def cmdDump(shell, cmdList, log):
    '''Dump commands to file.'''
    with open(log, 'ab') as f:
        f.write('Command to be run is:\n')
        f.write(shellify(cmdList) + '\n')
        f.write('========================================================\n')
        f.write('|  Fuzzing %s js shell builds\n' %
                     (shell.getRepoName() ))
        f.write('|  DATE: %s\n' % dateStr())
        f.write('========================================================\n\n')
{'fraction_non_alphanumeric': 0.15111610727477115, 'fraction_numerical': 0.005540388630158985, 'mean_word_length': 2.664313033245072, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5590062', 'n_tokens_mistral': 3794, 'n_tokens_neox': 3641, 'n_words': 998}