<commit_before>#!/usr/bin/env python

from __future__ import with_statement

import os
import pinpoint
import subprocess
import sys
import re

from optparse import OptionParser

import jsInteresting
import shellFlags

path0 = os.path.dirname(os.path.abspath(__file__))
path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util'))
sys.path.append(path1)
from subprocesses import shellify
import lithOps


lengthLimit = 1000000

def lastLine(err):
    lines = err.split("\n")
    if len(lines) >= 2:
        return lines[-2]
    return ""

# MallocScribble prints a line that includes the process's pid.  We don't want to include that pid in the comparison!
def ignoreMallocScribble(e):
    lines = e.split("\n")
    if lines[0].endswith("malloc: enabling scribbling to detect mods to free blocks"):
        return "\n".join(lines[1:])
    else:
        return e

# For use by loopjsfunfuzz.py
def compareJIT(jsEngine, flags, infilename, logPrefix, knownPath, repo, timeout, targetTime):
    lev = compareLevel(jsEngine, flags, infilename, logPrefix + "-initial", knownPath, timeout, False, True)

    if lev != jsInteresting.JS_FINE:
        itest = [__file__, "--flags="+' '.join(flags), "--minlevel="+str(lev), "--timeout="+str(timeout), knownPath]
        (lithResult, lithDetails) = pinpoint.pinpoint(itest, logPrefix, jsEngine, [], infilename, repo, targetTime)
        print infilename
        print compareLevel(jsEngine, flags, infilename, logPrefix + "-final", knownPath, timeout, True, False)
        return (lithResult, lithDetails)
    else:
        return (lithOps.HAPPY, None)


dvgRE = re.compile("TypeError\: .* is .*")

def compareLevel(jsEngine, flags, infilename, logPrefix, knownPath, timeout, showDetailedDiffs, quickMode):
    combos = shellFlags.basicFlagSets(jsEngine)

    if quickMode:
        # Only used during initial fuzzing. Allowed to have false negatives.
        combos = [combos[0]]

    if len(flags):
        combos.append(flags)

    commands = [[jsEngine] + combo + [infilename] for combo in combos]

    for i in range(0, len(commands)):
        prefix = logPrefix + "-r" + str(i)
        command = commands[i]
        (lev, issues, r) = jsInteresting.baseLevel(command, timeout, knownPath, prefix)

        with open(prefix + "-out.txt") as f:
            r.out = f.read()
        with open(prefix + "-err.txt") as f:
            r.err = f.read()

        if len(r.out) > lengthLimit:
            r.out = "VERYLONGOUT"
        r.err = ignoreMallocScribble(r.err)

        if (r.rc == 1 or r.rc == 2) and (r.err.find('usage: ') != -1 or r.out.find('Usage: ') != -1):
            print "Got usage error from:"<commit_msg>Change the way we detect shell usage errors, so usage errors from functions like schedulegc do not trip our detection.<commit_after>#!/usr/bin/env python

from __future__ import with_statement

import os
import pinpoint
import subprocess
import sys
import re

from optparse import OptionParser

import jsInteresting
import shellFlags

path0 = os.path.dirname(os.path.abspath(__file__))
path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util'))
sys.path.append(path1)
from subprocesses import shellify
import lithOps


lengthLimit = 1000000

def lastLine(err):
    lines = err.split("\n")
    if len(lines) >= 2:
        return lines[-2]
    return ""

# MallocScribble prints a line that includes the process's pid.  We don't want to include that pid in the comparison!
def ignoreMallocScribble(e):
    lines = e.split("\n")
    if lines[0].endswith("malloc: enabling scribbling to detect mods to free blocks"):
        return "\n".join(lines[1:])
    else:
        return e

# For use by loopjsfunfuzz.py
def compareJIT(jsEngine, flags, infilename, logPrefix, knownPath, repo, timeout, targetTime):
    lev = compareLevel(jsEngine, flags, infilename, logPrefix + "-initial", knownPath, timeout, False, True)

    if lev != jsInteresting.JS_FINE:
        itest = [__file__, "--flags="+' '.join(flags), "--minlevel="+str(lev), "--timeout="+str(timeout), knownPath]
        (lithResult, lithDetails) = pinpoint.pinpoint(itest, logPrefix, jsEngine, [], infilename, repo, targetTime)
        print infilename
        print compareLevel(jsEngine, flags, infilename, logPrefix + "-final", knownPath, timeout, True, False)
        return (lithResult, lithDetails)
    else:
        return (lithOps.HAPPY, None)


dvgRE = re.compile("TypeError\: .* is .*")

def compareLevel(jsEngine, flags, infilename, logPrefix, knownPath, timeout, showDetailedDiffs, quickMode):
    combos = shellFlags.basicFlagSets(jsEngine)

    if quickMode:
        # Only used during initial fuzzing. Allowed to have false negatives.
        combos = [combos[0]]

    if len(flags):
        combos.append(flags)

    commands = [[jsEngine] + combo + [infilename] for combo in combos]

    for i in range(0, len(commands)):
        prefix = logPrefix + "-r" + str(i)
        command = commands[i]
        (lev, issues, r) = jsInteresting.baseLevel(command, timeout, knownPath, prefix)

        with open(prefix + "-out.txt") as f:
            r.out = f.read()
        with open(prefix + "-err.txt") as f:
            r.err = f.read()

        if len(r.out) > lengthLimit:
            r.out = "VERYLONGOUT"
        r.err = ignoreMallocScribble(r.err)

        if (r.rc == 1 or r.rc == 2) and (r.out.find('[[script] scriptArgs*]') != -1 or r.err.find('[scriptfile] [scriptarg...]') != -1):
            print "Got usage error from:"
{'fraction_non_alphanumeric': 0.1273792093704246, 'fraction_numerical': 0.0076866764275256225, 'mean_word_length': 3.216820987654321, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1253770', 'n_tokens_mistral': 1784, 'n_tokens_neox': 1720, 'n_words': 504}