<commit_before>#!/usr/bin/env python

# Recognizes NS_ASSERTIONs based on condition, text, and filename (ignoring irrelevant parts of the path)
# Recognizes JS_ASSERT based on condition only :(
# Recognizes ObjC exceptions based on message, since there is no stack information available, at least on Tiger.

import os, sys

simpleIgnoreList = []
twoPartIgnoreList = []
ready = False

# Called directly by rundomfuzz.py
def scanLine(knownPath, line):
    global ignoreList
    if not ready:
        readIgnoreList(knownPath)

    line = line.strip("\x07").rstrip("\n")

    if assertiony(line) and not ignore(line):
        return True

    return False

def scanFile(knownPath, currentFile, verbose, ignoreKnownAssertions):
    global ignoreList
    if not ready:
        readIgnoreList(knownPath)

    foundSomething = False

    # map from (assertion message) to (true, if seen in the current file)
    seenInCurrentFile = {}

    for line in currentFile:
        line = line.strip("\x07").rstrip("\n")
        if (assertiony(line) and not (line in seenInCurrentFile)):
            seenInCurrentFile[line] = True
            if not (ignore(line)):
                print "! New assertion: "
                print line
                foundSomething = True
            elif not ignoreKnownAssertions:
                foundSomething = True
            elif verbose:
                print "@ Known assertion: "
                print line

    currentFile.close()

    return foundSomething

def assertiony(line):
    return (line.startswith("###!!!") or # NS_ASSERTION and also aborts
             line.startswith("Assertion failure:") or # spidermonkey; nss; new nanojit
             line.find("Mozilla has caught an Obj-C exception") != -1 or
             line.find("Assertion failed:") != -1 or # old nanojit; assert.h e.g. as used by harfbuzz
             line.find("failed assertion") != -1 # nanojit
            )

def readIgnoreList(knownPath):
    global simpleIgnoreList
    global twoPartIgnoreList
    global ready
    ignoreFile = file(os.path.join(knownPath, "assertions.txt"), "r")
    for line in ignoreFile:
        line = line.strip()
        if ((len(line) > 0) and not line.startswith("#")):
            mpi = line.find(", file ")  # NS_ASSERTION and friends use this format
            if (mpi == -1):
                mpi = line.find(": file ")  # NS_ABORT uses this format
            if (mpi == -1):
                simpleIgnoreList.append(line)
            else:
                twoPartIgnoreList.append((line[:mpi+7], line[mpi+7:]))
    ignoreFile.close()
    ready = True
    #print "detect_assertions is ready (ignoring %d strings without filenames and %d strings with filenames)" % (len(simpleIgnoreList), len(twoPartIgnoreList))

        
def ignore(assertion):
    global simpleIgnoreList
    for ig in simpleIgnoreList:
        if assertion.find(ig) != -1:
            return True
    for (part1, part2) in twoPartIgnoreList:
        if assertion.find(part1) != -1 and assertion.find(part2) != -1:
            return True
    return False


# For use by af_timed_run and jsunhappy.py
def amiss(knownPath, logPrefix, verbose, ignoreKnownAssertions=True):
    currentFile = file(logPrefix + "-err", "r")
    return scanFile(knownPath, currentFile, verbose, ignoreKnownAssertions)

# For standalone use
if __name__ == "__main__":
    knownPath = sys.argv[1]
    currentFile = file(sys.argv[2], "r")
    print scanFile(knownPath, currentFile, False, True)<commit_msg>Fix detect_assertions on Windows harder. Turns out __FILE__ sometimes uses backslash and sometimes uses forward slash depending on its mood.<commit_after>#!/usr/bin/env python

# Recognizes NS_ASSERTIONs based on condition, text, and filename (ignoring irrelevant parts of the path)
# Recognizes JS_ASSERT based on condition only :(
# Recognizes ObjC exceptions based on message, since there is no stack information available, at least on Tiger.

import os, sys

simpleIgnoreList = []
twoPartIgnoreList = []
ready = False

# Called directly by rundomfuzz.py
def scanLine(knownPath, line):
    global ignoreList
    if not ready:
        readIgnoreList(knownPath)

    line = line.strip("\x07").rstrip("\n")

    if assertiony(line) and not ignore(line):
        return True

    return False

def scanFile(knownPath, currentFile, verbose, ignoreKnownAssertions):
    global ignoreList
    if not ready:
        readIgnoreList(knownPath)

    foundSomething = False

    # map from (assertion message) to (true, if seen in the current file)
    seenInCurrentFile = {}

    for line in currentFile:
        line = line.strip("\x07").rstrip("\n")
        if (assertiony(line) and not (line in seenInCurrentFile)):
            seenInCurrentFile[line] = True
            if not (ignore(line)):
                print "! New assertion: "
                print line
                foundSomething = True
            elif not ignoreKnownAssertions:
                foundSomething = True
            elif verbose:
                print "@ Known assertion: "
                print line

    currentFile.close()

    return foundSomething

def assertiony(line):
    return (line.startswith("###!!!") or # NS_ASSERTION and also aborts
             line.startswith("Assertion failure:") or # spidermonkey; nss; new nanojit
             line.find("Mozilla has caught an Obj-C exception") != -1 or
             line.find("Assertion failed:") != -1 or # old nanojit; assert.h e.g. as used by harfbuzz
             line.find("failed assertion") != -1 # nanojit
            )

def readIgnoreList(knownPath):
    global simpleIgnoreList
    global twoPartIgnoreList
    global ready
    ignoreFile = file(os.path.join(knownPath, "assertions.txt"), "r")
    for line in ignoreFile:
        line = line.strip()
        if ((len(line) > 0) and not line.startswith("#")):
            mpi = line.find(", file ")  # NS_ASSERTION and friends use this format
            if (mpi == -1):
                mpi = line.find(": file ")  # NS_ABORT uses this format
            if (mpi == -1):
                simpleIgnoreList.append(line)
            else:
                twoPartIgnoreList.append((line[:mpi+7], line[mpi+7:]))
    ignoreFile.close()
    ready = True
    #print "detect_assertions is ready (ignoring %d strings without filenames and %d strings with filenames)" % (len(simpleIgnoreList), len(twoPartIgnoreList))

        
def ignore(assertion):
    global simpleIgnoreList
    for ig in simpleIgnoreList:
        if assertion.find(ig) != -1:
            return True
    for (part1, part2) in twoPartIgnoreList:
        if assertion.find(part1) != -1 and assertion.replace('\\', '/').find(part2) != -1:
            return True
    return False


# For use by af_timed_run and jsunhappy.py
def amiss(knownPath, logPrefix, verbose, ignoreKnownAssertions=True):
    currentFile = file(logPrefix + "-err", "r")
    return scanFile(knownPath, currentFile, verbose, ignoreKnownAssertions)

# For standalone use
if __name__ == "__main__":
    knownPath = sys.argv[1]
    currentFile = file(sys.argv[2], "r")
    print scanFile(knownPath, currentFile, False, True)
{'fraction_non_alphanumeric': 0.10086980920314254, 'fraction_numerical': 0.005892255892255892, 'mean_word_length': 2.646547314578005, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5812992', 'n_tokens_mistral': 2113, 'n_tokens_neox': 2006, 'n_words': 637}