<commit_before>import sys
import re
import argparse

## entr_filter.py <entrained file name> <whitelist file name>
##
## This loads the file of entrained variables, along with the whitelist, then prints out
## any entrained variables that are not covered by the whitelist.
##


entrainedByPatt = re.compile('Script ([^ ]+) \(([^:]+):(\d+)\) has variables entrained by ([^ ]+) \(([^:]+):(\d+)\) ::([^\r\n]*)\r?$')

# 1: outer function script name
# 2: outer function file name
# 3: outer function line number
# 4: inner function name
# 5: inner function file name
# 6: inner function line number
# 7: list of symbols


# For now, there's no ability to only whitelist for certain inner functions.
whitelistPatt = re.compile('outer ([^ ]+) \(([^:]+):(\d+)\) inner \* ::([^\r\n]*)\r?$')

# 1: outer function script name
# 2: outer function file name
# 3: outer function line number
# 4: list of symbols

# You can also start a line with '#' in the white list, and it will be ignored.



def scriptToString(s):
    return '{0} ({1}:{2})'.format(s[0], s[1], s[2])

wildcardWhitelist = set(['*'])

def cancelify (entrainedVars, whitelist):
    for outer, innerMap in entrainedVars.iteritems():
        innerWhitelist = whitelist.get(outer, set([]))

        # If this outer function is wildcarded
        if innerWhitelist == wildcardWhitelist:
            print 'WILD CARD!'
            continue
        print 'OUTER', outer
        print '\t', innerMap
        print '\t', innerWhitelist


#        sys.stdout.write('outer {0}\n'.format(scriptToString(outer)))
#        sys.stdout.write('inner {0}\n'.format(scriptToString(inner)))

#        print '\tentrained: ', evars



#########

def relativizePath(path, relativeTo):
    if path.startswith(relativeTo):
        return path[len(relativeTo):]
    print 'Script path', path, 'does not have base path', relativeTo, 'as a prefix'
    exit(-1)


def parseEntrainedFile(f, basePath):
    entrainedVars = {}

    for l in f:
        em = entrainedByPatt.match(l)
        if not em:
            print 'Failed to match line:'
            print l
            continue
        outer = (em.group(1), relativizePath(em.group(2), basePath), em.group(3))
        inner = (em.group(4), relativizePath(em.group(5), basePath), em.group(6))
        evars = set(em.group(7).split())

        if not outer in entrainedVars:
            entrainedVars[outer] = {}
        innerMap = entrainedVars[outer]
        if inner in innerMap:
            print 'Should not have gotten the outer/inner pair',
            print scriptToString(outer), scriptToString(inner), 'twice'
            exit(-1)
        innerMap[inner] = evars

    return entrainedVars


def loadEntrainedFile(entrainedFileName, basePath):
    try:
        entrFile = open(entrainedFileName, 'r')
    except:
        print 'Error opening entrained variable file', fname
        exit(-1)

    results = parseEntrainedFile(entrFile, basePath)
    entrFile.close()

    return results


###############

def parseWhitelistFile(f):
    whitelist = {}
    for l in f:
        wlm = whitelistPatt.match(l)
        if not wlm:
            # Ignore comments
            if l.startswith('#'):
                continue
            # We should also ignore lines that are entirely whitespace.
            print 'Failed to match line:'
            print l
            continue
        outer = (wlm.group(1), wlm.group(2), wlm.group(3))
        evars = set(wlm.group(4).split())

        if outer in whitelist:
            print 'We only support a single white list entry for an outer function because I am lazy'
            exit(-1)
        whitelist[outer] = evars

    return whitelist


def loadWhitelist(whitelistFileName):
    try:
        whitelistFile = open(whitelistFileName, 'r')
    except:
        print 'Error opening whitelist file', fname
        exit(-1)
    return parseWhitelistFile(whitelistFile)


###########

if len(sys.argv) < 3:
    print 'Not enough arguments.'
    exit()

basePath='/Users/amccreight/mz/'

entrainedVars = loadEntrainedFile(sys.argv[1], basePath)
whitelist = loadWhitelist(sys.argv[2])

cancelify(entrainedVars, whitelist)




<commit_msg>implement basic filtering and pretty printing<commit_after>import sys
import re
import argparse

## entr_filter.py <entrained file name> <whitelist file name>
##
## This loads the file of entrained variables, along with the whitelist, then prints out
## any entrained variables that are not covered by the whitelist.
##


entrainedByPatt = re.compile('Script ([^ ]+) \(([^:]+):(\d+)\) has variables entrained by ([^ ]+) \(([^:]+):(\d+)\) ::([^\r\n]*)\r?$')

# 1: outer function script name
# 2: outer function file name
# 3: outer function line number
# 4: inner function name
# 5: inner function file name
# 6: inner function line number
# 7: list of symbols


# For now, there's no ability to only whitelist for certain inner functions.
whitelistPatt = re.compile('outer ([^ ]+) \(([^:]+):(\d+)\) inner \* ::([^\r\n]*)\r?$')

# 1: outer function script name
# 2: outer function file name
# 3: outer function line number
# 4: list of symbols

# You can also start a line with '#' in the white list, and it will be ignored.



def scriptToString(s):
    return '{0} ({1}:{2})'.format(s[0], s[1], s[2])

def printEntrainedVars(entrainedVars):
    for outer, innerMap in entrainedVars.iteritems():
        for inner, evars in innerMap.iteritems():
            print 'OUTER:', scriptToString(outer), '; INNER:', scriptToString(inner), '; VARS:', ', '.join(evars)

##########

wildcardWhitelist = set(['*'])

def filterVars (entrainedVars, whitelist):
    filteredEntrainedVars = {}

    for outer, innerMap in entrainedVars.iteritems():
        innerWhitelist = whitelist.get(outer, set([]))

        # If this outer function is wildcarded
        if innerWhitelist == wildcardWhitelist:
            continue

        newInnerMap = {}

        for inner, evars in innerMap.iteritems():
            # Could complain when something in the white list isn't actually entrained.
            newEvars = evars - innerWhitelist
            if len(newEvars) == 0:
                continue
            newInnerMap[inner] = newEvars

        if len(newInnerMap) == 0:
            continue

        filteredEntrainedVars[outer] = newInnerMap

    return filteredEntrainedVars



#########

def relativizePath(path, relativeTo):
    if path.startswith(relativeTo):
        return path[len(relativeTo):]
    print 'Script path', path, 'does not have base path', relativeTo, 'as a prefix'
    exit(-1)


def parseEntrainedFile(f, basePath):
    entrainedVars = {}

    for l in f:
        em = entrainedByPatt.match(l)
        if not em:
            print 'Failed to match line:'
            print l
            continue
        outer = (em.group(1), relativizePath(em.group(2), basePath), em.group(3))
        inner = (em.group(4), relativizePath(em.group(5), basePath), em.group(6))
        evars = set(em.group(7).split())

        if not outer in entrainedVars:
            entrainedVars[outer] = {}
        innerMap = entrainedVars[outer]
        if inner in innerMap:
            print 'Should not have gotten the outer/inner pair',
            print scriptToString(outer), scriptToString(inner), 'twice'
            exit(-1)
        innerMap[inner] = evars

    return entrainedVars


def loadEntrainedFile(entrainedFileName, basePath):
    try:
        entrFile = open(entrainedFileName, 'r')
    except:
        print 'Error opening entrained variable file', fname
        exit(-1)

    results = parseEntrainedFile(entrFile, basePath)
    entrFile.close()

    return results


###############

def parseWhitelistFile(f):
    whitelist = {}
    for l in f:
        wlm = whitelistPatt.match(l)
        if not wlm:
            # Ignore comments
            if l.startswith('#'):
                continue
            # We should also ignore lines that are entirely whitespace.
            print 'Failed to match line:'
            print l
            continue
        outer = (wlm.group(1), wlm.group(2), wlm.group(3))
        evars = set(wlm.group(4).split())

        if outer in whitelist:
            print 'We only support a single white list entry for an outer function because I am lazy'
            exit(-1)
        whitelist[outer] = evars

    return whitelist


def loadWhitelist(whitelistFileName):
    try:
        whitelistFile = open(whitelistFileName, 'r')
    except:
        print 'Error opening whitelist file', fname
        exit(-1)
    return parseWhitelistFile(whitelistFile)


###########

if len(sys.argv) < 3:
    print 'Not enough arguments.'
    exit()

basePath='/Users/amccreight/mz/'

entrainedVars = loadEntrainedFile(sys.argv[1], basePath)
whitelist = loadWhitelist(sys.argv[2])

entrainedVars = filterVars(entrainedVars, whitelist)

printEntrainedVars(entrainedVars)
{'fraction_non_alphanumeric': 0.1272316384180791, 'fraction_numerical': 0.008587570621468926, 'mean_word_length': 2.753604749787956, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '453451', 'n_tokens_mistral': 2772, 'n_tokens_neox': 2662, 'n_words': 810}