<commit_before>#!/usr/bin/env python3

import os
import re
import subprocess
import sys
import yaml

index = {}

# demangles SunPro mangled names - partially. Arguments aren't parsed yet.
def demangle(mangled):
    if mangled.startswith("__1c"):
        # Looks like a SunPro compiler name
        ptr = 4;
        names = [];
        while True:
            if ptr > len(mangled)-1: break
            lengthCharacter = mangled[ptr]
            if ord(lengthCharacter) > ord("A") and ord(lengthCharacter) <= ord("Z"):
                symlen = ord(lengthCharacter) - ord('A') + 1;
                names.append(mangled[ptr+1:ptr+symlen])
                ptr += symlen
            else:
                break
        if len(names)==0:
            print("Zero-length name after demangling "+mangled)
            return mangled
        return "::".join(names)
    else:
        return mangled

def scanFile(directory, filename):
    global index
    print("Scanning file "+os.path.join(directory, filename))
    parser = os.environ['PARSE_LIB']
    packageName = filename
    if packageName.startswith("calls."): packageName = packageName[6:]
    if packageName.endswith(".a.cache"): packageName = packageName[:-8]
    yamlFile = open(packageName+".yaml", "wt")

    parserResult = subprocess.check_output([parser, os.path.join(directory, filename)], stdin=None).decode("utf-8")
    print("Parser returned %d bytes"%len(parserResult))
    symbolType = None
    symbolCalls = {}
    objectSymbols = {}
    dataSymbols = {}
    for l in parserResult.splitlines():
        m = re.match('^(defs|local defs|undefs|weak defs|local undefs):\s*$',l)
        if(m):
            symbolType = m.group(1)
            continue
        m = re.match('^\s+(\S+).o\s*$',l) # Object name
        if m:
            currentObject = m.group(1)
            if currentObject not in objectSymbols: objectSymbols[currentObject] = []
            continue
        m = re.match('^\s+(\S+) (\S+)$',l) # Plain symbol with no call
        if m:
            symbol = m.group(1)
            defType = m.group(2)
            if defType[0].lower() == 't' and symbolType != "local defs":
                if symbolType == "defs":
                    if symbol not in symbolCalls:
                        symbolCalls[symbol] = []
                        sys.stderr.write("Updating calls for %s\n"%symbol)
                    objectSymbols[currentObject].append(symbol)
            elif defType[0].lower() == 'd': # Data symbol, should be ignored
                dataSymbols[symbol] = 1
            print("Recording symbol %s of type %s\n"%(symbol,defType))
        m = re.match('^\s+(\S+) (\S+), caller: (\S+)$',l) # A call from a symbol in our object
        if m:
            calledSymbol = m.group(1)
            callingSymbol = m.group(3)
            if callingSymbol in dataSymbols or calledSymbol in dataSymbols: continue
            if callingSymbol not in symbolCalls: symbolCalls[callingSymbol] = []

            if calledSymbol not in index:
                callDest = "NULL:"+demangle(calledSymbol)
            else:
                calledPackageName = index[calledSymbol]
                callDest = "id:"+calledPackageName+":"+demangle(calledSymbol)
            symbolCalls[callingSymbol].append(callDest)
            objectSymbols[currentObject].append(callingSymbol)

    package = {'contains': [], '@id': "id:"+packageName, '@type':'sw:Package', 'name': packageName }
    yamlRoot = { '@context': ['http://localhost:8000/context.jsonld'], '@graph': package }

    for (objectName,objectContents) in objectSymbols.items():
        obj = { '@id:': "id:"+packageName+":"+objectName, '@type': 'sw:Object', 'name': objectName, 'contains': [] }
        for symbol in objectContents:
            if symbol=="":
                print("Zero-length symbol found in objectContents for "+objectName)
                exit(1)
            symbolYaml = { '@id': "id:"+packageName+":"+objectName+":"+demangle(symbol), 'name': demangle(symbol), '@type':'sw:Symbol', 'calls': symbolCalls[symbol] }
            obj['contains'].append(symbolYaml)
        yamlRoot['@graph']['contains'].append(obj)

    yamlFile.write(yaml.dump(yamlRoot))
    yamlFile.close()

def scanDirectory(directory):
    print("Scanning %s"%directory, file=sys.stderr)

    files = os.listdir(directory)
    for f in files:
        if f.endswith(".a.cache"):
            scanFile(directory, f)

def main():
    global index
    # Load the symbol directory

    if 'PARSE_LIB' not in os.environ or not os.path.exists(os.environ['PARSE_LIB']):
        print("PARSE_LIB must be set to a valid cache file parser.")
        exit(1)

    indexfile = open("alldefs_sorted_uniq")
    index = {}

    while True:
        l = indexfile.readline()
        if l == "": break
        (symbol, objectName, libraryName) = l.split(":")
        index[symbol]= "%s:%s"%(libraryName.strip(),objectName.strip())

    if len(sys.argv) > 1:
        if os.path.isdir(sys.argv[1]):
            scanDirectory(sys.argv[1])
        else:
            scanFile(".",sys.argv[1])
    else:
        scanDirectory(".")

if __name__=="__main__":
    main()
<commit_msg>Move scanfile's logic into a class to better share the data structures<commit_after>#!/usr/bin/env python3

import os
import re
import subprocess
import sys
import yaml

index = {}

# demangles SunPro mangled names - partially. Arguments aren't parsed yet.
def demangle(mangled):
    if mangled.startswith("__1c"):
        # Looks like a SunPro compiler name
        ptr = 4;
        names = [];
        while True:
            if ptr > len(mangled)-1: break
            lengthCharacter = mangled[ptr]
            if ord(lengthCharacter) > ord("A") and ord(lengthCharacter) <= ord("Z"):
                symlen = ord(lengthCharacter) - ord('A') + 1;
                names.append(mangled[ptr+1:ptr+symlen])
                ptr += symlen
            else:
                break
        if len(names)==0:
            print("Zero-length name after demangling "+mangled)
            return mangled
        return "::".join(names)
    else:
        return mangled

class ParseLibParser(object):
    def __init__(self, packageName):
        # The section we're currently processing (defs, undefs etc)
        self.symbolType = None

        #For each symbol, an array of the symbols it calls
        self.symbolCalls = {}

        # For each object, an array of the symbols it contains
        self.objectSymbols = {}

        # dataSymbols is a set of all the symbols which
        # are data rather than code (value is meaningless)
        self.dataSymbols = {}

        self.currentObject = None
        self.packageName = packageName

    def processPlainSymbol(self, symbol, defType):
        if defType[0].lower() == 't' and self.symbolType != "local defs":
            if self.symbolType == "defs": # Um
                if symbol not in self.symbolCalls:
                    self.symbolCalls[symbol] = []
                    self.objectSymbols[self.currentObject].append(symbol)
                    print("Symbol added via plan definition: "+symbol)

        elif defType[0].lower() == 'd': # Data symbol, should be ignored
            self.dataSymbols[symbol] = 1

    def processCallSymbols(self, calledSymbol, callingSymbol):
        global index
        if callingSymbol in self.dataSymbols or calledSymbol in self.dataSymbols: return
        if callingSymbol not in self.symbolCalls: self.symbolCalls[callingSymbol] = []
        if calledSymbol not in index:
            callDest = "NULL:"+demangle(calledSymbol)
        else:
            calledPackageName = index[calledSymbol]
            callDest = "id:"+calledPackageName+":"+demangle(calledSymbol)
        self.symbolCalls[callingSymbol].append(callDest)
        print("Symbol added via call: "+callingSymbol)
        if callingSymbol not in self.objectSymbols[self.currentObject]:
            self.objectSymbols[self.currentObject].append(callingSymbol)

    def parse(self, text):
        for l in text.splitlines():
            m = re.match('^(defs|local defs|undefs|weak defs|local undefs):\s*$',l)
            if(m):
                self.symbolType = m.group(1)
                continue
            m = re.match('^\s+(\S+).o\s*$',l) # Object name
            if m:
                self.currentObject = m.group(1)
                if self.currentObject not in self.objectSymbols: self.objectSymbols[self.currentObject] = []
                continue
            m = re.match('^\s+(\S+) (\S+)$',l) # Plain symbol with no call
            if m:
                self.processPlainSymbol(symbol = m.group(1), defType = m.group(2))
                continue
            m = re.match('^\s+(\S+) (\S+), caller: (\S+)$',l) # A call from a symbol in our object
            if m:
                self.processCallSymbols(calledSymbol = m.group(1), callingSymbol = m.group(3))

    def getYaml(self):
        package = {'contains': [], '@id': "id:"+self.packageName,
                   '@type':'sw:Package', 'name': self.packageName }
        yamlRoot = { '@context': ['http://localhost:8000/context.jsonld'],
                     '@graph': package }

        for (objectName,objectContents) in self.objectSymbols.items():
            objectIdentifier = "id:"+self.packageName+":"+objectName
            obj = { '@id:': objectIdentifier, '@type': 'sw:Object',
                    'name': objectName, 'contains': [] }
            for symbol in objectContents:
                if symbol=="":
                    print("Zero-length symbol found in objectContents for "+objectName)
                    exit(1)
                symbolIdentifier = objectIdentifier+":"+demangle(symbol)
                symbolYaml = { '@id': symbolIdentifier,
                               'name': demangle(symbol),
                               '@type':'sw:Symbol',
                               'calls': self.symbolCalls[symbol] }
                obj['contains'].append(symbolYaml)
            package['contains'].append(obj)
        return yaml.dump(yamlRoot)

def scanFile(directory, filename):
    global index
    print("Scanning file "+os.path.join(directory, filename))
    parser = os.environ['PARSE_LIB']
    packageName = filename
    if packageName.startswith("calls."): packageName = packageName[6:]
    if packageName.endswith(".a.cache"): packageName = packageName[:-8]

    parserResult = subprocess.check_output([parser, os.path.join(directory, filename)], stdin=None).decode("utf-8")
    print("Parser returned %d bytes"%len(parserResult))
    parser = ParseLibParser(packageName)
    parser.parse(parserResult)
    yaml = parser.getYaml()

    yamlFile = open(packageName+".yaml", "wt")
    yamlFile.write(yaml)
    yamlFile.close()

def scanDirectory(directory):
    print("Scanning %s"%directory, file=sys.stderr)

    files = os.listdir(directory)
    for f in files:
        if f.endswith(".a.cache"):
            scanFile(directory, f)

def main():
    global index
    # Load the symbol directory

    if 'PARSE_LIB' not in os.environ or not os.path.exists(os.environ['PARSE_LIB']):
        print("PARSE_LIB must be set to a valid cache file parser.")
        exit(1)

    # Load the index
    indexfile = open("alldefs_sorted_uniq")
    index = {}

    while True:
        l = indexfile.readline()
        if l == "": break
        (symbol, objectName, libraryName) = l.split(":")
        index[symbol]= "%s:%s"%(libraryName.strip(),objectName.strip())

    if len(sys.argv) > 1:
        if os.path.isdir(sys.argv[1]):
            scanDirectory(sys.argv[1])
        else:
            scanFile(".",sys.argv[1])
    else:
        scanDirectory(".")

if __name__=="__main__":
    main()

{'fraction_non_alphanumeric': 0.13253527148353997, 'fraction_numerical': 0.0049593843522873025, 'mean_word_length': 2.2863163810059004, 'pattern_counts': {'":': 21, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2845144', 'n_tokens_mistral': 3456, 'n_tokens_neox': 3354, 'n_words': 818}