usrbinenv python3 combinerefs path a helper for analyzing pythondumprefs output when the pythondumprefs envar is set in a debug build at python shutdown time pyfinalizeex prints the list of all live objects twice first it prints the repr of each object while the interpreter is still fully intact after cleaning up everything it can it prints all remaining live objects again but the second time just prints their addresses refcounts and type names because the interpreter has been torn down calling repr methods at this point can get into infinite loops or blow up save all this output into a file then run this script passing the path to that file the script finds both output chunks combines them then prints a line of output for each object still alive at the end address refcnt typename repr address is the address of the object in whatever format the platform c produces for a p format code refcnt is of the form ref when the object s refcount is the same in both pythondumprefs output blocks or refbefore refafter if the refcount changed typename is pytypeobjecttpname extracted from the second pythondumprefs output block repr is reprobject extracted from the first pythondumprefs output block caution if object is a container type it may not actually contain all the objects shown in the repr the repr was captured from the first output block and some of the containees may have been released since then for example it s common for the line showing the dict of interned strings to display strings that no longer exist at the end of pyfinalizeex this can be recognized albeit painfully because such containees don t have a line of their own the objects are listed in allocation order with mostrecently allocated printed first and the first object allocated printed last simple examples 00857060 14 str len the str object len is alive at shutdown time and both pythondumprefs output blocks said there were 14 references to it this is probably due to c modules that intern the string len and keep a reference to it in a file static 00857038 465 tuple 465 41 references to the empty tuple were removed by the cleanup actions between the times pythondumprefs produced output 00858028 10251456 str dummy key the string dummy key which is used in dictobject c to overwrite a real key that gets deleted grew several hundred references during cleanup it suggests that stuff did get removed from dicts by cleanup but that the dicts themselves are staying alive for some reason import re import sys generate lines from fileiter if whilematch is true continue reading while the regexp object pat matches line if whilematch is false lines are read so long as pat doesn t match them in any case the first line that doesn t match pat when whilematch is true or that does match pat when whilematch is false is lost and fileiter will resume at the line following it def readfileiter pat whilematch for line in fileiter if boolpat matchline whilematch yield line else break def combinefilef fi iterf for line in readfi re compiler remaining objects false pass crack re compiler azazd d addr2rc addr2guts before 0 for line in readfi re compiler remaining object addresses false m crack matchline if m addr addr2rcaddr addr2gutsaddr m groups before 1 else print skipped line after 0 for line in readfi crack true after 1 m crack matchline assert m addr rc guts m groups guts is type name here if addr not in addr2rc print new object created while tearing down line rstrip continue printaddr end if rc addr2rcaddr print s rc end else print ss addr2rcaddr rc end printguts addr2gutsaddr printd objects before d after before after def combinefname with openfname as f combinefilef if name main combinesys argv1 usr bin env python3 combinerefs path a helper for analyzing pythondumprefs output when the pythondumprefs envar is set in a debug build at python shutdown time py_finalizeex prints the list of all live objects twice first it prints the repr of each object while the interpreter is still fully intact after cleaning up everything it can it prints all remaining live objects again but the second time just prints their addresses refcounts and type names because the interpreter has been torn down calling repr methods at this point can get into infinite loops or blow up save all this output into a file then run this script passing the path to that file the script finds both output chunks combines them then prints a line of output for each object still alive at the end address refcnt typename repr address is the address of the object in whatever format the platform c produces for a p format code refcnt is of the form ref when the object s refcount is the same in both pythondumprefs output blocks or ref_before ref_after if the refcount changed typename is py_type object tp_name extracted from the second pythondumprefs output block repr is repr object extracted from the first pythondumprefs output block caution if object is a container type it may not actually contain all the objects shown in the repr the repr was captured from the first output block and some of the containees may have been released since then for example it s common for the line showing the dict of interned strings to display strings that no longer exist at the end of py_finalizeex this can be recognized albeit painfully because such containees don t have a line of their own the objects are listed in allocation order with most recently allocated printed first and the first object allocated printed last simple examples 00857060 14 str __len__ the str object __len__ is alive at shutdown time and both pythondumprefs output blocks said there were 14 references to it this is probably due to c modules that intern the string __len__ and keep a reference to it in a file static 00857038 46 5 tuple 46 5 41 references to the empty tuple were removed by the cleanup actions between the times pythondumprefs produced output 00858028 1025 1456 str dummy key the string dummy key which is used in dictobject c to overwrite a real key that gets deleted grew several hundred references during cleanup it suggests that stuff did get removed from dicts by cleanup but that the dicts themselves are staying alive for some reason generate lines from fileiter if whilematch is true continue reading while the regexp object pat matches line if whilematch is false lines are read so long as pat doesn t match them in any case the first line that doesn t match pat when whilematch is true or that does match pat when whilematch is false is lost and fileiter will resume at the line following it guts is type name here
import re
import sys







def read(fileiter, pat, whilematch):
    for line in fileiter:
        if bool(pat.match(line)) == whilematch:
            yield line
        else:
            break

def combinefile(f):
    fi = iter(f)

    for line in read(fi, re.compile(r'^Remaining objects:$'), False):
        pass

    crack = re.compile(r'([a-zA-Z\d]+) \[(\d+)\] (.*)')
    addr2rc = {}
    addr2guts = {}
    before = 0
    for line in read(fi, re.compile(r'^Remaining object addresses:$'), False):
        m = crack.match(line)
        if m:
            addr, addr2rc[addr], addr2guts[addr] = m.groups()
            before += 1
        else:
            print('??? skipped:', line)

    after = 0
    for line in read(fi, crack, True):
        after += 1
        m = crack.match(line)
        assert m
        addr, rc, guts = m.groups() 
        if addr not in addr2rc:
            print('??? new object created while tearing down:', line.rstrip())
            continue
        print(addr, end=' ')
        if rc == addr2rc[addr]:
            print('[%s]' % rc, end=' ')
        else:
            print('[%s->%s]' % (addr2rc[addr], rc), end=' ')
        print(guts, addr2guts[addr])

    print("%d objects before, %d after" % (before, after))

def combine(fname):
    with open(fname) as f:
        combinefile(f)

if __name__ == '__main__':
    combine(sys.argv[1])