utilities for comparing files and directories classes dircmp functions cmpf1 f2 shallowtrue int cmpfilesa b common clearcache clear the filecmp cache cache clear def cmpf1 f2 shallowtrue s1 sigos statf1 s2 sigos statf2 if s10 stat sifreg or s20 stat sifreg return false if shallow and s1 s2 return true if s11 s21 return false outcome cache getf1 f2 s1 s2 if outcome is none outcome docmpf1 f2 if lencache 100 limit the maximum size of the cache clearcache cachef1 f2 s1 s2 outcome return outcome def sigst return stat sifmtst stmode st stsize st stmtime def docmpf1 f2 bufsize bufsize with openf1 rb as fp1 openf2 rb as fp2 while true b1 fp1 readbufsize b2 fp2 readbufsize if b1 b2 return false if not b1 return true directory comparison class class dircmp def initself a b ignorenone hidenone initialize self left a self right b if hide is none self hide os curdir os pardir names never to be shown else self hide hide if ignore is none self ignore defaultignores else self ignore ignore def phase0self compare everything except common subdirectories self leftlist filteros listdirself left self hideself ignore self rightlist filteros listdirself right self hideself ignore self leftlist sort self rightlist sort def phase1self compute common names a dictzipmapos path normcase self leftlist self leftlist b dictzipmapos path normcase self rightlist self rightlist self common listmapa getitem filterb contains a self leftonly listmapa getitem filterfalseb contains a self rightonly listmapb getitem filterfalsea contains b def phase2self distinguish files directories funnies self commondirs self commonfiles self commonfunny for x in self common apath os path joinself left x bpath os path joinself right x ok true try astat os statapath except oserror print can t stat apath why args1 ok false try bstat os statbpath except oserror print can t stat bpath why args1 ok false if ok atype stat sifmtastat stmode btype stat sifmtbstat stmode if atype btype self commonfunny appendx elif stat sisdiratype self commondirs appendx elif stat sisregatype self commonfiles appendx else self commonfunny appendx else self commonfunny appendx def phase3self find out differences between common files xx cmpfilesself left self right self commonfiles self samefiles self difffiles self funnyfiles xx def phase4self find out differences between common subdirectories a new dircmp or mydircmp if dircmp was subclassed object is created for each common subdirectory these are stored in a dictionary indexed by filename the hide and ignore properties are inherited from the parent self subdirs for x in self commondirs ax os path joinself left x bx os path joinself right x self subdirsx self classax bx self ignore self hide def phase4closureself recursively call phase4 on subdirectories self phase4 for sd in self subdirs values sd phase4closure def reportself print a report on the differences between a and b output format is purposely lousy print diff self left self right if self leftonly self leftonly sort print only in self left self leftonly if self rightonly self rightonly sort print only in self right self rightonly if self samefiles self samefiles sort print identical files self samefiles if self difffiles self difffiles sort print differing files self difffiles if self funnyfiles self funnyfiles sort print trouble with common files self funnyfiles if self commondirs self commondirs sort print common subdirectories self commondirs if self commonfunny self commonfunny sort print common funny cases self commonfunny def reportpartialclosureself print reports on self and on subdirs self report for sd in self subdirs values print sd report def reportfullclosureself report on self and subdirs recursively self report for sd in self subdirs values print sd reportfullclosure methodmap dictsubdirsphase4 samefilesphase3 difffilesphase3 funnyfilesphase3 commondirsphase2 commonfilesphase2 commonfunnyphase2 commonphase1 leftonlyphase1 rightonlyphase1 leftlistphase0 rightlistphase0 def getattrself attr if attr not in self methodmap raise attributeerrorattr self methodmapattrself return getattrself attr classgetitem classmethodgenericalias def cmpfilesa b common shallowtrue res for x in common ax os path joina x bx os path joinb x rescmpax bx shallow appendx return res compare two files return 0 for equal 1 for different 2 for funny cases can t stat etc def cmpa b sh absabs cmpcmp try return not abscmpa b sh except oserror return 2 return a copy with items that occur in skip removed def filterflist skip return listfilterfalseskip contains flist demonstration and testing def demo import sys import getopt options args getopt getoptsys argv1 r if lenargs 2 raise getopt getopterror need exactly two args none dd dircmpargs0 args1 if r in options dd reportfullclosure else dd report if name main demo clear the filecmp cache compare two files arguments f1 first file name f2 second file name shallow treat files as identical if their stat signatures type size mtime are identical otherwise files are considered different if their sizes or contents differ default true return value true if the files are the same false otherwise this function uses a cache for past comparisons and the results with cache entries invalidated if their stat information changes the cache may be cleared by calling clear_cache limit the maximum size of the cache directory comparison class a class that manages the comparison of 2 directories dircmp a b ignore none hide none a and b are directories ignore is a list of names to ignore defaults to default_ignores hide is a list of names to hide defaults to os curdir os pardir high level usage x dircmp dir1 dir2 x report prints a report on the differences between dir1 and dir2 or x report_partial_closure prints report on differences between dir1 and dir2 and reports on common immediate subdirectories x report_full_closure like report_partial_closure but fully recursive attributes left_list right_list the files in dir1 and dir2 filtered by hide and ignore common a list of names in both dir1 and dir2 left_only right_only names only in dir1 dir2 common_dirs subdirectories in both dir1 and dir2 common_files files in both dir1 and dir2 common_funny names in both dir1 and dir2 where the type differs between dir1 and dir2 or the name is not stat able same_files list of identical files diff_files list of filenames which differ funny_files list of files which could not be compared subdirs a dictionary of dircmp instances or mydircmp instances if this object is of type mydircmp a subclass of dircmp keyed by names in common_dirs initialize names never to be shown compare everything except common subdirectories compute common names distinguish files directories funnies print can t stat a_path why args 1 print can t stat b_path why args 1 find out differences between common files find out differences between common subdirectories a new dircmp or mydircmp if dircmp was subclassed object is created for each common subdirectory these are stored in a dictionary indexed by filename the hide and ignore properties are inherited from the parent recursively call phase4 on subdirectories print a report on the differences between a and b output format is purposely lousy print reports on self and on subdirs report on self and subdirs recursively compare common files in two directories a b directory names common list of file names found in both directories shallow if true do comparison based solely on stat information returns a tuple of three lists files that compare equal files that are different filenames that aren t regular files compare two files return 0 for equal 1 for different 2 for funny cases can t stat etc return a copy with items that occur in skip removed demonstration and testing
import os
import stat
from itertools import filterfalse
from types import GenericAlias

__all__ = ['clear_cache', 'cmp', 'dircmp', 'cmpfiles', 'DEFAULT_IGNORES']

_cache = {}
BUFSIZE = 8*1024

DEFAULT_IGNORES = [
    'RCS', 'CVS', 'tags', '.git', '.hg', '.bzr', '_darcs', '__pycache__']

def clear_cache():
    
    _cache.clear()

def cmp(f1, f2, shallow=True):
    

    s1 = _sig(os.stat(f1))
    s2 = _sig(os.stat(f2))
    if s1[0] != stat.S_IFREG or s2[0] != stat.S_IFREG:
        return False
    if shallow and s1 == s2:
        return True
    if s1[1] != s2[1]:
        return False

    outcome = _cache.get((f1, f2, s1, s2))
    if outcome is None:
        outcome = _do_cmp(f1, f2)
        if len(_cache) > 100:      
            clear_cache()
        _cache[f1, f2, s1, s2] = outcome
    return outcome

def _sig(st):
    return (stat.S_IFMT(st.st_mode),
            st.st_size,
            st.st_mtime)

def _do_cmp(f1, f2):
    bufsize = BUFSIZE
    with open(f1, 'rb') as fp1, open(f2, 'rb') as fp2:
        while True:
            b1 = fp1.read(bufsize)
            b2 = fp2.read(bufsize)
            if b1 != b2:
                return False
            if not b1:
                return True



class dircmp:
    

    def __init__(self, a, b, ignore=None, hide=None): 
        self.left = a
        self.right = b
        if hide is None:
            self.hide = [os.curdir, os.pardir] 
        else:
            self.hide = hide
        if ignore is None:
            self.ignore = DEFAULT_IGNORES
        else:
            self.ignore = ignore

    def phase0(self): 
        self.left_list = _filter(os.listdir(self.left),
                                 self.hide+self.ignore)
        self.right_list = _filter(os.listdir(self.right),
                                  self.hide+self.ignore)
        self.left_list.sort()
        self.right_list.sort()

    def phase1(self): 
        a = dict(zip(map(os.path.normcase, self.left_list), self.left_list))
        b = dict(zip(map(os.path.normcase, self.right_list), self.right_list))
        self.common = list(map(a.__getitem__, filter(b.__contains__, a)))
        self.left_only = list(map(a.__getitem__, filterfalse(b.__contains__, a)))
        self.right_only = list(map(b.__getitem__, filterfalse(a.__contains__, b)))

    def phase2(self): 
        self.common_dirs = []
        self.common_files = []
        self.common_funny = []

        for x in self.common:
            a_path = os.path.join(self.left, x)
            b_path = os.path.join(self.right, x)

            ok = True
            try:
                a_stat = os.stat(a_path)
            except OSError:
                
                ok = False
            try:
                b_stat = os.stat(b_path)
            except OSError:
                
                ok = False

            if ok:
                a_type = stat.S_IFMT(a_stat.st_mode)
                b_type = stat.S_IFMT(b_stat.st_mode)
                if a_type != b_type:
                    self.common_funny.append(x)
                elif stat.S_ISDIR(a_type):
                    self.common_dirs.append(x)
                elif stat.S_ISREG(a_type):
                    self.common_files.append(x)
                else:
                    self.common_funny.append(x)
            else:
                self.common_funny.append(x)

    def phase3(self): 
        xx = cmpfiles(self.left, self.right, self.common_files)
        self.same_files, self.diff_files, self.funny_files = xx

    def phase4(self): 
        
        
        
        
        self.subdirs = {}
        for x in self.common_dirs:
            a_x = os.path.join(self.left, x)
            b_x = os.path.join(self.right, x)
            self.subdirs[x]  = self.__class__(a_x, b_x, self.ignore, self.hide)

    def phase4_closure(self): 
        self.phase4()
        for sd in self.subdirs.values():
            sd.phase4_closure()

    def report(self): 
        
        print('diff', self.left, self.right)
        if self.left_only:
            self.left_only.sort()
            print('Only in', self.left, ':', self.left_only)
        if self.right_only:
            self.right_only.sort()
            print('Only in', self.right, ':', self.right_only)
        if self.same_files:
            self.same_files.sort()
            print('Identical files :', self.same_files)
        if self.diff_files:
            self.diff_files.sort()
            print('Differing files :', self.diff_files)
        if self.funny_files:
            self.funny_files.sort()
            print('Trouble with common files :', self.funny_files)
        if self.common_dirs:
            self.common_dirs.sort()
            print('Common subdirectories :', self.common_dirs)
        if self.common_funny:
            self.common_funny.sort()
            print('Common funny cases :', self.common_funny)

    def report_partial_closure(self): 
        self.report()
        for sd in self.subdirs.values():
            print()
            sd.report()

    def report_full_closure(self): 
        self.report()
        for sd in self.subdirs.values():
            print()
            sd.report_full_closure()

    methodmap = dict(subdirs=phase4,
                     same_files=phase3, diff_files=phase3, funny_files=phase3,
                     common_dirs=phase2, common_files=phase2, common_funny=phase2,
                     common=phase1, left_only=phase1, right_only=phase1,
                     left_list=phase0, right_list=phase0)

    def __getattr__(self, attr):
        if attr not in self.methodmap:
            raise AttributeError(attr)
        self.methodmap[attr](self)
        return getattr(self, attr)

    __class_getitem__ = classmethod(GenericAlias)


def cmpfiles(a, b, common, shallow=True):
    
    res = ([], [], [])
    for x in common:
        ax = os.path.join(a, x)
        bx = os.path.join(b, x)
        res[_cmp(ax, bx, shallow)].append(x)
    return res








def _cmp(a, b, sh, abs=abs, cmp=cmp):
    try:
        return not abs(cmp(a, b, sh))
    except OSError:
        return 2




def _filter(flist, skip):
    return list(filterfalse(skip.__contains__, flist))




def demo():
    import sys
    import getopt
    options, args = getopt.getopt(sys.argv[1:], 'r')
    if len(args) != 2:
        raise getopt.GetoptError('need exactly two args', None)
    dd = dircmp(args[0], args[1])
    if ('-r', '') in options:
        dd.report_full_closure()
    else:
        dd.report()

if __name__ == '__main__':
    demo()