In a large directory structure it is easy to inadvertently leave unnecessary copies of files around, which can use considerable disk space and create confusion.



Task

Create a program which, given a minimum size and a folder/directory, will find all files of at least size bytes with duplicate contents under the directory and output or show the sets of duplicate files in order of decreasing size.

The program may be command-line or graphical, and duplicate content may be determined by direct comparison or by calculating a hash of the data.

Specify which filesystems or operating systems your program works with if it has any filesystem- or OS-specific requirements.

Identify hard links (filenames referencing the same content) in the output if applicable for the filesystem.

For extra points, detect when whole directory sub-trees are identical, or optionally remove or link identical files.




without js -- file i/o
integer min_size=1
sequence res = {}
atom t1 = time()+1
 
function store_res(string filepath, sequence dir_entry)
    if not match("backup",filepath) -- (example filter)
    and not find('d', dir_entry[D_ATTRIBUTES]) then
        atom size = dir_entry[D_SIZE]
        if size>=min_size then
            res = append(res,{size,filepath,dir_entry})
            if time()>t1 then
                printf(1,"%d files found\r",length(res))
                t1 = time()+1
            end if
        end if
    end if
    return 0 -- keep going
end function
integer exit_code = walk_dir("demo\\clocks\\love", store_res, true)
 
res = sort(res,DESCENDING)
printf(1,"%d files found\n",length(res))
 
integer duplicates = 0
for i=1 to length(res)-1 do
    for j=i+1 to length(res) do
        if res[i][1]!=res[j][1] then exit end if
        string si = join_path({res[i][2],res[i][3][D_NAME]}),
               sj = join_path({res[j][2],res[j][3][D_NAME]})
        integer fni = open(si,"rb"),
                fnj = open(sj,"rb"),
                size = res[i][1]
        bool same = true
        if fni=-1 or fnj=-1 then ?9/0 end if
        for k=1 to size+1 do    -- (check eof as well)
            if getc(fni)!=getc(fnj) then
                same = false
                exit
            end if
        end for
        close(fni)
        close(fnj)
        if same then
            -- prettifying the output left as an exercise...
            ?res[i]
            ?res[j]
            duplicates += 1
        end if
    end for
    if time()>t1 then
        printf(1,"processing %d/%d...\r",{i,length(res)})
        t1 = time()+1
    end if
end for
printf(1,"%d duplicates found\n",duplicates)
