<commit_before>#if 0
set -e -x
g++ -I../libpfkutil ../libpfkutil/md5.c deduplifier.cc -o d
exit 0
#endif

#include <dirent.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>

#include <iostream>
#include <string>
#include <map>
#include <list>

#include "md5.h"

using namespace std;

typedef map<string,string> digestFileMap_t;
typedef list<string> pendingDirs_t;

extern "C" int
deduplifier_main(int argc, char ** argv)
{
    digestFileMap_t digestFileMap;
    pendingDirs_t pendingDirs;

    if (argc != 2)
    {
        cerr << "usage : dedup /full/path/to/rootdir" << endl;
        return 1;
    }

    string rootdir = argv[1];

    if (rootdir[0] != '/')
    {
        cerr << "must use absolute path" << endl;
        return 1;
    }

    if (rootdir[rootdir.size()-1] == '/')
        // strip trailing slash
        rootdir.erase(rootdir.size()-1,1);

    pendingDirs.push_back(rootdir);

    while (!pendingDirs.empty())
    {
        const string &currentDir = pendingDirs.front();
        DIR * d = opendir(currentDir.c_str());
        if (d)
        {
            struct dirent * de;
            while ((de = readdir(d)) != NULL)
            {
                const string fname = de->d_name;
                if (fname == "." || fname == ".." || fname == ".git")
                    continue;
                const string fullPath = currentDir + "/" + fname;
                struct stat sb;
                if (lstat(fullPath.c_str(), &sb) == 0)
                {
                    if (S_ISDIR(sb.st_mode))
                        pendingDirs.push_back(fullPath);
                    else if (S_ISREG(sb.st_mode))
                    {
                        char buffer[48];
                        if (MD5File(fullPath.c_str(), buffer) != NULL)
                        {
                            string digest = (string)((char*)buffer);
                            pair<digestFileMap_t::iterator,bool> rv;
                            rv = digestFileMap.insert(
                                digestFileMap_t::value_type(
                                    digest,fullPath));
                            if (rv.second == false)
                            {
                                cout << rv.first->second
                                     << " same as " << fullPath << endl;
                                (void) unlink(fullPath.c_str());
                                if (symlink(rv.first->second.c_str(),
                                            fullPath.c_str()) < 0)
                                    cerr << "unable to symlink: "
                                         << strerror(errno) << endl;
                            }
                        }
                        else
                            cerr << "unable to md5 " << fullPath << endl;
                    }
                }
                else
                    cerr << "unable to stat " << fullPath << endl;
            }
            closedir(d);
        }
        else
            cerr << "unable to open " << currentDir << endl;
        pendingDirs.pop_front();
    }

    return 0;
}
<commit_msg>why did this compile on F25 but not F24?<commit_after>#if 0
set -e -x
g++ -I../libpfkutil ../libpfkutil/md5.c deduplifier.cc -o d
exit 0
#endif

#include <dirent.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>

#include <iostream>
#include <string>
#include <map>
#include <list>

#include "md5.h"

using namespace std;

typedef map<string,string> digestFileMap_t;
typedef list<string> pendingDirs_t;

extern "C" int
deduplifier_main(int argc, char ** argv)
{
    digestFileMap_t digestFileMap;
    pendingDirs_t pendingDirs;

    if (argc != 2)
    {
        cerr << "usage : dedup /full/path/to/rootdir" << endl;
        return 1;
    }

    string rootdir = argv[1];

    if (rootdir[0] != '/')
    {
        cerr << "must use absolute path" << endl;
        return 1;
    }

    if (rootdir[rootdir.size()-1] == '/')
        // strip trailing slash
        rootdir.erase(rootdir.size()-1,1);

    pendingDirs.push_back(rootdir);

    while (!pendingDirs.empty())
    {
        const string &currentDir = pendingDirs.front();
        DIR * d = opendir(currentDir.c_str());
        if (d)
        {
            struct dirent * de;
            while ((de = readdir(d)) != NULL)
            {
                const string fname = de->d_name;
                if (fname == "." || fname == ".." || fname == ".git")
                    continue;
                const string fullPath = currentDir + "/" + fname;
                struct stat sb;
                if (lstat(fullPath.c_str(), &sb) == 0)
                {
                    if (S_ISDIR(sb.st_mode))
                        pendingDirs.push_back(fullPath);
                    else if (S_ISREG(sb.st_mode))
                    {
                        char buffer[48];
                        if (MD5File(fullPath.c_str(), buffer) != NULL)
                        {
                            string digest = (string)((char*)buffer);
                            pair<digestFileMap_t::iterator,bool> rv;
                            rv = digestFileMap.insert(
                                digestFileMap_t::value_type(
                                    digest,fullPath));
                            if (rv.second == false)
                            {
                                cout << rv.first->second
                                     << " same as " << fullPath << endl;
                                (void) unlink(fullPath.c_str());
                                if (symlink(rv.first->second.c_str(),
                                            fullPath.c_str()) < 0)
                                    cerr << "unable to symlink: "
                                         << strerror(errno) << endl;
                            }
                        }
                        else
                            cerr << "unable to md5 " << fullPath << endl;
                    }
                }
                else
                    cerr << "unable to stat " << fullPath << endl;
            }
            closedir(d);
        }
        else
            cerr << "unable to open " << currentDir << endl;
        pendingDirs.pop_front();
    }

    return 0;
}

{'fraction_non_alphanumeric': 0.12040816326530612, 'fraction_numerical': 0.006593406593406593, 'mean_word_length': 1.1321954484605088, 'pattern_counts': {'":': 0, '<': 110, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4781251', 'n_tokens_mistral': 1882, 'n_tokens_neox': 1775, 'n_words': 392}