<commit_before>#include "CrashHandler.h"
#include "DMJsonWriter.h"
#include "DMSrcSink.h"
#include "OverwriteLine.h"
#include "ProcStats.h"
#include "SkBBHFactory.h"
#include "SkChecksum.h"
#include "SkCommonFlags.h"
#include "SkForceLinking.h"
#include "SkGraphics.h"
#include "SkInstCnt.h"
#include "SkMD5.h"
#include "SkOSFile.h"
#include "SkTDynamicHash.h"
#include "SkTaskGroup.h"
#include "Test.h"
#include "Timer.h"

DEFINE_string(src, "tests gm skp image subset", "Source types to test.");
DEFINE_bool(nameByHash, false,
            "If true, write to FLAGS_writePath[0]/<hash>.png instead of "
            "to FLAGS_writePath[0]/<config>/<sourceType>/<name>.png");
DEFINE_bool2(pathOpsExtended, x, false, "Run extended pathOps tests.");
DEFINE_string(matrix, "1 0 0 1",
              "2x2 scale+skew matrix to apply or upright when using "
              "'matrix' or 'upright' in config.");
DEFINE_bool(gpu_threading, false, "Allow GPU work to run on multiple threads?");

DEFINE_string(blacklist, "",
        "Space-separated config/src/name triples to blacklist.  '_' matches anything.  E.g. \n"
        "'--blacklist gpu skp _' will blacklist all SKPs drawn into the gpu config.\n"
        "'--blacklist gpu skp _ 8888 gm aarects' will also blacklist the aarects GM on 8888.");

DEFINE_string2(readPath, r, "", "If set check for equality with golden results in this directory.");

__SK_FORCE_IMAGE_DECODER_LINKING;
using namespace DM;

/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

SK_DECLARE_STATIC_MUTEX(gFailuresMutex);
static SkTArray<SkString> gFailures;

static void fail(ImplicitString err) {
    SkAutoMutexAcquire lock(gFailuresMutex);
    SkDebugf("\n\nFAILURE: %s\n\n", err.c_str());
    gFailures.push_back(err);
}

static int32_t gPending = 0;  // Atomic.

static void done(double ms,
                 ImplicitString config, ImplicitString src, ImplicitString name,
                 ImplicitString log) {
    if (!log.isEmpty()) {
        log.prepend("\n");
    }
    auto pending = sk_atomic_dec(&gPending)-1;
    SkDebugf("%s(%4dMB %5d) %s\t%s %s %s%s", FLAGS_verbose ? "\n" : kSkOverwriteLine
                                           , sk_tools::getMaxResidentSetSizeMB()
                                           , pending
                                           , HumanizeMs(ms).c_str()
                                           , config.c_str()
                                           , src.c_str()
                                           , name.c_str()
                                           , log.c_str());
    // We write our dm.json file every once in a while in case we crash.
    // Notice this also handles the final dm.json when pending == 0.
    if (pending % 500 == 0) {
        JsonWriter::DumpJson();
    }
}

/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

struct Gold : public SkString {
    Gold(ImplicitString sink, ImplicitString src, ImplicitString name, ImplicitString md5)
        : SkString("") {
        this->append(sink);
        this->append(src);
        this->append(name);
        this->append(md5);
        while (this->size() % 4) {
            this->append("!");  // Pad out if needed so we can pass this to Murmur3.
        }
    }
    static const Gold& GetKey(const Gold& g) { return g; }
    static uint32_t Hash(const Gold& g) {
        return SkChecksum::Murmur3((const uint32_t*)g.c_str(), g.size());
    }
};
static SkTDynamicHash<Gold, Gold> gGold;

static void add_gold(JsonWriter::BitmapResult r) {
    gGold.add(new Gold(r.config, r.sourceType, r.name, r.md5));  // We'll let these leak. Lazybones.
}

static void gather_gold() {
    if (!FLAGS_readPath.isEmpty()) {
        SkString path(FLAGS_readPath[0]);
        path.append("/dm.json");
        if (!JsonWriter::ReadJson(path.c_str(), add_gold)) {
            fail(SkStringPrintf("Couldn't read %s for golden results.", path.c_str()));
        }
    }
}

/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

template <typename T>
struct Tagged : public SkAutoTDelete<T> { const char* tag; };

static const bool kMemcpyOK = true;

static SkTArray<Tagged<Src>,  kMemcpyOK>  gSrcs;
static SkTArray<Tagged<Sink>, kMemcpyOK> gSinks;

static void push_src(const char* tag, Src* s) {
    SkAutoTDelete<Src> src(s);
    if (FLAGS_src.contains(tag) &&
        !SkCommandLineFlags::ShouldSkip(FLAGS_match, src->name().c_str())) {
        Tagged<Src>& s = gSrcs.push_back();
        s.reset(src.detach());
        s.tag = tag;
    }
}

static void gather_srcs() {
    for (const skiagm::GMRegistry* r = skiagm::GMRegistry::Head(); r; r = r->next()) {
        push_src("gm", new GMSrc(r->factory()));
    }
    for (int i = 0; i < FLAGS_skps.count(); i++) {
        const char* path = FLAGS_skps[i];
        if (sk_isdir(path)) {
            SkOSFile::Iter it(path, "skp");
            for (SkString file; it.next(&file); ) {
                push_src("skp", new SKPSrc(SkOSPath::Join(path, file.c_str())));
            }
        } else {
            push_src("skp", new SKPSrc(path));
        }
    }
    static const char* const exts[] = {
        "bmp", "gif", "jpg", "jpeg", "png", "webp", "ktx", "astc", "wbmp", "ico",
        "BMP", "GIF", "JPG", "JPEG", "PNG", "WEBP", "KTX", "ASTC", "WBMP", "ICO",
    };
    for (int i = 0; i < FLAGS_images.count(); i++) {
        const char* flag = FLAGS_images[i];
        if (sk_isdir(flag)) {
            for (size_t j = 0; j < SK_ARRAY_COUNT(exts); j++) {
                SkOSFile::Iter it(flag, exts[j]);
                for (SkString file; it.next(&file); ) {
                    SkString path = SkOSPath::Join(flag, file.c_str());
                    push_src("image", new ImageSrc(path));     // Decode entire image.
                    push_src("subset", new ImageSrc(path, 2)); // Decode into 2 x 2 subsets
                }
            }
        } else if (sk_exists(flag)) {
            // assume that FLAGS_images[i] is a valid image if it is a file.
            push_src("image", new ImageSrc(flag));     // Decode entire image.
            push_src("subset", new ImageSrc(flag, 2)); // Decode into 2 x 2 subsets
        }
    }
}

static GrGLStandard get_gpu_api() {
    if (FLAGS_gpuAPI.contains("gl"))   { return kGL_GrGLStandard; }
    if (FLAGS_gpuAPI.contains("gles")) { return kGLES_GrGLStandard; }
    return kNone_GrGLStandard;
}

static void push_sink(const char* tag, Sink* s) {
    SkAutoTDelete<Sink> sink(s);
    if (!FLAGS_config.contains(tag)) {
        return;
    }
    // Try a noop Src as a canary.  If it fails, skip this sink.
    struct : public Src {
        Error draw(SkCanvas*) const SK_OVERRIDE { return ""; }
        SkISize size() const SK_OVERRIDE { return SkISize::Make(16, 16); }
        Name name() const SK_OVERRIDE { return "noop"; }
    } noop;

    SkBitmap bitmap;
    SkDynamicMemoryWStream stream;
    SkString log;
    Error err = sink->draw(noop, &bitmap, &stream, &log);
    if (!err.isEmpty()) {
        SkDebugf("Skipping %s: %s\n", tag, err.c_str());
        return;
    }

    Tagged<Sink>& ts = gSinks.push_back();
    ts.reset(sink.detach());
    ts.tag = tag;
}

static bool gpu_supported() {
#if SK_SUPPORT_GPU
    return FLAGS_gpu;
#else
    return false;
#endif
}

static Sink* create_sink(const char* tag) {
#define SINK(t, sink, ...) if (0 == strcmp(t, tag)) { return new sink(__VA_ARGS__); }
    if (gpu_supported()) {
        typedef GrContextFactory Gr;
        const GrGLStandard api = get_gpu_api();
        SINK("gpunull",    GPUSink, Gr::kNull_GLContextType,   api,  0, false, FLAGS_gpu_threading);
        SINK("gpudebug",   GPUSink, Gr::kDebug_GLContextType,  api,  0, false, FLAGS_gpu_threading);
        SINK("gpu",        GPUSink, Gr::kNative_GLContextType, api,  0, false, FLAGS_gpu_threading);
        SINK("gpudft",     GPUSink, Gr::kNative_GLContextType, api,  0,  true, FLAGS_gpu_threading);
        SINK("msaa4",      GPUSink, Gr::kNative_GLContextType, api,  4, false, FLAGS_gpu_threading);
        SINK("msaa16",     GPUSink, Gr::kNative_GLContextType, api, 16, false, FLAGS_gpu_threading);
        SINK("nvprmsaa4",  GPUSink, Gr::kNVPR_GLContextType,   api,  4, false, FLAGS_gpu_threading);
        SINK("nvprmsaa16", GPUSink, Gr::kNVPR_GLContextType,   api, 16, false, FLAGS_gpu_threading);
    #if SK_ANGLE
        SINK("angle",      GPUSink, Gr::kANGLE_GLContextType,  api,  0, false, FLAGS_gpu_threading);
    #endif
    #if SK_MESA
        SINK("mesa",       GPUSink, Gr::kMESA_GLContextType,   api,  0, false, FLAGS_gpu_threading);
    #endif
    }

    if (FLAGS_cpu) {
        SINK("565",  RasterSink, kRGB_565_SkColorType);
        SINK("8888", RasterSink, kN32_SkColorType);
        SINK("pdf",  PDFSink);
        SINK("skp",  SKPSink);
        SINK("svg",  SVGSink);
        SINK("null", NullSink);
    }
#undef SINK
    return NULL;
}

static Sink* create_via(const char* tag, Sink* wrapped) {
#define VIA(t, via, ...) if (0 == strcmp(t, tag)) { return new via(__VA_ARGS__); }
    VIA("pipe",      ViaPipe,          wrapped);
    VIA("serialize", ViaSerialization, wrapped);
    VIA("tiles",     ViaTiles, 256, 256,               NULL, wrapped);
    VIA("tiles_rt",  ViaTiles, 256, 256, new SkRTreeFactory, wrapped);

    if (FLAGS_matrix.count() == 4) {
        SkMatrix m;
        m.reset();
        m.setScaleX((SkScalar)atof(FLAGS_matrix[0]));
        m.setSkewX ((SkScalar)atof(FLAGS_matrix[1]));
        m.setSkewY ((SkScalar)atof(FLAGS_matrix[2]));
        m.setScaleY((SkScalar)atof(FLAGS_matrix[3]));
        VIA("matrix",  ViaMatrix,  m, wrapped);
        VIA("upright", ViaUpright, m, wrapped);
    }
#undef VIA
    return NULL;
}

static void gather_sinks() {
    for (int i = 0; i < FLAGS_config.count(); i++) {
        const char* config = FLAGS_config[i];
        SkTArray<SkString> parts;
        SkStrSplit(config, "-", &parts);

        Sink* sink = NULL;
        for (int i = parts.count(); i-- > 0;) {
            const char* part = parts[i].c_str();
            Sink* next = (sink == NULL) ? create_sink(part) : create_via(part, sink);
            if (next == NULL) {
                SkDebugf("Skipping %s: Don't understand '%s'.\n", config, part);
                delete sink;
                sink = NULL;
                break;
            }
            sink = next;
        }
        if (sink) {
            push_sink(config, sink);
        }
    }
}

static bool match(const char* needle, const char* haystack) {
    return 0 == strcmp("_", needle) || NULL != strstr(haystack, needle);
}

static ImplicitString is_blacklisted(const char* sink, const char* src, const char* name) {
    for (int i = 0; i < FLAGS_blacklist.count() - 2; i += 3) {
        if (match(FLAGS_blacklist[i+0], sink) &&
            match(FLAGS_blacklist[i+1],  src) &&
            match(FLAGS_blacklist[i+2], name)) {
            return SkStringPrintf("%s %s %s",
                                  FLAGS_blacklist[i+0], FLAGS_blacklist[i+1], FLAGS_blacklist[i+2]);
        }
    }
    return "";
}

// The finest-grained unit of work we can run: draw a single Src into a single Sink,
// report any errors, and perhaps write out the output: a .png of the bitmap, or a raw stream.
struct Task {
    Task(const Tagged<Src>& src, const Tagged<Sink>& sink) : src(src), sink(sink) {}
    const Tagged<Src>&  src;
    const Tagged<Sink>& sink;

    static void Run(Task* task) {
        SkString name = task->src->name();
        SkString whyBlacklisted = is_blacklisted(task->sink.tag, task->src.tag, name.c_str());
        SkString log;
        WallTimer timer;
        timer.start();
        if (!FLAGS_dryRun && whyBlacklisted.isEmpty()) {
            SkBitmap bitmap;
            SkDynamicMemoryWStream stream;
            Error err = task->sink->draw(*task->src, &bitmap, &stream, &log);
            if (!err.isEmpty()) {
                fail(SkStringPrintf("%s %s %s: %s",
                                    task->sink.tag,
                                    task->src.tag,
                                    name.c_str(),
                                    err.c_str()));
            }
            SkAutoTDelete<SkStreamAsset> data(stream.detachAsStream());

            SkString md5;
            if (!FLAGS_writePath.isEmpty() || !FLAGS_readPath.isEmpty()) {
                SkMD5 hash;
                if (data->getLength()) {
                    hash.writeStream(data, data->getLength());
                    data->rewind();
                } else {
                    hash.write(bitmap.getPixels(), bitmap.getSize());
                }
                SkMD5::Digest digest;
                hash.finish(digest);
                for (int i = 0; i < 16; i++) {
                    md5.appendf("%02x", digest.data[i]);
                }
            }

            if (!FLAGS_readPath.isEmpty() &&
                !gGold.find(Gold(task->sink.tag, task->src.tag, name, md5))) {<commit_msg>Use simpler SkTHashSet in DM to store reference gold<commit_after>#include "CrashHandler.h"
#include "DMJsonWriter.h"
#include "DMSrcSink.h"
#include "OverwriteLine.h"
#include "ProcStats.h"
#include "SkBBHFactory.h"
#include "SkChecksum.h"
#include "SkCommonFlags.h"
#include "SkForceLinking.h"
#include "SkGraphics.h"
#include "SkInstCnt.h"
#include "SkMD5.h"
#include "SkOSFile.h"
#include "SkTHash.h"
#include "SkTaskGroup.h"
#include "Test.h"
#include "Timer.h"

DEFINE_string(src, "tests gm skp image subset", "Source types to test.");
DEFINE_bool(nameByHash, false,
            "If true, write to FLAGS_writePath[0]/<hash>.png instead of "
            "to FLAGS_writePath[0]/<config>/<sourceType>/<name>.png");
DEFINE_bool2(pathOpsExtended, x, false, "Run extended pathOps tests.");
DEFINE_string(matrix, "1 0 0 1",
              "2x2 scale+skew matrix to apply or upright when using "
              "'matrix' or 'upright' in config.");
DEFINE_bool(gpu_threading, false, "Allow GPU work to run on multiple threads?");

DEFINE_string(blacklist, "",
        "Space-separated config/src/name triples to blacklist.  '_' matches anything.  E.g. \n"
        "'--blacklist gpu skp _' will blacklist all SKPs drawn into the gpu config.\n"
        "'--blacklist gpu skp _ 8888 gm aarects' will also blacklist the aarects GM on 8888.");

DEFINE_string2(readPath, r, "", "If set check for equality with golden results in this directory.");

__SK_FORCE_IMAGE_DECODER_LINKING;
using namespace DM;

/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

SK_DECLARE_STATIC_MUTEX(gFailuresMutex);
static SkTArray<SkString> gFailures;

static void fail(ImplicitString err) {
    SkAutoMutexAcquire lock(gFailuresMutex);
    SkDebugf("\n\nFAILURE: %s\n\n", err.c_str());
    gFailures.push_back(err);
}

static int32_t gPending = 0;  // Atomic.

static void done(double ms,
                 ImplicitString config, ImplicitString src, ImplicitString name,
                 ImplicitString log) {
    if (!log.isEmpty()) {
        log.prepend("\n");
    }
    auto pending = sk_atomic_dec(&gPending)-1;
    SkDebugf("%s(%4dMB %5d) %s\t%s %s %s%s", FLAGS_verbose ? "\n" : kSkOverwriteLine
                                           , sk_tools::getMaxResidentSetSizeMB()
                                           , pending
                                           , HumanizeMs(ms).c_str()
                                           , config.c_str()
                                           , src.c_str()
                                           , name.c_str()
                                           , log.c_str());
    // We write our dm.json file every once in a while in case we crash.
    // Notice this also handles the final dm.json when pending == 0.
    if (pending % 500 == 0) {
        JsonWriter::DumpJson();
    }
}

/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

struct Gold : public SkString {
    Gold() : SkString("") {}
    Gold(ImplicitString sink, ImplicitString src, ImplicitString name, ImplicitString md5)
        : SkString("") {
        this->append(sink);
        this->append(src);
        this->append(name);
        this->append(md5);
        while (this->size() % 4) {
            this->append("!");  // Pad out if needed so we can pass this to Murmur3.
        }
    }
    static uint32_t Hash(const Gold& g) {
        return SkChecksum::Murmur3((const uint32_t*)g.c_str(), g.size());
    }
};
static SkTHashSet<Gold, Gold::Hash> gGold;

static void add_gold(JsonWriter::BitmapResult r) {
    gGold.add(Gold(r.config, r.sourceType, r.name, r.md5));
}

static void gather_gold() {
    if (!FLAGS_readPath.isEmpty()) {
        SkString path(FLAGS_readPath[0]);
        path.append("/dm.json");
        if (!JsonWriter::ReadJson(path.c_str(), add_gold)) {
            fail(SkStringPrintf("Couldn't read %s for golden results.", path.c_str()));
        }
    }
}

/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

template <typename T>
struct Tagged : public SkAutoTDelete<T> { const char* tag; };

static const bool kMemcpyOK = true;

static SkTArray<Tagged<Src>,  kMemcpyOK>  gSrcs;
static SkTArray<Tagged<Sink>, kMemcpyOK> gSinks;

static void push_src(const char* tag, Src* s) {
    SkAutoTDelete<Src> src(s);
    if (FLAGS_src.contains(tag) &&
        !SkCommandLineFlags::ShouldSkip(FLAGS_match, src->name().c_str())) {
        Tagged<Src>& s = gSrcs.push_back();
        s.reset(src.detach());
        s.tag = tag;
    }
}

static void gather_srcs() {
    for (const skiagm::GMRegistry* r = skiagm::GMRegistry::Head(); r; r = r->next()) {
        push_src("gm", new GMSrc(r->factory()));
    }
    for (int i = 0; i < FLAGS_skps.count(); i++) {
        const char* path = FLAGS_skps[i];
        if (sk_isdir(path)) {
            SkOSFile::Iter it(path, "skp");
            for (SkString file; it.next(&file); ) {
                push_src("skp", new SKPSrc(SkOSPath::Join(path, file.c_str())));
            }
        } else {
            push_src("skp", new SKPSrc(path));
        }
    }
    static const char* const exts[] = {
        "bmp", "gif", "jpg", "jpeg", "png", "webp", "ktx", "astc", "wbmp", "ico",
        "BMP", "GIF", "JPG", "JPEG", "PNG", "WEBP", "KTX", "ASTC", "WBMP", "ICO",
    };
    for (int i = 0; i < FLAGS_images.count(); i++) {
        const char* flag = FLAGS_images[i];
        if (sk_isdir(flag)) {
            for (size_t j = 0; j < SK_ARRAY_COUNT(exts); j++) {
                SkOSFile::Iter it(flag, exts[j]);
                for (SkString file; it.next(&file); ) {
                    SkString path = SkOSPath::Join(flag, file.c_str());
                    push_src("image", new ImageSrc(path));     // Decode entire image.
                    push_src("subset", new ImageSrc(path, 2)); // Decode into 2 x 2 subsets
                }
            }
        } else if (sk_exists(flag)) {
            // assume that FLAGS_images[i] is a valid image if it is a file.
            push_src("image", new ImageSrc(flag));     // Decode entire image.
            push_src("subset", new ImageSrc(flag, 2)); // Decode into 2 x 2 subsets
        }
    }
}

static GrGLStandard get_gpu_api() {
    if (FLAGS_gpuAPI.contains("gl"))   { return kGL_GrGLStandard; }
    if (FLAGS_gpuAPI.contains("gles")) { return kGLES_GrGLStandard; }
    return kNone_GrGLStandard;
}

static void push_sink(const char* tag, Sink* s) {
    SkAutoTDelete<Sink> sink(s);
    if (!FLAGS_config.contains(tag)) {
        return;
    }
    // Try a noop Src as a canary.  If it fails, skip this sink.
    struct : public Src {
        Error draw(SkCanvas*) const SK_OVERRIDE { return ""; }
        SkISize size() const SK_OVERRIDE { return SkISize::Make(16, 16); }
        Name name() const SK_OVERRIDE { return "noop"; }
    } noop;

    SkBitmap bitmap;
    SkDynamicMemoryWStream stream;
    SkString log;
    Error err = sink->draw(noop, &bitmap, &stream, &log);
    if (!err.isEmpty()) {
        SkDebugf("Skipping %s: %s\n", tag, err.c_str());
        return;
    }

    Tagged<Sink>& ts = gSinks.push_back();
    ts.reset(sink.detach());
    ts.tag = tag;
}

static bool gpu_supported() {
#if SK_SUPPORT_GPU
    return FLAGS_gpu;
#else
    return false;
#endif
}

static Sink* create_sink(const char* tag) {
#define SINK(t, sink, ...) if (0 == strcmp(t, tag)) { return new sink(__VA_ARGS__); }
    if (gpu_supported()) {
        typedef GrContextFactory Gr;
        const GrGLStandard api = get_gpu_api();
        SINK("gpunull",    GPUSink, Gr::kNull_GLContextType,   api,  0, false, FLAGS_gpu_threading);
        SINK("gpudebug",   GPUSink, Gr::kDebug_GLContextType,  api,  0, false, FLAGS_gpu_threading);
        SINK("gpu",        GPUSink, Gr::kNative_GLContextType, api,  0, false, FLAGS_gpu_threading);
        SINK("gpudft",     GPUSink, Gr::kNative_GLContextType, api,  0,  true, FLAGS_gpu_threading);
        SINK("msaa4",      GPUSink, Gr::kNative_GLContextType, api,  4, false, FLAGS_gpu_threading);
        SINK("msaa16",     GPUSink, Gr::kNative_GLContextType, api, 16, false, FLAGS_gpu_threading);
        SINK("nvprmsaa4",  GPUSink, Gr::kNVPR_GLContextType,   api,  4, false, FLAGS_gpu_threading);
        SINK("nvprmsaa16", GPUSink, Gr::kNVPR_GLContextType,   api, 16, false, FLAGS_gpu_threading);
    #if SK_ANGLE
        SINK("angle",      GPUSink, Gr::kANGLE_GLContextType,  api,  0, false, FLAGS_gpu_threading);
    #endif
    #if SK_MESA
        SINK("mesa",       GPUSink, Gr::kMESA_GLContextType,   api,  0, false, FLAGS_gpu_threading);
    #endif
    }

    if (FLAGS_cpu) {
        SINK("565",  RasterSink, kRGB_565_SkColorType);
        SINK("8888", RasterSink, kN32_SkColorType);
        SINK("pdf",  PDFSink);
        SINK("skp",  SKPSink);
        SINK("svg",  SVGSink);
        SINK("null", NullSink);
    }
#undef SINK
    return NULL;
}

static Sink* create_via(const char* tag, Sink* wrapped) {
#define VIA(t, via, ...) if (0 == strcmp(t, tag)) { return new via(__VA_ARGS__); }
    VIA("pipe",      ViaPipe,          wrapped);
    VIA("serialize", ViaSerialization, wrapped);
    VIA("tiles",     ViaTiles, 256, 256,               NULL, wrapped);
    VIA("tiles_rt",  ViaTiles, 256, 256, new SkRTreeFactory, wrapped);

    if (FLAGS_matrix.count() == 4) {
        SkMatrix m;
        m.reset();
        m.setScaleX((SkScalar)atof(FLAGS_matrix[0]));
        m.setSkewX ((SkScalar)atof(FLAGS_matrix[1]));
        m.setSkewY ((SkScalar)atof(FLAGS_matrix[2]));
        m.setScaleY((SkScalar)atof(FLAGS_matrix[3]));
        VIA("matrix",  ViaMatrix,  m, wrapped);
        VIA("upright", ViaUpright, m, wrapped);
    }
#undef VIA
    return NULL;
}

static void gather_sinks() {
    for (int i = 0; i < FLAGS_config.count(); i++) {
        const char* config = FLAGS_config[i];
        SkTArray<SkString> parts;
        SkStrSplit(config, "-", &parts);

        Sink* sink = NULL;
        for (int i = parts.count(); i-- > 0;) {
            const char* part = parts[i].c_str();
            Sink* next = (sink == NULL) ? create_sink(part) : create_via(part, sink);
            if (next == NULL) {
                SkDebugf("Skipping %s: Don't understand '%s'.\n", config, part);
                delete sink;
                sink = NULL;
                break;
            }
            sink = next;
        }
        if (sink) {
            push_sink(config, sink);
        }
    }
}

static bool match(const char* needle, const char* haystack) {
    return 0 == strcmp("_", needle) || NULL != strstr(haystack, needle);
}

static ImplicitString is_blacklisted(const char* sink, const char* src, const char* name) {
    for (int i = 0; i < FLAGS_blacklist.count() - 2; i += 3) {
        if (match(FLAGS_blacklist[i+0], sink) &&
            match(FLAGS_blacklist[i+1],  src) &&
            match(FLAGS_blacklist[i+2], name)) {
            return SkStringPrintf("%s %s %s",
                                  FLAGS_blacklist[i+0], FLAGS_blacklist[i+1], FLAGS_blacklist[i+2]);
        }
    }
    return "";
}

// The finest-grained unit of work we can run: draw a single Src into a single Sink,
// report any errors, and perhaps write out the output: a .png of the bitmap, or a raw stream.
struct Task {
    Task(const Tagged<Src>& src, const Tagged<Sink>& sink) : src(src), sink(sink) {}
    const Tagged<Src>&  src;
    const Tagged<Sink>& sink;

    static void Run(Task* task) {
        SkString name = task->src->name();
        SkString whyBlacklisted = is_blacklisted(task->sink.tag, task->src.tag, name.c_str());
        SkString log;
        WallTimer timer;
        timer.start();
        if (!FLAGS_dryRun && whyBlacklisted.isEmpty()) {
            SkBitmap bitmap;
            SkDynamicMemoryWStream stream;
            Error err = task->sink->draw(*task->src, &bitmap, &stream, &log);
            if (!err.isEmpty()) {
                fail(SkStringPrintf("%s %s %s: %s",
                                    task->sink.tag,
                                    task->src.tag,
                                    name.c_str(),
                                    err.c_str()));
            }
            SkAutoTDelete<SkStreamAsset> data(stream.detachAsStream());

            SkString md5;
            if (!FLAGS_writePath.isEmpty() || !FLAGS_readPath.isEmpty()) {
                SkMD5 hash;
                if (data->getLength()) {
                    hash.writeStream(data, data->getLength());
                    data->rewind();
                } else {
                    hash.write(bitmap.getPixels(), bitmap.getSize());
                }
                SkMD5::Digest digest;
                hash.finish(digest);
                for (int i = 0; i < 16; i++) {
                    md5.appendf("%02x", digest.data[i]);
                }
            }

            if (!FLAGS_readPath.isEmpty() &&
                !gGold.contains(Gold(task->sink.tag, task->src.tag, name, md5))) {
{'fraction_non_alphanumeric': 0.160868729579089, 'fraction_numerical': 0.009609840476648087, 'mean_word_length': 2.341811175337187, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6755750', 'n_tokens_mistral': 8759, 'n_tokens_neox': 8458, 'n_words': 2012}