<commit_before>/*
 * Copyright 2013 Google Inc.
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

#include "CrashHandler.h"
#include "DMJsonWriter.h"
#include "DMSrcSink.h"
#include "DMSrcSinkAndroid.h"
#include "OverwriteLine.h"
#include "ProcStats.h"
#include "SkBBHFactory.h"
#include "SkChecksum.h"
#include "SkCodec.h"
#include "SkCommonFlags.h"
#include "SkCommonFlagsConfig.h"
#include "SkFontMgr.h"
#include "SkForceLinking.h"
#include "SkGraphics.h"
#include "SkMD5.h"
#include "SkMutex.h"
#include "SkOSFile.h"
#include "SkTHash.h"
#include "SkTaskGroup.h"
#include "SkThreadUtils.h"
#include "Test.h"
#include "Timer.h"
#include "sk_tool_utils.h"

#ifdef SK_PDF_IMAGE_STATS
extern void SkPDFImageDumpStats();
#endif

#include "png.h"

#include <stdlib.h>

#ifndef SK_BUILD_FOR_WIN32
    #include <unistd.h>
#endif

DEFINE_string(src, "tests gm skp image", "Source types to test.");
DEFINE_bool(nameByHash, false,
            "If true, write to FLAGS_writePath[0]/<hash>.png instead of "
            "to FLAGS_writePath[0]/<config>/<sourceType>/<sourceOptions>/<name>.png");
DEFINE_bool2(pathOpsExtended, x, false, "Run extended pathOps tests.");
DEFINE_string(matrix, "1 0 0 1",
              "2x2 scale+skew matrix to apply or upright when using "
              "'matrix' or 'upright' in config.");
DEFINE_bool(gpu_threading, false, "Allow GPU work to run on multiple threads?");

DEFINE_string(blacklist, "",
        "Space-separated config/src/srcOptions/name quadruples to blacklist.  '_' matches anything.  E.g. \n"
        "'--blacklist gpu skp _ _' will blacklist all SKPs drawn into the gpu config.\n"
        "'--blacklist gpu skp _ _ 8888 gm _ aarects' will also blacklist the aarects GM on 8888.");

DEFINE_string2(readPath, r, "", "If set check for equality with golden results in this directory.");

DEFINE_string(uninterestingHashesFile, "",
        "File containing a list of uninteresting hashes. If a result hashes to something in "
        "this list, no image is written for that result.");

DEFINE_int32(shards, 1, "We're splitting source data into this many shards.");
DEFINE_int32(shard,  0, "Which shard do I run?");

__SK_FORCE_IMAGE_DECODER_LINKING;
using namespace DM;

/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

static double now_ms() { return SkTime::GetNSecs() * 1e-6; }

SK_DECLARE_STATIC_MUTEX(gFailuresMutex);
static SkTArray<SkString> gFailures;

static void fail(ImplicitString err) {
    SkAutoMutexAcquire lock(gFailuresMutex);
    SkDebugf("\n\nFAILURE: %s\n\n", err.c_str());
    gFailures.push_back(err);
}

static int32_t gPending = 0;  // Atomic.  Total number of running and queued tasks.

SK_DECLARE_STATIC_MUTEX(gRunningAndTallyMutex);
static SkTArray<SkString>        gRunning;
static SkTHashMap<SkString, int> gNoteTally;

static void done(double ms,
                 ImplicitString config, ImplicitString src, ImplicitString srcOptions,
                 ImplicitString name, ImplicitString note, ImplicitString log) {
    SkString id = SkStringPrintf("%s %s %s %s", config.c_str(), src.c_str(),
                                                srcOptions.c_str(), name.c_str());
    {
        SkAutoMutexAcquire lock(gRunningAndTallyMutex);
        for (int i = 0; i < gRunning.count(); i++) {
            if (gRunning[i] == id) {
                gRunning.removeShuffle(i);
                break;
            }
        }
        if (!note.isEmpty()) {
            if (int* tally = gNoteTally.find(note)) {
                *tally += 1;
            } else {
                gNoteTally.set(note, 1);
            }
        }
    }
    if (!log.isEmpty()) {
        log.prepend("\n");
    }
    auto pending = sk_atomic_dec(&gPending)-1;
    if (!FLAGS_quiet && note.isEmpty()) {
        SkDebugf("%s(%4d/%-4dMB %6d) %s\t%s%s", FLAGS_verbose ? "\n" : kSkOverwriteLine
                                           , sk_tools::getCurrResidentSetSizeMB()
                                           , sk_tools::getMaxResidentSetSizeMB()
                                           , pending
                                           , HumanizeMs(ms).c_str()
                                           , id.c_str()
                                           , log.c_str());
    }
    // We write our dm.json file every once in a while in case we crash.
    // Notice this also handles the final dm.json when pending == 0.
    if (pending % 500 == 0) {
        JsonWriter::DumpJson();
    }
}

static void start(ImplicitString config, ImplicitString src,
                  ImplicitString srcOptions, ImplicitString name) {
    SkString id = SkStringPrintf("%s %s %s %s", config.c_str(), src.c_str(),
                                                srcOptions.c_str(), name.c_str());
    SkAutoMutexAcquire lock(gRunningAndTallyMutex);
    gRunning.push_back(id);
}

/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

struct Gold : public SkString {
    Gold() : SkString("") {}
    Gold(ImplicitString sink, ImplicitString src, ImplicitString srcOptions,
         ImplicitString name, ImplicitString md5)
        : SkString("") {
        this->append(sink);
        this->append(src);
        this->append(srcOptions);
        this->append(name);
        this->append(md5);
    }
    struct Hash {
        uint32_t operator()(const Gold& g) const {
            return SkGoodHash()((const SkString&)g);
        }
    };
};
static SkTHashSet<Gold, Gold::Hash> gGold;

static void add_gold(JsonWriter::BitmapResult r) {
    gGold.add(Gold(r.config, r.sourceType, r.sourceOptions, r.name, r.md5));
}

static void gather_gold() {
    if (!FLAGS_readPath.isEmpty()) {
        SkString path(FLAGS_readPath[0]);
        path.append("/dm.json");
        if (!JsonWriter::ReadJson(path.c_str(), add_gold)) {
            fail(SkStringPrintf("Couldn't read %s for golden results.", path.c_str()));
        }
    }
}

/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

static SkTHashSet<SkString> gUninterestingHashes;

static void gather_uninteresting_hashes() {
    if (!FLAGS_uninterestingHashesFile.isEmpty()) {
        SkAutoTUnref<SkData> data(SkData::NewFromFileName(FLAGS_uninterestingHashesFile[0]));
        if (!data) {
            SkDebugf("WARNING: unable to read uninteresting hashes from %s\n",
                     FLAGS_uninterestingHashesFile[0]);
            return;
        }
        SkTArray<SkString> hashes;
        SkStrSplit((const char*)data->data(), "\n", &hashes);
        for (const SkString& hash : hashes) {
            gUninterestingHashes.add(hash);
        }
    }
}

/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

struct TaggedSrc : public SkAutoTDelete<Src> {
    ImplicitString tag;
    ImplicitString options;
};

struct TaggedSink : public SkAutoTDelete<Sink> {
    SkString tag;
};

static const bool kMemcpyOK = true;

static SkTArray<TaggedSrc,  kMemcpyOK>  gSrcs;
static SkTArray<TaggedSink, kMemcpyOK> gSinks;

static bool in_shard() {
    static int N = 0;
    return N++ % FLAGS_shards == FLAGS_shard;
}

static void push_src(ImplicitString tag, ImplicitString options, Src* s) {
    SkAutoTDelete<Src> src(s);
    if (in_shard() &&
        FLAGS_src.contains(tag.c_str()) &&
        !SkCommandLineFlags::ShouldSkip(FLAGS_match, src->name().c_str())) {<commit_msg>DM: add uninteresting hashes FYI<commit_after>/*
 * Copyright 2013 Google Inc.
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

#include "CrashHandler.h"
#include "DMJsonWriter.h"
#include "DMSrcSink.h"
#include "DMSrcSinkAndroid.h"
#include "OverwriteLine.h"
#include "ProcStats.h"
#include "SkBBHFactory.h"
#include "SkChecksum.h"
#include "SkCodec.h"
#include "SkCommonFlags.h"
#include "SkCommonFlagsConfig.h"
#include "SkFontMgr.h"
#include "SkForceLinking.h"
#include "SkGraphics.h"
#include "SkMD5.h"
#include "SkMutex.h"
#include "SkOSFile.h"
#include "SkTHash.h"
#include "SkTaskGroup.h"
#include "SkThreadUtils.h"
#include "Test.h"
#include "Timer.h"
#include "sk_tool_utils.h"

#ifdef SK_PDF_IMAGE_STATS
extern void SkPDFImageDumpStats();
#endif

#include "png.h"

#include <stdlib.h>

#ifndef SK_BUILD_FOR_WIN32
    #include <unistd.h>
#endif

DEFINE_string(src, "tests gm skp image", "Source types to test.");
DEFINE_bool(nameByHash, false,
            "If true, write to FLAGS_writePath[0]/<hash>.png instead of "
            "to FLAGS_writePath[0]/<config>/<sourceType>/<sourceOptions>/<name>.png");
DEFINE_bool2(pathOpsExtended, x, false, "Run extended pathOps tests.");
DEFINE_string(matrix, "1 0 0 1",
              "2x2 scale+skew matrix to apply or upright when using "
              "'matrix' or 'upright' in config.");
DEFINE_bool(gpu_threading, false, "Allow GPU work to run on multiple threads?");

DEFINE_string(blacklist, "",
        "Space-separated config/src/srcOptions/name quadruples to blacklist.  '_' matches anything.  E.g. \n"
        "'--blacklist gpu skp _ _' will blacklist all SKPs drawn into the gpu config.\n"
        "'--blacklist gpu skp _ _ 8888 gm _ aarects' will also blacklist the aarects GM on 8888.");

DEFINE_string2(readPath, r, "", "If set check for equality with golden results in this directory.");

DEFINE_string(uninterestingHashesFile, "",
        "File containing a list of uninteresting hashes. If a result hashes to something in "
        "this list, no image is written for that result.");

DEFINE_int32(shards, 1, "We're splitting source data into this many shards.");
DEFINE_int32(shard,  0, "Which shard do I run?");

__SK_FORCE_IMAGE_DECODER_LINKING;
using namespace DM;

/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

static double now_ms() { return SkTime::GetNSecs() * 1e-6; }

SK_DECLARE_STATIC_MUTEX(gFailuresMutex);
static SkTArray<SkString> gFailures;

static void fail(ImplicitString err) {
    SkAutoMutexAcquire lock(gFailuresMutex);
    SkDebugf("\n\nFAILURE: %s\n\n", err.c_str());
    gFailures.push_back(err);
}

static int32_t gPending = 0;  // Atomic.  Total number of running and queued tasks.

SK_DECLARE_STATIC_MUTEX(gRunningAndTallyMutex);
static SkTArray<SkString>        gRunning;
static SkTHashMap<SkString, int> gNoteTally;

static void done(double ms,
                 ImplicitString config, ImplicitString src, ImplicitString srcOptions,
                 ImplicitString name, ImplicitString note, ImplicitString log) {
    SkString id = SkStringPrintf("%s %s %s %s", config.c_str(), src.c_str(),
                                                srcOptions.c_str(), name.c_str());
    {
        SkAutoMutexAcquire lock(gRunningAndTallyMutex);
        for (int i = 0; i < gRunning.count(); i++) {
            if (gRunning[i] == id) {
                gRunning.removeShuffle(i);
                break;
            }
        }
        if (!note.isEmpty()) {
            if (int* tally = gNoteTally.find(note)) {
                *tally += 1;
            } else {
                gNoteTally.set(note, 1);
            }
        }
    }
    if (!log.isEmpty()) {
        log.prepend("\n");
    }
    auto pending = sk_atomic_dec(&gPending)-1;
    if (!FLAGS_quiet && note.isEmpty()) {
        SkDebugf("%s(%4d/%-4dMB %6d) %s\t%s%s", FLAGS_verbose ? "\n" : kSkOverwriteLine
                                           , sk_tools::getCurrResidentSetSizeMB()
                                           , sk_tools::getMaxResidentSetSizeMB()
                                           , pending
                                           , HumanizeMs(ms).c_str()
                                           , id.c_str()
                                           , log.c_str());
    }
    // We write our dm.json file every once in a while in case we crash.
    // Notice this also handles the final dm.json when pending == 0.
    if (pending % 500 == 0) {
        JsonWriter::DumpJson();
    }
}

static void start(ImplicitString config, ImplicitString src,
                  ImplicitString srcOptions, ImplicitString name) {
    SkString id = SkStringPrintf("%s %s %s %s", config.c_str(), src.c_str(),
                                                srcOptions.c_str(), name.c_str());
    SkAutoMutexAcquire lock(gRunningAndTallyMutex);
    gRunning.push_back(id);
}

/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

struct Gold : public SkString {
    Gold() : SkString("") {}
    Gold(ImplicitString sink, ImplicitString src, ImplicitString srcOptions,
         ImplicitString name, ImplicitString md5)
        : SkString("") {
        this->append(sink);
        this->append(src);
        this->append(srcOptions);
        this->append(name);
        this->append(md5);
    }
    struct Hash {
        uint32_t operator()(const Gold& g) const {
            return SkGoodHash()((const SkString&)g);
        }
    };
};
static SkTHashSet<Gold, Gold::Hash> gGold;

static void add_gold(JsonWriter::BitmapResult r) {
    gGold.add(Gold(r.config, r.sourceType, r.sourceOptions, r.name, r.md5));
}

static void gather_gold() {
    if (!FLAGS_readPath.isEmpty()) {
        SkString path(FLAGS_readPath[0]);
        path.append("/dm.json");
        if (!JsonWriter::ReadJson(path.c_str(), add_gold)) {
            fail(SkStringPrintf("Couldn't read %s for golden results.", path.c_str()));
        }
    }
}

/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

static SkTHashSet<SkString> gUninterestingHashes;

static void gather_uninteresting_hashes() {
    if (!FLAGS_uninterestingHashesFile.isEmpty()) {
        SkAutoTUnref<SkData> data(SkData::NewFromFileName(FLAGS_uninterestingHashesFile[0]));
        if (!data) {
            SkDebugf("WARNING: unable to read uninteresting hashes from %s\n",
                     FLAGS_uninterestingHashesFile[0]);
            return;
        }
        SkTArray<SkString> hashes;
        SkStrSplit((const char*)data->data(), "\n", &hashes);
        for (const SkString& hash : hashes) {
            gUninterestingHashes.add(hash);
        }
        SkDebugf("FYI: loaded %d distinct uninteresting hashes from %d lines\n",
                 gUninterestingHashes.count(), hashes.count());
    }
}

/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

struct TaggedSrc : public SkAutoTDelete<Src> {
    ImplicitString tag;
    ImplicitString options;
};

struct TaggedSink : public SkAutoTDelete<Sink> {
    SkString tag;
};

static const bool kMemcpyOK = true;

static SkTArray<TaggedSrc,  kMemcpyOK>  gSrcs;
static SkTArray<TaggedSink, kMemcpyOK> gSinks;

static bool in_shard() {
    static int N = 0;
    return N++ % FLAGS_shards == FLAGS_shard;
}

static void push_src(ImplicitString tag, ImplicitString options, Src* s) {
    SkAutoTDelete<Src> src(s);
    if (in_shard() &&
        FLAGS_src.contains(tag.c_str()) &&
        !SkCommandLineFlags::ShouldSkip(FLAGS_match, src->name().c_str())) {
{'fraction_non_alphanumeric': 0.16739101804293427, 'fraction_numerical': 0.0075069142631371, 'mean_word_length': 2.8584857723577235, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6378338', 'n_tokens_mistral': 4827, 'n_tokens_neox': 4669, 'n_words': 1113}