<commit_before>#include "WikiWalker.h"

#include <iostream>
#include <json/json.h>

//! represents continue operations
struct ContinueData
{
    //! Creates a new ContinueData object
    ContinueData() : moreData(false), continueString("") {};
    //! more data available
    bool moreData;
    //! string required for API operation
    std::string continueString;
};

Article getArticle(std::string json, ContinueData &contData)
{
    Json::Reader reader;
    Json::Value document;
    bool success = reader.parse(json, document, false);

    if(!success) {
        throw WalkerException("Error parsing JSON");
    }

    auto allPages = document.get("query", Json::Value::nullSingleton())
                            .get("pages", Json::Value::nullSingleton());

    // only get first page
    auto wantedPage = allPages.get(allPages.getMemberNames()[0],
                                   Json::Value::nullSingleton());

    Article wantedArticle(wantedPage.get("title", Json::Value::nullSingleton()).asString());

    // add links
    for(const auto &linked : wantedPage.get("links", Json::Value::nullSingleton()))
    {
        wantedArticle.addLink(
            new Article(linked.get("title", Json::Value::nullSingleton()).asString())
        );
    }

    if(!document.isMember("batchcomplete")) {
        contData.moreData = true;
        contData.continueString =
          document.get("continue", Json::Value::nullSingleton())
                  .get("plcontinue", Json::Value::nullSingleton())
                    .asString();
    }
    else
    {
        contData.moreData = false;
    }

    return wantedArticle;
}

void WikiWalker::startWalking(std::string url)
{
    std::string findUrl = "en.wikipedia.org/wiki/";
    size_t pos = url.find(findUrl);
    if(pos == std::string::npos) {
        throw WalkerException("Must be an English Wikipedia URL");
    }

    // extract Wikipedia title
    std::string title = url.substr(pos + findUrl.length());

    //! \todo: little bobby tables?
    std::string json = grabber.grabUrl("https://en.wikipedia.org/w/api.php?action=query&format=json&prop=links&pllimit=50&plnamespace=0&formatversion=1&titles=" + title);

    if(json != "")
    {
        ContinueData contData;
        Article article = getArticle(json, contData);

        while(contData.moreData && contData.continueString != "") {
            std::string json = grabber.grabUrl("https://en.wikipedia.org/w/api.php?action=query&format=json&prop=links&pllimit=50&plnamespace=0&formatversion=1&plcontinue=" + contData.continueString + "&titles=" + title);
            Article article2 = getArticle(json, contData);

            for(auto x = article2.linkBegin(); x != article2.linkEnd(); x++) {
                article.addLink(*x);
            }
        }

        std::cout << "Article " << article.getTitle() << " has " << article.getNumLinks() << " links" << std::endl;
    }
    else
    {
        std::cerr << "Error fetching article" << std::endl;
    }
};<commit_msg>Add articles to global collection instance<commit_after>#include "WikiWalker.h"

#include <iostream>
#include <json/json.h>

//! represents continue operations
struct ContinueData
{
    //! Creates a new ContinueData object
    ContinueData() : moreData(false), continueString("") {};
    //! more data available
    bool moreData;
    //! string required for API operation
    std::string continueString;
};

//! \todo really ugly workaround, passing in the ArticleCollection instance... :/
Article* getArticle(std::string json, ContinueData &contData, ArticleCollection* addArticle)
{
    Json::Reader reader;
    Json::Value document;
    bool success = reader.parse(json, document, false);

    if(!success) {
        throw WalkerException("Error parsing JSON");
    }

    auto allPages = document.get("query", Json::Value::nullSingleton())
                            .get("pages", Json::Value::nullSingleton());

    // only get first page
    auto wantedPage = allPages.get(allPages.getMemberNames()[0],
                                   Json::Value::nullSingleton());

    Article* wantedArticle = new Article(wantedPage.get("title", Json::Value::nullSingleton()).asString());
    addArticle->add(wantedArticle);

    // add links
    for(const auto &linked : wantedPage.get("links", Json::Value::nullSingleton()))
    {
        auto par = new Article(linked.get("title", Json::Value::nullSingleton()).asString());
        wantedArticle->addLink(
            par
        );
        addArticle->add(par);
    }

    if(!document.isMember("batchcomplete")) {
        contData.moreData = true;
        contData.continueString =
          document.get("continue", Json::Value::nullSingleton())
                  .get("plcontinue", Json::Value::nullSingleton())
                    .asString();
    }
    else
    {
        contData.moreData = false;
    }

    return wantedArticle;
}

void WikiWalker::startWalking(std::string url)
{
    std::string findUrl = "en.wikipedia.org/wiki/";
    size_t pos = url.find(findUrl);
    if(pos == std::string::npos) {
        throw WalkerException("Must be an English Wikipedia URL");
    }

    // extract Wikipedia title
    std::string title = url.substr(pos + findUrl.length());

    //! \todo: little bobby tables?
    std::string json = grabber.grabUrl("https://en.wikipedia.org/w/api.php?action=query&format=json&prop=links&pllimit=50&plnamespace=0&formatversion=1&titles=" + title);

    if(json != "")
    {
        ContinueData contData;
        Article* article = getArticle(json, contData, &articleSet);

        while(contData.moreData && contData.continueString != "") {
            std::string json = grabber.grabUrl("https://en.wikipedia.org/w/api.php?action=query&format=json&prop=links&pllimit=50&plnamespace=0&formatversion=1&plcontinue=" + contData.continueString + "&titles=" + title);
            Article* article2 = getArticle(json, contData, &articleSet);

            for(auto x = article2->linkBegin(); x != article2->linkEnd(); x++) {
                article->addLink(*x);
            }
            // delete duplicate article
            //! \todo cleanup problem: linked articles may also already exist in collection -> leak
            delete article2;
        }

        std::cout << "Article " << article->getTitle() << " has " << article->getNumLinks() << " links" << std::endl;
    }
    else
    {
        std::cerr << "Error fetching article" << std::endl;
    }
}
{'fraction_non_alphanumeric': 0.1487115802545793, 'fraction_numerical': 0.00388078236572493, 'mean_word_length': 2.8789885611077666, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 16, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '67744', 'n_tokens_mistral': 1888, 'n_tokens_neox': 1804, 'n_words': 370}