<commit_before>#include "WebViewParser.hpp"

#include "common/fs/Resources.hpp"
#include "utils/Managers.hpp"

#include "networking/WebServer.hpp" // FIXME remove dependency

#include "control/media/MediaResources.hpp"
#include "control/media/webview/WebViewResources.hpp"

#include <regex>
#include <fstream>
#include <boost/algorithm/string/replace.hpp>

const bool DefaultTransparency = true;
const int DefaultWebviewMode = 0;
const int NativeModeid = 1;

const std::regex DurationRegex("DURATION=([0-9]+)");
const std::string DefaultWebviewExtension = ".html";

int WebViewParser::durationFrom(const ptree_node& node)
{
    auto baseDuration = MediaParser::durationFrom(node);
    return parseDuration(uriFrom(node).path()).value_or(baseDuration);
}

ExtraOptions WebViewParser::extraOptionsImpl(const ptree_node& node)
{
    auto transparency = node.get<bool>(XlfResources::WebView::Transparency, DefaultTransparency);
    auto mode = node.get<int>(XlfResources::WebView::ModeId, DefaultWebviewMode);

    return {
        {XlfResources::WebView::Transparency, std::to_string(transparency)},
        {XlfResources::WebView::ModeId, std::to_string(mode)}
    };
}

std::optional<int> WebViewParser::parseDuration(const FilePath& path)
{
    std::ifstream in(path);

    std::smatch matchedGroups;
    std::string line;
    const int DurationGroup = 1;

    while(std::getline(in, line))
    {
        if(std::regex_search(line, matchedGroups, DurationRegex) && matchedGroups.size() > 1)
            break;
    }

    return matchedGroups.size() > 1 ? std::stoi(matchedGroups[DurationGroup].str()) : std::optional<int>{};
}

Uri WebViewParser::uriFrom(const ptree_node& node)
{
    auto mode = node.get<int>(XlfResources::WebView::ModeId, DefaultWebviewMode);

    if(mode != NativeModeid)
    {
        auto fileName = std::to_string(idFrom(node)) + DefaultWebviewExtension;
        return Uri{Managers::webserver().address() + fileName};
    }
    else
    {
        auto uri = node.get<std::string>(XlfResources::Media::Uri);
        return Uri{removeEscapedSymbolsFromUri(uri)};
    }
}

std::string WebViewParser::removeEscapedSymbolsFromUri(std::string url)
{
    boost::replace_all(url, "%2F", "/");
    boost::replace_all(url, "%3A", ":");
    boost::replace_all(url, "%3F", "?");
    boost::replace_all(url, "%3D", "=");
    boost::replace_all(url, "%26", "&");
<commit_msg>fix: extend escaping symbols in URI<commit_after>#include "WebViewParser.hpp"

#include "common/fs/Resources.hpp"
#include "utils/Managers.hpp"

#include "networking/WebServer.hpp" // FIXME remove dependency

#include "control/media/MediaResources.hpp"
#include "control/media/webview/WebViewResources.hpp"

#include <regex>
#include <fstream>
#include <boost/algorithm/string/replace.hpp>

const bool DefaultTransparency = true;
const int DefaultWebviewMode = 0;
const int NativeModeid = 1;

const std::regex DurationRegex("DURATION=([0-9]+)");
const std::string DefaultWebviewExtension = ".html";

int WebViewParser::durationFrom(const ptree_node& node)
{
    auto baseDuration = MediaParser::durationFrom(node);
    return parseDuration(uriFrom(node).path()).value_or(baseDuration);
}

ExtraOptions WebViewParser::extraOptionsImpl(const ptree_node& node)
{
    auto transparency = node.get<bool>(XlfResources::WebView::Transparency, DefaultTransparency);
    auto mode = node.get<int>(XlfResources::WebView::ModeId, DefaultWebviewMode);

    return {
        {XlfResources::WebView::Transparency, std::to_string(transparency)},
        {XlfResources::WebView::ModeId, std::to_string(mode)}
    };
}

std::optional<int> WebViewParser::parseDuration(const FilePath& path)
{
    std::ifstream in(path);

    std::smatch matchedGroups;
    std::string line;
    const int DurationGroup = 1;

    while(std::getline(in, line))
    {
        if(std::regex_search(line, matchedGroups, DurationRegex) && matchedGroups.size() > 1)
            break;
    }

    return matchedGroups.size() > 1 ? std::stoi(matchedGroups[DurationGroup].str()) : std::optional<int>{};
}

Uri WebViewParser::uriFrom(const ptree_node& node)
{
    auto mode = node.get<int>(XlfResources::WebView::ModeId, DefaultWebviewMode);

    if(mode != NativeModeid)
    {
        auto fileName = std::to_string(idFrom(node)) + DefaultWebviewExtension;
        return Uri{Managers::webserver().address() + fileName};
    }
    else
    {
        auto uri = node.get<std::string>(XlfResources::Media::Uri);
        return Uri{removeEscapedSymbolsFromUri(uri)};
    }
}

std::string WebViewParser::removeEscapedSymbolsFromUri(std::string url)
{
    boost::replace_all(url, "%21", "!");
    boost::replace_all(url, "%23", "#");
    boost::replace_all(url, "%24", "$");
    boost::replace_all(url, "%25", "%");
    boost::replace_all(url, "%26", "&");
    boost::replace_all(url, "%27", "'");
    boost::replace_all(url, "%28", "(");
    boost::replace_all(url, "%29", ")");
    boost::replace_all(url, "%2A", "*");
    boost::replace_all(url, "%2B", "+");
    boost::replace_all(url, "%2C", ",");
    boost::replace_all(url, "%2F", "/");
    boost::replace_all(url, "%3A", ":");
    boost::replace_all(url, "%3B", ";");
    boost::replace_all(url, "%3D", "=");
    boost::replace_all(url, "%3F", "?");
    boost::replace_all(url, "%40", "@");
    boost::replace_all(url, "%5B", "]");
    boost::replace_all(url, "%5D", "[");

{'fraction_non_alphanumeric': 0.17119970193740686, 'fraction_numerical': 0.00894187779433681, 'mean_word_length': 5.032584269662921, 'pattern_counts': {'":': 2, '<': 21, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4883834', 'n_tokens_mistral': 1796, 'n_tokens_neox': 1744, 'n_words': 291}