<commit_before>// Copyright 2016-2017 Directive Games Limited - All Rights Reserved

#include "DriftHttpPCH.h"

#include "FileHttpCache.h"
#include "HttpRequest.h"
#include "FileHttpCacheFactory.h"
#include "CachedHttpResponse.h"
#include "JsonArchive.h"
#include "Details/DateHelper.h"

#include "SecureHash.h"


DEFINE_LOG_CATEGORY(LogHttpCache);


const int32 MAX_INLINE_CACHED_CONTENT_SIZE = 1024;
const int32 HTTP_CACHE_INDEX_VERSION = 1;


FileHttpCache::FileHttpCache()
: cacheDir{ FPaths::Combine(*FPaths::GameSavedDir(), TEXT("HttpCache")) }
, cacheVersion{ HTTP_CACHE_INDEX_VERSION }
{
    UE_LOG(LogHttpCache, Log, TEXT("Initializing FileHttpCache at: %s"), *cacheDir);

    LoadCache();
}


void FileHttpCache::CacheResponse(const ResponseContext& context)
{
    auto cacheHeader = context.response->GetHeader(TEXT("Cache-Control"));
    if (!(cacheHeader.Contains(TEXT("no-cache")) || cacheHeader.Contains(TEXT("no-store"))))
    {
        int32 maxAge = 0;
        if (FParse::Value(*cacheHeader, TEXT("max-age="), maxAge))
        {
            if (maxAge > 0)
            {
                auto dateHeader = context.response->GetHeader(TEXT("Date"));
                FDateTime dateValue;
                if (!internal::ParseRfc7231DateTime(*dateHeader, dateValue))
                {
                    UE_LOG(LogHttpCache, Verbose, TEXT("Failed to parse 'Date:' header, not safe to cache"));
                    return;
                }

                auto url = context.request->GetURL();
                auto& entry = data.Emplace(url);
                entry.url = url;
                entry.headers = context.response->GetAllHeaders();
                entry.maxAge = maxAge;
                entry.date = dateValue;
                entry.requestTime = context.sent;
                entry.responseTime = context.received;
                entry.responseCode = context.responseCode;
                entry.contentType = context.response->GetContentType();

                entry.correctedInitialAge = CalculateCorrectedInitialAge(context.response, entry).GetTotalSeconds();
                
                FString urlHash;
                auto path = index.Find(url);
                if (path)
                {
                    urlHash = *path;
                }
                else
                {
                    urlHash = FMD5::HashAnsiString(*url);
                }

                entry.urlHash = urlHash;
                entry.contentHash = GetContentHash(context.response);
                
                if (context.response->GetContent().Num() < MAX_INLINE_CACHED_CONTENT_SIZE)
                {
                    entry.payload = context.response->GetContent();
                }
                else
                {
                    entry.onDisk = true;
                    if (!SaveBody(urlHash, context.response->GetContent()))
                    {
                        UE_LOG(LogHttpCache, Error, TEXT("Failed to save cache body as '%s'"), *urlHash);
                        return;
                    }
                }<commit_msg>Use actual lenght, not ContentLength<commit_after>// Copyright 2016-2017 Directive Games Limited - All Rights Reserved

#include "DriftHttpPCH.h"

#include "FileHttpCache.h"
#include "HttpRequest.h"
#include "FileHttpCacheFactory.h"
#include "CachedHttpResponse.h"
#include "JsonArchive.h"
#include "Details/DateHelper.h"

#include "SecureHash.h"


DEFINE_LOG_CATEGORY(LogHttpCache);


const int32 MAX_INLINE_CACHED_CONTENT_SIZE = 1024;
const int32 HTTP_CACHE_INDEX_VERSION = 1;


FileHttpCache::FileHttpCache()
: cacheDir{ FPaths::Combine(*FPaths::GameSavedDir(), TEXT("HttpCache")) }
, cacheVersion{ HTTP_CACHE_INDEX_VERSION }
{
    UE_LOG(LogHttpCache, Log, TEXT("Initializing FileHttpCache at: %s"), *cacheDir);

    LoadCache();
}


void FileHttpCache::CacheResponse(const ResponseContext& context)
{
    auto cacheHeader = context.response->GetHeader(TEXT("Cache-Control"));
    if (!(cacheHeader.Contains(TEXT("no-cache")) || cacheHeader.Contains(TEXT("no-store"))))
    {
        int32 maxAge = 0;
        if (FParse::Value(*cacheHeader, TEXT("max-age="), maxAge))
        {
            if (maxAge > 0)
            {
                auto dateHeader = context.response->GetHeader(TEXT("Date"));
                FDateTime dateValue;
                if (!internal::ParseRfc7231DateTime(*dateHeader, dateValue))
                {
                    UE_LOG(LogHttpCache, Verbose, TEXT("Failed to parse 'Date:' header, not safe to cache"));
                    return;
                }

                auto url = context.request->GetURL();
                auto& entry = data.Emplace(url);
                entry.url = url;
                entry.headers = context.response->GetAllHeaders();
                entry.maxAge = maxAge;
                entry.date = dateValue;
                entry.requestTime = context.sent;
                entry.responseTime = context.received;
                entry.responseCode = context.responseCode;
                entry.contentType = context.response->GetContentType();

                entry.correctedInitialAge = CalculateCorrectedInitialAge(context.response, entry).GetTotalSeconds();
                
                FString urlHash;
                auto path = index.Find(url);
                if (path)
                {
                    urlHash = *path;
                }
                else
                {
                    urlHash = FMD5::HashAnsiString(*url);
                }

                entry.urlHash = urlHash;
                entry.contentHash = GetContentHash(context.response);
                
                if (context.response->GetContentLength() < MAX_INLINE_CACHED_CONTENT_SIZE)
                {
                    entry.payload = context.response->GetContent();
                }
                else
                {
                    entry.onDisk = true;
                    if (!SaveBody(urlHash, context.response->GetContent()))
                    {
                        UE_LOG(LogHttpCache, Error, TEXT("Failed to save cache body as '%s'"), *urlHash);
                        return;
                    }
                }
{'fraction_non_alphanumeric': 0.10762042854841308, 'fraction_numerical': 0.00837763815047527, 'mean_word_length': 1.7838565022421524, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2648951', 'n_tokens_mistral': 1750, 'n_tokens_neox': 1579, 'n_words': 306}