<commit_before>/*
 * Copyright 2007-2017 Content Management AG
 * All rights reserved.
 *
 * author: <NAME> <<EMAIL>>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the
 * distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "Cache.hxx"
#include "Stock.hxx"
#include "Client.hxx"
#include "Handler.hxx"
#include "Istream.hxx"
#include "strmap.hxx"
#include "pool/pool.hxx"
#include "rubber.hxx"
#include "sink_rubber.hxx"
#include "istream_unlock.hxx"
#include "istream_rubber.hxx"
#include "istream/UnusedPtr.hxx"
#include "istream/istream_null.hxx"
#include "istream/istream_tee.hxx"
#include "AllocatorStats.hxx"
#include "cache.hxx"
#include "event/TimerEvent.hxx"
#include "io/Logger.hxx"
#include "util/Cancellable.hxx"

#include <boost/intrusive/list.hpp>

#include <sys/stat.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>

static constexpr struct timeval nfs_cache_compress_interval = { 600, 0 };

struct NfsCache;
struct NfsCacheItem;
struct NfsCacheStore;

struct NfsCacheStore final
    : public boost::intrusive::list_base_hook<boost::intrusive::link_mode<boost::intrusive::normal_link>>,
      RubberSinkHandler {

    struct pool &pool;

    NfsCache &cache;

    const char *key;

    struct stat stat;

    TimerEvent timeout_event;
    CancellablePointer cancel_ptr;

    NfsCacheStore(struct pool &_pool, NfsCache &_cache,
                  const char *_key, const struct stat &_st);

    /**
     * Release resources held by this request.
     */
    void Release();

    /**
     * Abort the request.
     */
    void Abort();

    void Put(RubberAllocation &&a);

    void OnTimeout() {
        /* reading the response has taken too long already; don't store
           this resource */
        LogConcat(4, "NfsCache", "timeout ", key);
        Abort();
    }

    /* virtual methods from class RubberSinkHandler */
    void RubberDone(RubberAllocation &&a, size_t size) override;
    void RubberOutOfMemory() override;
    void RubberTooLarge() override;
    void RubberError(std::exception_ptr ep) override;
};

struct NfsCache {
    struct pool &pool;

    NfsStock &stock;
    EventLoop &event_loop;

    Rubber rubber;

    Cache cache;

    TimerEvent compress_timer;

    /**
     * A list of requests that are currently saving their contents to
     * the cache.
     */
    boost::intrusive::list<NfsCacheStore,
                           boost::intrusive::constant_time_size<false>> requests;

    NfsCache(struct pool &_pool, size_t max_size, NfsStock &_stock,
             EventLoop &_event_loop);

    ~NfsCache() {
        compress_timer.Cancel();
        pool_unref(&pool);
    }

private:
    void OnCompressTimer() {
        rubber.Compress();
        compress_timer.Add(nfs_cache_compress_interval);
    }
};

struct NfsCacheRequest final : NfsStockGetHandler, NfsClientOpenFileHandler {
    struct pool &pool;

    NfsCache &cache;

    const char *key;
    const char *path;

    NfsCacheHandler &handler;
    CancellablePointer &cancel_ptr;

    NfsCacheRequest(struct pool &_pool, NfsCache &_cache,
                    const char *_key, const char *_path,
                    NfsCacheHandler &_handler,
                    CancellablePointer &_cancel_ptr)
        :pool(_pool), cache(_cache),
         key(_key), path(_path),
         handler(_handler),
         cancel_ptr(_cancel_ptr) {}

    void Error(std::exception_ptr ep) {
        handler.OnNfsCacheError(ep);
    }

    /* virtual methods from class NfsStockGetHandler */
    void OnNfsStockReady(NfsClient &client) override;
    void OnNfsStockError(std::exception_ptr ep) override;

    /* virtual methods from class NfsClientOpenFileHandler */
    void OnNfsOpen(NfsFileHandle *handle, const struct stat *st) override;
    void OnNfsOpenError(std::exception_ptr ep) override {
        Error(ep);
    }
};

struct NfsCacheHandle {
    NfsCache &cache;
    const char *key;

    NfsFileHandle *file;
    NfsCacheItem *item;
    const struct stat &stat;
};

struct NfsCacheItem final : CacheItem {
    const PoolPtr pool;

    struct stat stat;

    const RubberAllocation body;

    NfsCacheItem(PoolPtr &&_pool, const NfsCacheStore &store,
                 RubberAllocation &&_body) noexcept
        :CacheItem(std::chrono::minutes(1), store.stat.st_size),
         pool(std::move(_pool)), stat(store.stat),
         body(std::move(_body)) {
    }

    /* virtual methods from class CacheItem */
    void Destroy() override {
        pool_trash(pool);
        this->~NfsCacheItem();
    }
};

static constexpr off_t cacheable_size_limit = 512 * 1024;

static constexpr struct timeval nfs_cache_timeout = { 60, 0 };

static const char *
nfs_cache_key(struct pool &pool, const char *server,
              const char *_export, const char *path)
{
    return p_strcat(&pool, server, ":", _export, path, nullptr);
}

NfsCacheStore::NfsCacheStore(struct pool &_pool, NfsCache &_cache,
                             const char *_key, const struct stat &_st)
    :pool(_pool), cache(_cache),
     key(_key),
     stat(_st),
     timeout_event(cache.event_loop, BIND_THIS_METHOD(OnTimeout)) {}

void
NfsCacheStore::Release()
{
    assert(!cancel_ptr);

    timeout_event.Cancel();

    cache.requests.erase(cache.requests.iterator_to(*this));
    pool_unref(&pool);
}

void
NfsCacheStore::Abort()
{
    assert(cancel_ptr);

    cancel_ptr.CancelAndClear();
    Release();
}

void
NfsCacheStore::Put(RubberAllocation &&a)
{
    LogConcat(4, "NfsCache", "put ", key);

    const auto item = NewFromPool<NfsCacheItem>(PoolPtr(PoolPtr::donate,
                                                        *pool_new_libc(&cache.pool, "NfsCacheItem")),
                                                *this,
                                                std::move(a));
    cache.cache.Put(p_strdup(item->pool, key), *item);
}

/*
 * sink_rubber_handler
 *
 */

void
NfsCacheStore::RubberDone(RubberAllocation &&a, gcc_unused size_t size)
{
    assert((off_t)size == stat.st_size);
<commit_msg>nfs/Cache: use class PoolHolder<commit_after>/*
 * Copyright 2007-2017 Content Management AG
 * All rights reserved.
 *
 * author: <NAME> <<EMAIL>>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the
 * distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "Cache.hxx"
#include "Stock.hxx"
#include "Client.hxx"
#include "Handler.hxx"
#include "Istream.hxx"
#include "strmap.hxx"
#include "pool/pool.hxx"
#include "rubber.hxx"
#include "sink_rubber.hxx"
#include "istream_unlock.hxx"
#include "istream_rubber.hxx"
#include "istream/UnusedPtr.hxx"
#include "istream/istream_null.hxx"
#include "istream/istream_tee.hxx"
#include "AllocatorStats.hxx"
#include "cache.hxx"
#include "event/TimerEvent.hxx"
#include "io/Logger.hxx"
#include "util/Cancellable.hxx"

#include <boost/intrusive/list.hpp>

#include <sys/stat.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>

static constexpr struct timeval nfs_cache_compress_interval = { 600, 0 };

struct NfsCache;
struct NfsCacheItem;
struct NfsCacheStore;

struct NfsCacheStore final
    : public boost::intrusive::list_base_hook<boost::intrusive::link_mode<boost::intrusive::normal_link>>,
      RubberSinkHandler {

    struct pool &pool;

    NfsCache &cache;

    const char *key;

    struct stat stat;

    TimerEvent timeout_event;
    CancellablePointer cancel_ptr;

    NfsCacheStore(struct pool &_pool, NfsCache &_cache,
                  const char *_key, const struct stat &_st);

    /**
     * Release resources held by this request.
     */
    void Release();

    /**
     * Abort the request.
     */
    void Abort();

    void Put(RubberAllocation &&a);

    void OnTimeout() {
        /* reading the response has taken too long already; don't store
           this resource */
        LogConcat(4, "NfsCache", "timeout ", key);
        Abort();
    }

    /* virtual methods from class RubberSinkHandler */
    void RubberDone(RubberAllocation &&a, size_t size) override;
    void RubberOutOfMemory() override;
    void RubberTooLarge() override;
    void RubberError(std::exception_ptr ep) override;
};

struct NfsCache {
    struct pool &pool;

    NfsStock &stock;
    EventLoop &event_loop;

    Rubber rubber;

    Cache cache;

    TimerEvent compress_timer;

    /**
     * A list of requests that are currently saving their contents to
     * the cache.
     */
    boost::intrusive::list<NfsCacheStore,
                           boost::intrusive::constant_time_size<false>> requests;

    NfsCache(struct pool &_pool, size_t max_size, NfsStock &_stock,
             EventLoop &_event_loop);

    ~NfsCache() {
        compress_timer.Cancel();
        pool_unref(&pool);
    }

private:
    void OnCompressTimer() {
        rubber.Compress();
        compress_timer.Add(nfs_cache_compress_interval);
    }
};

struct NfsCacheRequest final : NfsStockGetHandler, NfsClientOpenFileHandler {
    struct pool &pool;

    NfsCache &cache;

    const char *key;
    const char *path;

    NfsCacheHandler &handler;
    CancellablePointer &cancel_ptr;

    NfsCacheRequest(struct pool &_pool, NfsCache &_cache,
                    const char *_key, const char *_path,
                    NfsCacheHandler &_handler,
                    CancellablePointer &_cancel_ptr)
        :pool(_pool), cache(_cache),
         key(_key), path(_path),
         handler(_handler),
         cancel_ptr(_cancel_ptr) {}

    void Error(std::exception_ptr ep) {
        handler.OnNfsCacheError(ep);
    }

    /* virtual methods from class NfsStockGetHandler */
    void OnNfsStockReady(NfsClient &client) override;
    void OnNfsStockError(std::exception_ptr ep) override;

    /* virtual methods from class NfsClientOpenFileHandler */
    void OnNfsOpen(NfsFileHandle *handle, const struct stat *st) override;
    void OnNfsOpenError(std::exception_ptr ep) override {
        Error(ep);
    }
};

struct NfsCacheHandle {
    NfsCache &cache;
    const char *key;

    NfsFileHandle *file;
    NfsCacheItem *item;
    const struct stat &stat;
};

struct NfsCacheItem final : PoolHolder, CacheItem {
    struct stat stat;

    const RubberAllocation body;

    NfsCacheItem(PoolPtr &&_pool, const NfsCacheStore &store,
                 RubberAllocation &&_body) noexcept
        :PoolHolder(std::move(_pool)),
         CacheItem(std::chrono::minutes(1), store.stat.st_size),
         stat(store.stat),
         body(std::move(_body)) {
    }

    using PoolHolder::GetPool;

    /* virtual methods from class CacheItem */
    void Destroy() override {
        pool_trash(pool);
        this->~NfsCacheItem();
    }
};

static constexpr off_t cacheable_size_limit = 512 * 1024;

static constexpr struct timeval nfs_cache_timeout = { 60, 0 };

static const char *
nfs_cache_key(struct pool &pool, const char *server,
              const char *_export, const char *path)
{
    return p_strcat(&pool, server, ":", _export, path, nullptr);
}

NfsCacheStore::NfsCacheStore(struct pool &_pool, NfsCache &_cache,
                             const char *_key, const struct stat &_st)
    :pool(_pool), cache(_cache),
     key(_key),
     stat(_st),
     timeout_event(cache.event_loop, BIND_THIS_METHOD(OnTimeout)) {}

void
NfsCacheStore::Release()
{
    assert(!cancel_ptr);

    timeout_event.Cancel();

    cache.requests.erase(cache.requests.iterator_to(*this));
    pool_unref(&pool);
}

void
NfsCacheStore::Abort()
{
    assert(cancel_ptr);

    cancel_ptr.CancelAndClear();
    Release();
}

void
NfsCacheStore::Put(RubberAllocation &&a)
{
    LogConcat(4, "NfsCache", "put ", key);

    const auto item = NewFromPool<NfsCacheItem>(PoolPtr(PoolPtr::donate,
                                                        *pool_new_libc(&cache.pool, "NfsCacheItem")),
                                                *this,
                                                std::move(a));
    cache.cache.Put(p_strdup(item->GetPool(), key), *item);
}

/*
 * sink_rubber_handler
 *
 */

void
NfsCacheStore::RubberDone(RubberAllocation &&a, gcc_unused size_t size)
{
    assert((off_t)size == stat.st_size);

{'fraction_non_alphanumeric': 0.10184099841412121, 'fraction_numerical': 0.0034475625732607045, 'mean_word_length': 3.151116199198626, 'pattern_counts': {'":': 2, '<': 31, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3553522', 'n_tokens_mistral': 4562, 'n_tokens_neox': 4313, 'n_words': 1225}