<commit_before>// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "crypto_engine.h"
#include <vector>
#include <chrono>
#include <thread>
#include <vespa/vespalib/xxhash/xxhash.h>
#include <vespa/vespalib/stllike/string.h>
#include <vespa/vespalib/net/tls/transport_security_options.h>
#include <vespa/vespalib/net/tls/transport_security_options_reading.h>
#include <vespa/vespalib/net/tls/tls_crypto_engine.h>
#include <assert.h>

namespace vespalib {

namespace {

struct HashState {
    using clock = std::chrono::high_resolution_clock;
    const void       *self;
    clock::time_point now;
    HashState() : self(this), now(clock::now()) {}
};

char gen_key() {
    HashState hash_state;
    std::this_thread::sleep_for(std::chrono::microseconds(42));
    return XXH64(&hash_state, sizeof(hash_state), 0);
}

class NullCryptoSocket : public CryptoSocket
{
private:
    SocketHandle _socket;
public:
    NullCryptoSocket(SocketHandle socket) : _socket(std::move(socket)) {}
    int get_fd() const override { return _socket.get(); }
    HandshakeResult handshake() override { return HandshakeResult::DONE; }
    size_t min_read_buffer_size() const override { return 1; }
    ssize_t read(char *buf, size_t len) override { return _socket.read(buf, len); }
    ssize_t drain(char *, size_t) override { return 0; }
    ssize_t write(const char *buf, size_t len) override { return _socket.write(buf, len); }
    ssize_t flush() override { return 0; }
};

class XorCryptoSocket : public CryptoSocket
{
private:
    static constexpr size_t CHUNK_SIZE = 4096;
    enum class OP { READ_KEY, WRITE_KEY };
    std::vector<OP> _op_stack;
    char              _my_key;
    char              _peer_key;
    std::vector<char> _readbuf;
    std::vector<char> _writebuf;    
    SocketHandle      _socket;

    bool is_blocked(ssize_t res, int error) const {
        return ((res < 0) && ((error == EWOULDBLOCK) || (error == EAGAIN)));
    }

    HandshakeResult try_read_key() {
        ssize_t res = _socket.read(&_peer_key, 1);
        if (is_blocked(res, errno)) {
            return HandshakeResult::NEED_READ;
        }
        return (res == 1)
            ? HandshakeResult::DONE
            : HandshakeResult::FAIL;
    }

    HandshakeResult try_write_key() {
        ssize_t res = _socket.write(&_my_key, 1);
        if (is_blocked(res, errno)) {
            return HandshakeResult::NEED_WRITE;
        }
        return (res == 1)
            ? HandshakeResult::DONE
            : HandshakeResult::FAIL;
    }

    HandshakeResult perform_hs_op(OP op) {
        if (op == OP::READ_KEY) {
            return try_read_key();
        } else {
            assert(op == OP::WRITE_KEY);
            return try_write_key();
        }
    }

public:
    XorCryptoSocket(SocketHandle socket, bool is_server)
        : _op_stack(is_server
                    ? std::vector<OP>({OP::WRITE_KEY, OP::READ_KEY})
                    : std::vector<OP>({OP::READ_KEY, OP::WRITE_KEY})),
          _my_key(gen_key()),
          _peer_key(0),
          _readbuf(),
          _writebuf(),
          _socket(std::move(socket)) {}
    int get_fd() const override { return _socket.get(); }
    HandshakeResult handshake() override {
        while (!_op_stack.empty()) {
            HandshakeResult partial_result = perform_hs_op(_op_stack.back());
            if (partial_result != HandshakeResult::DONE) {
                return partial_result;
            }
            _op_stack.pop_back();
        }
        return HandshakeResult::DONE;
    }
    size_t min_read_buffer_size() const override { return 1; }
    ssize_t read(char *buf, size_t len) override {
        if (_readbuf.empty()) {
            _readbuf.resize(CHUNK_SIZE);
            ssize_t res = _socket.read(&_readbuf[0], _readbuf.size());
            if (res > 0) {
                _readbuf.resize(res);
            } else {
                _readbuf.clear();
                return res;
            }
        }
        return drain(buf, len);
    }
    ssize_t drain(char *buf, size_t len) override {
        size_t frame = std::min(len, _readbuf.size());
        for (size_t i = 0; i < frame; ++i) {
            buf[i] = (_readbuf[i] ^ _my_key);
        }
        _readbuf.erase(_readbuf.begin(), _readbuf.begin() + frame);
        return frame;
    }
    ssize_t write(const char *buf, size_t len) override {
        ssize_t res = flush();
        while (res > 0) {
            res = flush();
        }
        if (res < 0) {
            return res;
        }
        size_t frame = std::min(len, CHUNK_SIZE);
        for (size_t i = 0; i < frame; ++i) {
            _writebuf.push_back(buf[i] ^ _peer_key);
        }
        return frame;
    }
    ssize_t flush() override {
        if (!_writebuf.empty()) {
            ssize_t res = _socket.write(&_writebuf[0], _writebuf.size());
            if (res > 0) {
                _writebuf.erase(_writebuf.begin(), _writebuf.begin() + res);                
            } else {
                assert(res < 0);
            }
            return res;
        }
        return 0;
    }
};

CryptoEngine::SP create_default_crypto_engine() {
    const char *env = getenv("VESPA_TLS_CONFIG_FILE");
    vespalib::string cfg_file = env ? env : "";
    if (cfg_file.empty()) {
        return std::make_shared<NullCryptoEngine>();
    }
    auto tls_opts = net::tls::read_options_from_json_file(cfg_file);
    return std::make_shared<TlsCryptoEngine>(*tls_opts);
}

} // namespace vespalib::<unnamed>

std::mutex CryptoEngine::_shared_lock;
CryptoEngine::SP CryptoEngine::_shared_default(nullptr);

CryptoEngine::~CryptoEngine() = default;

CryptoEngine::SP
CryptoEngine::get_default()
{
    std::lock_guard guard(_shared_lock);
    if (!_shared_default) {
        _shared_default = create_default_crypto_engine();
    }
    return _shared_default;
}<commit_msg>let xor crypto engine use smart buffers<commit_after>// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "crypto_engine.h"
#include <vector>
#include <chrono>
#include <thread>
#include <vespa/vespalib/xxhash/xxhash.h>
#include <vespa/vespalib/stllike/string.h>
#include <vespa/vespalib/net/tls/transport_security_options.h>
#include <vespa/vespalib/net/tls/transport_security_options_reading.h>
#include <vespa/vespalib/net/tls/tls_crypto_engine.h>
#include <vespa/vespalib/data/smart_buffer.h>
#include <assert.h>

namespace vespalib {

namespace {

struct HashState {
    using clock = std::chrono::high_resolution_clock;
    const void       *self;
    clock::time_point now;
    HashState() : self(this), now(clock::now()) {}
};

char gen_key() {
    HashState hash_state;
    std::this_thread::sleep_for(std::chrono::microseconds(42));
    return XXH64(&hash_state, sizeof(hash_state), 0);
}

class NullCryptoSocket : public CryptoSocket
{
private:
    SocketHandle _socket;
public:
    NullCryptoSocket(SocketHandle socket) : _socket(std::move(socket)) {}
    int get_fd() const override { return _socket.get(); }
    HandshakeResult handshake() override { return HandshakeResult::DONE; }
    size_t min_read_buffer_size() const override { return 1; }
    ssize_t read(char *buf, size_t len) override { return _socket.read(buf, len); }
    ssize_t drain(char *, size_t) override { return 0; }
    ssize_t write(const char *buf, size_t len) override { return _socket.write(buf, len); }
    ssize_t flush() override { return 0; }
};

class XorCryptoSocket : public CryptoSocket
{
private:
    static constexpr size_t CHUNK_SIZE = 16 * 1024;
    enum class OP { READ_KEY, WRITE_KEY };
    std::vector<OP> _op_stack;
    char            _my_key;
    char            _peer_key;
    SmartBuffer     _input;
    SmartBuffer     _output;
    SocketHandle    _socket;

    bool is_blocked(ssize_t res, int error) const {
        return ((res < 0) && ((error == EWOULDBLOCK) || (error == EAGAIN)));
    }

    HandshakeResult try_read_key() {
        ssize_t res = _socket.read(&_peer_key, 1);
        if (is_blocked(res, errno)) {
            return HandshakeResult::NEED_READ;
        }
        return (res == 1)
            ? HandshakeResult::DONE
            : HandshakeResult::FAIL;
    }

    HandshakeResult try_write_key() {
        ssize_t res = _socket.write(&_my_key, 1);
        if (is_blocked(res, errno)) {
            return HandshakeResult::NEED_WRITE;
        }
        return (res == 1)
            ? HandshakeResult::DONE
            : HandshakeResult::FAIL;
    }

    HandshakeResult perform_hs_op(OP op) {
        if (op == OP::READ_KEY) {
            return try_read_key();
        } else {
            assert(op == OP::WRITE_KEY);
            return try_write_key();
        }
    }

public:
    XorCryptoSocket(SocketHandle socket, bool is_server)
        : _op_stack(is_server
                    ? std::vector<OP>({OP::WRITE_KEY, OP::READ_KEY})
                    : std::vector<OP>({OP::READ_KEY, OP::WRITE_KEY})),
          _my_key(gen_key()),
          _peer_key(0),
          _input(CHUNK_SIZE * 2),
          _output(CHUNK_SIZE * 2),
          _socket(std::move(socket)) {}
    int get_fd() const override { return _socket.get(); }
    HandshakeResult handshake() override {
        while (!_op_stack.empty()) {
            HandshakeResult partial_result = perform_hs_op(_op_stack.back());
            if (partial_result != HandshakeResult::DONE) {
                return partial_result;
            }
            _op_stack.pop_back();
        }
        return HandshakeResult::DONE;
    }
    size_t min_read_buffer_size() const override { return 1; }
    ssize_t read(char *buf, size_t len) override {
        if (_input.obtain().size < CHUNK_SIZE) {
            auto dst = _input.reserve(CHUNK_SIZE);
            ssize_t res = _socket.read(dst.data, dst.size);
            if (res > 0) {
                _input.commit(res);
            } else {
                return res; // eof/error
            }
        }
        return drain(buf, len);
    }
    ssize_t drain(char *buf, size_t len) override {
        auto src = _input.obtain();
        size_t frame = std::min(len, src.size);
        for (size_t i = 0; i < frame; ++i) {
            buf[i] = (src.data[i] ^ _my_key);
        }
        _input.evict(frame);
        return frame;
    }
    ssize_t write(const char *buf, size_t len) override {
        if (_output.obtain().size >= CHUNK_SIZE) {
            if (flush() < 0) {
                return -1;
            }
            if (_output.obtain().size > 0) {
                errno = EWOULDBLOCK;
                return -1;
            }
        }
        size_t frame = std::min(len, CHUNK_SIZE);
        auto dst = _output.reserve(frame);
        for (size_t i = 0; i < frame; ++i) {
            dst.data[i] = (buf[i] ^ _peer_key);
        }
        _output.commit(frame);
        return frame;
    }
    ssize_t flush() override {
        auto pending = _output.obtain();
        if (pending.size > 0) {
            ssize_t res = _socket.write(pending.data, pending.size);
            if (res > 0) {
                _output.evict(res);
                return 1; // progress
            } else {
                assert(res < 0);
                return -1; // error
            }
        }
        return 0; // done
    }
};

CryptoEngine::SP create_default_crypto_engine() {
    const char *env = getenv("VESPA_TLS_CONFIG_FILE");
    vespalib::string cfg_file = env ? env : "";
    if (cfg_file.empty()) {
        return std::make_shared<NullCryptoEngine>();
    }
    auto tls_opts = net::tls::read_options_from_json_file(cfg_file);
    return std::make_shared<TlsCryptoEngine>(*tls_opts);
}

} // namespace vespalib::<unnamed>

std::mutex CryptoEngine::_shared_lock;
CryptoEngine::SP CryptoEngine::_shared_default(nullptr);

CryptoEngine::~CryptoEngine() = default;

CryptoEngine::SP
CryptoEngine::get_default()
{
    std::lock_guard guard(_shared_lock);
    if (!_shared_default) {
        _shared_default = create_default_crypto_engine();
    }
    return _shared_default;
}
{'fraction_non_alphanumeric': 0.12360755837940424, 'fraction_numerical': 0.006353659542866573, 'mean_word_length': 2.4131230639256547, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4885923', 'n_tokens_mistral': 4004, 'n_tokens_neox': 3900, 'n_words': 817}