<commit_before>#include "logging.hh"
#include "task.hh"
#include "net.hh"

#include <sstream>
#include <boost/utility.hpp>
#include <unordered_map>
#include <vector>
#include <functional>

namespace ten {

template <typename T> bool is_empty(const T &c) { return c.empty(); }

class cmd_server : public netsock_server {
public:
    typedef std::unordered_map<std::string, std::string> env_type;
    typedef std::vector<std::string> args_type;
    typedef std::function<void (netsock &s, const args_type &, env_type &)> callback_type;

    cmd_server(const std::string &welcome_="", size_t stacksize_=default_stacksize)
        : netsock_server("cmds", stacksize_), welcome(welcome_) 
    {
    }

    void add_command(const std::string &cmd,
            const callback_type &callback)
    {
        _cmds[cmd] = callback;
    }

    std::string welcome;
private:
    std::unordered_map<std::string, callback_type> _cmds;

    void on_shutdown() {
        // release any memory held by bound callbacks
        _cmds.clear();
    }

    void on_connection(netsock &s) {
        buffer buf(16*1024);

        s.s.setsockopt(IPPROTO_TCP, TCP_NODELAY, 1);
        std::stringstream ss;
        std::string line;

        env_type env;
        env["PROMPT"] = "$ ";

        ssize_t nw;
        buffer::slice rb = buf(0);
        if (!welcome.empty()) {
            nw = s.send(welcome.data(), welcome.size());
        }
        for (;;) {
            std::string prompt = env["PROMPT"];
            nw = s.send(prompt.data(), prompt.size());
            (void)nw;
            for (;;) {
                ssize_t nr = s.recv(rb.data(), rb.size());
                if (nr < 0) return;
                if (nr == 0) return;
                ss.write(rb.data(), nr);
                while (std::getline(ss, line)) {
                    VLOG(3) << "CMD LINE: " << line;
                    args_type args;
                    boost::split(args, line, boost::is_any_of(" \r"));
                    // remove empty args
                    auto end = std::remove_if(args.begin(), args.end(), is_empty<std::string>);
                    args.erase(end, args.end());

                    if (!args.empty()) {
                        if (args.front() == "\4") return; // ^D
                        auto it = _cmds.find(args.front());
                        if (it != _cmds.end()) {
                            try {
                                it->second(s, args, env);
                            } catch (std::exception &e) {
                                std::string reply = e.what();
                                reply += "\n";
                                nw = s.send(reply.data(), reply.size());<commit_msg>update cmd_server with new buffer api<commit_after>#include "logging.hh"
#include "task.hh"
#include "net.hh"

#include <sstream>
#include <boost/utility.hpp>
#include <unordered_map>
#include <vector>
#include <functional>

namespace ten {

template <typename T> bool is_empty(const T &c) { return c.empty(); }

class cmd_server : public netsock_server {
public:
    typedef std::unordered_map<std::string, std::string> env_type;
    typedef std::vector<std::string> args_type;
    typedef std::function<void (netsock &s, const args_type &, env_type &)> callback_type;

    cmd_server(const std::string &welcome_="", size_t stacksize_=default_stacksize)
        : netsock_server("cmds", stacksize_), welcome(welcome_) 
    {
    }

    void add_command(const std::string &cmd,
            const callback_type &callback)
    {
        _cmds[cmd] = callback;
    }

    std::string welcome;
private:
    std::unordered_map<std::string, callback_type> _cmds;

    void on_shutdown() {
        // release any memory held by bound callbacks
        _cmds.clear();
    }

    void on_connection(netsock &s) {
        buffer buf(4*1024);

        s.s.setsockopt(IPPROTO_TCP, TCP_NODELAY, 1);
        std::stringstream ss;
        std::string line;

        env_type env;
        env["PROMPT"] = "$ ";

        ssize_t nw;
        if (!welcome.empty()) {
            nw = s.send(welcome.data(), welcome.size());
        }
        for (;;) {
            std::string prompt = env["PROMPT"];
            nw = s.send(prompt.data(), prompt.size());
            (void)nw;
            for (;;) {
                buf.reserve(4*1024);
                ssize_t nr = s.recv(buf.back(), buf.available());
                if (nr < 0) return;
                buf.commit(nr);
                if (nr == 0) return;
                ss.write(buf.front(), buf.size());
                buf.remove(buf.size());
                while (std::getline(ss, line)) {
                    VLOG(3) << "CMD LINE: " << line;
                    args_type args;
                    boost::split(args, line, boost::is_any_of(" \r"));
                    // remove empty args
                    auto end = std::remove_if(args.begin(), args.end(), is_empty<std::string>);
                    args.erase(end, args.end());

                    if (!args.empty()) {
                        if (args.front() == "\4") return; // ^D
                        auto it = _cmds.find(args.front());
                        if (it != _cmds.end()) {
                            try {
                                it->second(s, args, env);
                            } catch (std::exception &e) {
                                std::string reply = e.what();
                                reply += "\n";
                                nw = s.send(reply.data(), reply.size());
{'fraction_non_alphanumeric': 0.1433996383363472, 'fraction_numerical': 0.00488245931283906, 'mean_word_length': 1.6694015444015444, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4573337', 'n_tokens_mistral': 1713, 'n_tokens_neox': 1609, 'n_words': 362}