<commit_before>/*
 *  AvanceDB - an in-memory database similar to Apache CouchDB
 *  Copyright (C) 2015-2017 Ripcord Software
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "http_server_log.h"

#include <iostream>
#include <cstdio>

#include <boost/format.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <boost/atomic.hpp>
#include <boost/thread.hpp>

#include "termcolor/termcolor.hpp"

#include "set_thread_name.h"

static const unsigned cumulativeSecsPerMonth[12] = { 2678400, 5097600, 7776000, 10368000, 13046400, 15638400, 18316800, 20995200, 23587200, 26265600, 28857600, 31536000 };
static const unsigned cumulativeSecsPerMonthLeap[12] = { 2678400, 5184000, 7862400, 10454400, 13132800, 15724800, 18403200, 21081600, 23673600, 26352000, 28944000, 31622400 };

struct LogRow {
    LogRow() : status(0) {
        data[0] = 0;
    }
    
    int status;
    char data[2048];
};

static const unsigned maxLogRows = 4096;
static LogRow logRows[maxLogRows];
static boost::atomic<unsigned> writeRowIndex;
static unsigned readRowIndex = 0;

static void streamWriterThread() {
    SetThreadName::Set("HttpServerLog");
    
    try {
        while (true) {
            boost::this_thread::sleep_for(boost::chrono::seconds(2));
            
            int lastStatus = 200;
            auto lastRowIndex = writeRowIndex.load();
            
            while (readRowIndex < lastRowIndex) {
                for (auto i = readRowIndex; i < lastRowIndex; ++i) {
                    auto& row = logRows[i % maxLogRows];

                    if (row.status != lastStatus) {
                        if (row.status >= 500) {
                            std::cout << termcolor::red;
                        } else if (row.status >= 400) {
                            std::cout << termcolor::grey << termcolor::bold;
                        } else if (row.status >= 300) {
                            std::cout << termcolor::yellow;
                        } else {
                            std::cout << termcolor::reset;
                        }

                        lastStatus = row.status;
                    }

                    std::cout << row.data << "\n";
                }

                readRowIndex = lastRowIndex;
                lastRowIndex = writeRowIndex.load();
            }                        
            
            std::cout << termcolor::reset << std::flush;                        
        }
    } catch (boost::thread_interrupted&) {
    }
}

static boost::thread logThread(streamWriterThread);

void HttpServerLog::Append(rs::httpserver::socket_ptr socket, rs::httpserver::request_ptr request, rs::httpserver::response_ptr response, const std::time_t& start, long duration) {
    
    int year, month, day, hour, min, sec;
    GetTimestamp(start, year, month, day, hour, min, sec);
    
    const auto queryString = request->getHeaders()->getQueryString();
    
    const auto localAddr = socket->getLocalEndpoint().address().to_string();
    const auto remoteAddr = socket->getRemoteEndpoint().address().to_string();
    
    auto index = writeRowIndex++;    
    index %= maxLogRows;
    
    // TODO: optimize the string formatting to reduce ad-hoc allocations and re-use already known string values (dates, etc.)
    logRows[index].status = response->getStatusCode();
    std::snprintf(logRows[index].data, sizeof(logRows[0].data), 
        "%04d-%02d-%02d %02d:%02d:%02dZ %s %s %s %s %u %s %s %d %u",
        year, month, day, hour, min, sec,
        localAddr.c_str(),
        request->getMethod().c_str(),
        request->getUri().c_str(),
        queryString.size() > 0 ? queryString.c_str() : "-",
        socket->getLocalEndpoint().port(),
        remoteAddr.c_str(),
        boost::replace_all_copy(request->getHeaders()->getUserAgent(), " ", "+").c_str(),
        response->getStatusCode(),
        duration);        <commit_msg>Start the log thread from Append not from a static * on fork() the thread is lost and no logging happens<commit_after>/*
 *  AvanceDB - an in-memory database similar to Apache CouchDB
 *  Copyright (C) 2015-2017 <NAME>
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "http_server_log.h"

#include <iostream>
#include <cstdio>

#include <boost/format.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <boost/atomic.hpp>
#include <boost/thread.hpp>

#include "termcolor/termcolor.hpp"

#include "set_thread_name.h"

static const unsigned cumulativeSecsPerMonth[12] = { 2678400, 5097600, 7776000, 10368000, 13046400, 15638400, 18316800, 20995200, 23587200, 26265600, 28857600, 31536000 };
static const unsigned cumulativeSecsPerMonthLeap[12] = { 2678400, 5184000, 7862400, 10454400, 13132800, 15724800, 18403200, 21081600, 23673600, 26352000, 28944000, 31622400 };

struct LogRow {
    LogRow() : status(0) {
        data[0] = 0;
    }
    
    int status;
    char data[2048];
};

static const unsigned maxLogRows = 4096;
static LogRow logRows[maxLogRows];
static boost::atomic<unsigned> writeRowIndex;
static unsigned readRowIndex = 0;

static boost::once_flag startThreadOnce = BOOST_ONCE_INIT;

static void StreamWriterThread() {
    SetThreadName::Set("avancedb-httplog");
    
    try {
        while (true) {
            boost::this_thread::sleep_for(boost::chrono::seconds(2));
            
            int lastStatus = 200;
            auto lastRowIndex = writeRowIndex.load();
            
            while (readRowIndex < lastRowIndex) {
                for (auto i = readRowIndex; i < lastRowIndex; ++i) {
                    auto& row = logRows[i % maxLogRows];

                    if (row.status != lastStatus) {
                        if (row.status >= 500) {
                            std::cout << termcolor::red;
                        } else if (row.status >= 400) {
                            std::cout << termcolor::grey << termcolor::bold;
                        } else if (row.status >= 300) {
                            std::cout << termcolor::yellow;
                        } else {
                            std::cout << termcolor::reset;
                        }

                        lastStatus = row.status;
                    }

                    std::cout << row.data << "\n";
                }

                readRowIndex = lastRowIndex;
                lastRowIndex = writeRowIndex.load();
            }                        
            
            std::cout << termcolor::reset << std::flush;                        
        }
    } catch (boost::thread_interrupted&) {
    }
}

static void StartStreamWriterThread() {
    boost::thread logThread(StreamWriterThread);
    logThread.detach();
}

void HttpServerLog::Append(rs::httpserver::socket_ptr socket, rs::httpserver::request_ptr request, rs::httpserver::response_ptr response, const std::time_t& start, long duration) {
    // start the log thread if it isn't already running
    boost::call_once(StartStreamWriterThread, startThreadOnce);
    
    int year, month, day, hour, min, sec;
    GetTimestamp(start, year, month, day, hour, min, sec);
    
    const auto queryString = request->getHeaders()->getQueryString();
    
    const auto localAddr = socket->getLocalEndpoint().address().to_string();
    const auto remoteAddr = socket->getRemoteEndpoint().address().to_string();
    
    auto index = writeRowIndex++;    
    index %= maxLogRows;
    
    // TODO: optimize the string formatting to reduce ad-hoc allocations and re-use already known string values (dates, etc.)
    logRows[index].status = response->getStatusCode();
    std::snprintf(logRows[index].data, sizeof(logRows[0].data), 
        "%04d-%02d-%02d %02d:%02d:%02dZ %s %s %s %s %u %s %s %d %u",
        year, month, day, hour, min, sec,
        localAddr.c_str(),
        request->getMethod().c_str(),
        request->getUri().c_str(),
        queryString.size() > 0 ? queryString.c_str() : "-",
        socket->getLocalEndpoint().port(),
        remoteAddr.c_str(),
        boost::replace_all_copy(request->getHeaders()->getUserAgent(), " ", "+").c_str(),
        response->getStatusCode(),
        duration);        
{'fraction_non_alphanumeric': 0.11636324922814863, 'fraction_numerical': 0.05067603534547003, 'mean_word_length': 2.3974683544303796, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5374065', 'n_tokens_mistral': 3041, 'n_tokens_neox': 2647, 'n_words': 742}