<commit_before>/*
 * This file is part of Bisq.
 *
 * Bisq is free software: you can redistribute it and/or modify it
 * under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or (at
 * your option) any later version.
 *
 * Bisq is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
 * License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
 */

package bisq.network.p2p.network;

import bisq.common.UserThread;
import bisq.common.proto.network.NetworkEnvelope;
import bisq.common.util.Utilities;

import javafx.beans.property.DoubleProperty;
import javafx.beans.property.IntegerProperty;
import javafx.beans.property.LongProperty;
import javafx.beans.property.SimpleDoubleProperty;
import javafx.beans.property.SimpleIntegerProperty;
import javafx.beans.property.SimpleLongProperty;

import java.util.Date;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;

import lombok.extern.slf4j.Slf4j;

/**
 * Network statistics per connection. As we are also interested in total network statistics
 * we use static properties to get traffic of all connections combined.
 */
@Slf4j
public class Statistic {

    ///////////////////////////////////////////////////////////////////////////////////////////
    // Static
    ///////////////////////////////////////////////////////////////////////////////////////////


    private final static long startTime = System.currentTimeMillis();
    private final static LongProperty totalSentBytes = new SimpleLongProperty(0);
    private final static DoubleProperty totalSentBytesPerSec = new SimpleDoubleProperty(0);
    private final static LongProperty totalReceivedBytes = new SimpleLongProperty(0);
    private final static DoubleProperty totalReceivedBytesPerSec = new SimpleDoubleProperty(0);
    private final static Map<String, Integer> totalReceivedMessages = new ConcurrentHashMap<>();
    private final static Map<String, Integer> totalSentMessages = new ConcurrentHashMap<>();
    private final static LongProperty numTotalSentMessages = new SimpleLongProperty(0);
    private final static DoubleProperty numTotalSentMessagesPerSec = new SimpleDoubleProperty(0);
    private final static LongProperty numTotalReceivedMessages = new SimpleLongProperty(0);
    private final static DoubleProperty numTotalReceivedMessagesPerSec = new SimpleDoubleProperty(0);

    static {
        UserThread.runPeriodically(() -> {
            numTotalSentMessages.set(totalSentMessages.values().stream().mapToInt(Integer::intValue).sum());
            numTotalReceivedMessages.set(totalReceivedMessages.values().stream().mapToInt(Integer::intValue).sum());

            long passed = (System.currentTimeMillis() - startTime) / 1000;
            numTotalSentMessagesPerSec.set(((double) numTotalSentMessages.get()) / passed);
            numTotalReceivedMessagesPerSec.set(((double) numTotalReceivedMessages.get()) / passed);

            totalSentBytesPerSec.set(((double) totalSentBytes.get()) / passed);
            totalReceivedBytesPerSec.set(((double) totalReceivedBytes.get()) / passed);
        }, 1);

        // We log statistics every 60 minutes
        UserThread.runPeriodically(() -> {
            String ls = System.lineSeparator();
            log.info("Accumulated network statistics:" + ls +
                            "Bytes sent: {};" + ls +
                            "Number of sent messages/Sent messages: {} / {};" + ls +
                            "Number of sent messages per sec: {};" + ls +
                            "Bytes received: {}" + ls +
                            "Number of received messages/Received messages: {} / {};" + ls +
                            "Number of received messages per sec: {};" + ls +
                            "Number of threads: {}" + ls,
                    Utilities.readableFileSize(totalSentBytes.get()),
                    numTotalSentMessages.get(), totalSentMessages,
                    numTotalSentMessagesPerSec.get(),
                    Utilities.readableFileSize(totalReceivedBytes.get()),
                    numTotalReceivedMessages.get(), totalReceivedMessages,
                    numTotalReceivedMessagesPerSec.get(),
                    Thread.getAllStackTraces().keySet().size());
        }, TimeUnit.MINUTES.toSeconds(60));
    }
<commit_msg>No. of threads is already logged every 10 minutes from printSystemLoad<commit_after>/*
 * This file is part of Bisq.
 *
 * Bisq is free software: you can redistribute it and/or modify it
 * under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or (at
 * your option) any later version.
 *
 * Bisq is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
 * License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
 */

package bisq.network.p2p.network;

import bisq.common.UserThread;
import bisq.common.proto.network.NetworkEnvelope;
import bisq.common.util.Utilities;

import javafx.beans.property.DoubleProperty;
import javafx.beans.property.IntegerProperty;
import javafx.beans.property.LongProperty;
import javafx.beans.property.SimpleDoubleProperty;
import javafx.beans.property.SimpleIntegerProperty;
import javafx.beans.property.SimpleLongProperty;

import java.util.Date;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;

import lombok.extern.slf4j.Slf4j;

/**
 * Network statistics per connection. As we are also interested in total network statistics
 * we use static properties to get traffic of all connections combined.
 */
@Slf4j
public class Statistic {

    ///////////////////////////////////////////////////////////////////////////////////////////
    // Static
    ///////////////////////////////////////////////////////////////////////////////////////////


    private final static long startTime = System.currentTimeMillis();
    private final static LongProperty totalSentBytes = new SimpleLongProperty(0);
    private final static DoubleProperty totalSentBytesPerSec = new SimpleDoubleProperty(0);
    private final static LongProperty totalReceivedBytes = new SimpleLongProperty(0);
    private final static DoubleProperty totalReceivedBytesPerSec = new SimpleDoubleProperty(0);
    private final static Map<String, Integer> totalReceivedMessages = new ConcurrentHashMap<>();
    private final static Map<String, Integer> totalSentMessages = new ConcurrentHashMap<>();
    private final static LongProperty numTotalSentMessages = new SimpleLongProperty(0);
    private final static DoubleProperty numTotalSentMessagesPerSec = new SimpleDoubleProperty(0);
    private final static LongProperty numTotalReceivedMessages = new SimpleLongProperty(0);
    private final static DoubleProperty numTotalReceivedMessagesPerSec = new SimpleDoubleProperty(0);

    static {
        UserThread.runPeriodically(() -> {
            numTotalSentMessages.set(totalSentMessages.values().stream().mapToInt(Integer::intValue).sum());
            numTotalReceivedMessages.set(totalReceivedMessages.values().stream().mapToInt(Integer::intValue).sum());

            long passed = (System.currentTimeMillis() - startTime) / 1000;
            numTotalSentMessagesPerSec.set(((double) numTotalSentMessages.get()) / passed);
            numTotalReceivedMessagesPerSec.set(((double) numTotalReceivedMessages.get()) / passed);

            totalSentBytesPerSec.set(((double) totalSentBytes.get()) / passed);
            totalReceivedBytesPerSec.set(((double) totalReceivedBytes.get()) / passed);
        }, 1);

        // We log statistics every 60 minutes
        UserThread.runPeriodically(() -> {
            String ls = System.lineSeparator();
            log.info("Accumulated network statistics:" + ls +
                            "Bytes sent: {};" + ls +
                            "Number of sent messages/Sent messages: {} / {};" + ls +
                            "Number of sent messages per sec: {};" + ls +
                            "Bytes received: {}" + ls +
                            "Number of received messages/Received messages: {} / {};" + ls +
                            "Number of received messages per sec: {}" + ls,
                    Utilities.readableFileSize(totalSentBytes.get()),
                    numTotalSentMessages.get(), totalSentMessages,
                    numTotalSentMessagesPerSec.get(),
                    Utilities.readableFileSize(totalReceivedBytes.get()),
                    numTotalReceivedMessages.get(), totalReceivedMessages,
                    numTotalReceivedMessagesPerSec.get());
        }, TimeUnit.MINUTES.toSeconds(60));
    }

{'fraction_non_alphanumeric': 0.12611932247275867, 'fraction_numerical': 0.004962779156327543, 'mean_word_length': 2.6946990833001196, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6984457', 'n_tokens_mistral': 2214, 'n_tokens_neox': 2153, 'n_words': 655}