<commit_before>package org.uct.cs.simplify.util;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;

public class StatRecorder implements AutoCloseable
{
    private static final int DEFAULT_INTERVAL_MS = 100;

    private final int intervalMs;
    private final List<Recording> recordings;
    private final Thread backgroundThread;
    private final long startTime;

    public StatRecorder()
    {
        this(DEFAULT_INTERVAL_MS);
    }

    public StatRecorder(int intervalMs)
    {
        this.startTime = System.nanoTime();
        if (intervalMs < 10) throw new IllegalArgumentException("intervalMS is too small!");
        this.intervalMs = intervalMs;
        this.recordings = new ArrayList<>(1000);
        this.backgroundThread = new Thread(new MemRecorderThread(this));
        this.backgroundThread.start();
    }

    @Override
    public void close() throws InterruptedException
    {
        this.backgroundThread.interrupt();
        this.backgroundThread.join();

        NumberSummary ns = new NumberSummary(this.recordings.size());

        for (Recording p : this.recordings)
        {
            ns.add(p.usage);
        }

        long elapsed = System.nanoTime() - this.startTime;

        System.out.printf("%n=== Memory Stats =========%n");
        System.out.printf("Mem : Min :      (%12d) %s%n", (long) ns.min, Useful.formatBytes((long) ns.min));
        System.out.printf("Mem : P50 :      (%12d) %s%n", (long) ns.p50, Useful.formatBytes((long) ns.p50));
        System.out.printf("Mem : Max :      (%12d) %s%n", (long) ns.max, Useful.formatBytes((long) ns.max));
        System.out.printf("Mem : Mean :     (%12d) %s%n", (long) ns.mean, Useful.formatBytes((long) ns.mean));
        System.out.printf("Time:            %s%n", Useful.formatTime(elapsed));
        System.out.printf("==========================%n");
    }

    public int getInterval()
    {
        return this.intervalMs;
    }

    private void add(long ms, long used)
    {
        this.recordings.add(new Recording(ms, used));
    }
<commit_msg>Print raw seconds as well<commit_after>package org.uct.cs.simplify.util;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;

public class StatRecorder implements AutoCloseable
{
    private static final int DEFAULT_INTERVAL_MS = 100;

    private final int intervalMs;
    private final List<Recording> recordings;
    private final Thread backgroundThread;
    private final long startTime;

    public StatRecorder()
    {
        this(DEFAULT_INTERVAL_MS);
    }

    public StatRecorder(int intervalMs)
    {
        this.startTime = System.nanoTime();
        if (intervalMs < 10) throw new IllegalArgumentException("intervalMS is too small!");
        this.intervalMs = intervalMs;
        this.recordings = new ArrayList<>(1000);
        this.backgroundThread = new Thread(new MemRecorderThread(this));
        this.backgroundThread.start();
    }

    @Override
    public void close() throws InterruptedException
    {
        this.backgroundThread.interrupt();
        this.backgroundThread.join();

        NumberSummary ns = new NumberSummary(this.recordings.size());

        for (Recording p : this.recordings)
        {
            ns.add(p.usage);
        }

        long elapsed = System.nanoTime() - this.startTime;

        System.out.printf("%n=== Memory Stats =========%n");
        System.out.printf("Mem : Min :      (%12d) %s%n", (long) ns.min, Useful.formatBytes((long) ns.min));
        System.out.printf("Mem : P50 :      (%12d) %s%n", (long) ns.p50, Useful.formatBytes((long) ns.p50));
        System.out.printf("Mem : Max :      (%12d) %s%n", (long) ns.max, Useful.formatBytes((long) ns.max));
        System.out.printf("Mem : Mean :     (%12d) %s%n", (long) ns.mean, Useful.formatBytes((long) ns.mean));
        System.out.printf("Time:            (%12s) %s%n", elapsed / Useful.NANOSECONDS_PER_SECOND, Useful.formatTime(elapsed));
        System.out.printf("==========================%n");
    }

    public int getInterval()
    {
        return this.intervalMs;
    }

    private void add(long ms, long used)
    {
        this.recordings.add(new Recording(ms, used));
    }

{'fraction_non_alphanumeric': 0.14383561643835616, 'fraction_numerical': 0.010958904109589041, 'mean_word_length': 2.8667255075022067, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '520729', 'n_tokens_mistral': 1362, 'n_tokens_neox': 1299, 'n_words': 293}