<commit_before>from __future__ import print_function
import test_utils
import time
import os
import pexpect
import csv
import numpy as np

"""
Crude measurement of the compression ratio achieved.
Carried out by monitoring network bytesin and bytesout vs time and integrating the curve.

Requires ifstat to be installed
"""


class Ifstat:

    def __init__(self, network_interface):
        self.ifstat = pexpect.spawn("ifstat -n -i " + network_interface)

    def get_output(self):
        # Get all output
        result = ""
        try:
            for n in range(1, 10):
                result = result + self.ifstat.read_nonblocking(size=16777216, timeout=2)
        except:
            print("Error reading from ifstat, buffer too small?")
        return result

    def __del__(self):
        self.ifstat.close(force=True)<commit_msg>Re #5 remove import of unused csv package<commit_after>from __future__ import print_function
import test_utils
import time
import os
import pexpect
import numpy as np

"""
Crude measurement of the compression ratio achieved.
Carried out by monitoring network bytesin and bytesout vs time and integrating the curve.

Requires ifstat to be installed
"""


class Ifstat:

    def __init__(self, network_interface):
        self.ifstat = pexpect.spawn("ifstat -n -i " + network_interface)

    def get_output(self):
        # Get all output
        result = ""
        try:
            for n in range(1, 10):
                result = result + self.ifstat.read_nonblocking(size=16777216, timeout=2)
        except:
            print("Error reading from ifstat, buffer too small?")
        return result

    def __del__(self):
        self.ifstat.close(force=True)
{'fraction_non_alphanumeric': 0.07348618459729571, 'fraction_numerical': 0.014697236919459141, 'mean_word_length': 2.868181818181818, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5377921', 'n_tokens_mistral': 498, 'n_tokens_neox': 470, 'n_words': 170}