<commit_before>#!/usr/bin/env python3

import subprocess
import shutil # cp/mv files
import os.path # check file exist
import glob # ls *
import threading
import sys
import argparse

sem = threading.Semaphore()

class Compression():
    """Compression is a wrapper class of a compression algorithm"""

    def __init__(self, command, input_file, output_file, replace, temp=False):
        """Init the compress algorithm

        :command: The command to build the output file
        :input_file: Input file name
        :output_file: Output file name
        :replace: Does the command remove the input

        """
        self.command = command
        self._input_file = input_file
        self.output_file = output_file
        self._replace = replace
        self._output = ""
        self._execed = False
        self._temp = temp
        self.fs = 0
        self.return_code = 0

    def format_command(self, input_file):
        return self.command.format(input_file, self.output_file)

    def exe(self):
        if not os.path.isfile(self._input_file):
            raise FileNotFoundError(f"No such file or directory: '{self._input_file}'")
        if self._replace:
            shutil.copyfile(self._input_file, self._input_file+".bak")
        input_file = self._input_file
        if self._temp:
            sem.acquire()
            input_file = "temp.temp"
            shutil.copyfile(self._input_file, input_file)
        self._process = subprocess.Popen(
            [self.format_command(input_file)],
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE
        )
        self._execed = True
        self.return_code = self._process.wait()
        self.stdout, self.stderr = self._process.communicate()
        if self._temp:
            files = glob.glob("temp.temp.*")
            shutil.move(files[0], self.output_file)
            if not self._replace:
                os.remove("temp.temp")
            sem.release()
        self.fs = os.path.getsize(self.output_file)
        if self._replace:
            shutil.move(self._input_file+".bak", self._input_file)
        print(self)

    def __str__(self):
        if self._execed:
            c = self.command.split(' ')[0]
            return f"{c},{self.output_file},{self.fs},{self.return_code},"
        return f"The command '{self.format_command(self._input_file)}' has not been run yet"

def generate_compression(input_file):
    return [
                Compression("7z a -t7z {1} {0}", input_file, f"{input_file}.7z", False),
                Compression("bzip2 {}", input_file, f"{input_file}.bz2", True, temp=True),
                Compression("compress -f {}", input_file, f"{input_file}.Z", True, temp=True),
                Compression("lzma {}", input_file, f"{input_file}.lzma", True, temp=True),
                Compression("pack {} {}", input_file, f"{input_file}.packed", False),
                Compression("rar a -r {1} {0}", input_file, f"{input_file}.rar", False),
                Compression("tar cfz {1} {0}", input_file, f"{input_file}.tar", False),
                Compression("xz {0}", input_file, f"{input_file}.xz", True, temp=True),
                Compression("zip {1} {0}", input_file, f"{input_file}.zip", False),
            ]

def run(input_file, count, thread=False):
    algos = generate_compression(input_file)
    for algo in algos:
        if thread:
            thr = threading.Thread(target=algo.exe)
            thr.start()
        else:
            algo.exe()
        if count > 0:
            run(algo.output_file, count-1)
        if thread:
            thr.join()
        os.remove(algo.output_file)

def create_parser():
    parser = argparse.ArgumentParser(description="Benchmark compressions algorithms")
    parser.add_argument("-f", "--file", help="Specify the input file name")
    parser.add_argument("-c", "--count", default=0,
            help="How deep is your recurtion")
    return parser

def main(argv):
    args = create_parser().parse_args(argv[1:])
    if not args.file:
        raise ValueError("No file provided, see --help")
    print("algo,command,file size,return code")
    print(f"none,'none',{os.path.getsize(args.file)},0,")
    run(args.file, count=int(args.count))

if __name__ == "__main__":
    main(sys.argv)
<commit_msg>bench adding time computing<commit_after>#!/usr/bin/env python3

import subprocess
import shutil # cp/mv files
import os.path # check file exist
import glob # ls *
import threading
import sys
import argparse
import time

sem = threading.Semaphore()

class Compression():
    """Compression is a wrapper class of a compression algorithm"""

    def __init__(self, command, input_file, output_file, replace, temp=False):
        """Init the compress algorithm

        :command: The command to build the output file
        :input_file: Input file name
        :output_file: Output file name
        :replace: Does the command remove the input

        """
        self.command = command
        self._input_file = input_file
        self.output_file = output_file
        self._replace = replace
        self._output = ""
        self._execed = False
        self._temp = temp
        self.fs = 0
        self.return_code = 0

    def format_command(self, input_file):
        return self.command.format(input_file, self.output_file)

    def exe(self):
        if not os.path.isfile(self._input_file):
            raise FileNotFoundError(f"No such file or directory: '{self._input_file}'")
        if self._replace:
            shutil.copyfile(self._input_file, self._input_file+".bak")
        input_file = self._input_file
        if self._temp:
            sem.acquire()
            input_file = "temp.temp"
            shutil.copyfile(self._input_file, input_file)
        self._time_start = time.time()
        self._process = subprocess.Popen(
            [self.format_command(input_file)],
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE
        )
        self._time_end = time.time()
        self._execed = True
        self.return_code = self._process.wait()
        self.stdout, self.stderr = self._process.communicate()
        if self._temp:
            files = glob.glob("temp.temp.*")
            shutil.move(files[0], self.output_file)
            if not self._replace:
                os.remove("temp.temp")
            sem.release()
        self.fs = os.path.getsize(self.output_file)
        if self._replace:
            shutil.move(self._input_file+".bak", self._input_file)
        print(self)

    def __str__(self):
        if self._execed:
            c = self.command.split(' ')[0]
            return f"{c},{self.output_file},{self.fs},{self.return_code},{self._time_end-self._time_start},"
        return f"The command '{self.format_command(self._input_file)}' has not been run yet"

def generate_compression(input_file):
    return [
                Compression("7z a -t7z {1} {0}", input_file, f"{input_file}.7z", False),
                Compression("bzip2 {}", input_file, f"{input_file}.bz2", True, temp=True),
                Compression("compress -f {}", input_file, f"{input_file}.Z", True, temp=True),
                Compression("lzma {}", input_file, f"{input_file}.lzma", True, temp=True),
                Compression("pack {} {}", input_file, f"{input_file}.packed", False),
                Compression("rar a -r {1} {0}", input_file, f"{input_file}.rar", False),
                Compression("tar cfz {1} {0}", input_file, f"{input_file}.tar", False),
                Compression("xz {0}", input_file, f"{input_file}.xz", True, temp=True),
                Compression("zip {1} {0}", input_file, f"{input_file}.zip", False),
            ]

def run(input_file, count, thread=False):
    algos = generate_compression(input_file)
    for algo in algos:
        if thread:
            thr = threading.Thread(target=algo.exe)
            thr.start()
        else:
            algo.exe()
        if count > 0:
            run(algo.output_file, count-1)
        if thread:
            thr.join()
        os.remove(algo.output_file)

def create_parser():
    parser = argparse.ArgumentParser(description="Benchmark compressions algorithms")
    parser.add_argument("-f", "--file", help="Specify the input file name")
    parser.add_argument("-c", "--count", default=0,
            help="How deep is your recurtion")
    return parser

def main(argv):
    args = create_parser().parse_args(argv[1:])
    if not args.file:
        raise ValueError("No file provided, see --help")
    print("algo,command,file size,return code,exec time,")
    print(f"none,none,{os.path.getsize(args.file)},0,0,")
    run(args.file, count=int(args.count))

if __name__ == "__main__":
    main(sys.argv)

{'fraction_non_alphanumeric': 0.1284623275960333, 'fraction_numerical': 0.005585318591131882, 'mean_word_length': 2.4166666666666665, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3217290', 'n_tokens_mistral': 2693, 'n_tokens_neox': 2657, 'n_words': 602}