<commit_before>#!/usr/bin/env python

import os
import json
import subprocess as sp
from colorama import Fore
import argparse

parser = argparse.ArgumentParser(description='Process command-line arguments')
parser.add_argument('--file', '-f', metavar='path/to/file', type=str, nargs='?', help='Path to file to use for test')
parser.add_argument('--level', '-l', metavar='#', type=int, nargs='?', help='Compression level')
parser.add_argument('--no-delete', const=True, default=False, nargs='?', help='Don\'t delete files produced for test')

args = parser.parse_args()

allPassed = True

testDir = 'test-files'
outDir = 'test-outs'

"""
Convenience function for running a command bash-like

@param command- string version of a command to run on
@param shell- Whether to run this through the shell; used in subprocess.Popen (default: true)
@return Object with properties 'returncode', 'stdout', and 'stderr'
"""
def run_cmd(command, shell=True):
	process = sp.Popen(command, shell=shell, stdout = sp.PIPE, stderr = sp.PIPE)
	stdout, stderr = process.communicate()
	returncode = process.returncode
	return {'returncode' : returncode, 'stdout' : stdout, 'stderr' : stderr}

"""
Run a single test

@param tFile- required; the full path to the file to run
@param level- optional (default: all); the compression level [1-9]
@param delete- optional (default: True); whether to delete the gzipped files
@return True if all tests passed; False if at least one test failed
"""
def runTest(tFile, level=None, delete=True):
	passed = True
	if level == None:
		for x in range(1, 10):
			if runTest(tFile, x, delete) == False:
				passed = False

		return passed

	out1 = os.path.join(outDir, os.path.basename(tFile) + '.gz')
	out2 = os.path.join(outDir, os.path.basename(tFile) + '.out.gz')

	run_cmd('gzip -c -%(level)d %(file)s > %(outfile)s' % {'level' : level, 'file' : tFile, 'outfile' : out1})
	run_cmd('../bin/runner.js --level %(level)d --file %(file)s --output %(output)s' % {'level' : level, 'file' : tFile, 'output' : out2})

	result = run_cmd('diff %(file1)s %(file2)s' % {'file1' : out1, 'file2' : out2})
	if result['returncode'] == 0:
		status = Fore.GREEN + 'PASSED' + Fore.RESET
	else:
		passed = False<commit_msg>Added level to output file for debugging purposes<commit_after>#!/usr/bin/env python

import os
import json
import subprocess as sp
from colorama import Fore
import argparse

parser = argparse.ArgumentParser(description='Process command-line arguments')
parser.add_argument('--file', '-f', metavar='path/to/file', type=str, nargs='?', help='Path to file to use for test')
parser.add_argument('--level', '-l', metavar='#', type=int, nargs='?', help='Compression level')
parser.add_argument('--no-delete', const=True, default=False, nargs='?', help='Don\'t delete files produced for test')

args = parser.parse_args()

allPassed = True

testDir = 'test-files'
outDir = 'test-outs'

"""
Convenience function for running a command bash-like

@param command- string version of a command to run on
@param shell- Whether to run this through the shell; used in subprocess.Popen (default: true)
@return Object with properties 'returncode', 'stdout', and 'stderr'
"""
def run_cmd(command, shell=True):
	process = sp.Popen(command, shell=shell, stdout = sp.PIPE, stderr = sp.PIPE)
	stdout, stderr = process.communicate()
	returncode = process.returncode
	return {'returncode' : returncode, 'stdout' : stdout, 'stderr' : stderr}

"""
Run a single test

@param tFile- required; the full path to the file to run
@param level- optional (default: all); the compression level [1-9]
@param delete- optional (default: True); whether to delete the gzipped files
@return True if all tests passed; False if at least one test failed
"""
def runTest(tFile, level=None, delete=True):
	passed = True
	if level == None:
		for x in range(1, 10):
			if runTest(tFile, x, delete) == False:
				passed = False

		return passed

	out1 = os.path.join(outDir, '%(file)s.%(level)d.gz' % {'file': os.path.basename(tFile), 'level' : level})
	out2 = os.path.join(outDir, '%(file)s.%(level)d.out.gz' % {'file': os.path.basename(tFile), 'level' : level})

	run_cmd('gzip -c -%(level)d %(file)s > %(outfile)s' % {'level' : level, 'file' : tFile, 'outfile' : out1})
	run_cmd('../bin/runner.js --level %(level)d --file %(file)s --output %(output)s' % {'level' : level, 'file' : tFile, 'output' : out2})

	result = run_cmd('diff %(file1)s %(file2)s' % {'file1' : out1, 'file2' : out2})
	if result['returncode'] == 0:
		status = Fore.GREEN + 'PASSED' + Fore.RESET
	else:
		passed = False
{'fraction_non_alphanumeric': 0.16147433084686266, 'fraction_numerical': 0.007020623080298377, 'mean_word_length': 4.800254452926208, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6764152', 'n_tokens_mistral': 1597, 'n_tokens_neox': 1547, 'n_words': 490}