<commit_before>import argparse
import sys
import re
import subprocess
import os

import aff3ct_help_reader as ahr


# read all the lines from the given file and set them in a list of string lines with striped \n \r
def readFileInTable(filename):
	aFile = open(filename, "r")
	lines = []
	for line in aFile:
		line = re.sub('\r','',line.rstrip('\n'))
		if len(line) > 0:
			lines.append(line)
	aFile.close()

	return lines;

def get_keys(filename):
	lines = readFileInTable(filename)

	list_keys = []

	for l in lines:
		if l.startswith(".. |"):
			start_pos = 4
			end_pos = l.find("|", start_pos)
			list_keys.append(l[start_pos:end_pos])

	return list_keys

def run_aff3ct(args_list):
	try:
		processAFFECT = subprocess.Popen(args_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		(stdoutAFFECT, stderrAFFECT) = processAFFECT.communicate()
	except KeyboardInterrupt:
		os.kill(processAFFECT.pid, signal.SIGINT)
		(stdoutAFFECT, stderrAFFECT) = processAFFECT.communicate()

	err = stderrAFFECT.decode(encoding='UTF-8')
	std = stdoutAFFECT.decode(encoding='UTF-8').split("\n")

	return std, err


def aff3ct_helpmap_to_keys_list(help_map, aff3ct_keys): # fill aff3ct_keys from help_map
	# ahr.print_help_map(help_map)
	for m in help_map: # module
		for a in help_map[m]: # argument
			if type(help_map[m][a]) is dict:
				key = help_map[m][a]["key"]

				if key != "":
					try:
						aff3ct_keys.index(key)
					except Exception as e:
						aff3ct_keys.append(key)
					else:
						pass


def get_aff3ct_help_keys(aff3ct_path):
	aff3ct_keys = []

	codesList = ["BCH", "LDPC", "POLAR", "RA", "REP", "RS", "RSC", "RSC_DB", "TURBO", "TURBO_DB", "TPC", "UNCODED"]
	simList = ["BFER", "BFERI", "EXIT"]

	for c in codesList:
		for s in simList:
			args_list = [aff3ct_path, "-C", c, "-H", "-k", "--sim-type", s]
			std, err  = run_aff3ct(args_list)

			helpMap = ahr.help_to_map(std)

			aff3ct_helpmap_to_keys_list(helpMap, aff3ct_keys)

	return aff3ct_keys


def check_keys(keys_file, aff3ct_path):

	list_keys   = get_keys(keys_file)
	aff3ct_keys = get_aff3ct_help_keys(aff3ct_path)

	not_in_aff3ct_keys = []

	for k in list_keys:
		try:
			idx = aff3ct_keys.index(k)
			del aff3ct_keys[idx]
		except Exception as e:
			not_in_aff3ct_keys.append(k)

	print("keys left in aff3ct help:")
	print(aff3ct_keys)
	print()<commit_msg>Auto generate the code and simulation types lists in check_keys_dochelp script<commit_after>import argparse
import sys
import re
import subprocess
import os

import aff3ct_help_reader as ahr


# read all the lines from the given file and set them in a list of string lines with striped \n \r
def readFileInTable(filename):
	aFile = open(filename, "r")
	lines = []
	for line in aFile:
		line = re.sub('\r','',line.rstrip('\n'))
		if len(line) > 0:
			lines.append(line)
	aFile.close()

	return lines;

def get_keys(filename):
	lines = readFileInTable(filename)

	list_keys = []

	for l in lines:
		if l.startswith(".. |"):
			start_pos = 4
			end_pos = l.find("|", start_pos)
			list_keys.append(l[start_pos:end_pos])

	return list_keys

def run_aff3ct(args_list):
	try:
		processAFFECT = subprocess.Popen(args_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		(stdoutAFFECT, stderrAFFECT) = processAFFECT.communicate()
	except KeyboardInterrupt:
		os.kill(processAFFECT.pid, signal.SIGINT)
		(stdoutAFFECT, stderrAFFECT) = processAFFECT.communicate()

	err = stderrAFFECT.decode(encoding='UTF-8')
	std = stdoutAFFECT.decode(encoding='UTF-8').split("\n")

	return std, err


def aff3ct_helpmap_to_keys_list(help_map, aff3ct_keys): # fill aff3ct_keys from help_map
	# ahr.print_help_map(help_map)
	for m in help_map: # module
		for a in help_map[m]: # argument
			if type(help_map[m][a]) is dict:
				key = help_map[m][a]["key"]

				if key != "":
					try:
						aff3ct_keys.index(key)
					except Exception as e:
						aff3ct_keys.append(key)
					else:
						pass


def get_aff3ct_help_keys(aff3ct_path):

	# get the available codes and simulation types
	args_list = [aff3ct_path, "-h"]
	std, err  = run_aff3ct(args_list)
	helpMap = ahr.help_to_map(std)

	codesList = helpMap["Simulation"]["--sim-cde-type, -C"]["limits"] [1:-1].split("|")
	simList   = helpMap["Simulation"]["--sim-type"        ]["limits"] [1:-1].split("|")


	# try to run all codes ans simu to get their helps
	aff3ct_keys = []
	for c in codesList:
		for s in simList:
			args_list = [aff3ct_path, "-C", c, "-H", "-k", "--sim-type", s]
			std, err  = run_aff3ct(args_list)

			helpMap = ahr.help_to_map(std)

			aff3ct_helpmap_to_keys_list(helpMap, aff3ct_keys)

	return aff3ct_keys


def check_keys(keys_file, aff3ct_path):

	list_keys   = get_keys(keys_file)
	aff3ct_keys = get_aff3ct_help_keys(aff3ct_path)

	not_in_aff3ct_keys = []

	for k in list_keys:
		try:
			idx = aff3ct_keys.index(k)
			del aff3ct_keys[idx]
		except Exception as e:
			not_in_aff3ct_keys.append(k)

	print("keys left in aff3ct help:")
	print(aff3ct_keys)
	print()
{'fraction_non_alphanumeric': 0.12957347887608653, 'fraction_numerical': 0.01293713361633313, 'mean_word_length': 4.275053304904051, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3803810', 'n_tokens_mistral': 2141, 'n_tokens_neox': 1971, 'n_words': 438}