<commit_before>import argparse
import collections
import inspect
import json
import logging
import multiprocessing as mp
import numpy as np
import os
import re
import sys
from datetime import datetime, timedelta
from keras import backend as K
from os import path, environ
from textwrap import wrap

PARALLEL_PROCESS_NUM = mp.cpu_count()
TIMESTAMP_REGEX = r'(\d{4}_\d{2}_\d{2}_\d{6})'
ASSET_PATH = path.join(path.dirname(__file__), 'asset')
PROBLEMS = json.loads(open(
    path.join(ASSET_PATH, 'problems.json')).read())
EXPERIMENT_SPECS = json.loads(open(
    path.join(ASSET_PATH, 'experiment_specs.json')).read())
for experiment_name in EXPERIMENT_SPECS:
    EXPERIMENT_SPECS[experiment_name]['experiment_name'] = experiment_name

# parse_args to add flag
parser = argparse.ArgumentParser(description='Set flag for functions')
parser.add_argument("-d", "--debug",
                    help="activate debug log",
                    action="store_const",
                    dest="loglevel",
                    const=logging.DEBUG,<commit_msg>add a quiet flag, useful for dev<commit_after>import argparse
import collections
import inspect
import json
import logging
import multiprocessing as mp
import numpy as np
import os
import re
import sys
from datetime import datetime, timedelta
from keras import backend as K
from os import path, environ
from textwrap import wrap

PARALLEL_PROCESS_NUM = mp.cpu_count()
TIMESTAMP_REGEX = r'(\d{4}_\d{2}_\d{2}_\d{6})'
ASSET_PATH = path.join(path.dirname(__file__), 'asset')
PROBLEMS = json.loads(open(
    path.join(ASSET_PATH, 'problems.json')).read())
EXPERIMENT_SPECS = json.loads(open(
    path.join(ASSET_PATH, 'experiment_specs.json')).read())
for experiment_name in EXPERIMENT_SPECS:
    EXPERIMENT_SPECS[experiment_name]['experiment_name'] = experiment_name

# parse_args to add flag
parser = argparse.ArgumentParser(description='Set flag for functions')
parser.add_argument("-d", "--debug",
                    help="activate debug log",
                    action="store_const",
                    dest="loglevel",
                    const=logging.DEBUG,
                    default=logging.INFO)
parser.add_argument("-q", "--quiet",
                    help="activate debug log",
                    action="store_const",
                    dest="loglevel",
                    const=logging.WARNING,
{'fraction_non_alphanumeric': 0.10615711252653928, 'fraction_numerical': 0.003397027600849257, 'mean_word_length': 3.4452830188679244, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4244674', 'n_tokens_mistral': 705, 'n_tokens_neox': 697, 'n_words': 178}