<commit_before>#!/usr/bin/env python
#
# This program is a parameter file sanity checker used by psims to determine is a run should proceed
#
import argparse
import datetime
import gzip
import netCDF4
import os
import sys
import yaml
from colorama import init, Fore, Style
import configuration.configuration
import translators.translator_factory as translator_factory
import checkers.checker_factory as checker_factory

# Exit with error if test fails
def assert_true(passed):
    check = Fore.GREEN + u'\u2713' + Style.RESET_ALL
    fail  = Fore.RED + u'\u2718' + Style.RESET_ALL
    if datetime.datetime.today().weekday() >= 5:
        fail = u'\U0001F4A9' + " "
    if passed[0]:
        print check + " " + passed[1]
    else:
        sys.exit(fail + " " + passed[1])

# Is a parameter defined?
def is_defined(name, value):
    if value:
        return (True, "%s is defined" % name)
    else:
        return (False, "%s is not defined!" % name)

# Is a list empty?
def is_not_empty(name, value):
    if len(value) > 0:
        return (True, "%s is defined" % name)
    else:
        return (False, "%s is undefined!" % name)

# Does the parameter value/directory exist?
def is_directory(name, value):
    if os.path.isdir(value):
        return (True, "%s directory exists" % name)
    else:
        return (False, "%s directory %s does not exist!" % (name, value))
<commit_msg>Fix sanity checker unicode problem, issue #9<commit_after>#!/usr/bin/env python
#
# This program is a parameter file sanity checker used by psims to determine is a run should proceed
#
import argparse
import datetime
import gzip
import netCDF4
import os
import sys
import yaml
from colorama import init, Fore, Style
import configuration.configuration
import translators.translator_factory as translator_factory
import checkers.checker_factory as checker_factory

# Exit with error if test fails
def assert_true(passed):
    check = Fore.GREEN + 'o' + Style.RESET_ALL
    fail  = Fore.RED + 'x' + Style.RESET_ALL
    if passed[0]:
        print check + " " + passed[1]
    else:
        sys.exit(fail + " " + passed[1])

# Is a parameter defined?
def is_defined(name, value):
    if value:
        return (True, "%s is defined" % name)
    else:
        return (False, "%s is not defined!" % name)

# Is a list empty?
def is_not_empty(name, value):
    if len(value) > 0:
        return (True, "%s is defined" % name)
    else:
        return (False, "%s is undefined!" % name)

# Does the parameter value/directory exist?
def is_directory(name, value):
    if os.path.isdir(value):
        return (True, "%s directory exists" % name)
    else:
        return (False, "%s directory %s does not exist!" % (name, value))

{'fraction_non_alphanumeric': 0.10699740836727138, 'fraction_numerical': 0.009626064420584968, 'mean_word_length': 3.282091917591125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1019353', 'n_tokens_mistral': 858, 'n_tokens_neox': 832, 'n_words': 305}