<commit_before>from arrange_schedule import *


def test_read_system_setting():
    keys = ['board_py_dir','shutdown','max_db_log','min_db_activity']
    system_setting = read_system_setting()
    for key in keys:
        assert key in system_setting
    return system_setting

def test_read_arrange_mode():
    keys = ['arrange_sn','arrange_mode','condition']
    receive_msg = read_arrange_mode()
    for key in keys:
        assert key in receive_msg

def test_crawler_cwb_img(system_setting):
    send_msg = {}
    send_msg['server_dir'] = system_setting['board_py_dir']
    send_msg['user_id'] = 1

    receive_msg = crawler_cwb_img(send_msg)
    assert receive_msg['result'] == 'success'

if __name__ == "__main__":
    system_setting = test_read_system_setting()
    test_read_arrange_mode()
    test_crawler_cwb_img(system_setting)
    print("All test passed")
<commit_msg>Add test case for forum crawler<commit_after>from arrange_schedule import *


def test_read_system_setting():
    keys = ['board_py_dir','shutdown','max_db_log','min_db_activity']
    system_setting = read_system_setting()
    for key in keys:
        assert key in system_setting
    return system_setting

def test_read_arrange_mode():
    keys = ['arrange_sn','arrange_mode','condition']
    receive_msg = read_arrange_mode()
    for key in keys:
        assert key in receive_msg

def test_crawler_cwb_img(system_setting):
    send_msg = {}
    send_msg['server_dir'] = system_setting['board_py_dir']
    send_msg['user_id'] = 1

    receive_msg = crawler_cwb_img(send_msg)
    assert receive_msg['result'] == 'success'

def test_crawler_inside_img():
    receive_msg = crawler_inside_news()
    assert receive_msg['result'] == 'success'

def test_crawler_techorange_news():
    receive_msg = crawler_techorange_news()
    assert receive_msg['result'] == 'success'

def test_crawler_medium_news():
    receive_msg = crawler_medium_news()
    assert receive_msg['result'] == 'success'

if __name__ == "__main__":
    system_setting = test_read_system_setting()
    test_read_arrange_mode()
    test_crawler_cwb_img(system_setting)
    test_crawler_inside_img()
    test_crawler_techorange_news()
    test_crawler_medium_news()
    print("All test passed")

{'fraction_non_alphanumeric': 0.10178571428571428, 'fraction_numerical': 0.0008928571428571428, 'mean_word_length': 4.851174934725848, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3482228', 'n_tokens_mistral': 759, 'n_tokens_neox': 748, 'n_words': 139}