def divide_db_by_random(dataPath='./thor_DB_msdn.json', test_rate=0.2):
    print('-- divide DB --')
    with open(dataPath, 'r') as f:
        DB = json.load(f)
    num_data = len(DB)
    test_idx = random.sample(list(range(num_data)), int((num_data * test_rate)))
    test_idx.sort()
    train_DB = []
    test_DB = []
    for (idx, data) in enumerate(DB):
        if ((idx % 10000) == 0):
            print('data preprocessing... ({}/{})'.format(idx, len(DB)))
        if (idx in test_idx):
            test_DB.append(data)
        else:
            train_DB.append(data)
    print('train.json#:', len(train_DB))
    print('test.json#:', len(test_DB))
    with open('./train.json', 'w') as f:
        json.dump(train_DB, f, indent='\t')
    print('generate train.json')
    with open('./test.json', 'w') as f:
        json.dump(test_DB, f, indent='\t')
    print('generate test.json')
def divide_db_by_random(dataPath='./thor_DB_msdn.json', test_rate=0.2):
    print('-- divide DB --')
    with open(dataPath, 'r') as f:
        DB = json.load(f)
    num_data = len(DB)
    test_idx = random.sample(list(range(num_data)), int((num_data * test_rate)))
    test_idx.sort()
    train_DB = []
    test_DB = []
    for (idx, data) in enumerate(DB):
        if ((idx % 10000) == 0):
            print('data preprocessing... ({}/{})'.format(idx, len(DB)))
        if (idx in test_idx):
            test_DB.append(data)
        else:
            train_DB.append(data)
    print('train.json#:', len(train_DB))
    print('test.json#:', len(test_DB))
    with open('./train.json', 'w') as f:
        json.dump(train_DB, f, indent='\t')
    print('generate train.json')
    with open('./test.json', 'w') as f:
        json.dump(test_DB, f, indent='\t')
    print('generate test.json')