def build_data_frame(data_dir):
    dirs = next(os.walk(data_dir))[1]
    class_names = []
    total_amount = []
    train_amount = []
    test_amount = []
    train_data = DataFrame({'value': [], 'class': []})
    test_data = DataFrame({'value': [], 'class': []})
    for d in dirs:
        tmp_dir = os.path.join(data_dir, d)
        rows = []
        index = []
        for f in glob.glob(os.path.join(tmp_dir, '*.txt')):
            with open(f, encoding='latin1') as fc:
                value = [line.replace('\n', '').replace('\r', '').replace('\t', '') for line in fc.readlines()]
                value = '. '.join(value)
                rows.append({'value': value, 'class': d})
                index.append(f)
        tmp_df = DataFrame(rows, index=index)
        size = int((len(tmp_df) * TRAIN_SIZE))
        (train_df, test_df) = (tmp_df.iloc[:size], tmp_df.iloc[size:])
        train_data = train_data.append(train_df)
        test_data = test_data.append(test_df)
        class_names.append(d)
        total_amount.append(len(os.listdir(tmp_dir)))
        train_amount.append(len(train_df))
        test_amount.append(len(test_df))
    tmp_arr = np.array([total_amount, train_amount, test_amount])
    print(DataFrame(tmp_arr, ['Total', 'Train', 'Test'], class_names))
    train_data = train_data.reindex(np.random.permutation(train_data.index))
    test_data = test_data.reindex(np.random.permutation(test_data.index))
    return (train_data, test_data, class_names)
def build_data_frame(data_dir):
    dirs = next(os.walk(data_dir))[1]
    class_names = []
    total_amount = []
    train_amount = []
    test_amount = []
    train_data = DataFrame({'value': [], 'class': []})
    test_data = DataFrame({'value': [], 'class': []})
    for d in dirs:
        tmp_dir = os.path.join(data_dir, d)
        rows = []
        index = []
        for f in glob.glob(os.path.join(tmp_dir, '*.txt')):
            with open(f, encoding='latin1') as fc:
                value = [line.replace('\n', ).replace('\r', ).replace('\t', ) for line in fc.readlines()]
                value = '. '.join(value)
                rows.append({'value': value, 'class': d})
                index.append(f)
        tmp_df = DataFrame(rows, index=index)
        size = int((len(tmp_df) * TRAIN_SIZE))
        (train_df, test_df) = (tmp_df.iloc[:size], tmp_df.iloc[size:])
        train_data = train_data.append(train_df)
        test_data = test_data.append(test_df)
        class_names.append(d)
        total_amount.append(len(os.listdir(tmp_dir)))
        train_amount.append(len(train_df))
        test_amount.append(len(test_df))
    tmp_arr = np.array([total_amount, train_amount, test_amount])
    print(DataFrame(tmp_arr, ['Total', 'Train', 'Test'], class_names))
    train_data = train_data.reindex(np.random.permutation(train_data.index))
    test_data = test_data.reindex(np.random.permutation(test_data.index))
    return (train_data, test_data, class_names)