<jupyter_start><jupyter_text>start training samples scanning..... training samples: (15000, 890)RF Layer training:(15000, 890)Layer 1acc: 0.9556Going to another layer(15000, 5890)Layer 2acc: 0.9658Going to another layer(15000, 10000)Layer 3acc: 0.982Going to another layer(15000, 10000)Layer 4acc: 0.9882Converged! Stopping building layerstraining time: 597.6242744922638Loading in testing dataLoad overtraining time: 39.19164228439331Statistics:The accuracy was:0.951219512195Params:num_tres in each layer = 1000md =Nonemss = 21/dstart training samples scanning..... training samples: (15000, 890)RF Layer training:(15000, 890)Layer 1acc: 0.9494Going to another layer(15000, 10890)Layer 2acc: 0.9714Going to another layer(15000, 20000)Layer 3acc: 0.985Going to another layer(15000, 20000)Layer 4acc: 0.9894Converged! Stopping building layerstraining time: 1865.0062873363495Loading in testing dataLoad overtraining time: 138.57394194602966Statistics:The accuracy was:0.952961672474Params:num_tres in each layer = 2000md =Nonzhuanymss = 21<jupyter_code># load in testing data, free up memory of the training data
print ('Loading in testing data')
# X_test = X_test(10000,784)
# X_t_curr = X_t_curr.astype('uint8')
# y_t = y_t.astype('uint8')
import time
st = time.time()
X_a_scan = np.hstack([scaner.scan_predict(X_test)
                             for scaner in [Scaner1,Scaner2]])
print ('Load over')
# push test data thru FTDRF layers
for i in range(len(Layers.keys())-1):
    X_a_scan = Layers[i].push_thru_data(X_a_scan)
last = len(Layers.keys())-1
y_pred = Layers[last].predict(X_a_scan)

et = time.time()
print('testing time:',et - st)

print ('Statistics:')
print ('The accuracy was:')
print (accuracy_score(y_pred, y_test))
print ('Params:')
print ('num_tres in each layer = ' + str(n))
print ('md =' + str(md))
print ('mss = ' + str(mss))<jupyter_output>Loading in testing data
Load over
testing time: 11.196585893630981
Statistics:
The accuracy was:
0.9425087108013938
Params:
num_tres in each layer = 500
md =None
mss = 21<jupyter_text>cifar10 对比start training samples scanning..... training samples: (50000, 500)RF Layer training:(50000, 500)Layer 1acc: 0.464870702586Going to another layer(50000, 5500)Layer 2acc: 0.782264354713Going to another layer(50000, 10000)Layer 3acc: 0.915281694366Going to another layer(50000, 10000)Layer 4acc: 0.966340673187Going to another layer(50000, 10000)Layer 5acc: 0.983680326393Going to another layertraining time: 2297.896358013153Loading in testing dataLoad overtraining time: 128.4914653301239Statistics:The accuracy was:0.94699Params:num_tres in each layer = 500md =Nonemss = 21<jupyter_code>X_train = 0
X_train_scan = 0
y_train = 0 # memory
fenge41 = fenge4[:100000]
fenge41.shape
fenge42 = fenge4[100000:200000]
fenge43 = fenge4[200000:]
X_a_scan1 = np.hstack([scaner.scan_predict(fenge41.reshape((len(fenge41),28,28,4)))
                             for scaner in [Scaner1,Scaner2]])
X_a_scan2 = np.hstack([scaner.scan_predict(fenge42.reshape((len(fenge42),28,28,4)))
                             for scaner in [Scaner1,Scaner2]])
X_a_scan3 = np.hstack([scaner.scan_predict(fenge43.reshape((len(fenge43),28,28,4)))
                             for scaner in [Scaner1,Scaner2]])
# X_a_scan = np.concatenate((X_a_scan1,X_a_scan2,X_a_scan3),axis = 0)
# X_a_scan.shape
X_a_scan.shape
for i in range(len(Layers.keys())-1):
    X_a_scan1 = Layers[i].push_thru_data(X_a_scan1)
last = len(Layers.keys())-1
y_pred1 = Layers[last].predict(X_a_scan1)
print('已完成三分之一')
X_a_scan1 = 0
X_a_scan = 0
for i in range(len(Layers.keys())-1):
    X_a_scan2 = Layers[i].push_thru_data(X_a_scan2)
last = len(Layers.keys())-1
y_pred2 = Layers[last].predict(X_a_scan2)
X_a_scan2 = 0
print('已完成三分之二')
for i in range(len(Layers.keys())-1):
    X_a_scan3 = Layers[i].push_thru_data(X_a_scan3)
last = len(Layers.keys())-1
y_pred3 = Layers[last].predict(X_a_scan3)
X_a_scan3 = 0
print('完成预测')
y_pred = np.concatenate((y_pred1,y_pred2,y_pred3),axis = 0)

class_predict = np.array(y_pred).reshape(row_side,loc_side)
img = np.zeros([row_side,loc_side,3])
for i in range(class_predict.shape[0]):
        for j in range(class_predict.shape[1]):
                if class_predict[i][j]  == 0 :  # 背景
                        img[i][j][0] = 0
                        img[i][j][1] = 0
                        img[i][j][2] = 0
                elif class_predict[i][j] == 1: # 沙漠
                        img[i][j][0] = 0
                        img[i][j][1] = 255
                        img[i][j][2] = 255
                elif class_predict[i][j] == 2: # 戈壁
                        img[i][j][0] = 190
                        img[i][j][1] = 190
                        img[i][j][2] = 190
                elif class_predict[i][j] == 3: # 绿洲
                        img[i][j][0] = 0
                        img[i][j][1] = 255
                        img[i][j][2] = 0
                elif class_predict[i][j] == 4: # 水系
                        img[i][j][0] = 255
                        img[i][j][1] = 250
                        img[i][j][2] = 87



        if i % 100 == 0:
            print("({0},:)".format(i))

cv.imwrite(savepath,img)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13235577850665636, 'fraction_numerical': 0.12174416361180783, 'mean_word_length': 3.3380753138075314, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '513828', 'license': 'MIT', 'n_tokens_mistral': 2363, 'n_tokens_neox': 1928, 'n_words': 405, 'path': 'mgrforest.ipynb', 'repo_name': 'qianmingduowan/A-Multi-dimensional-Multi-grained-Residual-Forest', 'size': '5183'}