<jupyter_start><jupyter_text>分类0.   其他1.   较低油分2.   低油分3.    中油分4.    高油分<jupyter_code>def loadData(flieName, dataIndex, temp_split=4):
    
    print("------------  loadData  ", dataIndex)
    # 原始数据路径
    DATA_PATH = os.path.join(os.getcwd(), flieName)

    index = str(dataIndex)
    data = envi.open( os.path.join(DATA_PATH, "{}.hdr".format(index)) ,os.path.join(DATA_PATH, "{}.dat".format(index)))
    mask_data = envi.open( os.path.join(DATA_PATH, "mask_{}.hdr".format(index)) ,os.path.join(DATA_PATH, "mask_{}.tiff".format(index)))

    HEIGHT = data.shape[0] //temp_split
    WIDTH = data.shape[1] //temp_split
    BAND = data.shape[2]
#     BAND = BAND_SIZE
    new_shape=(BAND,HEIGHT,WIDTH)
    new_data = np.zeros(new_shape, dtype = float)
    label = np.zeros((HEIGHT, WIDTH), dtype = int)
    

    sample_count = 0
    for h in range(HEIGHT): 
        for w in range(WIDTH):
            x = h*temp_split
            y = w*temp_split
            for b in range(BAND):
                new_data[b][h][w] = data[x,y][b]

            if(sum(mask_data[x, y])  > 0.01 ):
                label[h][w] = dataIndex 
                sample_count += 1
            else:
                label[h][w] = 0
    
    
    new_data = np.transpose(new_data, (1, 2, 0))  # 将通道数提前，便于数组处理操作
    print("sample_count = {} ".format(sample_count))
    print("data shape : ", new_data.shape)
    print("label shape : ", label.shape)
    return new_data, label<jupyter_output><empty_output><jupyter_text>PATCH  样本数量 * 通道 * 高 * 宽<jupyter_code>def create_sample_data(index):
    
    data, label = loadData("dataset", index)

    height = data.shape[0]
    width = data.shape[1]
    band = data.shape[2]
    
    
    print("band    : ", band)
    print("height  : ", height)
    print("width   : ", width)
    sample_count = 0
    for h in range(height):
        for w in range(width):
            if label[h][w] == index:
                sample_count += 1

    print("count   : ", sample_count)
    new_shape= (sample_count, band)
    temp_data = np.zeros(new_shape, dtype = float) 
    
    count = 0 
    for h in range(height):
        for w in range(width):
            if label[h][w] == index:
                for b in range(band):
                    temp_data[count][b] = data[h][w][b]
                count += 1
                
    return temp_data
    
    
%%time


new_data1  =  create_sample_data(1)
new_data2  =  create_sample_data(2)
new_data3  =  create_sample_data(3)
new_data4  =  create_sample_data(4)
def drawLine(new_data):

    plt.figure(figsize=(22, 3))
    new_data1.shape
    size = new_data1.shape[0]
    split = size // 4
    

    #第一行第一列图形
    ax1 = plt.subplot(1,4,1)
    ax2 = plt.subplot(1,4,2)
    ax3 = plt.subplot(1,4,3)
    ax4 = plt.subplot(1,4,4)
    x= np.linspace(0, 100 ,new_data.shape[1])

    
    plt.sca(ax1)
    plt.plot(x,new_data[split * 0])
    
    
    plt.sca(ax2)
    plt.plot(x,new_data[split * 1])
    
    
    plt.sca(ax3)
    plt.plot(x,new_data[split * 2])
    
    
    plt.sca(ax4)
    plt.plot(x,new_data[split * 3])

    plt.show()
    

drawLine(new_data1)
drawLine(new_data2)
drawLine(new_data3)
drawLine(new_data4)
def drawLine(d1, d2):

    plt.figure(figsize=(22, 5))
    size_1 = d1.shape[0]
    split_1 = size_1 // 4
    
    size_2 = d2.shape[0]
    split_2 = size_2 // 4
    

    #第一行第一列图形
    ax1 = plt.subplot(1,4,1)
    ax2 = plt.subplot(1,4,2)
    ax3 = plt.subplot(1,4,3)
    ax4 = plt.subplot(1,4,4)
    x= np.linspace(0, d1.shape[1] ,d1.shape[1])

    
    plt.sca(ax1)
    plt.plot(x,d1[split_1 * 0])
    plt.plot(x,d2[split_2 * 0], color='red')
    
    
    plt.sca(ax2)
    plt.plot(x,d1[split_1 * 1])
    plt.plot(x,d2[split_2 * 1], color='red')
    
    
    plt.sca(ax3)
    plt.plot(x,d1[split_1 * 2])
    plt.plot(x,d2[split_2 * 2], color='red')
    
    
    plt.sca(ax4)
    plt.plot(x,d1[split_1 * 3])
    plt.plot(x,d2[split_2 * 3], color='red')
    
    plt.show()<jupyter_output><empty_output><jupyter_text>中油分和高油分<jupyter_code>drawLine(new_data3, new_data4)<jupyter_output><empty_output><jupyter_text>低油分和较低油分<jupyter_code>drawLine(new_data1, new_data2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15297790585975024, 'fraction_numerical': 0.03434197886647455, 'mean_word_length': 2.3916938110749184, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '653753', 'license': 'Apache-2.0', 'n_tokens_mistral': 1792, 'n_tokens_neox': 1759, 'n_words': 269, 'path': 'explore_data.ipynb', 'repo_name': 'dikers/DeepHyper', 'size': '4164'}