<jupyter_start><jupyter_text>--- 🌄 I : How to identify two photos , differnt or the same<jupyter_code>import sys
import pyautogui
import time
from PIL import ImageChops
from PIL import Image
import os


file_one = input("Please input the files : thread_main_game_win/run_298__155.png\n")
file_two = input("Please input the files : thread_main_game_win/run_298__155.png\n")

_im_one = Image.open(file_one).convert('RGB')
_im_two = Image.open(file_two).convert('RGB')

diff = ImageChops.difference(_im_one,_im_two)
diff_bbox = diff.getbbox()

if diff_bbox is not None:
    print("This two images is different")
else:
    print("This two images is the same")<jupyter_output>Please input the files : thread_main_game_win/run_298__155.png
thread_main_game_win/run_298__155.png
Please input the files : thread_main_game_win/run_298__155.png
thread_main_game_win/run_298__155.png
This two images is the same<jupyter_text>--- 🌄 II : How to list all name under folder<jupyter_code>import os
folder_name = input("Please input the 📕 folder name , e.g. thread_main_game_win")
entries = os.listdir(folder_name)
print(entries)<jupyter_output>Please input the 📕 folder name , e.g. thread_main_game_winthread_main_game_win
['run_410__219.png', 'run_379__201.png', 'run_461__241.png', 'run_917__445.png', 'run_510__261.png', 'run_645__322.png', 'run_1248__594.png', 'run_1023__492.png', 'run_166__91.png', 'run_247__126.png', 'run_879__427.png', 'run_492__252.png', 'run_822__403.png', 'run_878__426.png', 'run_1198__567.png', 'run_348__181.png', 'run_111__62.png', 'run_85__47.png', 'run_402__214.png', 'run_1114__531.png', 'run_670__335.png', 'run_898__439.png', 'run_652__325.png', 'run_964__467.png', 'run_843__411.png', 'run_732__366.png', 'run_301__156.png', 'run_378__200.png', 'run_456__239.png', 'run_999__482.png', 'run_1001__483.png', 'run_1106__529.png', 'run_1065__511.png', 'run_319__165.png', 'run_1228__584.png', 'run_702__350.png', 'run_23__17.png', 'run_439__235.png', 'run_744__371.png', 'run_1193__566.png', 'run_1079__516.png', 'run_775__386.png', 'run_1147__547.png', 'run_1084__518.png', 'run_620__310.png', 'run_9__7.png'[...]<jupyter_text>--- 🌄 Combine I et II : On peut faire the unique images into training_folder.<jupyter_code>import sys
import pyautogui
import time
from PIL import ImageChops
from PIL import Image
import os

# Load all files
folder_name = input("Please input the 📕 folder name , e.g. thread_main_game_win")
entries = os.listdir(folder_name)

# Remove .keep files...
try:
    entries.remove('.keep')
except:
    pass
try:
    entries.remove('.DS_Store')
except:
    pass

# Initial unigue_imag_list
unique_images_list = []

for any_file_name in entries:
    # Load 
    _tem_im = Image.open("%s/%s"%(folder_name,any_file_name)).convert('RGB')
    
    # The image must different with all unique_images_list
    diff_with_all_unique_imgs_flag = True
    
    # 1st run, empty list, save without comparing.
    if len(unique_images_list) == 0:
        unique_images_list.append(_tem_im)
    # Start to compare with unique_image_list
    else:
        for any_unique_im in unique_images_list:
            diff = ImageChops.difference(any_unique_im,_tem_im)
            diff_bbox = diff.getbbox()
            
            if diff_bbox is not None:
                continue       
            else:
                diff_with_all_unique_imgs_flag = False
                break
                
        # Check flag
        if diff_with_all_unique_imgs_flag == True:
            # print("Found one unique image")
            
            # append
            unique_images_list.append(_tem_im)
        else:
            continue
        
        

print("The final len of images = ",len(unique_images_list))
            
        
        
# Save:
saving_folder = input("Please set saving folder e.g. sample\n")
for index, any_im in enumerate(unique_images_list):
    any_im.save("%s/sample_%s.jpg"%(saving_folder,index),"JPEG")

print("Finihed !")<jupyter_output>Please input the 📕 folder name , e.g. thread_main_game_winthread_main_game_win
The final len of images =  30
Please set saving folder e.g. sample
sample
Finihed !<jupyter_text>--- 🌄 III:  Mapping the image to score<jupyter_code>print("Please open the folder under sample, and select all to view all images")
_nothing = input("Are you ready ?\n")

unique_score_mapping = {}
for index, image in enumerate(unique_images_list):
    print("index  =",index,"   Please input the score")
    tem_v = float(input())
    unique_score_mapping[index] = tem_v

print("Finish, You can modified the mapping by coding here")
print(unique_score_mapping)

# You can modified the mapping by coding here !!<jupyter_output>Please open the folder under sample, and select all to view all images
Are you ready ?

index  = 0    Please input the score
1.35
index  = 1    Please input the score<jupyter_text>--- 🌄 IV:  Transform Image to Digits<jupyter_code># Load all files
folder_name = input("Please input the 📕 folder name , e.g. thread_main_game_win")
entries = os.listdir(folder_name)

# Remove .keep files...
try:
    entries.remove('.keep')
except:
    pass
try:
    entries.remove('.DS_Store')
except:
    pass

print(unique_score_mapping)
input()

total_gain = 0.0
total_gain_list = []

for any_file_name in entries:
    # Load 
    _tem_im = Image.open("%s/%s"%(folder_name,any_file_name)).convert('RGB')
    
  
    for index, any_unique_im in enumerate(unique_images_list):
        diff = ImageChops.difference(any_unique_im,_tem_im)
        diff_bbox = diff.getbbox()
            
        if diff_bbox is not None:
            continue       
        else:
            tem_gain = unique_score_mapping[index]
            total_gain += tem_gain
            total_gain_list.append(tem_gain)
            print(tem_gain)
            
print("Fianl total_gain = ",total_gain)
print("Total_gain_list = ")
input()
print(total_gain_list)
       
!python3 01_mkdir_train_models_by_sample_folder.py
!python3 02_obtenir_class_index_name_map_by_train_models_folder.py
!python3 03_image_generator_train_by_sample_folder.py
!python3 04_image_generator_validation_by_sample_folder.py
!python3 train_all_models_by_pick_best.py<jupyter_output>[System]: In folder sample, you put the following information:   
 sample_0.jpg
sample_1.jpg
sample_10.jpg
sample_11.jpg
sample_12.jpg
sample_13.jpg
sample_2.jpg
sample_3.jpg
sample_4.jpg
sample_5.jpg
sample_6.jpg
sample_7.jpg
sample_8.jpg
sample_9.jpg
 , type = <class 'str'>
['sample_0', 'sample_1', 'sample_10', 'sample_11', 'sample_12', 'sample_13', 'sample_2', 'sample_3', 'sample_4', 'sample_5', 'sample_6', 'sample_7', 'sample_8', 'sample_9']
{0: 'sample_0', 1: 'sample_1', 2: 'sample_10', 3: 'sample_11', 4: 'sample_12', 5: 'sample_13', 6: 'sample_2', 7: 'sample_3', 8: 'sample_4', 9: 'sample_5', 10: 'sample_6', 11: 'sample_7', 12: 'sample_8', 13: 'sample_9'}
Please remove all the files under train_models 
Do you clean it (type : y for continue)
^C
Traceback (most recent call last):
  File "01_mkdir_train_models_by_sample_folder.py", line 70, in <module>
    res = input()
KeyboardInterrupt<jupyter_text>How to cut the 500 samples into small pices base on the map_norm:---<jupyter_code>map_norm = { 0: 'dix', 1: 'onze', 2: 'cherry',3:'lamp',4:'kid',5:'house',6:'a',7:'douze'}
sample   = 500
small_sample_size = 500 / len(map_norm)
small_sample_size
import random
for v in range(sample):
    norm_index =  int(v // small_sample_size) # 0, 1, 2, 3, 4, 5, 6, 7
    
    method = random.randrange(0,3)
    # print(method) # 0, 1, 2
    
    
for any_norm_index in map_norm:
    print(any_norm_index)
    
    total_list = [ v for v in  range(len(map_norm))]
    print(total_list)
    total_list.remove(any_norm_index)
    print(total_list)<jupyter_output>0
[0, 1, 2, 3, 4, 5, 6, 7]
[1, 2, 3, 4, 5, 6, 7]
1
[0, 1, 2, 3, 4, 5, 6, 7]
[0, 2, 3, 4, 5, 6, 7]
2
[0, 1, 2, 3, 4, 5, 6, 7]
[0, 1, 3, 4, 5, 6, 7]
3
[0, 1, 2, 3, 4, 5, 6, 7]
[0, 1, 2, 4, 5, 6, 7]
4
[0, 1, 2, 3, 4, 5, 6, 7]
[0, 1, 2, 3, 5, 6, 7]
5
[0, 1, 2, 3, 4, 5, 6, 7]
[0, 1, 2, 3, 4, 6, 7]
6
[0, 1, 2, 3, 4, 5, 6, 7]
[0, 1, 2, 3, 4, 5, 7]
7
[0, 1, 2, 3, 4, 5, 6, 7]
[0, 1, 2, 3, 4, 5, 6]<jupyter_text>obtenir_class_index_name_map_by_sample_folder , PD Example---<jupyter_code>import pandas as pd
import os

current_loc = 'pwd'
res = os.popen(current_loc).read()
res
map_norm = { 0: 'dix', 1: 'onze', 2: 'cherry',3:'lamp',4:'kid',5:'house',6:'a',7:'douze'}
map_norm_reverse = {v:k for k, v in map_norm.items()}

cmd_ls = 'ls train_models'
res = os.popen(cmd_ls).read()
print(res)
for any_norm in map_norm_reverse:
    cmd_ls = 'ls train_models/%s_dir/Train'%any_norm
    res = os.popen(cmd_ls).read()
    item_list = res.splitlines()
    
    for index , class_name in enumerate(item_list):
        print("(index, class_name) = (%d, %s)"%(index,class_name))<jupyter_output>(index, class_name) = (0, dix)
(index, class_name) = (1, not_dix)
(index, class_name) = (0, not_onze)
(index, class_name) = (1, onze)
(index, class_name) = (0, cherry)
(index, class_name) = (1, not_cherry)
(index, class_name) = (0, lamp)
(index, class_name) = (1, not_lamp)
(index, class_name) = (0, kid)
(index, class_name) = (1, not_kid)
(index, class_name) = (0, house)
(index, class_name) = (1, not_house)
(index, class_name) = (0, a)
(index, class_name) = (1, not_a)
(index, class_name) = (0, douze)
(index, class_name) = (1, not_douze)<jupyter_text>Pandas DataFrame Format:---Table Example: Comme ca|index | 0 | 1 |:--- |:---|:---|is_dix | True | False |___is_onze___ | False | True | is_cherry | True | False<jupyter_code>index_col = []
zero_col = []
un_col   = []

def __name_conver_boolean(input_class_name, current_name):
    compare_value   = "%s"%input_class_name
    compare_value_2 = "not_%s"%input_class_name
    if current_name == compare_value:
        return True
    elif current_name == compare_value_2:
        return False
    else:
        print("Error: Unexpected current_name = %s"%current_name)
        

for any_norm in map_norm_reverse:
    cmd_ls = 'ls train_models/%s_dir/Train'%any_norm
    res = os.popen(cmd_ls).read()
    item_list = res.splitlines()
    
    # index_col faire ici:
    index_col.append("is_%s"%any_norm)
    
    for index , item_name in enumerate(item_list):
        print("(index, item_name) = (%d, %s)"%(index,item_name))
        
        if index == 0:
            tem_true_or_false = __name_conver_boolean(any_norm, item_name)
            zero_col.append(tem_true_or_false)  
        elif index == 1:
            tem_true_or_false = __name_conver_boolean(any_norm, item_name)
            un_col.append(tem_true_or_false)
        else:
            print("[Error]: Unexpected index %d"%index)
            

# Save to PD
index_name_map_df = pd.DataFrame({"class_name":index_col,'class_0':zero_col,'class_1':un_col})

# Set 'class_name' as inex
index_name_map_df = index_name_map_df.set_index('class_name')
print(index_name_map_df)

# Save to csv
index_name_map_df.to_csv('class_index_et_name.csv')
# Read Csv

load_csv_pd = pd.read_csv('class_index_et_name.csv')
load_csv_pd.head()
# Set index for further use
done_load_pd = load_csv_pd.set_index('class_name')
done_load_pd.head()

# Read
for any_norm in map_norm_reverse:
    index_name = 'is_%s'%any_norm
    print(index_name)
    zero = done_load_pd['class_0'][index_name]
    one  = done_load_pd['class_1'][index_name]
    print(zero,one)
    print("type = ",type(zero))<jupyter_output>is_dix
True False
type =  <class 'numpy.bool_'>
is_onze
False True
type =  <class 'numpy.bool_'>
is_cherry
True False
type =  <class 'numpy.bool_'>
is_lamp
True False
type =  <class 'numpy.bool_'>
is_kid
True False
type =  <class 'numpy.bool_'>
is_house
True False
type =  <class 'numpy.bool_'>
is_a
True False
type =  <class 'numpy.bool_'>
is_douze
True False
type =  <class 'numpy.bool_'><jupyter_text>Obtenir full images , split to col_index, row_index---<jupyter_code>import re
my_str = 'test/5_3.jpg'

tem_list = re.split('\.jpg',my_str)
# [test/1_1 , '']
tem_list = re.split('/',tem_list[0])
# ['test', '1_1']
my_list  = re.split('_',tem_list[1])


# [1_1]
col_index = my_list[0]
row_index = my_list[1]

print(col_index,row_index)

col_index_data = ['1','1','1']
row_index_data = ['1','2','3']

run_pd = pd.DataFrame({'col':col_index_data,'row':row_index_data})
print(run_pd)
run_pd['col'][0]
run_pd['row'][1]
run_pd
run_pd.loc[0,'a'] = 0
run_pd
value = 0.3
value_reverse = abs(value-1)
value_reverse
a_data = [1.0,1.0,1.0]
cherry_data = [0, 0,1.0]
dix_data = [1.0, 1.0, 0.0]

prediction_pd = pd.DataFrame({'a':a_data,'cherry':cherry_data,'dix':dix_data})
print(prediction_pd)
# Obtenir min value , add to result col
# get the column name of min values in every row
# minValueIndexObj = dfObj.idxmin(axis=1)
min_name_pd = prediction_pd[['a','cherry','dix']].idxmin(axis=1)
min_name_pd
for any_index, value in enumerate(min_name_pd):
    prediction_pd.loc[any_index,'predicted_result'] = value
prediction_pd<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13520388499190628, 'fraction_numerical': 0.05002697911046019, 'mean_word_length': 4.158648111332008, 'pattern_counts': {'":': 1, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '375913', 'license': 'MIT', 'n_tokens_mistral': 5455, 'n_tokens_neox': 5019, 'n_words': 1146, 'path': '.ipynb_checkpoints/97_Automation_SLOT_Training_Model-checkpoint.ipynb', 'repo_name': 'wolfmib/ja_ocr_lab', 'size': '12973'}