<jupyter_start><jupyter_text>Library Load<jupyter_code>import os
import pickle<jupyter_output><empty_output><jupyter_text>SynthText<jupyter_code># for SegLink and TextBoxes++

from data_synthtext import GTUtility
gt_util = GTUtility('data/SynthText/SynthText/', polygon=True)

file_name = 'gt_util_synthtext_seglink.pkl'
pickle.dump(gt_util, open(file_name,'wb'))
with open(file_name, 'rb') as f:
    gt_util = pickle.load(f)

print(gt_util)
from thirdparty import get_image_size
import glob

for img_path in glob.glob('data/K-SIGN/HighQuality/*.jpg'):
    width = get_image_size.get_image_size(img_path)[0]
    height = get_image_size.get_image_size(img_path)[1]
    
    if width <512 and height < 512:
        print("width : {}, height = {}".format(width,height))
        print("w,h")
    elif width < 512:
        print("width : {}, height = {}".format(width,height))
        print("w")
    elif height < 512:
        print("width : {}, height = {}".format(width,height))
        print("h")<jupyter_output><empty_output><jupyter_text>COCO-Text<jupyter_code>from data_cocotext import GTUtility

gt_util_train = GTUtility('data/COCO-TEXT', polygon = True, validation = False)
gt_util_val = GTUtility('data/COCO-TEXT', polygon = True, validation = True)

print(gt_util_train)
print(gt_util_val)<jupyter_output>Background              0
Text                68736

images              14708
objects             68736
per image            4.67
no annotation           0

Background              0
Text                15679

images               3424
objects             15679
per image            4.58
no annotation           0<jupyter_text>coco-text pickle<jupyter_code>#Trainset 피클파일 
file_name = 'gt_util_cocotext_train.pkl'
pickle.dump(gt_util_train, open(file_name,'wb'))

#Validation 피클파일
file_name = 'gt_util_cocotext_val.pkl'
pickle.dump(gt_util_val, open(file_name,'wb'))<jupyter_output><empty_output><jupyter_text>K-Sign data   ** quality = high / mid / low에 대한 파라미터 값  ** 이미지 데이터가 있는 경로를 준다 (이때도 퀄리티 값에 따라 다른 경로를 준다)<jupyter_code>from data_KSign import GTUtility

gt_util_high = GTUtility('data/K-SIGN/Annotation/', quality='cr')
file_name = 'gt_util_ksign_cr.pkl'
pickle.dump(gt_util_high, open(file_name,'wb'))
print(gt_util_high)
print(gt_util_mid)
print(gt_util_low)
#HighQuality 피클파일 
file_name = 'gt_util_ksign_high.pkl'
pickle.dump(gt_util_high, open(file_name,'wb'))

#MidQuality 피클파일 
file_name = 'gt_util_ksign_mid.pkl'
pickle.dump(gt_util_mid, open(file_name,'wb'))

#LowQuality 피클파일 
file_name = 'gt_util_ksign_low.pkl'
pickle.dump(gt_util_low, open(file_name,'wb'))
import pickle
from data_KSign import GTUtility
gt_util_ai_cr = GTUtility('data/K-SIGN/Annotation/', quality = 'cr')
from data_KSign_seglink import GTUtility
gt_util_high = GTUtility('data/K-SIGN/HighQuality/', quality = 'high')
from data_cracker import GTUtility
gt_util_cr = GTUtility('data/KSIGNBOARD/')
gt_util_merge = gt_util_cr.merge(gt_util_ai_cr)
gt_util_merge = gt_util_merge.merge(gt_util_high)

file_name = 'gt_util_merge_data.pkl'
pickle.dump(gt_util_merge, open(file_name,'wb'))<jupyter_output>/home/ina/anaconda3/envs/ina/lib/python3.7/site-packages/PIL/TiffImagePlugin.py:768: UserWarning: Possibly corrupt EXIF data.  Expecting to read 8 bytes but only got 2. Skipping tag 41493
  " Skipping tag %s" % (size, len(data), tag))
/home/ina/anaconda3/envs/ina/lib/python3.7/site-packages/PIL/TiffImagePlugin.py:768: UserWarning: Possibly corrupt EXIF data.  Expecting to read 8 bytes but only got 0. Skipping tag 41988
  " Skipping tag %s" % (size, len(data), tag))<jupyter_text>AI CR Data<jupyter_code>from data_KSign import GTUtility
import pickle

gt_util_ai_cr = GTUtility('data/K-SIGN/Annotation/', quality = 'cr')
gt_util_high = GTUtility('data/K-SIGN/HighQuality/', quality = 'high')

print(gt_util_ai_cr)
print(gt_util_high)
from data_KSign import GTUtility
import pickle

gt_util_ai_cr = GTUtility('data/K-SIGN/Annotation/', quality = 'cr')
gt_util_high = GTUtility('data/K-SIGN/HighQuality/', quality = 'high')

print(gt_util_ai_cr)
print(gt_util_high)<jupyter_output>Background              0
Text                  250

images                141
objects               250
per image            1.77
no annotation           0

Background              0
Text                  514

images                294
objects               514
per image            1.75
no annotation           0<jupyter_text>Merge<jupyter_code>from data_cracker import GTUtility

gt_util_cr = GTUtility('data/KSIGNBOARD/')

print(gt_util_cr)
from data_cracker import GTUtility

gt_util_cr = GTUtility('data/KSIGNBOARD/')

print(gt_util_cr)
gt_util_merge = gt_util_cr.merge(gt_util_ai_cr)
gt_util_merge = gt_util_merge.merge(gt_util_high)
print(gt_util_merge)
import pickle
file_name = 'gt_util_merge_data_v2.pkl'
pickle.dump(gt_util_merge, open(file_name,'wb'))<jupyter_output><empty_output><jupyter_text>cracker's<jupyter_code>import os
from data_cracker import GTUtility

gt_util_cr = GTUtility('data/KSIGNBOARD/')
print(gt_util_cr)
import os
from data_cracker import GTUtility

gt_util_cr = GTUtility('data/KSIGNBOARD/')
print(gt_util_cr)
import pickle
file_name = 'gt_util_ksignboard.pkl'
pickle.dump(gt_util_cr, open(file_name,'wb'))
%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.font_manager as fm

plt.rcParams['axes.labelsize'] = 18
plt.rcParams['xtick.labelsize'] = 16
plt.rcParams['ytick.labelsize'] = 16

#matplotlib 한글 폰트 설정

path = '/usr/share/fonts/truetype/nanum/NanumGothicEco.ttf'
font_name = fm.FontProperties(fname=path, size=14).get_name()
print(font_name)
plt.rc('font', family=font_name)
mpl.rcParams['axes.unicode_minus'] = False
fm._rebuild()
text = gt_util_cr.text
from itertools import chain
text = list(chain(*text))
key = []
value = []
for k,v in count.most_common():
    key.append(k)
    value.append(v)
count.most_common
import matplotlib.pyplot as plt

plt.figure(figsize=(20,20))
plt.bar(key[1:], value[1:])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11405702851425713, 'fraction_numerical': 0.020343505085876273, 'mean_word_length': 4.553703703703704, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '116486', 'license': 'MIT', 'n_tokens_mistral': 2378, 'n_tokens_neox': 2359, 'n_words': 418, 'path': 'load_datasets.ipynb', 'repo_name': 'PrimadonnaGit/tpgr_ssd_detectors', 'size': '5997'}