<jupyter_start><jupyter_text>ini<jupyter_code>BASE_FOLDER = '../'
%run -i ..\utility\feature_extractor\JupyterLoad_feature_extractor.py
%run -i ..\utility\extractor_batch.py
print('-'*44)
%run -i .\extdia_v0
print('-'*44)
target_folder = r'\dataset\extdia_v0'
n_jobs = 7
def process_set(FileFindDict, FileCountLimit=None):
    extractor_batch(base_folder= BASE_FOLDER, 
                    target_folder=target_folder, 
                    extdia = extdia_v0, 
                    FileFindDict = FileFindDict,
                    n_jobs = n_jobs,
                    target_class_map = {'abnormal':1, 'normal': 0},
                    FileCountLimit = FileCountLimit,
                    datset_folder_from_base = 'dataset')<jupyter_output><empty_output><jupyter_text>Test Small set<jupyter_code>process_set({'SNR': '6dB','machine': 'pump','ID': ['00','02','04','06']},2)<jupyter_output>2020-04-29 23:13:22: Target folder will be: A:\Dev\NF_Prj_MIMII_Dataset\dataset\extdia_v0
2020-04-29 23:13:22: Extractor diagram is fof type: <class '__main__.extdia_v0'>
2020-04-29 23:13:22: --------------------------------------------
2020-04-29 23:13:22: Working on machinepart:pump SNR:6dB ID:00
2020-04-29 23:13:22: Files to process: 4
2020-04-29 23:13:22: multithread mode filling the queue<jupyter_text>Full Batch<jupyter_code>ts = time.time()

process_set({'SNR': '6dB','machine': 'pump','ID': ['00','02','04','06']}) 
process_set({'SNR': 'min6dB','machine': 'pump','ID': ['00','02','04','06']})

process_set({'SNR': '6dB','machine': 'fan','ID': ['00','02','04','06']})
process_set({'SNR': 'min6dB','machine': 'fan','ID': ['00','02','04','06']})

process_set({'SNR': '6dB','machine': 'slider','ID': ['00','02','04','06']})
process_set({'SNR': 'min6dB','machine': 'slider','ID': ['00','02','04','06']})

process_set({'SNR': '6dB','machine': 'valve','ID': ['00','02','04','06']})
process_set({'SNR': 'min6dB','machine': 'valve','ID': ['00','02','04','06']})


#process_set({'SNR': '0dB','machine': 'pump','ID': ['00','02','04','06']})
#process_set({'SNR': '0dB','machine': 'vave','ID': ['00','02','04','06']})
#process_set({'SNR': '0dB','machine': 'slider','ID': ['00','02','04','06']})
#process_set({'SNR': '0dB','machine': 'fan','ID': ['00','02','04','06']})


print('#####> total time needed: ' + str(np.round(time.time()-ts/60,1))

str(np.round(time.time()-ts/60,1))<jupyter_output><empty_output><jupyter_text>Verfication<jupyter_code>df_filename = 'pump6dB00_EDiaV0_pandaDisc.pkl'
df = pd.read_pickle(os.path.abspath(BASE_FOLDER+target_folder+'\\'+df_filename)
# simple check read back
#target_folder=r'\feature_extraction_diagrams\extdia_v0_testout2'
df = pd.read_pickle(os.path.abspath(BASE_FOLDER+target_folder+r'\pump6dB00_EDiaV0_pandaDisc.pkl'))
i = 1
op = 'MEL_den'
fp = os.path.abspath(BASE_FOLDER+df[op][i])
dd = pickle.load( open( fp , "rb" ))
print(dd[i]['para_dict']['wave_filepath'])
fe = feature_extractor_from_dict(dd[i], BASE_FOLDER)
fe.plot()
384/60<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.24724356832609423, 'fraction_numerical': 0.08018710324089542, 'mean_word_length': 5.594713656387666, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '606572', 'license': 'MIT', 'n_tokens_mistral': 1352, 'n_tokens_neox': 1187, 'n_words': 161, 'path': 'feature_extraction_diagrams/extdia_v0_batch.ipynb', 'repo_name': 'BA-HanseML/NF_Prj_MIMII_Dataset', 'size': '2993'}