<jupyter_start><jupyter_text>Choose One folder and EnsambleModeMode types* "mean of logits, then softmax"* "softmax of logits, then mean"* "sigmoid, then mean"* "sigmoid, then median"* "sigmoid, then geo mean"<jupyter_code># folder, EnsambleMODE = "MULTI_CLASS_VERSION/", "mean of logits, then softmax"
folder, EnsambleMODE = "MULTI_LABEL_VERSION/", "sigmoid, then median"
model_paths = sorted(glob.glob(folder+"/*.best_val.h5"))
model_paths
config = json.load(open(folder+"/model_config.json",'r'))
config
models = list(map(tf.keras.models.load_model, model_paths))<jupyter_output><empty_output><jupyter_text>Show ith_data<jupyter_code>import IPython.display as idp
x, sr, label, remark = data_utils.load_wav("private_test/private_00017.wav")

data_utils.wav_to_mel(x, sr, display=True)

idp.Audio(x,rate=sr, autoplay=False)<jupyter_output><empty_output><jupyter_text>Create Private/Public Test Data<jupyter_code># private_wav_names = sorted(glob.glob("./private_test/*.wav"))
# public_wav_names = sorted(glob.glob("./public_test/*.wav"))
# def map_fun(path):
#   return data_utils.process(path)["mel"]
# %%time
# import multiprocessing
# pool_map = multiprocessing.Pool(processes=multiprocessing.cpu_count())
# mels = pool_map.map(map_fun, public_wav_names)
# data = [*zip(public_wav_names, mels)]
# pickle.dump(data, open("public_test.pickle","wb"))
# %%time
# import multiprocessing
# pool_map = multiprocessing.Pool(processes=multiprocessing.cpu_count())
# mels = pool_map.map(map_fun, private_wav_names)
# data = [*zip(private_wav_names, mels)]
# pickle.dump(data, open("private_test.pickle","wb"))<jupyter_output><empty_output><jupyter_text>Start Predict<jupyter_code>public_test_from_pickle = pickle.load(open("public_test.pickle", "rb"))
private_test_from_pickle = pickle.load(open("private_test.pickle", "rb"))
PUBLIC_INPUTs = []
PUBLIC_INPUT_NAMES = []
PRIVATE_INPUTS = []
PRIVATE_INPUT_NAMES = []
for wav_name, mel in tqdm(public_test_from_pickle):
  PUBLIC_INPUTs.append(tf.transpose(mel))
  PUBLIC_INPUT_NAMES.append(wav_name)

for wav_name, mel in tqdm(private_test_from_pickle):
  PRIVATE_INPUTS.append(tf.transpose(mel))
  PRIVATE_INPUT_NAMES.append(wav_name)
INPUT_TENSORS_PUBLIC = tf.stack(PUBLIC_INPUTs)
INPUT_TENSORS_PRIVATE = tf.stack(PRIVATE_INPUTS)
INPUT_TENSORS = tf.concat([INPUT_TENSORS_PUBLIC, INPUT_TENSORS_PRIVATE], axis=0)
all_names = PUBLIC_INPUT_NAMES+PRIVATE_INPUT_NAMES
ret = []
for m in tqdm(models):
  ret.append(m.predict(INPUT_TENSORS))
np.array(ret).shape
if EnsambleMODE == "mean of logits, then softmax":
  probiblity = tf.math.softmax(tf.reduce_mean(tf.stack(ret), axis=0),axis=-1)
elif EnsambleMODE == "softmax of logits, then mean":
  probiblity = tf.reduce_mean(tf.math.softmax(tf.stack(ret), axis=-1), axis=0)
elif EnsambleMODE == "sigmoid, then mean":
  probiblity = tf.reduce_mean(tf.math.sigmoid(tf.stack(ret)), axis=0)
elif EnsambleMODE == "sigmoid, then median":
  probiblity = np.median(tf.math.sigmoid(tf.stack(ret)), axis=0)
elif EnsambleMODE == "sigmoid, then geo mean":
  probiblity = tf.reduce_prod(tf.math.sigmoid(tf.stack(ret)), axis=0)**(1/len(ret))
else:
  raise

EnsambleMODE
probiblity.shape
import pandas as pd
df_prob_data = np.array(probiblity)
df_prob_data.shape
only_file_names = list(
  map(
    lambda string:string.split("/")[-1].split(".")[0],
    all_names,
  ),
)
only_file_names[:5]
len(only_file_names)
df_data = np.concatenate(
  [
    np.array(only_file_names).reshape(-1,1),
    df_prob_data
  ],axis=-1
)
df = pd.DataFrame(data=df_data, columns=['Filename', 'Barking', 'Howling', 'Crying', 'COSmoke', 'GlassBreaking','Other'])
df = df.sort_values(["Filename"])
df.head(20)
folder
csv_path = folder+"/"+EnsambleMODE+"_privite_test.csv"
csv_path
df.to_csv(
  csv_path,
  index=False,
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.150764364786505, 'fraction_numerical': 0.006062203479177649, 'mean_word_length': 7.433333333333334, 'pattern_counts': {'":': 5, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '468295', 'license': 'MIT', 'n_tokens_mistral': 1530, 'n_tokens_neox': 1492, 'n_words': 230, 'path': 'go_test.ipynb', 'repo_name': 'mistake0316/Audio_Tagging_TF_1D_CNN', 'size': '3794'}