<jupyter_start><jupyter_text>Load CSV<jupyter_code>df = pd.read_csv("train_relationships.csv")<jupyter_output><empty_output><jupyter_text>Split Train X,y<jupyter_code>parents_path =[("./train/"+row)for row in df["p1"]]
childs_path =[("./train/"+row)for row in df["p2"]]
df.iloc[121:133]<jupyter_output><empty_output><jupyter_text>Product n of Parent * child<jupyter_code>dataset_path=[]


def prep_load_map(parents_path,childs_path):
    trainX=[]
    trainY=[]
    for  x,y  in zip(parents_path,childs_path):
        parents = [ ".\\"+str(filename) for filename  in pathlib.Path(x).glob('*.jpg')]
        childs = [ ".\\"+str(filename) for filename  in pathlib.Path(y).glob('*.jpg')]
        family = list(product(parents,childs))
        if len(parents)==0 or len(childs)==0:
            print("NOT FOUND parent {0} child {1}   {2} and {3}".format(len(parents),len(childs),str(x),str(y)))
            continue
            # data set link broken
        parent , child = zip(*family)

        #print("\n")
        trainX.extend(parent)
        trainY.extend(child)
    print(len(trainX), len(trainY))
    return trainX,trainY


trainX,trainY = prep_load_map(parents_path,childs_path)<jupyter_output>NOT FOUND parent 4 child 0   ./train/F0039/MID1 and ./train/F0039/MID3
NOT FOUND parent 13 child 0   ./train/F0041/MID1 and ./train/F0041/MID7
NOT FOUND parent 0 child 2   ./train/F0041/MID3 and ./train/F0041/MID4
NOT FOUND parent 0 child 0   ./train/F0041/MID5 and ./train/F0041/MID7
NOT FOUND parent 0 child 5   ./train/F0051/MID5 and ./train/F0051/MID1
NOT FOUND parent 0 child 1   ./train/F0051/MID5 and ./train/F0051/MID3
NOT FOUND parent 0 child 4   ./train/F0051/MID5 and ./train/F0051/MID4
NOT FOUND parent 0 child 6   ./train/F0058/MID2 and ./train/F0058/MID4
NOT FOUND parent 0 child 1   ./train/F0058/MID2 and ./train/F0058/MID5
NOT FOUND parent 0 child 14   ./train/F0058/MID2 and ./train/F0058/MID1
NOT FOUND parent 8 child 0   ./train/F0070/MID1 and ./train/F0070/MID3
NOT FOUND parent 8 child 0   ./train/F0070/MID2 and ./train/F0070/MID3
NOT FOUND parent 0 child 2   ./train/F0070/MID3 and ./train/F0070/MID4
NOT FOUND parent 0 child 3   ./train/F0070/MID3 and ./train/F0070/MID5
NOT [...]<jupyter_text>Shuffle Dataset<jupyter_code>temp = list(zip(trainX, trainY))
print([i for i in temp[30]])
np.random.shuffle(temp)
print([i for i in temp[30]])
trainX,trainY =zip(*temp)<jupyter_output>['.\\train\\F0002\\MID1\\P00012_face2.jpg', '.\\train\\F0002\\MID3\\P00017_face1.jpg']
['.\\train\\F0754\\MID3\\P07965_face2.jpg', '.\\train\\F0754\\MID5\\P07951_face3.jpg']<jupyter_text>Observe Random Sample<jupyter_code>plt.figure(figsize=(15,15))
col = 1
for n in range(4):
    index = random.choice(range(len(trainX)))
    a = trainX[index]
    b = trainY[index]
    plt.subplot(2,4,col)
    a = cv2.cvtColor(cv2.imread(a ), cv2.COLOR_BGR2RGB)
    plt.imshow(a)
    plt.grid(False)
    plt.xticks([])
    plt.yticks([])
    plt.xlabel("A")
    col=col+1

    b = cv2.cvtColor(cv2.imread(b ), cv2.COLOR_BGR2RGB)
    plt.subplot(2,4,col)
    plt.imshow(b)
    plt.grid(False)
    plt.xticks([])
    plt.yticks([])
    plt.xlabel("B")
    col=col+1<jupyter_output><empty_output><jupyter_text>Reshape (n,2) to n,3<jupyter_code>temp = np.array(list(zip(trainX, trainY)))
print(temp.shape)
true_positive = np.append(temp, np.ones((temp.shape[0],1), dtype=int), axis=1)<jupyter_output>(165179, 2)<jupyter_text>generate negative (false positive)<jupyter_code>parents_path.sort(reverse = True) 
n_trainX,n_trainY = prep_load_map(parents_path,childs_path)
temp = np.array(list(zip(n_trainX, n_trainY)))
false_positive = np.append(temp, np.zeros((temp.shape[0],1), dtype=int), axis=1)
ds =  np.append(true_positive,false_positive, axis=0)
ds.shape
np.random.shuffle(ds) ## shuffle

import tensorflow as tf
from tensorflow.keras.layers import Input, Concatenate, GlobalMaxPool2D,Dropout,Multiply,Dense,GlobalAvgPool2D 
from tensorflow.keras.applications.resnet50 import ResNet50, preprocess_input
from tensorflow.keras.models import Model
def model():
    input_1 = Input(shape=(224, 224, 3))
    input_2 = Input(shape=(224, 224, 3))
    base_model = ResNet50(weights='imagenet', include_top=False)
    for x in base_model.layers[:-3]:
        x.trainable = True
    l1 =base_model(input_1)
    l2 =base_model(input_2)
    l1 = Concatenate(axis=-1)([GlobalMaxPool2D()(l1), GlobalAvgPool2D()(l1)])
    l2 = Concatenate(axis=-1)([GlobalMaxPool2D()(l2), GlobalAvgPool2D()(l2)])
    #l3 = Subtract()([l1, l2])
    l3 = tf.subtract(l1, l2)
    l3 = Multiply()([l3, l3])
    l4 = Multiply()([l1, l2])
    l4 = Concatenate(axis=-1)([l4, l3])
    
    l4 = Dense(100, activation="relu")(l4)
    l4 = Dropout(0.01)(l4)
    out = Dense(1, activation="sigmoid")(l4)

    model = Model(input=[input_1, input_2],output= out)

    model.compile(loss="binary_crossentropy", metrics=['acc'], optimizer=Adam())

    model.summary()

    return model
model = model()
model.fit([ds[],], labels, epochs=10, batch_size=32)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16152775014919435, 'fraction_numerical': 0.06823154963198727, 'mean_word_length': 4.104568527918782, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '337530', 'license': 'MIT', 'n_tokens_mistral': 2199, 'n_tokens_neox': 2021, 'n_words': 370, 'path': 'train_stash_REV2.ipynb', 'repo_name': 'Telexine/kinship-classifier-Playground-Prediction-Competition-', 'size': '5027'}