<jupyter_start><jupyter_text>Now, for the inter-GAN comparisons:<jupyter_code>lr_progan.score(X_stylegan_train, Y_stylegan_train), lr_progan.score(X_stylegan_val, Y_stylegan_val) 
print(
    np.mean([guess for guess, true in zip(lr_progan.predict(X_stylegan_train), Y_stylegan_train) if true == 1]),
    np.mean([1. - guess for guess, true in zip(lr_progan.predict(X_stylegan_train), Y_stylegan_train) if true == 0])
)
print('ProGAN is overconfident on StyleGAN quality')
lr_progan.score(X_began_train, Y_began_train), lr_progan.score(X_began_val, Y_began_val) 
lr_stylegan.score(X_progan_train, Y_progan_train), lr_stylegan.score(X_progan_val, Y_progan_val) 
print(
    np.mean([guess for guess, true in zip(lr_stylegan.predict(X_progan_train), Y_progan_train) if true == 1]),
    np.mean([1. - guess for guess, true in zip(lr_stylegan.predict(X_progan_train), Y_progan_train) if true == 0])
)
print('StyleGAN is overconfident on ProGAN quality')
lr_stylegan.score(X_began_train, Y_began_train), lr_stylegan.score(X_began_val, Y_began_val) 
lr_began.score(X_progan_train, Y_progan_train), lr_began.score(X_progan_val, Y_progan_val) 
lr_began.score(X_stylegan_train, Y_stylegan_train), lr_began.score(X_stylegan_val, Y_stylegan_val)<jupyter_output><empty_output><jupyter_text>And now to train on 2 datasets at a time<jupyter_code>lr_progan_stylegan = LogisticRegression().fit(X_progan_train + X_stylegan_train, Y_progan_train + Y_stylegan_train)
print(lr_progan_stylegan.score(X_progan_train + X_stylegan_train, Y_progan_train + Y_stylegan_train))
print(lr_progan_stylegan.score(X_progan_val + X_stylegan_val, Y_progan_val + Y_stylegan_val))

print('\nAnd now on BeGAN')
print(lr_progan_stylegan.score(X_began_train, Y_began_train))
print(lr_progan_stylegan.score(X_began_val, Y_began_val))

lr_progan_began = LogisticRegression().fit(X_progan_train + X_began_train, Y_progan_train + Y_began_train)
print(lr_progan_began.score(X_progan_train + X_began_train, Y_progan_train + Y_began_train))
print(lr_progan_began.score(X_progan_val + X_began_val, Y_progan_val + Y_began_val))

print('\nAnd now on stylegan')
print(lr_progan_began.score(X_stylegan_train, Y_stylegan_train))
print(lr_progan_began.score(X_stylegan_val, Y_stylegan_val))

lr_stylegan_began = LogisticRegression().fit(X_stylegan_train + X_began_train, Y_stylegan_train + Y_began_train)
print(lr_stylegan_began.score(X_stylegan_train + X_began_train, Y_stylegan_train + Y_began_train))
print(lr_stylegan_began.score(X_stylegan_val + X_began_val, Y_stylegan_val + Y_began_val))

print('\nAnd now on progan')
print(lr_stylegan_began.score(X_progan_train, Y_progan_train))
print(lr_stylegan_began.score(X_progan_val, Y_progan_val))

with open('expanded_progan_val_set.csv') as f:
    human_scores = {l.strip().split(',')[0][5:] : float(l.strip().split(',')[1]) for l in f.readlines()[1:] if 'fake' in l}

print(len(human_scores))
extended_progan_val_files = set(human_scores.keys())


progan_val_files_short = set([os.path.split(x)[-1] for x in progan_val_files])
print(len(extended_progan_val_files.intersection(progan_val_files_short)))
print(len(progan_val_files_short))
# for k,v in human_scores.items():
#     print(k,v)
expanded_progan_val_files_filtered = extended_progan_val_files.intersection(progan_val_files_short)
len(expanded_progan_val_files_filtered)
#vote of 1 is correctly spotting the fakeness, vote of 0 is being fooled
X_progan_val_extended = []
Y_progan_val_extended = []
for f in tqdm_notebook(expanded_progan_val_files_filtered):
    cur_feats = progan_features_by_file['imgs_by_label/progan_labeled/progan_labeled_imgs/' + f]
    
    human_score = human_scores[f]
    print(cur_feats, human_score)
    
    X_progan_val_extended.append(cur_feats)
    Y_progan_val_extended.append(human_score)
    

len(X_progan_val_extended), len(Y_progan_val_extended)
progan_prob_real = [x[0] for x in lr_progan.predict_proba(X_progan_val_extended)]
plt.scatter(progan_prob_real, Y_progan_val_extended)
plt.show()

np.corrcoef(progan_prob_real, Y_progan_val_extended)
scipy.stats.spearmanr(progan_prob_real, Y_progan_val_extended)
1. - lr_progan.score(X_progan_val_extended, (np.array(Y_progan_val_extended) > .5)*1)
stylegan_prob_real = [x[0] for x in lr_stylegan.predict_proba(X_progan_val_extended)]
plt.scatter(stylegan_prob_real, Y_progan_val_extended)
plt.show()

np.corrcoef(stylegan_prob_real, Y_progan_val_extended)
scipy.stats.spearmanr(stylegan_prob_real, Y_progan_val_extended)
img_yes_votes = defaultdict(int)
img_all_votes = defaultdict(int)

with open('stylegan_human_scores_cleaned.csv', newline='') as csvfile:
    spamreader = csv.reader(csvfile)
    for row in islice(spamreader,1,None):
        filename = row[0]
        if 'styleGAN' not in filename: continue
            
        is_correct = row[2]
        
        img_all_votes[filename] += 1
        if is_correct == 'True':
            img_yes_votes[filename] += 1
len(img_yes_votes.keys()), len(img_all_votes.keys())
stylegan_val_files
extended_stylegan_val_files = ['imgs_by_label/stylegan_labeled/stylegan_labeled_imgs/' + x[5:] for x in img_all_votes.keys()]
extended_stylegan_val_files = set(extended_stylegan_val_files).intersection(stylegan_val_files)
len(extended_stylegan_val_files)
#vote of 1 is correctly spotting the fakeness, vote of 0 is being fooled
X_stylegan_val_extended = []
Y_stylegan_val_extended = []
for f in tqdm_notebook(extended_stylegan_val_files):
    cur_feats = stylegan_features_by_file[f]
    
    short_f = 'fake_' + os.path.split(f)[-1]
    
    yes_votes = img_yes_votes[short_f] if short_f in img_yes_votes else 0
    all_votes = img_all_votes[short_f]
    #print(short_f)
    frac = yes_votes / all_votes
    
    #human_score = human_scores[f]
    print(cur_feats, frac)
    
    X_stylegan_val_extended.append(cur_feats)
    Y_stylegan_val_extended.append(frac)
    

len(X_stylegan_val_extended), len(Y_stylegan_val_extended)
stylegan_prob_real_2 = [x[0] for x in lr_stylegan.predict_proba(X_stylegan_val_extended)]
plt.scatter(stylegan_prob_real_2, Y_stylegan_val_extended)
plt.show()

np.corrcoef(stylegan_prob_real_2, Y_stylegan_val_extended)
scipy.stats.spearmanr(stylegan_prob_real_2, Y_stylegan_val_extended)
progan_prob_real_2 = [x[0] for x in lr_progan.predict_proba(X_stylegan_val_extended)]
plt.scatter(progan_prob_real_2, Y_stylegan_val_extended)
plt.show()

np.corrcoef(progan_prob_real_2, Y_stylegan_val_extended)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10026385224274406, 'fraction_numerical': 0.005897873661337886, 'mean_word_length': 7.390625, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '202145', 'license': 'MIT', 'n_tokens_mistral': 2721, 'n_tokens_neox': 2592, 'n_words': 366, 'path': 'distances/All_experiments.ipynb', 'repo_name': 'kolchinski/humanception-score', 'size': '6443'}