<jupyter_start><jupyter_text>remove all photos with more than one face in them<jupyter_code>multi_face = df_merge.loc[df_merge.faceid!='FaceId-0','url']
multi_face = multi_face.unique().tolist()
df_merge_one = df_merge.loc[~df_merge.url.isin(multi_face),:]
df_merge_one.shape[0]/df_merge.shape[0]
df_merge_one['name_ct'] = df_merge_one.groupby('MID').cumcount()
df_merge_one['save_nm'] = df_merge_one.short + '-' + df_merge_one.name_ct.astype(str).str.zfill(4)
df_merge_one.head()
many_ct = df_merge_one.loc[df_merge_one.name_ct==30,'MID'].tolist()
df_many = df_merge_one.loc[df_merge_one.MID.isin(many_ct),:]
df_many.shape
for i,v in df_many.iterrows():
    savePath = os.path.join(PATH,'images',f'{v.save_nm}.jpg')
    with open(savePath, 'wb') as f:
        f.write(base64.b64decode(v.face_data))  
df_save = df_many.loc[:,['MID','url','short','name_ct','save_nm']]
df_save.to_csv(f'{PATH}msceleb_img_ref.csv',index=False)
curr_name =''
name_ct = 0
for i,v in df_merge_one.iterrows():
    if v.short != curr_name:
        name_ct = 0
        curr_name = v.short
    else:
        name_ct += 1
    name_ct_str = str(name_ct).zfill(3)
    savePath = os.path.join(PATH,'images',f'{v.short}_{name_ct_str}.jpg')
    with open(savePath, 'wb') as f:
        f.write(base64.b64decode(v.face_data))   
row_num=1000
row = df.iloc[row_num,:]
MID, imgSearchRank, faceID, data = row[0], row[1], row[4], base64.b64decode(row[6])
savePath = "{}-{}.jpg".format(imgSearchRank, faceID)
with open(savePath, 'wb') as f:
    f.write(data)
savePath<jupyter_output><empty_output><jupyter_text>Correct for name counting at the beginning and end of every file<jupyter_code>df_short = pd.read_table(f'{PATH}FaceImageCroppedWithOutAlignment.tsv', nrows=2*10**3, skiprows=(10**6 - 10**3), delimiter='\t', names=['MID','rank','url','page_url','faceid','FaceRectangle','face_data'])
df_short.head()<jupyter_output><empty_output><jupyter_text>Remove first and last person for list<jupyter_code>MID_ignore = [df_short.MID[0], df_short.MID[df_short.shape[0]-1]]
df_short = df_short.loc[~df_short.MID.isin(MID_ignore),:]
df_short.shape
df_merge = df_short.merge(df_names_en,how='left')
multi_face = df_merge.loc[df_merge.faceid!='FaceId-0','url']
multi_face = multi_face.unique().tolist()
df_merge_one = df_merge.loc[~df_merge.url.isin(multi_face),:]
df_merge_one.shape[0]/df_merge.shape[0]
curr_name =''
name_ct = 0
for i,v in df_merge_one.iterrows():
    if v.short != curr_name:
        name_ct = 0
        curr_name = v.short
    else:
        name_ct += 1
    name_ct_str = str(name_ct).zfill(3)
    savePath = os.path.join(PATH,'images',f'{v.short}_{name_ct_str}.jpg')
    with open(savePath, 'wb') as f:
        f.write(base64.b64decode(v.face_data))<jupyter_output><empty_output><jupyter_text>Create a list of all the names<jupyter_code>1058604
df_nrw2 = pd.read_table(f'{PATH}FaceImageCroppedWithOutAlignment.tsv', nrows=(5*10**5+58604), skiprows=5*10**5, delimiter='\t', names=['MID','url','faceid'],usecols=[0,2,4])
df_nrw2.head()
df_merge2 = df_nrw2.merge(df_names_en,how='left')
df_merge2.head()
multi_face = df_merge2.loc[df_merge2.faceid!='FaceId-0','url']
multi_face = multi_face.unique().tolist()
df_merge_one2 = df_merge2.loc[~df_merge2.url.isin(multi_face),:]
df_merge_one2.shape[0]/df_merge2.shape[0]
df_merge_one2.shape
nm_ct_vct2 = nm_ct_vct
nm_ct = -1
cur_nm = ''
# nm_ct_vct = []
for i,v in df_merge_one2.iterrows():
    if v.short == cur_nm:
        nm_ct += 1
    else:
        cur_nm = v.short
        nm_ct = 0
    nm_ct_vct.append(nm_ct)
df_merge_one.shape[0] + df_merge_one2.shape[0]
len(nm_ct_vct)
len(nm_ct_vct)
df_merge_comb = df_merge_one.append(df_merge_one2, ignore_index=True)
df_merge_comb['name_ct'] = nm_ct_vct
df_merge_comb.shape
df_merge_comb.drop(columns=['faceid','name'],inplace=True)
df_merge_comb.head()
df_merge_comb['name_ct'] = df_merge_comb.groupby('MID').cumcount()
df_merge_comb.to_csv(f'{PATH}img_files.csv',index=False)
(df_merge_comb.short=='Scott_Baio-GB').sum()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16898954703832753, 'fraction_numerical': 0.02538576406172225, 'mean_word_length': 6.088183421516755, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '119953', 'license': 'MIT', 'n_tokens_mistral': 1753, 'n_tokens_neox': 1695, 'n_words': 216, 'path': 'notebooks/image_extract.ipynb', 'repo_name': 'MichoelSnow/data_science', 'size': '4018'}