<jupyter_start><jupyter_text>Checksum 1st degree<jupyter_code>first_degree_file = [f for f in files if f.startswith('1st')][0]
first_degree_file
print('Total records: ')
df_first = pd.read_csv(os.path.join(PATH, first_degree_file))
total = df_first.shape
total
print('Total active records: ')
total_active = df_first[df_first['active'] == 1].shape
total_active
print('Processed: ')
processed = df_first[df_first['processed']].shape
processed
assert total == processed
print('ALL RECORDS HAVE BEEN PROCESSED')<jupyter_output>ALL RECORDS HAVE BEEN PROCESSED<jupyter_text>Checksum 2nd degree<jupyter_code>second_degree_files = [f for f in files if f.startswith('2nd') and f.find('SKIPPED') == -1]
second_degree_files
second_degree_skipped = [f for f in files if f.startswith('2nd') and f.find('SKIPPED') != -1]
second_degree_skipped
second_degree_files
df = pd.concat([pd.read_csv(os.path.join(PATH, f)) for f in second_degree_files])
df.shape     
df.head()
print('Checksum total')
checksum_total = df['A_id'].unique().shape
checksum_total
print('Skipped records')

df_skipped = pd.concat([pd.read_csv(os.path.join(PATH, f)) for f in second_degree_skipped]) \
if len(second_degree_skipped) > 0 else pd.DataFrame()
    
df_skipped.shape
assert checksum_total[0] + df_skipped.shape[0] == total_active[0]
print('ALL ACTIVE RECORDS HAVE BEEN PROCESSED')
# df[['A_name', 'B_id']].groupby('A_name', axis=0).count().sort_values('B_id', ascending=False)['B_id'].sum()
# df[['A_name', 'B_id']].groupby('A_name', axis=0).count().sort_values('B_id', ascending=False).reset_index()<jupyter_output><empty_output><jupyter_text>Summary<jupyter_code>print('Hola, \n Tus datos pasaron el proceso de chequeo. Has finalizado tu trabajo.\n\n Resumen:')
print('\tTotal records: %.d' % total[0])
print('\tTotal active records: %.d - Skipped: %d' % (total_active[0], df_skipped.shape[0]))
print('\tTotal 2nd degree records: %.d' % df.shape[0])
df_first.loc[0, 'A_id']
# https://www.facebook.com/100005543827430/friends?source_ref=pb_friends_tl<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15673828125, 'fraction_numerical': 0.017578125, 'mean_word_length': 7.329268292682927, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '144431', 'license': 'MIT', 'n_tokens_mistral': 796, 'n_tokens_neox': 772, 'n_words': 149, 'path': 'Checksum process.ipynb', 'repo_name': 'santteegt/facebook-scraper', 'size': '2048'}