<jupyter_start><jupyter_text>check phase 2 output<jupyter_code>import gzip
kwaku_fpath = '/home/shaypalachy/Dropbox/data/twikwak17/phase_2_output/kwak10_unames.txt.gz'
kwu = gzip.open(kwaku_fpath, 'rt')
lines = []
for i in range(20):
    lines.append(kwu.readline())
lines
kwak_map_fpath = '/home/shaypalachy/Dropbox/data/twikwak17/phase_2_output/kwak10_uname_to_id.txt.gz'
kwmap = gzip.open(kwak_map_fpath, 'rt')
lines = []
for i in range(20):
    lines.append(kwmap.readline())
lines<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13980582524271845, 'fraction_numerical': 0.02912621359223301, 'mean_word_length': 6.478260869565218, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '149759', 'license': 'MIT', 'n_tokens_mistral': 234, 'n_tokens_neox': 224, 'n_words': 29, 'path': 'notebooks/twikwak17_phase2.ipynb', 'repo_name': 'shaypal5/gender_augmented_twitter_network', 'size': '515'}