<jupyter_start><jupyter_text>Combine page id and redirect to correct the links<jupyter_code>import pandas as pd
import time
# Load the data
start_t = time.time()
redirect_df = pd.read_pickle('/home/benjamin/wikipedia/Wikipedia/redirect_parsed0.gz')
print('redirect loaded')
pageid_df = pd.read_pickle('/home/benjamin/wikipedia/Wikipedia/pageid_parsed0.gz')
print('page loaded')
pagelinks_df = pd.read_pickle('/home/benjamin/wikipedia/Wikipedia/pagelinks_parsed0.gz')
print('pagelinks loaded')
print('Load Time:', time.time() - start_t)
pagelinks_df.head()<jupyter_output><empty_output><jupyter_text>Change source Ids to titles in redirect<jupyter_code>print(pageid_df.columns)
pageid_df.set_index('pageId', inplace=True)
print('Old name',redirect_df.columns)
redirect_df.rename(columns={'pageId' : 'initial_id', 'pageTitle' : 'fix_target'}, inplace=True)
print('New name',redirect_df.columns)
redirect_df = redirect_df.join(pageid_df, on='initial_id', how='left')
redirect_df.rename(columns={'pageTitle' : 'initial_target'}, inplace=True)
redirect_df = redirect_df.drop('initial_id', axis=1)
redirect_df = redirect_df.set_index('initial_target')<jupyter_output><empty_output><jupyter_text>Check for NaN values<jupyter_code>len(redirect_df)
redirect_df.isnull().sum()
redirect_df[redirect_df.isnull().any(axis=1)]
#redirect_df = redirect_df.reindex(columns=['initial_id','initial_target','fix_target'])
redirect_df.head()<jupyter_output><empty_output><jupyter_text>Replacing target in pagelinks by the redirected one<jupyter_code>pagelinks_df.head()
#pagelinks_df.set_index('sourceId', inplace=True)
len(pagelinks_df)
old_size = len(pagelinks_df)
pagelinks_df.rename(columns={'pageTitle' : 'target'}, inplace=True)
pagelinks_df = pagelinks_df.join(pageid_df, on='pageId', how='outer')
pagelinks_df.rename(columns={'pageTitle' : 'source'}, inplace=True)
new_size = len(pagelinks_df)
print(old_size,new_size)
pagelinks_df.isnull().sum()
pagelinks_df[pagelinks_df.isnull().any(axis=1)]
# Id list of unique pages that have an id but do not have a title
pagelinks_df['pageId'][pagelinks_df['source'].isnull()].unique()
pagelinks_df.head()
pagelinks_df['fix_target'] = pagelinks_df['target'].copy()
pagelinks_df = pagelinks_df.set_index('target')
#redirect_df.rename(columns={'r_t_T' : 'fix_target'}, inplace=True)
pagelinks_df.update(redirect_df)
pagelinks_df.reset_index(inplace = True)
pagelinks_df = pagelinks_df.drop('pageId', axis=1)
pagelinks_df['is_redirect'] = (pagelinks_df['target'] != pagelinks_df['fix_target'])
pagelinks_df = pagelinks_df.reindex(columns=['source','target','fix_target','is_redirect'])
print(len(pagelinks_df))
pagelinks_df.head()
len(pagelinks_df)
pagelinks_df.describe()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14856933235509906, 'fraction_numerical': 0.0025678650036683784, 'mean_word_length': 11.013215859030836, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '130235', 'license': 'Apache-2.0', 'n_tokens_mistral': 1011, 'n_tokens_neox': 1014, 'n_words': 139, 'path': 'test_joining_df.ipynb', 'repo_name': 'bricaud/WikipediaTools', 'size': '2726'}