<jupyter_start><jupyter_text>get original data that was fetched from ghtorrent Original was fetched from ghtorrent - all repos with (check previous notebook):     - Python as a main language,      - Updated in 2019,      - Not deleted     - Not forkedThe output of this notebook is to get all repos that has a readme file and has at least 50 watchers - the output of this notebook is saved here:*data/processed/repositories_with-readme.csv*<jupyter_code>import pandas as pd
import time
import numpy as np
import os
import os.path


from github import Github


data_path = '../../data/ghtorrent_python_notdeleted_repos_updated2019/repositories.csv'

    
original = pd.read_csv(data_path, na_values="\\N", sep=',')
repos = original.copy()
print(len(original))
print(len(original[original['watchers'] >  10]))
print(len(original[original['watchers'] >  50]))
print(len(original[original['watchers'] >  500]))
repos = repos[repos['watchers'] >  50]
repos= repos[['id', 'url', 'owner_id', 'name', 'description', 'language',
       'created_at',   'updated_at',  'watchers']]
repos.head()<jupyter_output><empty_output><jupyter_text>Fetch readme files for all repos with at least 50 watchers  PLEASE DO NOT RUN THIS CODE. The processed repos are already saved under *data/processed* and the readme files are saved under *data/readme_files*<jupyter_code>def add_save_readme(row, github):
    folder = "../../data/readme_files"
    row['has_readme'] = False
    try:
        if row['url'] is None or len(row['url'].strip()) == 0:            
            row['readme_stat'] = 'no_url'
            return row
        repo_fullname = row['url'].split('/')[-2:]
        # debug


        if repo_fullname[0].find(".") > -1 or repo_fullname[1].find(".") > -1:
            row['readme_name'] = 'many_dots'

        f_name = '{}/{}.md'.format(folder, '.'.join(repo_fullname))
        
        if os.path.isfile(f_name):
            row['has_readme'] = True
            row['readme_file_name'] ='.'.join(row['url'].split('/')[-2:])
            with open(f_name, 'r', encoding='utf-8', errors='backslashreplace') as file:
                data = file.read()
                row['readme_words_num'] = 0
                if len(data) > 0 : 
                    data.replace('\n', ' ')
                    row['readme_words_num'] = len(data.split(' '))
            
        else:    
            current_repo = github.get_repo('/'.join(repo_fullname))
            readme = current_repo.get_readme()

            readme_content = readme.decoded_content.decode("utf-8")
            row['readme_words_num'] = len(readme_content.split())

            if row['readme_words_num']> 0 :
                with open(f_name, 'w', encoding="utf-8", errors='backslashreplace') as w:
                    w.write(readme.decoded_content.decode("utf-8") )
                row['has_readme'] = True 
                row['readme_file_name'] ='.'.join(row['url'].split('/')[-2:])

                
    except Exception as e:
        print("error: {} for repo {}".format(e , row['url']))
        row['readme_stat'] = "error: {} ".format(e)

    return row
    

# Github api blocks us after 5000 attepmts - so the following function is called repeatidly until all files are fetched
def retry_save_readme(row, github):
    folder = "../../data/readme_files"
    if row['has_readme'] == False and  row['readme_stat'].find('API rate limit exceeded') >-1:
        try:

            if row['url'] is None or len(row['url'].strip()) == 0:            
                row['readme_stat'] = 'no_url'
                return row
            repo_fullname = row['url'].split('/')[-2:]
            # debug


            if repo_fullname[0].find(".") > -1 or repo_fullname[1].find(".") > -1:
                row['readme_name'] = 'many_dots'

            f_name = '{}/{}.md'.format(folder, '.'.join(repo_fullname))


            if os.path.isfile(f_name):
                row['has_readme'] = True
                with open(f_name, 'r', encoding='utf-8', errors='backslashreplace') as file:
                    data = file.read()
                    row['readme_words_num'] = 0
                    if len(data) > 0 : 
                        data.replace('\n', ' ')
                        row['readme_words_num'] = len(data.split(' '))

            else:    
                current_repo = github.get_repo('/'.join(repo_fullname))
                readme = current_repo.get_readme()

                time.sleep(0.5) 
                readme_content = readme.decoded_content.decode("utf-8")
                row['readme_words_num'] = len(readme_content.split())

                if row['readme_words_num']> 0 :
                    with open(f_name, 'w', encoding="utf-8") as w:
                        w.write(readme.decoded_content.decode("utf-8") )
                    row['has_readme'] = True 
        except Exception as e:
            #print("error: {} for repo {}".format(e , row['url']))
            row['readme_stat'] = "error: {} ".format(e)


    return row
    

github = Github(os.getenv('PAT'))
#repos = repos.apply(add_save_readme, axis=1, args=(github, ))

#repos = repos.apply(retry_save_readme, axis=1, args=(github, ))<jupyter_output><empty_output><jupyter_text>Removing duplicates<jupyter_code># Check for duplicate
urls_value_counts_df = (repos['url'].value_counts().to_frame())
duplicate_urls = urls_value_counts_df[urls_value_counts_df['url']>1].index.tolist()
duplicate_urls
repos.sort_values(by=['url', 'watchers'], ascending=False, inplace=True)
repos.drop_duplicates('url', keep='first', inplace=True)<jupyter_output><empty_output><jupyter_text>Number of fetched readme after removing duplicates<jupyter_code>repos['has_readme'].value_counts()<jupyter_output><empty_output><jupyter_text>Type of errors for failed readme files<jupyter_code>repos[repos['has_readme']==False]['readme_stat'].value_counts().to_frame()<jupyter_output><empty_output><jupyter_text>Saving the repo info that has readme files to *'data/processed/repositories_with-readme.csv'*<jupyter_code>repos_with_readme = repos[repos['has_readme'] == True]
#repos_with_readme.to_csv('../../data/processed/repositories_with-readme.csv', encoding="utf-8")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1496697277267601, 'fraction_numerical': 0.009666505558240696, 'mean_word_length': 2.748792270531401, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '246430', 'license': 'MIT', 'n_tokens_mistral': 2077, 'n_tokens_neox': 1952, 'n_words': 398, 'path': 'code/0_data-preparation/2_fetching-readme.ipynb', 'repo_name': 'DLR-SC/repository-synergy', 'size': '6207'}