<jupyter_start><jupyter_text>Let's prepare the environment<jupyter_code>import datetime
from dateutil.relativedelta import relativedelta
import pandas as pd
import pprint
import numpy
#let's get the GitHub token so we can use the API
import getpass

try:
    from secret import GITHUB_TOKEN
except ModuleNotFoundError:
    GITHUB_TOKEN = getpass.getpass("Introduce your personal access token to acces the GitHub API: ")
#!pip install PyGithub
from github import Github

g = Github(GITHUB_TOKEN)
print(g)<jupyter_output><github.MainClass.Github object at 0x0000022D7B810848><jupyter_text>Let's define some functions<jupyter_code>def get_repo_description(repo):
    try:
        license = (repo.get_license().decoded_content.decode().split('\n')[0]).strip()
    except:
        license = ''
    return [repo.full_name, repo.created_at, repo.pushed_at, repo.stargazers_count, license]
def commits_processing(repo, bh_start, bh_end):
    
    '''
    input: repo PyGitHub object
        bh_start Initial date of the event
        bh_end final date of the event
    Output: 1. list with overall commit days 
            2. set with all authors that contributed
    
    '''

    commit_lst = repo.get_commits()

    total_commits = [0, 0, 0, 0] #before, during, after, total
    #authors = set()
    
    for item in commit_lst:
        #print(item.commit.author.name, item.commit.author.date)
        #authors.add(str(item.commit.author.name))
        time_Obj = item.commit.author.date
        if time_Obj < bh_start :
            total_commits[0] = total_commits[0] + 1
        elif bh_start <= time_Obj and time_Obj <= bh_end : 
            total_commits[1] = total_commits[1] + 1
        else :
            total_commits[2] = total_commits[2] + 1
        total_commits[3] = total_commits[3] + 1

        #timeStamp = timeObj.strftime("%d-%m-%Y")

    #return sorted(commit_days, key = lambda date: datetime.strptime(date, "%d-%m-%Y")) , authors
    return total_commits
def process_repo_file(file_path, start, end) :
    matrix = []
    with open(file_path) as file :
        for line in file :        
            try :             
                path = line.strip()
                repo = g.get_repo(path)
                desc = get_repo_description(repo)
                commits = commits_processing(repo, start, end)
                row = desc + commits
                matrix.append(row)
            except:
                pass
    return matrix
def save_matrix_as_csv(file_path, matrix) :
    a = numpy.asarray(matrix)
    numpy.savetxt(file_path, a, delimiter=",")<jupyter_output><empty_output><jupyter_text>Let's get info for BH-EU 2019<jupyter_code>BH_2019_START = datetime.datetime.strptime('2019-11-18 00:00:00', '%Y-%m-%d %H:%M:%S')
BH_2019_END = datetime.datetime.strptime('2019-11-22 23:59:59', '%Y-%m-%d %H:%M:%S')
matrix = []
with open('./data/bh_2019_repos.txt') as file :
    for line in file :        
        try :             
            path = line.strip()
            repo = g.get_repo(path)
            desc = get_repo_description(repo)
            commits = commits_processing(repo, BH_2019_START, BH_2019_END)
            row = desc + commits
            matrix.append(row)
        except:
            pass

df = pd.DataFrame(matrix, columns=['name','created_at', 'last_push', 'stars', 'license', 'before', 'during', 'after', 'total'])
df
#pd.to_csv('./data/bh_2019.csv', index=False)<jupyter_output><empty_output><jupyter_text>Let's get info for BH-EU 2020<jupyter_code>df = pd.DataFrame(matrix, columns=['name','created_at', 'last_push', 'stars', 'license', 'before', 'during', 'after', 'total'])
df
#save_matrix_as_csv('./data/bh_2020.csv', matrix)<jupyter_output><empty_output><jupyter_text>And finally let's get some info for the Covid-10 BH<jupyter_code>df = pd.DataFrame(matrix, columns=['name','created_at', 'last_push', 'stars', 'license', 'before', 'during', 'after', 'total'])
df
#save_matrix_as_csv('./data/bh_covid_2020.csv', matrix)<jupyter_output><empty_output><jupyter_text>Ideas for later<jupyter_code>#default branch (either master or main) but we do not need to go for a branch, we can go for the repo
#branch = repo.default_branch

#Example
repo = g.get_repo('elixir-europe/BioHackathon-projects-2020')

#For each of the following, we will need to do some aggregation
#By now let's go only with commits
#lst = repo.get_downloads()
commit_lst = repo.get_commits()
#lst = repo.get_forks()
#lst = repo.get_releases()
#lst = repo.get_issues()

#Response is a paginated list, will a simple loop work? 
#It looks like but if too many we will run above the GitHub API limit... 
#ToDo: find a better way to traverse the list
commit_lst.__dict__
#commit is what we need, from it we get the author username and the date
print(commit_lst[0].commit.author.name, commit_lst[0].commit.author.date) 

#next we want to aggregate committers and commits by dates
#at the end we want to get a dataframe with the following columns
#name	created_at	last_push	license	total_stars	total_commits	commits_before	commits_during	commits_after	total_committers	committers_before	committers_during	committers_after
#Let's get first the description columns and the ones for commits
#Let's get second (but the same dataframe) columns for committers (those need an extra aggregation by commiter)<jupyter_output><empty_output><jupyter_text>Result with added Contributors in DataFrame<jupyter_code>repo = g.get_repo('elixir-europe/BioHackathon-projects-2019')

BH_2019_START = datetime.datetime.strptime('2019-11-18 00:00:00', '%Y-%m-%d %H:%M:%S')
BH_2019_END = datetime.datetime.strptime('2019-11-22 23:59:59', '%Y-%m-%d %H:%M:%S')

statistics_hackaton = commits_processing(repo, BH_2019_START, BH_2019_END)
def commits_processing_contributors(repo, bh_start, bh_end):
    
    '''
    input: repo PyGitHub object
        bh_start Initial date of the event
        bh_end final date of the event
    Output: 1. list with overall commit days 
            2. set with all authors that contributed
    
    '''

    commits = repo.get_commits()

    total_commits = [0, 0, 0, 0] #before, during, after, total
    overall_contributors = set() 
    total_contributors = [set(), set(), set()]#before, during, after
    for commit in commits:
        author = (str(commit.commit.author.name))
        overall_contributors.add(author)
        time_Obj = commit.commit.author.date
        if time_Obj < bh_start :
            total_commits[0] = total_commits[0] + 1
            total_contributors[0].add(author)
        elif bh_start <= time_Obj and time_Obj <= bh_end : 
            total_commits[1] = total_commits[1] + 1
            total_contributors[1].add(author)
        else :
            total_commits[2] = total_commits[2] + 1
            total_contributors[2].add(author)
        total_commits[3] = total_commits[3] + 1
        #timeStamp = timeObj.strftime("%d-%m-%Y")

    #return sorted(commit_days, key = lambda date: datetime.strptime(date, "%d-%m-%Y")) , authors
    return total_commits, total_contributors, overall_contributors
statistics_hackaton = commits_processing_contributors(repo, BH_2019_START, BH_2019_END)
contributors_before = len(statistics_hackaton[1][0])
contributors_during = len(statistics_hackaton[1][1])
contributors_after = len(statistics_hackaton[1][2])
contributors_overall = len(statistics_hackaton[2])
#DataFrame data

repo_name = get_repo_description(repo)[0]
license = get_repo_description(repo)[4]


before_hackaton = statistics_hackaton[0][0]
during_hackaton = statistics_hackaton[0][1]
after_hackaton = statistics_hackaton[0][2]
total_hackaton = statistics_hackaton[0][3]<jupyter_output><empty_output><jupyter_text>Output<jupyter_code>#Dataframe
exportable_dataframe = pd.DataFrame({"BioHackaton Repository" : [repo_name], "License" : license, 
                                     "Commits before BH": before_hackaton, "Commits during BH": during_hackaton, 
                                     "Commits after BH": after_hackaton, "Total Commits" : total_hackaton, 
                                     "Contributors before BH" : contributors_before, "Contributors during BH" : contributors_during, 
                                     "Contributors after BH" : contributors_after, "Total Contributors" : contributors_overall})
exportable_dataframe<jupyter_output><empty_output><jupyter_text>Export to CSV<jupyter_code>exportable_dataframe.to_csv('BioHackaton_stats_with_contributors.csv', index = False, header=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11018344308560678, 'fraction_numerical': 0.02398871119473189, 'mean_word_length': 3.624796084828711, 'pattern_counts': {'":': 3, '<': 43, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '93897', 'license': 'Apache-2.0', 'n_tokens_mistral': 2988, 'n_tokens_neox': 2738, 'n_words': 659, 'path': 'Testing.ipynb', 'repo_name': 'zbmed/BioHackOutcomes', 'size': '8504'}