<commit_before>#How to run this:

#Python libraries needed to run this file: Flask, Git Python, SQLAlchemy

#You will need to have Git installed, and it will need to be in your path.
#For example, on Windows you should be able to run a command like 'git pull' from the
#ordinary Windows command prompt and not just from Git Bash.

#You will need a MySQL server with the MSR14 datasource or other GHTorrent database with the same schema.
#Edit the line in this code that says db = sqlalchemy.create_engine to match your username:password@hostname:port/database.

#This file is hardcoded to download the ghdata repository.
#Since it is a preliminary example, each time it runs, 
#it deletes the local ghdata repo and re-downloads it (though this might not be a good option for the future).
#Because of this: if you have a folder named ghdata whose contents you do not want deleted, 
#do not place this file in the same folder as your ghdata folder.

#to run this, type "python pythonBlameHistoryTree.py" into the command prompt
#You will see some output about running on 127.0.0.1:5000 in the command prompt
#Open a web browser and navigate to 127.0.0.1:5000.
#This page will load for quite a while.  At least several minutes is expected.
#You can see it is still running due to the testing output in the command prompt Outer loop: commit# Inner loop: commit#
#When the testing output stops running you should see some output in the browser tab.

#the output shows the commit number and date, the total lines of code and other files (for example, the readme)
#and the percentage written by each organization.
#expected output for ghdata should show only the spdx-tools organization (Matt is a member)
#Number of lines corresponds to the lines written by Matt.

#You can see that earlier commits are lower on the page, and chronologically later ones appear higher up.

#An "error" I expect us to encounter when testing other repos:
#The way my sql query works right now, a user can be a member of multiple organizations.
#For a simple case of expected output problems:
#User1 wrote the entire repository (100%)
#User1 is a member of Microsoft and IBM
#Microsoft wrote 100% of the repository.  IBM also wrote 100% of the repository for a total of 200%

#Future changes planned for this file:
#Code cleanup for better readability
#Code commenting for each portion
#Thorough testing for various potential cases we might encounter
#Deciding for certain how to decide whether a user is a member of an organization
#A better method of dealing with local repository rather than deleting each time and re-downloading
#Not having the database password directly in the code
#Look into improving code efficiency where possible for faster runtime

from flask import Flask
from git import *
import sqlalchemy
from sqlalchemy import text
import shutil
import os
import stat
import time

app = Flask(__name__)

@app.route("/")
def pythonBlameHistory():
    path = './ghdata'
    if os.path.exists(path):
        for root, directories, files in os.walk(path):  
            for directory in directories:  
                os.chmod(os.path.join(root, directory), stat.S_IWRITE)
            for file in files:
                os.chmod(os.path.join(root, file), stat.S_IWRITE)
        os.chmod(path, stat.S_IWRITE)
    
        shutil.rmtree(path)
    
    db = sqlalchemy.create_engine('mysql+pymysql://root:password@localhost:3306/msr14')
    schema = sqlalchemy.MetaData()
    schema.reflect(bind=db)
    
    repo = Repo.init('ghdata')
    origin = repo.create_remote('origin','https://github.com/OSSHealth/ghdata.git')
    origin.fetch()
    origin.pull(origin.refs[0].remote_head)
    
    emailsAndOrganizations = {}
    organizationTotals = {}
    lineTotal = 0
    percentage = 0
    repoTotalLines = 0
    repoOrganizationTotals = {}
    
    outputString = ""
    for thisCommit in repo.iter_commits('master'):
        repoOrganizationTotals = {}
        repoTotalLines = 0
        print("Outer loop: " + str(thisCommit))
        for entry in thisCommit.tree.traverse():
            organizationTotals = {}
            lineTotal = 0
            if entry.type == 'blob':
                for commit, lines in repo.blame(thisCommit, entry.path):
                    blameLineCount = 0
                    for line in lines:
                        blameLineCount += 1
                        lineTotal += 1
                    print("Inner loop: " + str(commit))
                    if commit.author.email in emailsAndOrganizations:
                        for organization in emailsAndOrganizations[commit.author.email]:
                            if organization not in organizationTotals:
                                organizationTotals[organization] = blameLineCount
                            else:
                                organizationTotals[organization] += blameLineCount
                    else:
                        sql = text('select orgUser.login as org_name '
                                   'from users as thisUser join organization_members '
                                   'on organization_members.user_id = thisUser.id '
                                   'join users as orgUser on organization_members.org_id = orgUser.id '
                                   'where thisUser.email = "' + commit.author.email + '"')
                        result = db.engine.execute(sql)
                        emailsAndOrganizations[commit.author.email] = []
                        for row in result:
                            emailsAndOrganizations[commit.author.email] = emailsAndOrganizations[commit.author.email] + [row[0]]
                            if row[0] not in organizationTotals:
                                organizationTotals[row[0]] = blameLineCount
                            else:
                                organizationTotals[row[0]] += blameLineCount
                if lineTotal > 0:
                    repoTotalLines += lineTotal
                    for key in organizationTotals:
                        if key not in repoOrganizationTotals:
                            repoOrganizationTotals[key] = organizationTotals[key]
                        else:
                            repoOrganizationTotals[key] += organizationTotals[key]
                        percentage = organizationTotals[key] / lineTotal * 100
        outputString = outputString + "REPO TOTALS FOR COMMIT: " + str(thisCommit) + " authored at " + time.strftime("%I:%M %p, %b %d, %Y", time.gmtime(thisCommit.authored_date)) + " <br>" 
        outputString = outputString + "TOTAL REPO LINES: " + str(repoTotalLines) + "<br>"
        for key in repoOrganizationTotals:
            percentage = repoOrganizationTotals[key] / repoTotalLines * 100
            outputString = outputString + " ORGANIZATION: " + str(key) + " ORG TOTAL LINES: " + str(repoOrganizationTotals[key]) + " PERCENTAGE OF REPO: " + str(percentage) + "%<br>"
        outputString = outputString + "----------------------------------------------------------------------------<br>"
    return outputString

if __name__ == "__main__":
    app.run()<commit_msg>Add code comments to pythonBlameHistoryTree.py<commit_after>#How to run this:

#Python libraries needed to run this file: Flask, Git Python, SQLAlchemy

#You will need to have Git installed, and it will need to be in your path.
#For example, on Windows you should be able to run a command like 'git pull' from the
#ordinary Windows command prompt and not just from Git Bash.

#You will need a MySQL server with the MSR14 datasource or other GHTorrent database with the same schema.
#Edit the line in this code that says db = sqlalchemy.create_engine to match your username:password@hostname:port/database.

#This file is hardcoded to download the ghdata repository.
#Since it is a preliminary example, each time it runs, 
#it deletes the local ghdata repo and re-downloads it (though this might not be a good option for the future).
#Because of this: if you have a folder named ghdata whose contents you do not want deleted, 
#do not place this file in the same folder as your ghdata folder.

#to run this, type "python pythonBlameHistoryTree.py" into the command prompt
#You will see some output about running on 127.0.0.1:5000 in the command prompt
#Open a web browser and navigate to 127.0.0.1:5000.
#This page will load for quite a while.  At least several minutes is expected.
#You can see it is still running due to the testing output in the command prompt Outer loop: commit# Inner loop: commit#
#When the testing output stops running you should see some output in the browser tab.

#the output shows the commit number and date, the total lines of code and other files (for example, the readme)
#and the percentage written by each organization.
#expected output for ghdata should show only the spdx-tools organization (Matt is a member)
#Number of lines corresponds to the lines written by Matt.

#You can see that earlier commits are lower on the page, and chronologically later ones appear higher up.

#An "error" I expect us to encounter when testing other repos:
#The way my sql query works right now, a user can be a member of multiple organizations.
#For a simple case of expected output problems:
#User1 wrote the entire repository (100%)
#User1 is a member of Microsoft and IBM
#Microsoft wrote 100% of the repository.  IBM also wrote 100% of the repository for a total of 200%

#Future changes planned for this file:
#Code cleanup for better readability
#Code commenting for each portion
#Thorough testing for various potential cases we might encounter
#Deciding for certain how to decide whether a user is a member of an organization
#A better method of dealing with local repository rather than deleting each time and re-downloading
#Not having the database password directly in the code
#Look into improving code efficiency where possible for faster runtime

from flask import Flask
from git import *
import sqlalchemy
from sqlalchemy import text
import shutil
import os
import stat
import time

app = Flask(__name__)

@app.route("/")
def pythonBlameHistory():
    #path is the hardcoded folder for the last download of ghdata
    path = './ghdata'
    #We must remove the old ghdata if we want to download a new copy.
    #In order to delete it, we must first change the permissions
    #To be writable for all files and directories.
    if os.path.exists(path):
        for root, directories, files in os.walk(path):  
            for directory in directories:  
                os.chmod(os.path.join(root, directory), stat.S_IWRITE)
            for file in files:
                os.chmod(os.path.join(root, file), stat.S_IWRITE)
        os.chmod(path, stat.S_IWRITE)
    
        #delete the old ghdata
        shutil.rmtree(path)
    
    #connect to the database username:password@hostname:port/databasename
    db = sqlalchemy.create_engine('mysql+pymysql://root:password@localhost:3306/msr14')
    schema = sqlalchemy.MetaData()
    schema.reflect(bind=db)
    
    #Get the ghdata repository from GitHub
    repo = Repo.init('ghdata')
    origin = repo.create_remote('origin','https://github.com/OSSHealth/ghdata.git')
    origin.fetch()
    origin.pull(origin.refs[0].remote_head)
    
    #Dictionary to store results of sql queries
    #associating emails with organizations.
    #Without this dictionary, we would have to repeat
    #the same query over and over, which on my local machine
    #meant a runtime of over 24 hours (as opposed to several minutes using the dictionary)
    emailsAndOrganizations = {}
    #This dictionary keeps track of the lines written per organization for a single file.
    organizationTotals = {}
    #This is the total number of lines in a single file
    lineTotal = 0
    #this is used later to hold percentage results for output
    percentage = 0
    #This is the total number of lines in an entire repo
    repoTotalLines = 0
    #This dictionary keeps track of the lines written per organization for the entire repo.
    repoOrganizationTotals = {}
    
    #The output string will be displayed to the screen once everything is done running.
    outputString = ""
    #Outer loop: loop through each commit in the master branch.
    #This corresponds to the history of commits over time.
    for thisCommit in repo.iter_commits('master'):
        #Since we want to see the change over time in repo percentage by organization,
        #clear the variables for total lines and organization lines for each new commit
        #we examine.
        repoOrganizationTotals = {}
        repoTotalLines = 0
        #Testing output: only purpose is to show you it's still running :)
        print("Outer loop: " + str(thisCommit))
        #Now loop through every file in the repo.
        #You cannot use the os library file/directory loop for this part.
        #(as was used above to change file permissions)
        #That is because some files do not exist in every commit.
        #You must loop through the commit tree, not the ghdata directory.
        for entry in thisCommit.tree.traverse():
            #For each file, we want to clear out the total lines and organization totals per file.
            #That's because we're starting over with a new file.
            organizationTotals = {}
            lineTotal = 0
            #Files are of the blob type.  This if statement prevents us from trying
            #to examine 'lines' in a directory.
            if entry.type == 'blob':
                #Now for each file, perform git blame.  This will traverse
                #the lines in the file.
                #You can see there are now two variables of type commit:
                #thisCommit and commit (will improve variable naming in a future update)
                #thisCommit is the commit with respect to the overall repo history.
                #commit is the commit in which this line was most recently changed
                #as obtained through git blame.  We use the "commit" variable
                #to obtain the author of the commit for when the lines were last changed.
                for commit, lines in repo.blame(thisCommit, entry.path):
                    #Git blame does not always return one line at a time.
                    #Sometimes we are returned several lines committed by the same author.
                    #In that case, we must count how many lines there are or our
                    #total will not match the actual file.
                    blameLineCount = 0
                    for line in lines:
                        #increment lines to later attribute to an organization.
                        blameLineCount += 1
                        #increment lines in the file as a whole
                        lineTotal += 1
                    #Testing output: only shows that things are still running.
                    print("Inner loop: " + str(commit))
                    #Get the email address of the author of this commit.
                    #If we already have it in our dictionary, increase the total
                    #lines for the associated organization by blameLineCount
                    if commit.author.email in emailsAndOrganizations:
                        for organization in emailsAndOrganizations[commit.author.email]:
                            if organization not in organizationTotals:
                                organizationTotals[organization] = blameLineCount
                            else:
                                organizationTotals[organization] += blameLineCount
                    #If the email address is not in our dictionary, we must query
                    #the database to get any associated organizations.
                    else:
                        sql = text('select orgUser.login as org_name '
                                   'from users as thisUser join organization_members '
                                   'on organization_members.user_id = thisUser.id '
                                   'join users as orgUser on organization_members.org_id = orgUser.id '
                                   'where thisUser.email = "' + commit.author.email + '"')
                        result = db.engine.execute(sql)
                        #add the email to the dictionary
                        emailsAndOrganizations[commit.author.email] = []
                        #if there are organizations in the result, associate those organizations with the
                        #user email in the dictionary.
                        #Then, set or add blameLineCount to the organization total.
                        for row in result:
                            emailsAndOrganizations[commit.author.email] = emailsAndOrganizations[commit.author.email] + [row[0]]
                            if row[0] not in organizationTotals:
                                organizationTotals[row[0]] = blameLineCount
                            else:
                                organizationTotals[row[0]] += blameLineCount
                #If there is more than one line in this file
                if lineTotal > 0:
                    #Add the total lines in this file to the total lines in the repo.
                    repoTotalLines += lineTotal
                    #Loop through the organization total lines for this file.
                    #Add each organization to the repo's organization total lines.
                    for key in organizationTotals:
                        if key not in repoOrganizationTotals:
                            repoOrganizationTotals[key] = organizationTotals[key]
                        else:
                            repoOrganizationTotals[key] += organizationTotals[key]
                        #Calculate the percentage for this file by organization (no longer used: former testing output)
                        percentage = organizationTotals[key] / lineTotal * 100
        #Construct output for this commit.  First output the commit, date, and total lines in the repo.
        outputString = outputString + "REPO TOTALS FOR COMMIT: " + str(thisCommit) + " authored at " + time.strftime("%I:%M %p, %b %d, %Y", time.gmtime(thisCommit.authored_date)) + " <br>" 
        outputString = outputString + "TOTAL REPO LINES: " + str(repoTotalLines) + "<br>"
        #Now loop through the organizations and calculate the percentage of the repo for each.
        #Output a line for each organization showing organization name, lines from that organization, percentage of the file
        for key in repoOrganizationTotals:
            percentage = repoOrganizationTotals[key] / repoTotalLines * 100
            outputString = outputString + " ORGANIZATION: " + str(key) + " ORG TOTAL LINES: " + str(repoOrganizationTotals[key]) + " PERCENTAGE OF REPO: " + str(percentage) + "%<br>"
        #Output line between each commit in the history for easier legibility.
        outputString = outputString + "----------------------------------------------------------------------------<br>"
    #Show the outputString in the browser.
    return outputString

if __name__ == "__main__":
    app.run()
{'fraction_non_alphanumeric': 0.06890256624243689, 'fraction_numerical': 0.006572084289588984, 'mean_word_length': 2.2540733197556007, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5900874', 'n_tokens_mistral': 4844, 'n_tokens_neox': 4568, 'n_words': 2092}