<commit_before>from __future__ import print_function
import os
from github3 import login
import pandas as pd
import numpy as np
import codecs
import getpass
import json
import requests
from tinydb import TinyDB, Query
from bokeh.models import HoverTool, ColumnDataSource
from bokeh.plotting import figure
from bokeh.embed import components


class KpiStats(object):
    """**Gathers repo statistics from a list of github urls into a TinyDB**

    The class uses github3.py to create an authenticated Github session.
    Session is authenticaed either by:

    #. Setting up a `github-token
       <https://help.github.com/articles/creating-an-access-token-for-command-line-use/>`_
       and copying the key into a local file in the cwd called 'secret_key'.
    #. If no 'secret_key' file is detected, the user is prompted to
       enter a username and password.
    #. A github-token can be set as an environment variable
       'GHUB_API_TOKEN' (used this on Travis).

    public repo stats can be retrieved). Two methods can be used to log in.
    Firstly using a github api token: The program will look for this in a file
    called 'secret_key' in the local folder. If this is not found the software
    will default to asking for a username and password. The class should be
    instansiated with a list of github repo url strings.

    :param urls: list of url strings ['https://github.com/<user>/<repo>',]

    :returns: KpiStats() object

    :Example:

    The follwing use case shows how to feed a list of urls to KpiStats
    and then read the TinyDB object produced (by default) into pandas.

    >>> from DashPykpi.kpistats import KpiStats, GitURLs, GraphKPIs
    >>> url_fetch = GitURLs()
    >>> urls = url_fetch.urls
    # If looking through all UCL associated repos need to remove the following
    # lines:
    # urls.remove('https://github.com/UCL/ucl')
    # urls.remove('https://github.com/UCL-RITS/ucl-rits')
    >>> test = KpiStats(urls=urls)
    >>> test.work(status=True)
    >>> db = TinyDB('tinydb_for_KPI.json')
    >>> df = pd.DataFrame(db.all())
    """
    def __init__(self, urls):
        if os.path.isfile('secret_key'):
            fn = open("secret_key")
            # Locally, with a secret_key file
            self.gh = login(token=fn.read().split()[0])
        elif os.environ.get('GHUB_API_TOKEN'):
            # On Travis? (GHUB_API_TOKEN could be set...)
            self.gh = login(token=os.environ['GHUB_API_TOKEN'])
        else:
            # Or just use username/password method
            self.gh_name = input("Username to access github with:")
            pss = getpass.getpass(prompt='Ghub pswd {0}:'.format(self.gh_name))
            self.gh = login(self.gh_name, pss)
        self.urls = urls  # A list of URL strings
        self.repo = None
        self.stats = None
        self.db = TinyDB('tinydb_for_KPI.json')  # create new or open existing

    def __str__(self):
        print("A KPI back-end to extract data from Github.")

    def get_repo_object_from_url(self, url):
        """Get a repository object for a given github url

        Retrieves a github3.py.Repository() object from a url string under an
        authenticated session.

        :param url: a string of format 'https://github.com/<user>/<repo>'

        :returns: github3.py.Repository() object as self.repo()
        """
        demo = 'https://github.com/<user>/<repo>'
        er1 = "Error: url should be a string in format of "
        er2 = "Error: {0} isn't valid ".format(url)
        assert type(url) == str, er1 + demo
        assert url.split('/')[-3] == 'github.com', er2
        user_str, repo_str = url.split('/')[-2:]
        self.repo = self.gh.repository(user_str, repo_str)
        return

    def get_repo_stats(self, debug=False):
        """Identify the statistics of an individual repo

        Examines self.repo() to identify key statistics from a repository.

        :param: self.repo()
        :rtype: A dictionary object as self.stats

        :Example:

        >>> from DashPykpi.kpistats import KpiStats
        >>> test = KpiStats(urls=["https://github.com/UCL-RITS/RSD-Dashboard"])
        >>> test.get_repo_object_from_url(url=test.urls[0])
        >>> test.get_repo_stats()
        >>> test.stats # print a dictionary of retrieved stats
        """
        if debug:
            print('\nExamining stats of {0}'.format(self.repo))
        contribs = [(str(contrib.author), contrib.total)
                    for contrib in self.repo.iter_contributor_statistics()]
        total = sum([user_num[1] for user_num in contribs])
        branch_count = len([branch for branch in self.repo.iter_branches()])
        self.stats = {
            'stargazers': self.repo.stargazers,
            'fork_count': self.repo.fork_count,
            'commits_by_author': contribs,
            'num_contributors': len(contribs),
            'total_commits': total,
            'repo_owner': self.repo.owner.login,
            'repo_name': self.repo.name,
            'branches': branch_count,
            'language': self.repo.language,
            }
        return

    def add_db_row(self):
        """KpiStats.add_db_row(self)

        Checks if there is a database and entry already present, if there isn't<commit_msg>Added calculation of commits by quarters.<commit_after>from __future__ import print_function
import os
from github3 import login
import pandas as pd
import numpy as np
import codecs
import getpass
import json
import requests
from tinydb import TinyDB, Query
from bokeh.models import HoverTool, ColumnDataSource
from bokeh.plotting import figure
from bokeh.embed import components


class KpiStats(object):
    """**Gathers repo statistics from a list of github urls into a TinyDB**

    The class uses github3.py to create an authenticated Github session.
    Session is authenticaed either by:

    #. Setting up a `github-token
       <https://help.github.com/articles/creating-an-access-token-for-command-line-use/>`_
       and copying the key into a local file in the cwd called 'secret_key'.
    #. If no 'secret_key' file is detected, the user is prompted to
       enter a username and password.
    #. A github-token can be set as an environment variable
       'GHUB_API_TOKEN' (used this on Travis).

    public repo stats can be retrieved). Two methods can be used to log in.
    Firstly using a github api token: The program will look for this in a file
    called 'secret_key' in the local folder. If this is not found the software
    will default to asking for a username and password. The class should be
    instansiated with a list of github repo url strings.

    :param urls: list of url strings ['https://github.com/<user>/<repo>',]

    :returns: KpiStats() object

    :Example:

    The follwing use case shows how to feed a list of urls to KpiStats
    and then read the TinyDB object produced (by default) into pandas.

    >>> from DashPykpi.kpistats import KpiStats, GitURLs, GraphKPIs
    >>> url_fetch = GitURLs()
    >>> urls = url_fetch.urls
    # If looking through all UCL associated repos need to remove the following
    # lines:
    # urls.remove('https://github.com/UCL/ucl')
    # urls.remove('https://github.com/UCL-RITS/ucl-rits')
    >>> test = KpiStats(urls=urls)
    >>> test.work(status=True)
    >>> db = TinyDB('tinydb_for_KPI.json')
    >>> df = pd.DataFrame(db.all())
    """
    def __init__(self, urls):
        if os.path.isfile('secret_key'):
            fn = open("secret_key")
            # Locally, with a secret_key file
            self.gh = login(token=fn.read().split()[0])
        elif os.environ.get('GHUB_API_TOKEN'):
            # On Travis? (GHUB_API_TOKEN could be set...)
            self.gh = login(token=os.environ['GHUB_API_TOKEN'])
        else:
            # Or just use username/password method
            self.gh_name = input("Username to access github with:")
            pss = getpass.getpass(prompt='Ghub pswd {0}:'.format(self.gh_name))
            self.gh = login(self.gh_name, pss)
        self.urls = urls  # A list of URL strings
        self.repo = None
        self.stats = None
        self.db = TinyDB('tinydb_for_KPI.json')  # create new or open existing

    def __str__(self):
        print("A KPI back-end to extract data from Github.")

    def get_repo_object_from_url(self, url):
        """Get a repository object for a given github url

        Retrieves a github3.py.Repository() object from a url string under an
        authenticated session.

        :param url: a string of format 'https://github.com/<user>/<repo>'

        :returns: github3.py.Repository() object as self.repo()
        """
        demo = 'https://github.com/<user>/<repo>'
        er1 = "Error: url should be a string in format of "
        er2 = "Error: {0} isn't valid ".format(url)
        assert type(url) == str, er1 + demo
        assert url.split('/')[-3] == 'github.com', er2
        user_str, repo_str = url.split('/')[-2:]
        self.repo = self.gh.repository(user_str, repo_str)
        return

    def get_repo_stats(self, debug=False):
        """Identify the statistics of an individual repo

        Examines self.repo() to identify key statistics from a repository.

        :param: self.repo()
        :rtype: A dictionary object as self.stats

        :Example:

        >>> from DashPykpi.kpistats import KpiStats
        >>> test = KpiStats(urls=["https://github.com/UCL-RITS/RSD-Dashboard"])
        >>> test.get_repo_object_from_url(url=test.urls[0])
        >>> test.get_repo_stats()
        >>> test.stats # print a dictionary of retrieved stats
        """
        if debug:
            print('\nExamining repo {0}'.format(self.repo))
        contribs = [(str(contrib.author), contrib.total)
                    for contrib in self.repo.iter_contributor_statistics()]
        total = sum([user_num[1] for user_num in contribs])
        branch_count = len([branch for branch in self.repo.iter_branches()])
        l = [commit for commit in self.repo.iter_commit_activity()]
        q1 = 0
        q2 = 0
        q3 = 0
        q4 = 0
        for i, week in enumerate(l):
            if i < 13:
                q1 += week['total']
            if i >= 13 and i <= 25:
                q2 += week['total']
            if i > 26 and i <= 39:
                q3 += week['total']
            if i > 39:
                q4 += week['total']
        self.stats = {
            'stargazers': self.repo.stargazers,
            'fork_count': self.repo.fork_count,
            'commits_by_author': contribs,
            'num_contributors': len(contribs),
            'total_commits': total,
            'repo_owner': self.repo.owner.login,
            'repo_name': self.repo.name,
            'branches': branch_count,
            'language': self.repo.language,
            "Q1": q1,
            "Q2": q2,
            'Q3': q3,
            "Q4": q4,
            }
        return

    def add_db_row(self):
        """KpiStats.add_db_row(self)

        Checks if there is a database and entry already present, if there isn't
{'fraction_non_alphanumeric': 0.10841307095143528, 'fraction_numerical': 0.0057772161039898896, 'mean_word_length': 2.4215565163681285, 'pattern_counts': {'":': 3, '<': 20, '<?xml version=': 0, '>': 92, 'https://': 14, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1031275', 'n_tokens_mistral': 3435, 'n_tokens_neox': 3287, 'n_words': 1052}