<commit_before>import itertools
import json
import logging
import math
import os
import posixpath
import re
import sys
import tempfile
from collections import defaultdict
from functools import partial
from multiprocessing.dummy import Pool as ThreadPool

import requests

PROJECT_NAME = 'Tattle'

ITEMS_PER_PAGE = 100
NO_THREAD_LIMIT = sys.maxint

GITHUB_API_URL = 'https://api.github.com/'
ORGS = 'orgs'
REPOS = 'repos'
BRANCHES = 'branches'

PUBLIC_REPOS = 'public_repos'
TOTAL_PRIVATE_REPOS = 'total_private_repos'

JIRA_ISSUE_API_URL_TEMPLATE = 'https://{0}.atlassian.net/rest/api/2/issue/'
JIRA_ISSUE = 'jira_issue'

logger = logging.getLogger('model')
logger.setLevel(logging.DEBUG)
ish = logging.StreamHandler(sys.stdout)
ish.setLevel(logging.INFO)
info_formatter = \
    logging.Formatter('%(asctime)s - %(message)s...', '%Y-%m-%d %H:%M:%S')
ish.setFormatter(info_formatter)
logger.addHandler(ish)


def determine_num_of_threads(max_threads, num_of_items, per_page=1):
    requested_threads = int(math.ceil(num_of_items / float(per_page)))
    return min(max_threads, requested_threads)


def create_thread_pool(number_of_threads):
    return ThreadPool(number_of_threads)


def get_json(url, auth=None):

    response = requests.get(url, auth=auth)
    return json.loads(response.text)


def pagination_format(page_number):
    return '?page={0}&per_page={1}'.format(page_number, ITEMS_PER_PAGE)


def generate_github_api_url(request_type,
                            org_name='',
                            repo_name='',
                            branch_name='',
                            page_number=None,
                            ):

    urls = {'organization':    posixpath.join(ORGS,
                                              org_name
                                              ),

            'repos':           posixpath.join(ORGS,
                                              org_name,
                                              REPOS
                                              ) +
                               pagination_format(page_number),

            'list_branches':   posixpath.join(REPOS,
                                              org_name,
                                              repo_name,
                                              BRANCHES
                                              ),

            'detailed_branch': posixpath.join(REPOS,
                                              org_name,
                                              repo_name,
                                              BRANCHES,
                                              branch_name)
            }

    return posixpath.join(GITHUB_API_URL,
                          urls[request_type])


class GitHubObject(object):

    def __init__(self, name):
        self.name = name

    def __eq__(self, other):
        if type(other) is type(self):
            return self.__dict__ == other.__dict__
        return False

    def __ne__(self, other):
        return not self.__eq__(other)

    def __lt__(self, other):
        return self.name < other.name


class Organization(GitHubObject):

    def __init__(self, name):
        super(Organization, self).__init__(name)

    def __str__(self):
        return self.name

    @staticmethod
    def get_num_of_repos(org):

        url = generate_github_api_url('organization', org_name=org.name)

        json_org = get_json(url, auth=QueryConfig.github_credentials())

        return json_org[PUBLIC_REPOS] + json_org[TOTAL_PRIVATE_REPOS]


class Repo(GitHubObject):

    def __init__(self, name, org=None):
        super(Repo, self).__init__(name)
        self.organization = org

    def __str__(self):
        return self.name

    def __repr__(self):
        return 'Repo(name={0},org={1})'.format(self.name,
                                                        self.organization)

    @classmethod
    def from_json(cls, json_repo):
        name = json_repo['name']
        organization = Organization(json_repo['owner']['login'])
        return cls(name, organization)

    @classmethod
    def get_repos(cls, org, max_threads=NO_THREAD_LIMIT):
        logger.info('retrieving github repositories for the {0} organization'
                    .format(org))

        num_of_repos = Organization.get_num_of_repos(org)
        num_of_threads = determine_num_of_threads(max_threads,
                                                  num_of_repos,
                                                  per_page=ITEMS_PER_PAGE)
        pool = create_thread_pool(num_of_threads)

        json_repos = pool.map(partial(cls.get_json_repos, org=org),
                              range(1, num_of_threads + 1)
                              )
        repos = []
        for list_of_json_repos in json_repos:
            for json_repo in list_of_json_repos:
                repos.append(cls.from_json(json_repo))

        return repos

    @staticmethod
    def get_json_repos(page_number, org):

        url = generate_github_api_url('repos',
                                      org_name=org.name,
                                      page_number=page_number,
                                      )

        return get_json(url, auth=QueryConfig.github_credentials())


class Branch(GitHubObject):

    def __init__(self,
                 name,
                 repo,
                 jira_issue=None):
        super(Branch, self).__init__(name)
        self.repo = repo
        self.jira_issue = jira_issue
        self.committer_email = None
        # TODO maybe add date

    def __str__(self):
        return self.name

    @classmethod
    def from_json(cls, json_branch):

        name = json_branch['name']
        (repo_name, organization) = cls.extract_repo_data(json_branch['commit']['url'])
        repo = Repo(repo_name, org=organization)

        return cls(name, repo)

    @staticmethod
    def extract_repo_data(branch_url):
        url_regex = re.compile(r'https://api.github.com/repos/(.*)/(.*)/commits/(.*)')
        groups = url_regex.findall(branch_url)
        name = groups[0][1]
        organization = Organization(groups[0][0])
        return name, organization

    @classmethod
    def get_org_branches(cls, repos, org, max_threads=NO_THREAD_LIMIT):
        logger.info('retrieving basic github branch info '
                    'for the {0} organization'
                    .format(org))
        num_of_threads = determine_num_of_threads(max_threads, len(repos))
        pool = create_thread_pool(num_of_threads)<commit_msg>Add `committer_email` as a keyword argument of model.Branch<commit_after>import itertools
import json
import logging
import math
import os
import posixpath
import re
import sys
import tempfile
from collections import defaultdict
from functools import partial
from multiprocessing.dummy import Pool as ThreadPool

import requests

PROJECT_NAME = 'Tattle'

ITEMS_PER_PAGE = 100
NO_THREAD_LIMIT = sys.maxint

GITHUB_API_URL = 'https://api.github.com/'
ORGS = 'orgs'
REPOS = 'repos'
BRANCHES = 'branches'

PUBLIC_REPOS = 'public_repos'
TOTAL_PRIVATE_REPOS = 'total_private_repos'

JIRA_ISSUE_API_URL_TEMPLATE = 'https://{0}.atlassian.net/rest/api/2/issue/'
JIRA_ISSUE = 'jira_issue'

logger = logging.getLogger('model')
logger.setLevel(logging.DEBUG)
ish = logging.StreamHandler(sys.stdout)
ish.setLevel(logging.INFO)
info_formatter = \
    logging.Formatter('%(asctime)s - %(message)s...', '%Y-%m-%d %H:%M:%S')
ish.setFormatter(info_formatter)
logger.addHandler(ish)


def determine_num_of_threads(max_threads, num_of_items, per_page=1):
    requested_threads = int(math.ceil(num_of_items / float(per_page)))
    return min(max_threads, requested_threads)


def create_thread_pool(number_of_threads):
    return ThreadPool(number_of_threads)


def get_json(url, auth=None):

    response = requests.get(url, auth=auth)
    return json.loads(response.text)


def pagination_format(page_number):
    return '?page={0}&per_page={1}'.format(page_number, ITEMS_PER_PAGE)


def generate_github_api_url(request_type,
                            org_name='',
                            repo_name='',
                            branch_name='',
                            page_number=None,
                            ):

    urls = {'organization':    posixpath.join(ORGS,
                                              org_name
                                              ),

            'repos':           posixpath.join(ORGS,
                                              org_name,
                                              REPOS
                                              ) +
                               pagination_format(page_number),

            'list_branches':   posixpath.join(REPOS,
                                              org_name,
                                              repo_name,
                                              BRANCHES
                                              ),

            'detailed_branch': posixpath.join(REPOS,
                                              org_name,
                                              repo_name,
                                              BRANCHES,
                                              branch_name)
            }

    return posixpath.join(GITHUB_API_URL,
                          urls[request_type])


class GitHubObject(object):

    def __init__(self, name):
        self.name = name

    def __eq__(self, other):
        if type(other) is type(self):
            return self.__dict__ == other.__dict__
        return False

    def __ne__(self, other):
        return not self.__eq__(other)

    def __lt__(self, other):
        return self.name < other.name


class Organization(GitHubObject):

    def __init__(self, name):
        super(Organization, self).__init__(name)

    def __str__(self):
        return self.name

    @staticmethod
    def get_num_of_repos(org):

        url = generate_github_api_url('organization', org_name=org.name)

        json_org = get_json(url, auth=QueryConfig.github_credentials())

        return json_org[PUBLIC_REPOS] + json_org[TOTAL_PRIVATE_REPOS]


class Repo(GitHubObject):

    def __init__(self, name, org=None):
        super(Repo, self).__init__(name)
        self.organization = org

    def __str__(self):
        return self.name

    def __repr__(self):
        return 'Repo(name={0},org={1})'.format(self.name,
                                                        self.organization)

    @classmethod
    def from_json(cls, json_repo):
        name = json_repo['name']
        organization = Organization(json_repo['owner']['login'])
        return cls(name, organization)

    @classmethod
    def get_repos(cls, org, max_threads=NO_THREAD_LIMIT):
        logger.info('retrieving github repositories for the {0} organization'
                    .format(org))

        num_of_repos = Organization.get_num_of_repos(org)
        num_of_threads = determine_num_of_threads(max_threads,
                                                  num_of_repos,
                                                  per_page=ITEMS_PER_PAGE)
        pool = create_thread_pool(num_of_threads)

        json_repos = pool.map(partial(cls.get_json_repos, org=org),
                              range(1, num_of_threads + 1)
                              )
        repos = []
        for list_of_json_repos in json_repos:
            for json_repo in list_of_json_repos:
                repos.append(cls.from_json(json_repo))

        return repos

    @staticmethod
    def get_json_repos(page_number, org):

        url = generate_github_api_url('repos',
                                      org_name=org.name,
                                      page_number=page_number,
                                      )

        return get_json(url, auth=QueryConfig.github_credentials())


class Branch(GitHubObject):

    def __init__(self,
                 name,
                 repo,
                 jira_issue=None,
                 committer_email=None):
        super(Branch, self).__init__(name)
        self.repo = repo
        self.jira_issue = jira_issue
        self.committer_email = committer_email
        # TODO maybe add date

    def __str__(self):
        return self.name

    @classmethod
    def from_json(cls, json_branch):

        name = json_branch['name']
        (repo_name, organization) = cls.extract_repo_data(json_branch['commit']['url'])
        repo = Repo(repo_name, org=organization)

        return cls(name, repo)

    @staticmethod
    def extract_repo_data(branch_url):
        url_regex = re.compile(r'https://api.github.com/repos/(.*)/(.*)/commits/(.*)')
        groups = url_regex.findall(branch_url)
        name = groups[0][1]
        organization = Organization(groups[0][0])
        return name, organization

    @classmethod
    def get_org_branches(cls, repos, org, max_threads=NO_THREAD_LIMIT):
        logger.info('retrieving basic github branch info '
                    'for the {0} organization'
                    .format(org))
        num_of_threads = determine_num_of_threads(max_threads, len(repos))
        pool = create_thread_pool(num_of_threads)
{'fraction_non_alphanumeric': 0.08953752843062926, 'fraction_numerical': 0.0027293404094010615, 'mean_word_length': 1.7613564998953317, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 70, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2306431', 'n_tokens_mistral': 3773, 'n_tokens_neox': 3590, 'n_words': 671}