<commit_before>"""Module for accessing github.com projects."""

import base64
import httplib
import re
import sys
import traceback
import yaml

from mimic.__mimic import common

import fetcher
import model
import shared

from . import collection

from google.appengine.api import urlfetch_errors


_GITHUB_URL_RE = re.compile(
    '^https?://(?:[^/]+.)?github.com/(.+)$'
)

# projects which should not be shown in the cloud playground by default
_PROJECT_URL_SKIP_LIST = [
    # Java users only
    'https://github.com/GoogleCloudPlatform/appengine-enable-remote-api-python',
    # (deprecated) master/slave apps only
    'https://github.com/GoogleCloudPlatform/appengine-recover-unapplied-writes-python',
]


# https://api.github.com/
# {
#   "current_user_url": "https://api.github.com/user",
#   "authorizations_url": "https://api.github.com/authorizations",
#   "emails_url": "https://api.github.com/user/emails",
#   "emojis_url": "https://api.github.com/emojis",
#   "events_url": "https://api.github.com/events",
#   "following_url": "https://api.github.com/user/following{/target}",
#   "gists_url": "https://api.github.com/gists{/gist_id}",
#   "hub_url": "https://api.github.com/hub",
#   "issue_search_url": "https://api.github.com/legacy/issues/search/{owner}/{repo}/{state}/{keyword}",
#   "issues_url": "https://api.github.com/issues",
#   "keys_url": "https://api.github.com/user/keys",
#   "notifications_url": "https://api.github.com/notifications",
#   "organization_repositories_url": "https://api.github.com/orgs/{org}/repos/{?type,page,per_page,sort}",
#   "organization_url": "https://api.github.com/orgs/{org}",
#   "public_gists_url": "https://api.github.com/gists/public",
#   "rate_limit_url": "https://api.github.com/rate_limit",
#   "repository_url": "https://api.github.com/repos/{owner}/{repo}",
#   "repository_search_url": "https://api.github.com/legacy/repos/search/{keyword}{?language,start_page}",
#   "current_user_repositories_url": "https://api.github.com/user/repos{?type,page,per_page,sort}",
#   "starred_url": "https://api.github.com/user/starred{/owner}{/repo}",
#   "starred_gists_url": "https://api.github.com/gists/starred",
#   "team_url": "https://api.github.com/teams",
#   "user_url": "https://api.github.com/users/{user}",
#   "user_organizations_url": "https://api.github.com/user/orgs",
#   "user_repositories_url": "https://api.github.com/users/{user}/repos{?type,page,per_page,sort}",
#   "user_search_url": "https://api.github.com/legacy/user/search/{keyword}"
# }

# https://api.github.com/users/GoogleCloudPlatform/repos
# [
#   {
#     "id": 6730561,
#     "name": "appengine-guestbook-python",
#     ...
#     ...
#     "full_name": "GoogleCloudPlatform/appengine-guestbook-python",
#     "html_url": "https://github.com/GoogleCloudPlatform/appengine-guestbook-python",
#     "description": "Guestbook is an example application showing basic usage of Google App Engine",
#     "url": "https://api.github.com/repos/GoogleCloudPlatform/appengine-guestbook-python",
#     "branches_url": "https://api.github.com/repos/GoogleCloudPlatform/appengine-guestbook-python/branches{/branch}",
#     "tags_url": "https://api.github.com/repos/GoogleCloudPlatform/appengine-guestbook-python/tags{/tag}",
#     "trees_url": "https://api.github.com/repos/GoogleCloudPlatform/appengine-guestbook-python/git/trees{/sha}",
#     "contents_url": "https://api.github.com/repos/GoogleCloudPlatform/appengine-guestbook-python/contents/{+path}",
#     "git_url": "git://github.com/GoogleCloudPlatform/appengine-guestbook-python.git",
#     "homepage": "https://developers.google.com/appengine/docs/python/gettingstartedpython27/",
#     "language": "Python",
#     "master_branch": "master",
#     "default_branch": "master"
#   }
# ]
class Info(object):

  def __init__(self, user, repo=None, branch=None, path=''):
    self.user = self.owner = user
    self.repo = repo
    self.branch = branch
    self.path = path

  def repository_url(self):
    return 'https://api.github.com/repos/{owner}/{repo}'.format(**self.__dict__)

  def branches_url(self):
    if self.branch == None:
      fetched = FetchAsyncWithAuth(self.repository_url())
      data = fetched.json_content
      self.branch = data['master_branch']
    return ('https://api.github.com/repos/{owner}/{repo}/branches/{branch}'
            .format(**self.__dict__))


def GetInfo(html_url):
  """Get Info object based on the provided HTML URL.

  For example:
  - https://api.github.com/users/GoogleCloudPlatform/repos
  - https://github.com/GoogleCloudPlatform/appengine-guestbook-python
  - https://github.com/GoogleCloudPlatform/appengine-guestbook-python/tree/part6-staticfiles
  - https://api.github.com/repos/GoogleCloudPlatform/appengine-guestbook-python/branches/part6-staticfiles
  """
  matcher = _GITHUB_URL_RE.match(html_url)
  if not matcher:
    return None
  components = matcher.group(1).split('/')
  components.extend([None, None])
  kwargs = {}
  if components[0] in ('users', 'repos'):
    components.pop(0)
  kwargs['user'] = components.pop(0)
  kwargs['repo'] = components.pop(0)
  if components[0] in ('tree', 'branches'):
    components.pop(0)
    kwargs['branch'] = components.pop(0)
  return Info(**kwargs)


def IsValidUrl(url):
  return _GITHUB_URL_RE.match(url)


def FetchAsyncWithAuth(*args, **kwargs):
  credential = model.GetOAuth2Credential('github')
  if credential:
    url_auth_suffix = ('?client_id={0}&client_secret={1}'
                       .format(credential.client_id,
                               credential.client_secret))<commit_msg>support github .git URLs<commit_after>"""Module for accessing github.com projects."""

import base64
import httplib
import re
import sys
import traceback
import yaml

from mimic.__mimic import common

import fetcher
import model
import shared

from . import collection

from google.appengine.api import urlfetch_errors


_GITHUB_URL_RE = re.compile(
    '^(?:https?|git)://(?:[^/]+.)?github.com/(.+?)(?:\.git)?$'
)

# projects which should not be shown in the cloud playground by default
_PROJECT_URL_SKIP_LIST = [
    # Java users only
    'https://github.com/GoogleCloudPlatform/appengine-enable-remote-api-python',
    # (deprecated) master/slave apps only
    'https://github.com/GoogleCloudPlatform/appengine-recover-unapplied-writes-python',
]


# https://api.github.com/
# {
#   "current_user_url": "https://api.github.com/user",
#   "authorizations_url": "https://api.github.com/authorizations",
#   "emails_url": "https://api.github.com/user/emails",
#   "emojis_url": "https://api.github.com/emojis",
#   "events_url": "https://api.github.com/events",
#   "following_url": "https://api.github.com/user/following{/target}",
#   "gists_url": "https://api.github.com/gists{/gist_id}",
#   "hub_url": "https://api.github.com/hub",
#   "issue_search_url": "https://api.github.com/legacy/issues/search/{owner}/{repo}/{state}/{keyword}",
#   "issues_url": "https://api.github.com/issues",
#   "keys_url": "https://api.github.com/user/keys",
#   "notifications_url": "https://api.github.com/notifications",
#   "organization_repositories_url": "https://api.github.com/orgs/{org}/repos/{?type,page,per_page,sort}",
#   "organization_url": "https://api.github.com/orgs/{org}",
#   "public_gists_url": "https://api.github.com/gists/public",
#   "rate_limit_url": "https://api.github.com/rate_limit",
#   "repository_url": "https://api.github.com/repos/{owner}/{repo}",
#   "repository_search_url": "https://api.github.com/legacy/repos/search/{keyword}{?language,start_page}",
#   "current_user_repositories_url": "https://api.github.com/user/repos{?type,page,per_page,sort}",
#   "starred_url": "https://api.github.com/user/starred{/owner}{/repo}",
#   "starred_gists_url": "https://api.github.com/gists/starred",
#   "team_url": "https://api.github.com/teams",
#   "user_url": "https://api.github.com/users/{user}",
#   "user_organizations_url": "https://api.github.com/user/orgs",
#   "user_repositories_url": "https://api.github.com/users/{user}/repos{?type,page,per_page,sort}",
#   "user_search_url": "https://api.github.com/legacy/user/search/{keyword}"
# }

# https://api.github.com/users/GoogleCloudPlatform/repos
# [
#   {
#     "id": 6730561,
#     "name": "appengine-guestbook-python",
#     ...
#     ...
#     "full_name": "GoogleCloudPlatform/appengine-guestbook-python",
#     "html_url": "https://github.com/GoogleCloudPlatform/appengine-guestbook-python",
#     "description": "Guestbook is an example application showing basic usage of Google App Engine",
#     "url": "https://api.github.com/repos/GoogleCloudPlatform/appengine-guestbook-python",
#     "branches_url": "https://api.github.com/repos/GoogleCloudPlatform/appengine-guestbook-python/branches{/branch}",
#     "tags_url": "https://api.github.com/repos/GoogleCloudPlatform/appengine-guestbook-python/tags{/tag}",
#     "trees_url": "https://api.github.com/repos/GoogleCloudPlatform/appengine-guestbook-python/git/trees{/sha}",
#     "contents_url": "https://api.github.com/repos/GoogleCloudPlatform/appengine-guestbook-python/contents/{+path}",
#     "git_url": "git://github.com/GoogleCloudPlatform/appengine-guestbook-python.git",
#     "homepage": "https://developers.google.com/appengine/docs/python/gettingstartedpython27/",
#     "language": "Python",
#     "master_branch": "master",
#     "default_branch": "master"
#   }
# ]
class Info(object):

  def __init__(self, user, repo=None, branch=None, path=''):
    self.user = self.owner = user
    self.repo = repo
    self.branch = branch
    self.path = path

  def repository_url(self):
    return 'https://api.github.com/repos/{owner}/{repo}'.format(**self.__dict__)

  def branches_url(self):
    if self.branch == None:
      fetched = FetchAsyncWithAuth(self.repository_url())
      data = fetched.json_content
      self.branch = data['master_branch']
    return ('https://api.github.com/repos/{owner}/{repo}/branches/{branch}'
            .format(**self.__dict__))


def GetInfo(html_url):
  """Get Info object based on the provided HTML URL.

  For example:
  - https://api.github.com/users/GoogleCloudPlatform/repos
  - https://github.com/GoogleCloudPlatform/appengine-guestbook-python
  - https://github.com/GoogleCloudPlatform/appengine-guestbook-python.git
  - git://github.com/GoogleCloudPlatform/appengine-guestbook-python.git
  - https://github.com/GoogleCloudPlatform/appengine-guestbook-python/tree/part6-staticfiles
  - https://api.github.com/repos/GoogleCloudPlatform/appengine-guestbook-python/branches/part6-staticfiles
  """
  matcher = _GITHUB_URL_RE.match(html_url)
  if not matcher:
    return None
  components = matcher.group(1).split('/')
  components.extend([None, None])
  kwargs = {}
  if components[0] in ('users', 'repos'):
    components.pop(0)
  kwargs['user'] = components.pop(0)
  kwargs['repo'] = components.pop(0)
  if components[0] in ('tree', 'branches'):
    components.pop(0)
    kwargs['branch'] = components.pop(0)
  return Info(**kwargs)


def IsValidUrl(url):
  return _GITHUB_URL_RE.match(url)


def FetchAsyncWithAuth(*args, **kwargs):
  credential = model.GetOAuth2Credential('github')
  if credential:
    url_auth_suffix = ('?client_id={0}&client_secret={1}'
                       .format(credential.client_id,
                               credential.client_secret))
{'fraction_non_alphanumeric': 0.18077601410934743, 'fraction_numerical': 0.004232804232804233, 'mean_word_length': 4.228676809589673, 'pattern_counts': {'":': 82, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 87, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1956470', 'n_tokens_mistral': 3946, 'n_tokens_neox': 3828, 'n_words': 510}