<commit_before># -*- coding: utf-8 -*-

"""
    pickup.git
    ~~~~~~~~~~

    Implements various git methods

    :author:    Feei <<EMAIL>>
    :homepage:  https://github.com/wufeifei/cobra
    :license:   MIT, see LICENSE for more details.
    :copyright: Copyright (c) 2016 Feei. All rights reserved
"""

import os
import re
import subprocess
import logging
from urllib import quote
from utils import config

logging = logging.getLogger(__name__)

"""
usage and example.

#!/usr/bin/env python
from pickup.git import Git
repo_address = 'your repo address here'

# create a git object.
# if the repo_address is a public repo:
gg = Git(repo_address)
# if the repo_address is a private repo:
gg = Git(repo_address, username='your username here', password='<PASSWORD>')
# if the code on the other branch rather than master, try this:
gg = Git(repo_address, branch='dev')
# By default, if no branch param supply, it will be 'master'.

# use get_repo() method to clone the repo, if already cloned, it will pull the latest version.
gg.get_repo()

# you can also call the special method manually.
# clone() will clone the repo to local.
# pull() will pull the local repo to the latest version.
# gg.clone()
# gg.pull()

# Of course you can checkout to special branch
gg.checkout('master')
gg.checkout('dev')

# diff(new_version, old_version, raw_output) method will diff the two version.
# and return the diff result in str.
diff_result = gg.diff('ef8ab030a54e3', '4640bc08a08f4')
print diff_result

# the third param "raw_output" will control the return format. default is False.
# If it set True, it will return the "git diff" raw output, if it set to False (default is False),
# it will return a dict which stores the formatted increment content.
#
# example output
# {'bb.txt': ['hhhhhhh'], 'aa.txt': ['ccccc', 'ddddd']}
# bb.txt add a line, the content is 'hhhhhhh'.
# aa.txt add two line, the content is 'ccccc' and 'ddddd'.
"""


class Git:
    """
    A Git class.
    You can clone, pull, diff the repo via this class.

    repo_address: the repo's url
    repo_directory: the repo's local path
    repo_username: the username for the repo's url
    repo_password: the password for the repo's password
    repo_branch: the repo branch
    """

    repo_address = None
    repo_directory = None
    repo_username = None
    repo_password = None
    repo_branch = None
    repo_author = None
    repo_name = None

    # https://github.com/<username>/<reponame>

    def __init__(self, repo_address, branch='master', username=None, password=<PASSWORD>):

        # get upload directory
        self.upload_directory = os.path.join(config.Config('upload', 'directory').value, 'versions')
        if os.path.isdir(self.upload_directory) is False:
            os.makedirs(self.upload_directory)

        self.repo_address = repo_address
        self.repo_username = username
        self.repo_password = password
        self.repo_branch = branch
        repo_user = self.repo_address.split('/')[-2]
        repo_name = self.repo_address.split('/')[-1].replace('.git', '')
        self.repo_author = repo_user
        self.repo_name = repo_name
        if '.git' not in self.repo_address:
            self.repo_address += '.git'
        else:
            repo_name = repo_name.split('.')[0]

        self.repo_directory = os.path.join(os.path.join(self.upload_directory, repo_user), repo_name)

        logging.info('Git class init.')

    def pull(self):
        """Pull a repo from repo_address and repo_directory"""
        logging.info('Start Pull Repo...')

        if not self.__check_exist():
            logging.info('No local repo exist. Please clone first.')
            return False

        # change work directory to the repo
        repo_dir = self.repo_directory
        logging.debug('cd directory: {0}'.format(repo_dir))
        os.chdir(repo_dir)

        cmd = 'git pull origin master'
        p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
        (pull_out, pull_err) = p.communicate()
        logging.info(pull_out)
        logging.info(pull_err)

        # change work directory back.
        os.chdir(repo_dir)

        if 'Updating' in pull_out or 'up-to-date' in pull_out:
            logging.info('Pull done.')
            return True
        else:<commit_msg>fix gitlab.com/user/project.something.else.git case<commit_after># -*- coding: utf-8 -*-

"""
    pickup.git
    ~~~~~~~~~~

    Implements various git methods

    :author:    Feei <<EMAIL>>
    :homepage:  https://github.com/wufeifei/cobra
    :license:   MIT, see LICENSE for more details.
    :copyright: Copyright (c) 2016 Feei. All rights reserved
"""

import os
import re
import subprocess
import logging
from urllib import quote
from utils import config

logging = logging.getLogger(__name__)

"""
usage and example.

#!/usr/bin/env python
from pickup.git import Git
repo_address = 'your repo address here'

# create a git object.
# if the repo_address is a public repo:
gg = Git(repo_address)
# if the repo_address is a private repo:
gg = Git(repo_address, username='your username here', password='<PASSWORD>')
# if the code on the other branch rather than master, try this:
gg = Git(repo_address, branch='dev')
# By default, if no branch param supply, it will be 'master'.

# use get_repo() method to clone the repo, if already cloned, it will pull the latest version.
gg.get_repo()

# you can also call the special method manually.
# clone() will clone the repo to local.
# pull() will pull the local repo to the latest version.
# gg.clone()
# gg.pull()

# Of course you can checkout to special branch
gg.checkout('master')
gg.checkout('dev')

# diff(new_version, old_version, raw_output) method will diff the two version.
# and return the diff result in str.
diff_result = gg.diff('ef8ab030a54e3', '4640bc08a08f4')
print diff_result

# the third param "raw_output" will control the return format. default is False.
# If it set True, it will return the "git diff" raw output, if it set to False (default is False),
# it will return a dict which stores the formatted increment content.
#
# example output
# {'bb.txt': ['hhhhhhh'], 'aa.txt': ['ccccc', 'ddddd']}
# bb.txt add a line, the content is 'hhhhhhh'.
# aa.txt add two line, the content is 'ccccc' and 'ddddd'.
"""


class Git:
    """
    A Git class.
    You can clone, pull, diff the repo via this class.

    repo_address: the repo's url
    repo_directory: the repo's local path
    repo_username: the username for the repo's url
    repo_password: the password for the repo's password
    repo_branch: the repo branch
    """

    repo_address = None
    repo_directory = None
    repo_username = None
    repo_password = None
    repo_branch = None
    repo_author = None
    repo_name = None

    # https://github.com/<username>/<reponame>

    def __init__(self, repo_address, branch='master', username=None, password=None):

        # get upload directory
        self.upload_directory = os.path.join(config.Config('upload', 'directory').value, 'versions')
        if os.path.isdir(self.upload_directory) is False:
            os.makedirs(self.upload_directory)

        self.repo_address = repo_address
        self.repo_username = username
        self.repo_password = password
        self.repo_branch = branch
        repo_user = self.repo_address.split('/')[-2]
        repo_name = self.repo_address.split('/')[-1].replace('.git', '')
        self.repo_author = repo_user
        self.repo_name = repo_name

        self.repo_directory = os.path.join(os.path.join(self.upload_directory, repo_user), repo_name)

        logging.info('Git class init.')

    def pull(self):
        """Pull a repo from repo_address and repo_directory"""
        logging.info('Start Pull Repo...')

        if not self.__check_exist():
            logging.info('No local repo exist. Please clone first.')
            return False

        # change work directory to the repo
        repo_dir = self.repo_directory
        logging.debug('cd directory: {0}'.format(repo_dir))
        os.chdir(repo_dir)

        cmd = 'git pull origin master'
        p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
        (pull_out, pull_err) = p.communicate()
        logging.info(pull_out)
        logging.info(pull_err)

        # change work directory back.
        os.chdir(repo_dir)

        if 'Updating' in pull_out or 'up-to-date' in pull_out:
            logging.info('Pull done.')
            return True
        else:
{'fraction_non_alphanumeric': 0.10930178069353327, 'fraction_numerical': 0.005740393626991565, 'mean_word_length': 3.04405495026054, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7097733', 'n_tokens_mistral': 2648, 'n_tokens_neox': 2559, 'n_words': 855}