<commit_before># -- coding: utf-8 --
from __future__ import absolute_import
import logging
from os import mkdir, remove
from os.path import join, split, exists, isdir, sep
from git import Repo
from git.cmd import GitCommandError
import yaml
from re import match, search
import json
import random

from . import edit_functions, google_api_functions

TASK_METADATA_FILENAME = u'_task.yml'
BRANCH_NAME_LENGTH = 9
DESCRIPTION_MAX_LENGTH = 15
ACTIVITY_CREATED_MESSAGE = u'activity was started'
ACTIVITY_UPDATED_MESSAGE = u'activity was updated'
ACTIVITY_DELETED_MESSAGE = u'activity was deleted'
COMMENT_COMMIT_PREFIX = u'Provided feedback.'
REVIEW_STATE_COMMIT_PREFIX = u'Updated review state.'
ACTIVITY_FEEDBACK_MESSAGE = u'requested feedback on this activity.'
ACTIVITY_ENDORSED_MESSAGE = u'endorsed this activity.'
ACTIVITY_PUBLISHED_MESSAGE = u'published this activity.'

# the different categories and types of messages that can be displayed in the activity overview
MESSAGE_CATEGORY_INFO = u'info'
MESSAGE_TYPE_ACTIVITY_UPDATE = u'activity update'
MESSAGE_TYPE_REVIEW_UPDATE = u'review update'

MESSAGE_CATEGORY_EDIT = u'edit'
MESSAGE_TYPE_EDIT = u'edit'

MESSAGE_CATEGORY_COMMENT = u'comment'
MESSAGE_TYPE_COMMENT = u'comment'

# the different review states for an activity
# no changes have yet been made to the activity
REVIEW_STATE_FRESH = u'fresh'
# there are un-reviewed edits in the activity (or no edits at all)
REVIEW_STATE_EDITED = u'unreviewed edits'
# there are un-reviewed edits in the activity and a review has been requested
REVIEW_STATE_FEEDBACK = u'feedback requested'
# a review has happened and the site is ready to be published
REVIEW_STATE_ENDORSED = u'edits endorsed'
# the site has been published
REVIEW_STATE_PUBLISHED = u'changes published'

# the working state of the activity
WORKING_STATE_ACTIVE = u'active'
WORKING_STATE_PUBLISHED = u'published'
WORKING_STATE_DELETED = u'deleted'

# Name of file in running state dir that signals a need to push upstream.
NEEDS_PUSH_FILE = 'needs-push'

# actions for merge conflict descriptions
CONFLICT_ACTION_CREATED = u'created'
CONFLICT_ACTION_DELETED = u'deleted'
CONFLICT_ACTION_EDITED = u'edited'

class MergeConflict (Exception):
    def __init__(self, remote_commit, local_commit):
        self.remote_commit = remote_commit
        self.local_commit = local_commit

    def files(self):
        diffs = self.remote_commit.diff(self.local_commit)

        created_files = [{"actions": CONFLICT_ACTION_CREATED, "path": d.b_blob.path} for d in diffs if d.new_file]
        deleted_files = [{"actions": CONFLICT_ACTION_DELETED, "path": d.a_blob.path} for d in diffs if d.deleted_file]
        edited_files = [{"actions": CONFLICT_ACTION_EDITED, "path": d.a_blob.path} for d in diffs if not (d.deleted_file or d.new_file)]

        return created_files + deleted_files + edited_files

    def __str__(self):
        return 'MergeConflict(%s, %s)' % (self.remote_commit, self.local_commit)

class ChimeRepo(Repo):

    def full_path(self, *args):
        return join(self.working_dir, self.repo_path(*args))

    def repo_path(self, *args):
        if len(args) >= 2:
            return self.canonicalize_path(*args) # no idea why; probably belongs up in web-handling code
        return join(*args)

    def canonicalize_path(self, *args):
        ''' Return a slugified version of the passed path
        '''
        result = join((args[0] or '').rstrip('/'), *args[1:])
        split_path = split(result) # also probably belongs up in request handling code
        result = join(edit_functions.make_slug_path(split_path[0]), split_path[1])

        return result

    def exists(self, path_in_repo):
        return exists(self.full_path(path_in_repo))

    def create_directories_if_necessary(self, path):
        ''' Creates any necessary directories for a path, ignoring any file component.
            If you pass "foo/bar.txt" or "foo/bar" it'll create directory "foo".
            If you pass "foo/bar/" it will create directories "foo" and "foo/bar".
        '''
        dirs = self.dirs_for_path(path)

        # Create directory tree.
        #
        for i in range(len(dirs)):
            build_path = join(self.working_dir, sep.join(dirs[:i + 1]))

            if not isdir(build_path):
                mkdir(build_path)

    def dirs_for_path(self, path):
        head, dirs = split(path)[0], []
        while head:
            head, check_dir = split(head)
            dirs.insert(0, check_dir)
        if '..' in dirs:
            raise Exception('Encountered an invalid component in this path: {}'.format(path))

        return dirs

def _origin(branch_name):
    ''' Format the branch name into a origin path and return it.
    '''
    return 'origin/' + branch_name

def get_activity_working_state(repo, branch_name):
    ''' Get whether the activity is active, published, or deleted.
    '''
    ls_remote_output = repo.git.ls_remote("origin", branch_name)
    if 'refs/heads/{}'.format(branch_name) not in ls_remote_output and 'refs/tags/{}'.format(branch_name) in ls_remote_output:
        return WORKING_STATE_PUBLISHED

    if 'refs/heads/{}'.format(branch_name) not in ls_remote_output and 'refs/tags/{}'.format(branch_name) not in ls_remote_output:
        return WORKING_STATE_DELETED

    return WORKING_STATE_ACTIVE

def get_branch_start_point(clone, default_branch_name, new_branch_name):
    ''' Return the last commit on the branch
    '''
    if _origin(new_branch_name) in clone.refs:
        return clone.refs[_origin(new_branch_name)].commit

    if _origin(default_branch_name) in clone.refs:
        return clone.refs[_origin(default_branch_name)].commit

    return clone.branches[default_branch_name].commit

def get_existing_branch(clone, default_branch_name, new_branch_name):
    ''' Return an existing branch with the passed name, if it exists.
    '''
    clone.git.fetch('origin')<commit_msg>splitting ls-remote output<commit_after># -- coding: utf-8 --
from __future__ import absolute_import
import logging
from os import mkdir, remove
from os.path import join, split, exists, isdir, sep
from git import Repo
from git.cmd import GitCommandError
import yaml
from re import match, search
import json
import random

from . import edit_functions, google_api_functions

TASK_METADATA_FILENAME = u'_task.yml'
BRANCH_NAME_LENGTH = 9
DESCRIPTION_MAX_LENGTH = 15
ACTIVITY_CREATED_MESSAGE = u'activity was started'
ACTIVITY_UPDATED_MESSAGE = u'activity was updated'
ACTIVITY_DELETED_MESSAGE = u'activity was deleted'
COMMENT_COMMIT_PREFIX = u'Provided feedback.'
REVIEW_STATE_COMMIT_PREFIX = u'Updated review state.'
ACTIVITY_FEEDBACK_MESSAGE = u'requested feedback on this activity.'
ACTIVITY_ENDORSED_MESSAGE = u'endorsed this activity.'
ACTIVITY_PUBLISHED_MESSAGE = u'published this activity.'

# the different categories and types of messages that can be displayed in the activity overview
MESSAGE_CATEGORY_INFO = u'info'
MESSAGE_TYPE_ACTIVITY_UPDATE = u'activity update'
MESSAGE_TYPE_REVIEW_UPDATE = u'review update'

MESSAGE_CATEGORY_EDIT = u'edit'
MESSAGE_TYPE_EDIT = u'edit'

MESSAGE_CATEGORY_COMMENT = u'comment'
MESSAGE_TYPE_COMMENT = u'comment'

# the different review states for an activity
# no changes have yet been made to the activity
REVIEW_STATE_FRESH = u'fresh'
# there are un-reviewed edits in the activity (or no edits at all)
REVIEW_STATE_EDITED = u'unreviewed edits'
# there are un-reviewed edits in the activity and a review has been requested
REVIEW_STATE_FEEDBACK = u'feedback requested'
# a review has happened and the site is ready to be published
REVIEW_STATE_ENDORSED = u'edits endorsed'
# the site has been published
REVIEW_STATE_PUBLISHED = u'changes published'

# the working state of the activity
WORKING_STATE_ACTIVE = u'active'
WORKING_STATE_PUBLISHED = u'published'
WORKING_STATE_DELETED = u'deleted'

# Name of file in running state dir that signals a need to push upstream.
NEEDS_PUSH_FILE = 'needs-push'

# actions for merge conflict descriptions
CONFLICT_ACTION_CREATED = u'created'
CONFLICT_ACTION_DELETED = u'deleted'
CONFLICT_ACTION_EDITED = u'edited'

class MergeConflict (Exception):
    def __init__(self, remote_commit, local_commit):
        self.remote_commit = remote_commit
        self.local_commit = local_commit

    def files(self):
        diffs = self.remote_commit.diff(self.local_commit)

        created_files = [{"actions": CONFLICT_ACTION_CREATED, "path": d.b_blob.path} for d in diffs if d.new_file]
        deleted_files = [{"actions": CONFLICT_ACTION_DELETED, "path": d.a_blob.path} for d in diffs if d.deleted_file]
        edited_files = [{"actions": CONFLICT_ACTION_EDITED, "path": d.a_blob.path} for d in diffs if not (d.deleted_file or d.new_file)]

        return created_files + deleted_files + edited_files

    def __str__(self):
        return 'MergeConflict(%s, %s)' % (self.remote_commit, self.local_commit)

class ChimeRepo(Repo):

    def full_path(self, *args):
        return join(self.working_dir, self.repo_path(*args))

    def repo_path(self, *args):
        if len(args) >= 2:
            return self.canonicalize_path(*args) # no idea why; probably belongs up in web-handling code
        return join(*args)

    def canonicalize_path(self, *args):
        ''' Return a slugified version of the passed path
        '''
        result = join((args[0] or '').rstrip('/'), *args[1:])
        split_path = split(result) # also probably belongs up in request handling code
        result = join(edit_functions.make_slug_path(split_path[0]), split_path[1])

        return result

    def exists(self, path_in_repo):
        return exists(self.full_path(path_in_repo))

    def create_directories_if_necessary(self, path):
        ''' Creates any necessary directories for a path, ignoring any file component.
            If you pass "foo/bar.txt" or "foo/bar" it'll create directory "foo".
            If you pass "foo/bar/" it will create directories "foo" and "foo/bar".
        '''
        dirs = self.dirs_for_path(path)

        # Create directory tree.
        #
        for i in range(len(dirs)):
            build_path = join(self.working_dir, sep.join(dirs[:i + 1]))

            if not isdir(build_path):
                mkdir(build_path)

    def dirs_for_path(self, path):
        head, dirs = split(path)[0], []
        while head:
            head, check_dir = split(head)
            dirs.insert(0, check_dir)
        if '..' in dirs:
            raise Exception('Encountered an invalid component in this path: {}'.format(path))

        return dirs

def _origin(branch_name):
    ''' Format the branch name into a origin path and return it.
    '''
    return 'origin/' + branch_name

def get_activity_working_state(repo, branch_name):
    ''' Get whether the activity is active, published, or deleted.
    '''
    ls_remote_output = repo.git.ls_remote("origin", branch_name).split()
    if 'refs/heads/{}'.format(branch_name) not in ls_remote_output and 'refs/tags/{}'.format(branch_name) in ls_remote_output:
        return WORKING_STATE_PUBLISHED

    if 'refs/heads/{}'.format(branch_name) not in ls_remote_output and 'refs/tags/{}'.format(branch_name) not in ls_remote_output:
        return WORKING_STATE_DELETED

    return WORKING_STATE_ACTIVE

def get_branch_start_point(clone, default_branch_name, new_branch_name):
    ''' Return the last commit on the branch
    '''
    if _origin(new_branch_name) in clone.refs:
        return clone.refs[_origin(new_branch_name)].commit

    if _origin(default_branch_name) in clone.refs:
        return clone.refs[_origin(default_branch_name)].commit

    return clone.branches[default_branch_name].commit

def get_existing_branch(clone, default_branch_name, new_branch_name):
    ''' Return an existing branch with the passed name, if it exists.
    '''
    clone.git.fetch('origin')
{'fraction_non_alphanumeric': 0.0901950848745883, 'fraction_numerical': 0.0020268558398783888, 'mean_word_length': 3.923908523908524, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2813806', 'n_tokens_mistral': 3761, 'n_tokens_neox': 3627, 'n_words': 1055}