<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" .draft -- a carrot for open science """

from __future__ import (division, print_function, absolute_import,
                        unicode_literals)

__all__ = ["webhook"]

import logging
import os
import requests
import signal
import subprocess
import yaml
from collections import deque
from random import choice
from re import search as re_search
from string import (ascii_uppercase, digits)
from tempfile import mkdtemp

# This is so dumb.
import github 
import pygithub3 as pygithub

logger = logging.getLogger(__name__)

GH_TOKEN = os.environ["GH_TOKEN"]
HEROKU_URL = os.environ["HEROKU_URL"] # There must be a better way..

# General utilities.
class Alarm(Exception):
    pass

def alarm_handler(signum, frame):
    raise Alarm

def random_string(N=10):
    """ Return a random string of alphanumeric characters of length `N`. """
    return ''.join(choice(ascii_uppercase + digits) for _ in range(N))


# Specific methods.
def git(command, cwd=None, shell=True, **kwargs):
    """
    Execute a git command (e.g., git `command`).

    :param command:
        The git command to execute.

    :param cwd: [optional]
        The current working directory where to execute the git command.

    :param shell: [optional]
        Whether to execute the command in a shell.

    :returns:
        The output from the git command.
    """
    
    try:
        result = subprocess.check_output("git {}".format(command), 
            cwd=cwd, shell=shell, **kwargs)

    except subprocess.CalledProcessError:
        raise

    else:
        return result


def is_valid_github_request(request):
    """
    Check whether a HTTP request is a valid ping from GitHub.

    :param request:
        The Django WSGI request.
    """
    print("checking request", request)
    print("request dict keys", request.__dict__.keys())
    print("request dict", request.__dict__)

    # Check the metadata headers.
    required_meta_headers = {
        "HTTP_X_GITHUB_EVENT": ("pull_request", "push"),
        "HTTP_X_GITHUB_DELIVERY": None
    }
    for key, acceptable_values in required_meta_headers.items():
        print("Check", key, request.environ, acceptable_values)
        
        # Only a key required?
        if key not in request.environ:
            return False

        # Specific values are acceptable.
        if  acceptable_values is not None \
        and request.environ[key] not in acceptable_values:
            return False


    # Check the payload.
    logger.info("Valid GitHub push request detected.")
    return True


def get_commit_comparisons(payload, repository_path=None):
    """
    Get the hashs of two commits that will be compared. A previous commit hash
    can be specified in the commit message, or will default to the previous
    commit on the same branch.

    :param payload:
        An event payload sent by GitHub.
        See https://developer.github.com/v3/activity/events/types/ for details.

    :param repository_path: [optional]
        A local folder containing the head of the repository that triggered the
        GitHub `payload`. This is necessary to check if a specific hash is valid
        (e.g., one that was specified by a commit message).

    :returns:
        A two-length tuple of the commit hashes to compare against (prev, now).
        If there is no previous commit, then the first item in the tuple will be
        `None`.
    """

    # We are on a commit.
    after_hash = payload["after"]
    prev_hash = payload["before"]   if len(payload["commits"]) == 1 \
                                    else payload["commits"][-2]["id"]

    # Check the commit message for something like [dd <hash>] or just [dd].
    m = re_search(
        "\[[d|D]{2}(\s[a-zA-Z0-9]+)*\]", payload["commits"][-1]["message"])

    if m is not None:
        if repository_path is None:
            raise ValueError(
                "a clone of the repository is necessary to check "
                "for valid commits")

        # Need to check that the hash is valid.
        hash_stub = m.group(0).rstrip("]").split(" ")
        if len(hash_stub) > 1:
            hash_stub = hash_stub[-1]

            try:
                r = git("cat-file -t {}".format(hash_stub), cwd=repository_path)
            
            except subprocess.CalledProcessError:
                # Invalid hash stub. Use the previous commit.
                logger.exception(
                    "Invalid hash stub given '{}':".format(hash_stub))
                
            else:
                prev_hash = hash_stub
    
    else:
        return (None, after_hash)

    # When this is the initial commit, GitHub gives a before hash of
    # '0000000000000000000000000000000000000000'
    if prev_hash == ("0"*40): prev_hash = None

    return (prev_hash, after_hash)


# TODO: This can probably go into a GHPayload class.
def get_manuscript_path(repository_path):
    """
    Get the path of the manuscript TeX file that will be used for comparisons.
    This defaults to the most edited `*.tex` file in the repository.

    :param repository_path:
        A local folder containing the head of the repository.

    :returns:
        The basename of the most edited `*.tex` file in the repository, 
        or `None` if no `*.tex` files could be found.
    """

    try:
        r = git('log --pretty=format: --name-only | sort | uniq -c |'\
                'sort -rg | grep ".tex$"', cwd=repository_path)
    
    except subprocess.CalledProcessError:
        logger.exception(
            "Cannot find any TeX files in repo at {}".format(repository_path))
        return None

    # Just in case the manuscript has spaces in it..
    basename = " ".join(str(r).split("\n")[0].strip().split()[1:])
    return basename




def load_settings(repository_path=None):
    """
    Load settings, optionally from a repository.

    :param repository_path: [optional]
        A local folder containing the head of the repository to look for a
        `.draft.yaml` file. If `None` is given, then the default options will
        be returned.

    :returns:
        A dictionary containing settings to use when performing the revision
        checks.
    """

    # Load defaults.
    with open(os.path.join(os.path.dirname(__file__), ".draft.yml"), "r") as fp:
        settings = yaml.load(fp)

    if repository_path is not None:
        path = os.path.join(repository_path, ".draft.yml")
        if os.path.exists(path):
            logger.info("Loading settings from {}".format(path))

            try:
                with open(path, "r") as fp:
                    given_settings = yaml.load(fp)

            except yaml.YAMLError:
                logger.exception("Cannot parse settings from {}".format(path))

            else:
                given_settings = given_settings or {}
                given_settings \
                    = dict((k.lower(), v) for k, v in given_settings.items())

                # Validate the settings so that we don't allow trickery.
                acceptable = {
                    "latex": ("pdflatex", ),
                }
                for key in set(given_settings).intersection(acceptable):
                    if given_settings[key] in acceptable[key]:
                        settings[key] = given_settings[key]

    return settings





def clone_repository(payload, branch=None):
    """
    Clone the repository that triggered the given event payload.

    :param payload:
        An event payload sent by GitHub. 
        See https://developer.github.com/v3/activity/events/types/ for details.

    :param branch: [optional]
        Specify the branch to clone.

    :returns:
        A temporary working directory where the repository has been cloned to.
    """

    twd = mkdtemp()
    branch = "" if branch is None else "-b {}".format(branch)

    r = git("clone {} {} {}".format(
        branch, payload["repository"]["clone_url"], twd))

    return twd


def latexdiff(old_path, new_path, **kwargs):
    """
    Run LaTeX-diff between an old manuscript and a new manuscript, and save the
    difference to a random (unused) filename in the path.

    :param old_path:
        The complete path of the old (before) manuscript.

    :param new_path:
        The complete path of the new (after) manuscript.

    :returns:
        The complete path of a new file that contains the LaTeX difference of
        the old and new files.
    """

    # Generate a temporary unused path for the difference file.<commit_msg>Repo failing to clone #debug<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" .draft -- a carrot for open science """

from __future__ import (division, print_function, absolute_import,
                        unicode_literals)

__all__ = ["webhook"]

import logging
import os
import requests
import signal
import subprocess
import yaml
from collections import deque
from random import choice
from re import search as re_search
from string import (ascii_uppercase, digits)
from tempfile import mkdtemp

# This is so dumb.
import github 
import pygithub3 as pygithub

logger = logging.getLogger(__name__)

GH_TOKEN = os.environ["GH_TOKEN"]
HEROKU_URL = os.environ["HEROKU_URL"] # There must be a better way..

# General utilities.
class Alarm(Exception):
    pass

def alarm_handler(signum, frame):
    raise Alarm

def random_string(N=10):
    """ Return a random string of alphanumeric characters of length `N`. """
    return ''.join(choice(ascii_uppercase + digits) for _ in range(N))


# Specific methods.
def git(command, cwd=None, shell=True, **kwargs):
    """
    Execute a git command (e.g., git `command`).

    :param command:
        The git command to execute.

    :param cwd: [optional]
        The current working directory where to execute the git command.

    :param shell: [optional]
        Whether to execute the command in a shell.

    :returns:
        The output from the git command.
    """
    
    try:
        result = subprocess.check_output("git {}".format(command), 
            cwd=cwd, shell=shell, **kwargs)

    except subprocess.CalledProcessError:
        raise

    else:
        return result


def is_valid_github_request(request):
    """
    Check whether a HTTP request is a valid ping from GitHub.

    :param request:
        The Django WSGI request.
    """
    print("checking request", request)
    print("request dict keys", request.__dict__.keys())
    print("request dict", request.__dict__)

    # Check the metadata headers.
    required_meta_headers = {
        "HTTP_X_GITHUB_EVENT": ("pull_request", "push"),
        "HTTP_X_GITHUB_DELIVERY": None
    }
    for key, acceptable_values in required_meta_headers.items():
        print("Check", key, request.environ, acceptable_values)
        
        # Only a key required?
        if key not in request.environ:
            return False

        # Specific values are acceptable.
        if  acceptable_values is not None \
        and request.environ[key] not in acceptable_values:
            return False


    # Check the payload.
    logger.info("Valid GitHub push request detected.")
    return True


def get_commit_comparisons(payload, repository_path=None):
    """
    Get the hashs of two commits that will be compared. A previous commit hash
    can be specified in the commit message, or will default to the previous
    commit on the same branch.

    :param payload:
        An event payload sent by GitHub.
        See https://developer.github.com/v3/activity/events/types/ for details.

    :param repository_path: [optional]
        A local folder containing the head of the repository that triggered the
        GitHub `payload`. This is necessary to check if a specific hash is valid
        (e.g., one that was specified by a commit message).

    :returns:
        A two-length tuple of the commit hashes to compare against (prev, now).
        If there is no previous commit, then the first item in the tuple will be
        `None`.
    """

    # We are on a commit.
    after_hash = payload["after"]
    prev_hash = payload["before"]   if len(payload["commits"]) == 1 \
                                    else payload["commits"][-2]["id"]

    # Check the commit message for something like [dd <hash>] or just [dd].
    m = re_search(
        "\[[d|D]{2}(\s[a-zA-Z0-9]+)*\]", payload["commits"][-1]["message"])

    if m is not None:
        if repository_path is None:
            raise ValueError(
                "a clone of the repository is necessary to check "
                "for valid commits")

        # Need to check that the hash is valid.
        hash_stub = m.group(0).rstrip("]").split(" ")
        if len(hash_stub) > 1:
            hash_stub = hash_stub[-1]

            try:
                r = git("cat-file -t {}".format(hash_stub), cwd=repository_path)
            
            except subprocess.CalledProcessError:
                # Invalid hash stub. Use the previous commit.
                logger.exception(
                    "Invalid hash stub given '{}':".format(hash_stub))
                
            else:
                prev_hash = hash_stub
    
    else:
        return (None, after_hash)

    # When this is the initial commit, GitHub gives a before hash of
    # '0000000000000000000000000000000000000000'
    if prev_hash == ("0"*40): prev_hash = None

    return (prev_hash, after_hash)


# TODO: This can probably go into a GHPayload class.
def get_manuscript_path(repository_path):
    """
    Get the path of the manuscript TeX file that will be used for comparisons.
    This defaults to the most edited `*.tex` file in the repository.

    :param repository_path:
        A local folder containing the head of the repository.

    :returns:
        The basename of the most edited `*.tex` file in the repository, 
        or `None` if no `*.tex` files could be found.
    """

    try:
        r = git('log --pretty=format: --name-only | sort | uniq -c |'\
                'sort -rg | grep ".tex$"', cwd=repository_path)
    
    except subprocess.CalledProcessError:
        logger.exception(
            "Cannot find any TeX files in repo at {}".format(repository_path))
        return None

    # Just in case the manuscript has spaces in it..
    basename = " ".join(str(r).split("\n")[0].strip().split()[1:])
    return basename




def load_settings(repository_path=None):
    """
    Load settings, optionally from a repository.

    :param repository_path: [optional]
        A local folder containing the head of the repository to look for a
        `.draft.yaml` file. If `None` is given, then the default options will
        be returned.

    :returns:
        A dictionary containing settings to use when performing the revision
        checks.
    """

    # Load defaults.
    with open(os.path.join(os.path.dirname(__file__), ".draft.yml"), "r") as fp:
        settings = yaml.load(fp)

    if repository_path is not None:
        path = os.path.join(repository_path, ".draft.yml")
        if os.path.exists(path):
            logger.info("Loading settings from {}".format(path))

            try:
                with open(path, "r") as fp:
                    given_settings = yaml.load(fp)

            except yaml.YAMLError:
                logger.exception("Cannot parse settings from {}".format(path))

            else:
                given_settings = given_settings or {}
                given_settings \
                    = dict((k.lower(), v) for k, v in given_settings.items())

                # Validate the settings so that we don't allow trickery.
                acceptable = {
                    "latex": ("pdflatex", ),
                }
                for key in set(given_settings).intersection(acceptable):
                    if given_settings[key] in acceptable[key]:
                        settings[key] = given_settings[key]

    return settings





def clone_repository(payload, branch=None):
    """
    Clone the repository that triggered the given event payload.

    :param payload:
        An event payload sent by GitHub. 
        See https://developer.github.com/v3/activity/events/types/ for details.

    :param branch: [optional]
        Specify the branch to clone.

    :returns:
        A temporary working directory where the repository has been cloned to.
    """

    twd = mkdtemp()
    branch = "" if branch is None else "-b {}".format(branch)

    print("branch", branch)
    print("payload repo clone", payload["repository"]["clone_url"])
    print("twd", twd)

    r = git("clone {} {} {}".format(
        branch, payload["repository"]["clone_url"], twd))
    print(r)

    return twd


def latexdiff(old_path, new_path, **kwargs):
    """
    Run LaTeX-diff between an old manuscript and a new manuscript, and save the
    difference to a random (unused) filename in the path.

    :param old_path:
        The complete path of the old (before) manuscript.

    :param new_path:
        The complete path of the new (after) manuscript.

    :returns:
        The complete path of a new file that contains the LaTeX difference of
        the old and new files.
    """

    # Generate a temporary unused path for the difference file.
{'fraction_non_alphanumeric': 0.0960992291195734, 'fraction_numerical': 0.006955312119631368, 'mean_word_length': 2.309802417034337, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5483028', 'n_tokens_mistral': 4924, 'n_tokens_neox': 4637, 'n_words': 1718}