<commit_before># Webhooks for external integrations.

from __future__ import absolute_import

from django.conf import settings
from zerver.models import UserProfile, get_client, get_user_profile_by_email
from zerver.lib.actions import check_send_message
from zerver.lib.notifications import  convert_html_to_markdown
from zerver.lib.response import json_success, json_error
from zerver.lib.validator import check_dict
from zerver.decorator import authenticated_api_view, REQ, \
    has_request_variables, authenticated_rest_api_view, \
    api_key_only_webhook_view, to_non_negative_int, flexible_boolean
from zerver.views.messages import send_message_backend
from django.db.models import Q

from defusedxml.ElementTree import fromstring as xml_fromstring

import pprint
import base64
import logging
import re
import ujson
from functools import wraps

def github_generic_subject(noun, topic_focus, blob):
    # issue and pull_request objects have the same fields we're interested in
    return "%s: %s %d: %s" % (topic_focus, noun, blob['number'], blob['title'])

def github_generic_content(noun, payload, blob):
    action = payload['action']
    if action == 'synchronize':
        action = 'synchronized'

    # issue and pull_request objects have the same fields we're interested in
    content = ("%s %s [%s %s](%s)"
               % (payload['sender']['login'],
                  action,
                  noun,
                  blob['number'],
                  blob['html_url']))
    if payload['action'] in ('opened', 'reopened'):
        content += "\n\n~~~ quote\n%s\n~~~" % (blob['body'],)
    return content


def api_github_v1(user_profile, event, payload, branches, stream, **kwargs):
    """
    processes github payload with version 1 field specification
    `payload` comes in unmodified from github
    `stream` is set to 'commits' if otherwise unset
    """
    commit_stream = stream
    issue_stream = 'issues'
    return api_github_v2(user_profile, event, payload, branches, stream, commit_stream, issue_stream, **kwargs)


def api_github_v2(user_profile, event, payload, branches, default_stream, commit_stream, issue_stream, topic_focus = None):
    """
    processes github payload with version 2 field specification
    `payload` comes in unmodified from github
    `default_stream` is set to what `stream` is in v1 above
    `commit_stream` and `issue_stream` fall back to `default_stream` if they are empty
    This and allowing alternative endpoints is what distinguishes v1 from v2 of the github configuration
    """
    if not commit_stream:
        commit_stream = default_stream
    if not issue_stream:
        issue_stream = default_stream

    target_stream = commit_stream
    repository = payload['repository']

    if not topic_focus:
        topic_focus = repository['name']

    # Event Handlers
    if event == 'pull_request':
        pull_req = payload['pull_request']
        subject = github_generic_subject('pull request', topic_focus, pull_req)
        content = github_generic_content('pull request', payload, pull_req)
    elif event == 'issues':
        # in v1, we assume that this stream exists, since we only handle issues for CUSTOMER5 and ourselves
        target_stream = issue_stream
        issue = payload['issue']
        subject = github_generic_subject('issue', topic_focus, issue)
        content = github_generic_content('issue', payload, issue)
    elif event == 'issue_comment':
        # Comments on both issues and pull requests come in as issue_comment events
        issue = payload['issue']
        if 'pull_request' not in issue or issue['pull_request']['diff_url'] is None:
            # It's an issues comment
            target_stream = issue_stream
            noun = 'issue'
        else:
            # It's a pull request comment
            noun = 'pull request'

        subject = github_generic_subject(noun, topic_focus, issue)
        comment = payload['comment']
        content = ("%s [commented](%s) on [%s %d](%s)\n\n~~~ quote\n%s\n~~~"
                   % (comment['user']['login'],
                      comment['html_url'],
                      noun,
                      issue['number'],
                      issue['html_url'],
                      comment['body']))
    elif event == 'push':
        subject, content = build_message_from_gitlog(user_profile, topic_focus,
                                                     payload['ref'], payload['commits'],
                                                     payload['before'], payload['after'],
                                                     payload['compare'],
                                                     payload['pusher']['name'],
                                                     forced=payload['forced'],
                                                     created=payload['created'])
    elif event == 'commit_comment':
        comment = payload['comment']
        subject = "%s: commit %s" % (topic_focus, comment['commit_id'])

        content = ("%s [commented](%s)"
                   % (comment['user']['login'],
                      comment['html_url']))

        if comment['line'] is not None:
            content += " on `%s`, line %d" % (comment['path'], comment['line'])

        content += "\n\n~~~ quote\n%s\n~~~" % (comment['body'],)

    return (target_stream, subject, content)

@authenticated_api_view
@has_request_variables
def api_github_landing(request, user_profile, event=REQ,
                       payload=REQ(validator=check_dict([])),
                       branches=REQ(default=''),
                       stream=REQ(default=''),
                       version=REQ(converter=to_non_negative_int, default=1),
                       commit_stream=REQ(default=''),
                       issue_stream=REQ(default=''),
                       exclude_pull_requests=REQ(converter=flexible_boolean, default=False),
                       exclude_issues=REQ(converter=flexible_boolean, default=False),
                       exclude_commits=REQ(converter=flexible_boolean, default=False)
                       ):

    repository = payload['repository']

    # Special hook for capturing event data. If we see our special test repo, log the payload from github.
    try:
        if repository['name'] == 'zulip-test' and repository['id'] == 6893087 and settings.PRODUCTION:
            with open('/var/log/zulip/github-payloads', 'a') as f:
                f.write(ujson.dumps({'event': event,
                                     'payload': payload,
                                     'branches': branches,
                                     'stream': stream,
                                     'version': version,
                                     'commit_stream': commit_stream,
                                     'issue_stream': issue_stream,
                                     'exclude_pull_requests': exclude_pull_requests,
                                     'exclude_issues': exclude_issues,
                                     'exclude_commits': exclude_commits}))
                f.write("\n")
    except Exception:
        logging.exception("Error while capturing Github event")

    if not stream:
        stream = 'commits'

    short_ref = re.sub(r'^refs/heads/', '', payload.get('ref', ""))
    kwargs = dict()

    ### realm-specific logic
    domain = user_profile.realm.domain
    if domain == "customer26.invalid" and short_ref:
        kwargs['topic_focus'] = short_ref

    allowed_events = set()
    if not exclude_pull_requests:
        allowed_events.add('pull_request')

    if not exclude_issues:
        allowed_events.add("issues")
        allowed_events.add("issue_comment")

    if not exclude_commits:
        allowed_events.add("push")
        allowed_events.add("commit_comment")

    if event not in allowed_events:
        return json_success()

    # We filter issue_comment events for issue creation events<commit_msg>Add support for emphasize_branch_in_topic feature.<commit_after># Webhooks for external integrations.

from __future__ import absolute_import

from django.conf import settings
from zerver.models import UserProfile, get_client, get_user_profile_by_email
from zerver.lib.actions import check_send_message
from zerver.lib.notifications import  convert_html_to_markdown
from zerver.lib.response import json_success, json_error
from zerver.lib.validator import check_dict
from zerver.decorator import authenticated_api_view, REQ, \
    has_request_variables, authenticated_rest_api_view, \
    api_key_only_webhook_view, to_non_negative_int, flexible_boolean
from zerver.views.messages import send_message_backend
from django.db.models import Q

from defusedxml.ElementTree import fromstring as xml_fromstring

import pprint
import base64
import logging
import re
import ujson
from functools import wraps

def github_generic_subject(noun, topic_focus, blob):
    # issue and pull_request objects have the same fields we're interested in
    return "%s: %s %d: %s" % (topic_focus, noun, blob['number'], blob['title'])

def github_generic_content(noun, payload, blob):
    action = payload['action']
    if action == 'synchronize':
        action = 'synchronized'

    # issue and pull_request objects have the same fields we're interested in
    content = ("%s %s [%s %s](%s)"
               % (payload['sender']['login'],
                  action,
                  noun,
                  blob['number'],
                  blob['html_url']))
    if payload['action'] in ('opened', 'reopened'):
        content += "\n\n~~~ quote\n%s\n~~~" % (blob['body'],)
    return content


def api_github_v1(user_profile, event, payload, branches, stream, **kwargs):
    """
    processes github payload with version 1 field specification
    `payload` comes in unmodified from github
    `stream` is set to 'commits' if otherwise unset
    """
    commit_stream = stream
    issue_stream = 'issues'
    return api_github_v2(user_profile, event, payload, branches, stream, commit_stream, issue_stream, **kwargs)


def api_github_v2(user_profile, event, payload, branches, default_stream, commit_stream, issue_stream, topic_focus = None):
    """
    processes github payload with version 2 field specification
    `payload` comes in unmodified from github
    `default_stream` is set to what `stream` is in v1 above
    `commit_stream` and `issue_stream` fall back to `default_stream` if they are empty
    This and allowing alternative endpoints is what distinguishes v1 from v2 of the github configuration
    """
    if not commit_stream:
        commit_stream = default_stream
    if not issue_stream:
        issue_stream = default_stream

    target_stream = commit_stream
    repository = payload['repository']

    if not topic_focus:
        topic_focus = repository['name']

    # Event Handlers
    if event == 'pull_request':
        pull_req = payload['pull_request']
        subject = github_generic_subject('pull request', topic_focus, pull_req)
        content = github_generic_content('pull request', payload, pull_req)
    elif event == 'issues':
        # in v1, we assume that this stream exists, since we only handle issues for CUSTOMER5 and ourselves
        target_stream = issue_stream
        issue = payload['issue']
        subject = github_generic_subject('issue', topic_focus, issue)
        content = github_generic_content('issue', payload, issue)
    elif event == 'issue_comment':
        # Comments on both issues and pull requests come in as issue_comment events
        issue = payload['issue']
        if 'pull_request' not in issue or issue['pull_request']['diff_url'] is None:
            # It's an issues comment
            target_stream = issue_stream
            noun = 'issue'
        else:
            # It's a pull request comment
            noun = 'pull request'

        subject = github_generic_subject(noun, topic_focus, issue)
        comment = payload['comment']
        content = ("%s [commented](%s) on [%s %d](%s)\n\n~~~ quote\n%s\n~~~"
                   % (comment['user']['login'],
                      comment['html_url'],
                      noun,
                      issue['number'],
                      issue['html_url'],
                      comment['body']))
    elif event == 'push':
        subject, content = build_message_from_gitlog(user_profile, topic_focus,
                                                     payload['ref'], payload['commits'],
                                                     payload['before'], payload['after'],
                                                     payload['compare'],
                                                     payload['pusher']['name'],
                                                     forced=payload['forced'],
                                                     created=payload['created'])
    elif event == 'commit_comment':
        comment = payload['comment']
        subject = "%s: commit %s" % (topic_focus, comment['commit_id'])

        content = ("%s [commented](%s)"
                   % (comment['user']['login'],
                      comment['html_url']))

        if comment['line'] is not None:
            content += " on `%s`, line %d" % (comment['path'], comment['line'])

        content += "\n\n~~~ quote\n%s\n~~~" % (comment['body'],)

    return (target_stream, subject, content)

@authenticated_api_view
@has_request_variables
def api_github_landing(request, user_profile, event=REQ,
                       payload=REQ(validator=check_dict([])),
                       branches=REQ(default=''),
                       stream=REQ(default=''),
                       version=REQ(converter=to_non_negative_int, default=1),
                       commit_stream=REQ(default=''),
                       issue_stream=REQ(default=''),
                       exclude_pull_requests=REQ(converter=flexible_boolean, default=False),
                       exclude_issues=REQ(converter=flexible_boolean, default=False),
                       exclude_commits=REQ(converter=flexible_boolean, default=False),
                       emphasize_branch_in_topic=REQ(converter=flexible_boolean, default=False),
                       ):

    repository = payload['repository']

    # Special hook for capturing event data. If we see our special test repo, log the payload from github.
    try:
        if repository['name'] == 'zulip-test' and repository['id'] == 6893087 and settings.PRODUCTION:
            with open('/var/log/zulip/github-payloads', 'a') as f:
                f.write(ujson.dumps({'event': event,
                                     'payload': payload,
                                     'branches': branches,
                                     'stream': stream,
                                     'version': version,
                                     'commit_stream': commit_stream,
                                     'issue_stream': issue_stream,
                                     'exclude_pull_requests': exclude_pull_requests,
                                     'exclude_issues': exclude_issues,
                                     'exclude_commits': exclude_commits,
                                     'emphasize_branch_in_topic': emphasize_branch_in_topic,
                                     }))
                f.write("\n")
    except Exception:
        logging.exception("Error while capturing Github event")

    if not stream:
        stream = 'commits'

    short_ref = re.sub(r'^refs/heads/', '', payload.get('ref', ""))
    kwargs = dict()

    if (emphasize_branch_in_topic or user_profile.realm.domain == "customer26.invalid") and short_ref:
        kwargs['topic_focus'] = short_ref

    allowed_events = set()
    if not exclude_pull_requests:
        allowed_events.add('pull_request')

    if not exclude_issues:
        allowed_events.add("issues")
        allowed_events.add("issue_comment")

    if not exclude_commits:
        allowed_events.add("push")
        allowed_events.add("commit_comment")

    if event not in allowed_events:
        return json_success()

    # We filter issue_comment events for issue creation events
{'fraction_non_alphanumeric': 0.10025325838532337, 'fraction_numerical': 0.0027178948668849217, 'mean_word_length': 2.209117938553023, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1747413', 'n_tokens_mistral': 4269, 'n_tokens_neox': 4061, 'n_words': 1119}