<commit_before>from .. import app
from .. import db
from .. import queue
from .. import util
from ..models import Post, Mention, get_settings
from bs4 import BeautifulSoup
from flask import request, make_response, render_template, url_for, abort
from werkzeug.exceptions import NotFound
import datetime
import mf2py
import mf2util
import requests
import urllib.parse
import urllib.request

_app = None


def register():
    pass


@app.route('/webmention', methods=['GET', 'POST'])
def receive_webmention():
    if request.method == 'GET':
        return render_template('webmention.jinja2')

    source = request.form.get('source')
    target = request.form.get('target')
    callback = request.form.get('callback')

    if not source:
        return make_response(
            'webmention missing required source parameter', 400)

    if not target:
        return make_response(
            'webmention missing required target parameter', 400)

    app.logger.debug("Webmention from %s to %s received", source, target)

    key = queue.enqueue(process_webmention, source, target, callback)
    status_url = url_for('webmention_status', key=key, _external=True)

    return make_response(
        render_template('wm_received.jinja2', status_url=status_url), 202)


@app.route('/webmention/status/<key>')
def webmention_status(key):
    rv = queue.query(key)

    if not rv:
        rv = {
            'response_code': 400,
            'status': 'unknown',
            'reason': 'Job does not exist or its status has expired',
        }
    elif rv == 'queued':
        rv = {
            'response_code': 202,
            'status': 'queued',
            'reason': 'Mention has been queued for processing',
        }

    return make_response(
       render_template('wm_status.jinja2', **rv),
       rv.get('response_code', 400))


def process_webmention(source, target, callback):
    def call_callback(result):
        if callback:
            requests.post(callback, data=result)

    try:
        target_post, mention, delete, error = do_process_webmention(
            source, target)

        if error or not target_post or not mention:
            app.logger.warn("Failed to process webmention: %s", error)
            result = {
                'source': source,
                'target': target,
                'response_code': 400,
                'status': 'error',
                'reason': error
            }
            call_callback(result)
            return result

        if delete:
            target_post.mentions = [m for m in target_post.mentions if
                                    m.url != source]

        if not delete:
            target_post.mentions.append(mention)

        db.session.commit()
        app.logger.debug("saved mentions to %s", target_post.path)

        result = {
            'source': source,
            'target': target,
            'response_code': 200,
            'status': 'success',
            'reason': 'Deleted' if delete else 'Created'
        }

        call_callback(result)
        return result

    except Exception as e:
        app.logger.exception("exception while processing webmention")
        result = {
            'source': source,
            'target': target,
            'response_code': 400,
            'status': 'error',
            'reason': "exception while processing webmention {}".format(e)
        }
        call_callback(result)
        return result


def do_process_webmention(source, target):
    app.logger.debug("processing webmention from %s to %s", source, target)
    if target and target.strip('/') == get_settings().site_url.strip('/'):
        # received a domain-level mention
        app.logger.debug('received domain-level webmention from %s', source)
        target_post = None
        target_urls = (target,)
        # TODO save domain-level webmention somewhere
    else:
        # confirm that target is a valid link to a post
        target_post = find_target_post(target)

        if not target_post:
            app.logger.warn(
                "Webmention could not find target post: %s. Giving up", target)
            return (None, None, False,
                    "Webmention could not find target post: {}".format(target))
        target_urls = (target, target_post.permalink,)

    if source in target_urls:
        return (None, None, False,
                '{} and {} refer to the same post'.format(source, target))

    # confirm that source actually refers to the post
    source_response = util.fetch_html(source)
    app.logger.debug('received response from source %s', source_response)

    if source_response.status_code == 410:
        app.logger.debug("Webmention indicates original was deleted")
        return target_post, None, True, None

    if source_response.status_code // 100 != 2:
        app.logger.warn(
            "Webmention could not read source post: %s. Giving up", source)
        return target_post, None, False, \
            "Bad response when reading source post: {}, {}"\
            .format(source, source_response)

    source_length = source_response.headers.get('Content-Length')

    if source_length and int(source_length) > 2097152:
        app.logger.warn("Very large source. length=%s", source_length)
        return target_post, None, False,\
            "Source is very large. Length={}"\
            .format(source_length)

    link_to_target = find_link_to_target(source, source_response, target_urls)
    if not link_to_target:
        app.logger.warn(
            "Webmention source %s does not appear to link to target %s. "
            "Giving up", source, target)
        return target_post, None, False,\
            "Could not find any links from source to target"

    mention = create_mention(target_post, source, source_response)
    return target_post, mention, False, None


def find_link_to_target(source_url, source_response, target_urls):
    if source_response.status_code // 2 != 100:
        app.logger.warn(
            "Received unexpected response from webmention source: %s",
            source_response.text)
        return None

    # Don't worry about Microformats for now; just see if there is a
    # link anywhere that points back to the target
    soup = BeautifulSoup(source_response.text)
    for link in soup.find_all(['a', 'link']):<commit_msg>Added optional Pushover notifications in wm_receiver<commit_after>from .. import app
from .. import db
from .. import queue
from .. import util
from ..models import Post, Mention, get_settings
from bs4 import BeautifulSoup
from flask import request, make_response, render_template, url_for, abort
from werkzeug.exceptions import NotFound
import collections
import datetime
import mf2py
import mf2util
import requests
import urllib.parse
import urllib.request

ProcessResult = collections.namedtuple(
    'ProcessResult', 'post mention create delete error')

def register():
    pass


@app.route('/webmention', methods=['GET', 'POST'])
def receive_webmention():
    if request.method == 'GET':
        return render_template('webmention.jinja2')

    source = request.form.get('source')
    target = request.form.get('target')
    callback = request.form.get('callback')

    if not source:
        return make_response(
            'webmention missing required source parameter', 400)

    if not target:
        return make_response(
            'webmention missing required target parameter', 400)

    app.logger.debug("Webmention from %s to %s received", source, target)

    key = queue.enqueue(process_webmention, source, target, callback)
    status_url = url_for('webmention_status', key=key, _external=True)

    return make_response(
        render_template('wm_received.jinja2', status_url=status_url), 202)


@app.route('/webmention/status/<key>')
def webmention_status(key):
    rv = queue.query(key)

    if not rv:
        rv = {
            'response_code': 400,
            'status': 'unknown',
            'reason': 'Job does not exist or its status has expired',
        }
    elif rv == 'queued':
        rv = {
            'response_code': 202,
            'status': 'queued',
            'reason': 'Mention has been queued for processing',
        }

    return make_response(
       render_template('wm_status.jinja2', **rv),
       rv.get('response_code', 400))


def process_webmention(source, target, callback):
    def call_callback(result):
        if callback:
            requests.post(callback, data=result)

    try:
        result = do_process_webmention(source, target)

        if result.error:
            app.logger.warn("Failed to process webmention: %s", result.error)
            response = {
                'source': source,
                'target': target,
                'response_code': 400,
                'status': 'error',
                'reason': result.error
            }
            call_callback(response)
            return response

        if result.post and result.delete:
            result.post.mentions = [m for m in result.post.mentions if
                                    m.url != source]
        elif result.post and result.mention:
            result.post.mentions.append(result.mention)

        db.session.commit()
        app.logger.debug("saved mentions to %s", result.post.path)

        if result.post and result.mention and result.create:
            send_push_notification(result.post, result.mention)
        
        response = {
            'source': source,
            'target': target,
            'response_code': 200,
            'status': 'success',
            'reason': 'Deleted' if result.delete 
            else 'Created' if result.create
            else 'Updated'
        }

        call_callback(response)
        return response

    except Exception as e:
        app.logger.exception("exception while processing webmention")
        response = {
            'source': source,
            'target': target,
            'response_code': 400,
            'status': 'error',
            'reason': "exception while processing webmention {}".format(e)
        }
        call_callback(response)
        return response

    
def send_push_notification(post, mention):
    token = app.config.get('PUSHOVER_TOKEN')
    user = app.config.get('PUSHOVER_USER')
    if token and user:
        message = '{} from {} on {}{}'.format(
            mention.reftype, mention.author_name, post.title_or_fallback,
            (': ' + mention.content_plain[:256]) if mention.content_plain else '')

        requests.post('https://api.pushover.net/1/messages.json', data={
                'token': token,
                'user': user,
                'message': message,
                })
    

def do_process_webmention(source, target):
    app.logger.debug("processing webmention from %s to %s", source, target)
    if target and target.strip('/') == get_settings().site_url.strip('/'):
        # received a domain-level mention
        app.logger.debug('received domain-level webmention from %s', source)
        target_post = None
        target_urls = (target,)
        # TODO save domain-level webmention somewhere
    else:
        # confirm that target is a valid link to a post
        target_post = find_target_post(target)

        if not target_post:
            app.logger.warn(
                "Webmention could not find target post: %s. Giving up", target)
            return ProcessResult(
                post=None, mention=None, create=False, delete=False,
                error="Webmention could not find target post: {}".format(target))
        target_urls = (target, target_post.permalink,)

    if source in target_urls:
        return ProcessResult(
            post=None, mention=None, create=False, delete=False,
            error='{} and {} refer to the same post'.format(source, target))

    # confirm that source actually refers to the post
    source_response = util.fetch_html(source)
    app.logger.debug('received response from source %s', source_response)

    if source_response.status_code == 410:
        app.logger.debug("Webmention indicates original was deleted")
        return ProcessResult(
            post=target_post, mention=None, create=False, delete=True, error=None)

    if source_response.status_code // 100 != 2:
        app.logger.warn(
            "Webmention could not read source post: %s. Giving up", source)
        return ProcessResult(
            post=target_post, mention=None, create=False, delete=False,
            error="Bad response when reading source post: {}, {}".format(source, source_response))

    source_length = source_response.headers.get('Content-Length')

    if source_length and int(source_length) > 2097152:
        app.logger.warn("Very large source. length=%s", source_length)
        return ProcessResult(
            post=target_post, mention=None, create=False, delete=False,
            error="Source is very large. Length={}".format(source_length))

    link_to_target = find_link_to_target(source, source_response, target_urls)
    if not link_to_target:
        app.logger.warn(
            "Webmention source %s does not appear to link to target %s. "
            "Giving up", source, target)
        return target_post, None, False,\
            "Could not find any links from source to target"

    mention = create_mention(target_post, source, source_response)
    return ProcessResult(
        post=target_post, mention=mention, create=not mention.id,
        delete=False, error=None)


def find_link_to_target(source_url, source_response, target_urls):
    if source_response.status_code // 2 != 100:
        app.logger.warn(
            "Received unexpected response from webmention source: %s",
            source_response.text)
        return None

    # Don't worry about Microformats for now; just see if there is a
    # link anywhere that points back to the target
    soup = BeautifulSoup(source_response.text)
    for link in soup.find_all(['a', 'link']):
{'fraction_non_alphanumeric': 0.10066490312432974, 'fraction_numerical': 0.007578465718166869, 'mean_word_length': 2.422559334475165, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5913086', 'n_tokens_mistral': 3983, 'n_tokens_neox': 3817, 'n_words': 1112}