<commit_before>from app import app

from models import Post, Mention
from werkzeug.exceptions import NotFound
import urllib.parse
import requests

from bs4 import BeautifulSoup
import hentry_parser


def process_webmention(source, target):
    app.logger.debug("processing webmention from %s to %s", source, target)

    # confirm that source actually refers to the post
    source_response = requests.get(source)

    if source_response.status_code // 100 != 2:
        app.logger.warn(
            "Webmention could not read source post: %s. Giving up", source)
        return None

    # confirm that target is a valid link to a post
    target_post = find_target_post(target)

    if not target_post:
        app.logger.warn(
            "Webmention could not find target post: %s. Giving up", target)
        return None

    link_to_target = find_link_to_target(source, source_response,
                                         [target, target_post.permalink_url,
                                          target_post.short_permalink_url])
    if not link_to_target:
        app.logger.warn(
            "Webmention source %s does not appear to link to target %s. "
            "Giving up", source, target)
        return None

    hentry = hentry_parser.parse(source_response.text)

    if not hentry:
        app.logger.warn(
            "Webmention could not find h-entry on source page: %s. Giving up",
            source)
        return None

    reftypes = set()
    for ref in hentry.references:
        if (ref.url == target_post.permalink_url
                or ref.url == target_post.short_permalink_url):<commit_msg>handle comments on old-style permalinks<commit_after>from app import app

from models import Post, Mention
from werkzeug.exceptions import NotFound
import urllib.parse
import requests

from bs4 import BeautifulSoup
import hentry_parser


def process_webmention(source, target):
    app.logger.debug("processing webmention from %s to %s", source, target)

    # confirm that source actually refers to the post
    source_response = requests.get(source)

    if source_response.status_code // 100 != 2:
        app.logger.warn(
            "Webmention could not read source post: %s. Giving up", source)
        return None

    # confirm that target is a valid link to a post
    target_post = find_target_post(target)

    if not target_post:
        app.logger.warn(
            "Webmention could not find target post: %s. Giving up", target)
        return None

    #TODO remove after the old permalinks are out of circulation
    old_permalink = '{}/{}/{}/{}'\
        .format(app.config['SITE_URL'],
                target_post.post_type,
                target_post.pub_date.strftime("%y%m%d"),
                target_post.date_index)

    link_to_target = find_link_to_target(source, source_response,
                                         [target, target_post.permalink_url,
                                          target_post.short_permalink_url,
                                          old_permalink])
    if not link_to_target:
        app.logger.warn(
            "Webmention source %s does not appear to link to target %s. "
            "Giving up", source, target)
        return None

    hentry = hentry_parser.parse(source_response.text)

    if not hentry:
        app.logger.warn(
            "Webmention could not find h-entry on source page: %s. Giving up",
            source)
        return None

    reftypes = set()
    for ref in hentry.references:
        if (ref.url == target_post.permalink_url
                or ref.url == target_post.short_permalink_url):
{'fraction_non_alphanumeric': 0.07391064533921676, 'fraction_numerical': 0.0027578599007170436, 'mean_word_length': 2.201235657546337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5864284', 'n_tokens_mistral': 1042, 'n_tokens_neox': 1014, 'n_words': 315}