<commit_before>import logging
import os.path

import bottle
import lxml.html

from pyroxy import app, config


log = logging.getLogger(__name__)


def pred_filter_internal_download_links(href, title):
    allowed_extensions = config.get('allowed_extensions')
    if allowed_extensions is None:
        return True

    _, _, extension = title.rpartition('.')
    return extension in allowed_extensions


def pred_filter_home_pages(href, title):
    return href.startswith("http") and "home_page" in title


def pred_filter_external_download_links(href, title):
    return "download_url" in title


def filter_index(index_path):
    try:
        fd = open(index_path, "r")
    except IOError:
        log.exception("Could not open simple index for filtering.")
        bottle.abort(404)

    html_tree = lxml.html.parse(fd)
    html_tree = remove_links(html_tree)
    return lxml.html.tostring(html_tree)


def remove_links(html_tree):
    external_download_links = []
    home_pages = []
    internal_download_links = []
    unknown_links = []

    for element in html_tree.iterfind(".//a"):
        href = element.get("href")
        title = element.text_content()

        if pred_filter_internal_download_links(href, title):
            internal_download_links.append(element)
        elif pred_filter_home_pages(href, title):
            home_pages.append(element)
        elif pred_filter_external_download_links(href, title):
            external_download_links.append(element)
        else:
            log.warning("Unknown link %s (%s)", title, href)
            unknown_links.append(element)

    if internal_download_links:
        log.debug("Internal download links used.")
        to_be_removed = external_download_links + home_pages + unknown_links
    elif external_download_links:
        log.debug("External download links used.")
        to_be_removed = home_pages + unknown_links
    elif home_pages:
        to_be_removed = unknown_links
    else:
        log.debug("Home pages used.")
        to_be_removed = []

    for element in to_be_removed:
        log.debug("Filtering %s", title)
        element.getnext().drop_tree()
        element.drop_tree()

    return html_tree


@app.route("/simple/<package_name>/")
@app.route("/simple/<package_name>/index.html")
def package_list(package_name):
    package_index_path = os.path.join(
        config['pypi_web_path'], "simple", package_name, "index.html")
    return filter_index(package_index_path)
<commit_msg>Changing a warn to debug<commit_after>import logging
import os.path

import bottle
import lxml.html

from pyroxy import app, config


log = logging.getLogger(__name__)


def pred_filter_internal_download_links(href, title):
    allowed_extensions = config.get('allowed_extensions')
    if allowed_extensions is None:
        return True

    _, _, extension = title.rpartition('.')
    return extension in allowed_extensions


def pred_filter_home_pages(href, title):
    return href.startswith("http") and "home_page" in title


def pred_filter_external_download_links(href, title):
    return "download_url" in title


def filter_index(index_path):
    try:
        fd = open(index_path, "r")
    except IOError:
        log.exception("Could not open simple index for filtering.")
        bottle.abort(404)

    html_tree = lxml.html.parse(fd)
    html_tree = remove_links(html_tree)
    return lxml.html.tostring(html_tree)


def remove_links(html_tree):
    external_download_links = []
    home_pages = []
    internal_download_links = []
    unknown_links = []

    for element in html_tree.iterfind(".//a"):
        href = element.get("href")
        title = element.text_content()

        if pred_filter_internal_download_links(href, title):
            internal_download_links.append(element)
        elif pred_filter_home_pages(href, title):
            home_pages.append(element)
        elif pred_filter_external_download_links(href, title):
            external_download_links.append(element)
        else:
            log.debug("Filtering out link: %s (%s)", title, href)
            unknown_links.append(element)

    if internal_download_links:
        log.debug("Internal download links used.")
        to_be_removed = external_download_links + home_pages + unknown_links
    elif external_download_links:
        log.debug("External download links used.")
        to_be_removed = home_pages + unknown_links
    elif home_pages:
        to_be_removed = unknown_links
    else:
        log.debug("Home pages used.")
        to_be_removed = []

    for element in to_be_removed:
        log.debug("Filtering %s", title)
        element.getnext().drop_tree()
        element.drop_tree()

    return html_tree


@app.route("/simple/<package_name>/")
@app.route("/simple/<package_name>/index.html")
def package_list(package_name):
    package_index_path = os.path.join(
        config['pypi_web_path'], "simple", package_name, "index.html")
    return filter_index(package_index_path)

{'fraction_non_alphanumeric': 0.09738955823293173, 'fraction_numerical': 0.0012048192771084338, 'mean_word_length': 3.3013816925734023, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4905378', 'n_tokens_mistral': 1491, 'n_tokens_neox': 1472, 'n_words': 315}