<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
from packaging.version import parse

import changelogs


def get_metadata(session, name):
    """
    Gets meta data from pypi for the given package.
    :param session: requests Session instance
    :param name: str, package
    :return: dict, meta data
    """
    resp = session.get("https://pypi.python.org/pypi/{}/json".format(name))
    if resp.status_code == 200:
        return resp.json()
    return {}


def get_releases(data, **kwargs):
    """
    Gets all releases from pypi meta data.
    :param data: dict, meta data
    :return: list, str releases
    """
    if "releases" in data:
        return sorted(data["releases"].keys(), key=lambda v: parse(v), reverse=True)
    return []


def get_urls(session, name, data, find_changelogs_fn, **kwargs):
    """
    Gets URLs to changelogs.
    :param session: requests Session instance
    :param name: str, package name
    :param data: dict, meta data
    :param find_changelogs_fn: function, find_changelogs
    :return: tuple, (set(changelog URLs), set(repo URLs))
    """
    # if this package has valid meta data, build up a list of URL candidates we can possibly
    # search for changelogs on
    if "info" in data:
        # add all URLs in pypi's meta data:
        # {
        #   "info": {
        #       "home_page":
        #       "docs_url":
        #       "bugtrack_url":
        #   }
        # }
        candidates = [
            url for url in
            [data["info"].get(attr) for attr in ("home_page", "docs_url", "bugtrack_url")]
            if url
        ]
        # the latest release page on pypi might also contain links, add it
        candidates.append("https://pypi.python.org/pypi/{name}/{latest_release}".format(
            name=name,
            latest_release=next(iter(get_releases(data)))
        ))
        for url in changelogs.url_re.findall(data["info"]["description"]):
            candidates.append(url)
        return find_changelogs_fn(session=session, name=name, candidates=candidates)
    return set(), set()<commit_msg>Only check for URL in the description if we cannot find any in the normal fields.<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
from packaging.version import parse

import changelogs


def get_metadata(session, name):
    """
    Gets meta data from pypi for the given package.
    :param session: requests Session instance
    :param name: str, package
    :return: dict, meta data
    """
    resp = session.get("https://pypi.python.org/pypi/{}/json".format(name))
    if resp.status_code == 200:
        return resp.json()
    return {}


def get_releases(data, **kwargs):
    """
    Gets all releases from pypi meta data.
    :param data: dict, meta data
    :return: list, str releases
    """
    if "releases" in data:
        return sorted(data["releases"].keys(), key=lambda v: parse(v), reverse=True)
    return []


def get_urls(session, name, data, find_changelogs_fn, **kwargs):
    """
    Gets URLs to changelogs.
    :param session: requests Session instance
    :param name: str, package name
    :param data: dict, meta data
    :param find_changelogs_fn: function, find_changelogs
    :return: tuple, (set(changelog URLs), set(repo URLs))
    """
    # if this package has valid meta data, build up a list of URL candidates we can possibly
    # search for changelogs on
    if "info" in data:
        # add all URLs in pypi's meta data:
        # {
        #   "info": {
        #       "home_page":
        #       "docs_url":
        #       "bugtrack_url":
        #   }
        # }
        candidates = [
            url for url in
            [data["info"].get(attr) for attr in ("home_page", "docs_url", "bugtrack_url")]
            if url
        ]
        # the latest release page on pypi might also contain links, add it
        candidates.append("https://pypi.python.org/pypi/{name}/{latest_release}".format(
            name=name,
            latest_release=next(iter(get_releases(data)))
        ))
        changelog, repo = find_changelogs_fn(session=session, name=name,
                                             candidates=candidates)
        if changelog or repo:
            return changelog, repo
        # If we could not find any repos or changelogs in the
        # standard fields. Look for URLs in the description.
        new_candidates = changelogs.url_re.findall(data["info"]["description"])
        return find_changelogs_fn(session=session, name=name,
                                  candidates=new_candidates)
    return set(), set()
{'fraction_non_alphanumeric': 0.11002340923600766, 'fraction_numerical': 0.0017024898914662695, 'mean_word_length': 2.4381858083394294, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1731228', 'n_tokens_mistral': 1405, 'n_tokens_neox': 1359, 'n_words': 423}