<commit_before>import re

import requests

# Titles regex slightly adapted from https://github.com/md-y/mangadex-full-api
# Thanks!
TITLES_PATTERN = re.compile(
    r"""<a[^>]*href=["']\/title\/(\d+)\/\S+["'][^>]*manga_title[^>]*>([^<]*)<"""
)


def _parse_chapter_number(string):
    if string == "":
        # most likely a oneshot
        return {"number": ""}
    nums = string.split(".")
    count = len(nums)
    assert count == 1 or count == 2
    result = {"number": string}
    result["num_major"] = int(nums[0])
    if count == 2:
        result["num_minor"] = int(nums[1])
    return result


def _extract_groups(chap):
    return [
        group.strip()
        for group in [chap["group_name"], chap["group_name_2"], chap["group_name_3"]]
        if group
    ]


def get_title(title_id):
    url = f"https://mangadex.org/api/?id={title_id}&type=manga"
    md_resp = requests.get(url)
    assert md_resp.status_code == 200, md_resp.text
    md_json = md_resp.json()
    assert md_json["status"] == "OK"

    cover = md_json["manga"]["cover_url"].split("/")[-1]
    cover_ext = cover[cover.find(".") + 1 : cover.rfind("?")]

    title = {
        "id": title_id,
        "name": md_json["manga"]["title"],
        "cover_ext": cover_ext,
        "alt_names": md_json["manga"]["alt_names"],
        "descriptions": md_json["manga"]["description"].split("\r\n\r\n"),
        "chapters": [
            {
                "id": str(chap_id),
                "name": chap["title"],
                "volume": int(chap["volume"]) if chap["volume"] else None,
                "groups": _extract_groups(chap),
                **_parse_chapter_number(chap["chapter"]),
            }
            for chap_id, chap in md_json["chapter"].items()
            if chap["lang_code"] == "gb" and chap["group_name"] != "MangaPlus"
        ],
    }
    return title


def get_chapter(chapter_id):
    md_resp = requests.get(
        f"https://mangadex.org/api/?id={chapter_id}&type=chapter&saver=0"
    )
    assert md_resp.status_code == 200, md_resp.text
    md_json = md_resp.json()
    assert md_json["status"] == "OK"

    server = md_json.get("server_fallback") or md_json["server"]
    img_path = f"{server}{md_json['hash']}"

    chapter = {
        "id": chapter_id,<commit_msg>gracefully handle empty chapter list<commit_after>import re

import requests

# Titles regex slightly adapted from https://github.com/md-y/mangadex-full-api
# Thanks!
TITLES_PATTERN = re.compile(
    r"""<a[^>]*href=["']\/title\/(\d+)\/\S+["'][^>]*manga_title[^>]*>([^<]*)<"""
)


def _parse_chapter_number(string):
    if string == "":
        # most likely a oneshot
        return {"number": ""}
    nums = string.split(".")
    count = len(nums)
    assert count == 1 or count == 2
    result = {"number": string}
    result["num_major"] = int(nums[0])
    if count == 2:
        result["num_minor"] = int(nums[1])
    return result


def _extract_groups(chap):
    return [
        group.strip()
        for group in [chap["group_name"], chap["group_name_2"], chap["group_name_3"]]
        if group
    ]


def get_title(title_id):
    url = f"https://mangadex.org/api/?id={title_id}&type=manga"
    md_resp = requests.get(url)
    assert md_resp.status_code == 200, md_resp.text
    md_json = md_resp.json()
    assert md_json["status"] == "OK"

    cover = md_json["manga"]["cover_url"].split("/")[-1]
    cover_ext = cover[cover.find(".") + 1 : cover.rfind("?")]

    title = {
        "id": title_id,
        "name": md_json["manga"]["title"],
        "cover_ext": cover_ext,
        "alt_names": md_json["manga"]["alt_names"],
        "descriptions": md_json["manga"]["description"].split("\r\n\r\n"),
        "chapters": [
            {
                "id": str(chap_id),
                "name": chap["title"],
                "volume": int(chap["volume"]) if chap["volume"] else None,
                "groups": _extract_groups(chap),
                **_parse_chapter_number(chap["chapter"]),
            }
            for chap_id, chap in md_json.get("chapter", {}).items()
            if chap["lang_code"] == "gb" and chap["group_name"] != "MangaPlus"
        ],
    }
    return title


def get_chapter(chapter_id):
    md_resp = requests.get(
        f"https://mangadex.org/api/?id={chapter_id}&type=chapter&saver=0"
    )
    assert md_resp.status_code == 200, md_resp.text
    md_json = md_resp.json()
    assert md_json["status"] == "OK"

    server = md_json.get("server_fallback") or md_json["server"]
    img_path = f"{server}{md_json['hash']}"

    chapter = {
        "id": chapter_id,
{'fraction_non_alphanumeric': 0.18574231523871812, 'fraction_numerical': 0.00697623719206453, 'mean_word_length': 2.721005677210057, 'pattern_counts': {'":': 28, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5291476', 'n_tokens_mistral': 1637, 'n_tokens_neox': 1576, 'n_words': 284}