<commit_before>"""
Interface that uses Blizzard's pages as the source.
"""
import functools
import logging

import asyncio
from lxml import etree

import aiohttp
from kyoukai.context import HTTPRequestContext

from owapi import util

B_BASE_URL = "https://playoverwatch.com/en-us/"
B_PAGE_URL = B_BASE_URL + "career/{platform}{region}/{btag}"

logger = logging.getLogger("OWAPI")


async def get_page_body(ctx: HTTPRequestContext, url: str, cache_time=300) -> str:
    """
    Downloads page body from MasterOverwatch and caches it.
    """
    session = aiohttp.ClientSession(headers={"User-Agent": "OWAPI Scraper/1.0.0"})

    async def _real_get_body(_, url: str):
        # Real function.
        logger.info("GET => {}".format(url))
        async with session.get(url) as req:
            assert isinstance(req, aiohttp.ClientResponse)
            if req.status != 200:
                return None
            return (await req.read()).decode()

    result = await util.with_cache(ctx, _real_get_body, url, expires=cache_time)
    session.close()
    return result


def _parse_page(content: str) -> etree._Element:
    """
    Internal function to parse a page and return the data.
    """
    if content:
        data = etree.HTML(content)
        return data


async def get_user_page(ctx: HTTPRequestContext, battletag: str, platform: str="pc", region: str = "us", extra="",
                        cache_time=300) -> etree._Element:
    """
    Downloads the BZ page for a user, and parses it.
    """
    if platform != "pc":
        region = ""
    built_url = B_PAGE_URL.format(
        region=region, btag=battletag.replace("#", "-"), platform=platform) + "{}".format(extra)
    page_body = await get_page_body(ctx, built_url, cache_time=cache_time)

    if not page_body:
        return None

    # parse the page
    parse_partial = functools.partial(_parse_page, page_body)
    loop = asyncio.get_event_loop()
    parsed = await loop.run_in_executor(None, parse_partial)

    return parsed


async def region_helper(ctx: HTTPRequestContext, battletag: str, platform="pc", region=None, extra=""):
    """
    Downloads the correct page for a user in the right region.

    This will return either (etree._Element, region) or (None, None).
    """
    if region is None:
        reg_l = ["/eu", "/us", "/cn", "/kr"]
    else:
        if not region.startswith("/"):
            # ugh
            region = "/" + region
        reg_l = [region]

    for reg in reg_l:
        # Get the user page.
        page = await get_user_page(ctx, battletag, platform, reg, extra)
        # Check if the page was returned successfully.
        # If it was, return it.<commit_msg>Remove cn region - it wasn't even used anyway<commit_after>"""
Interface that uses Blizzard's pages as the source.
"""
import functools
import logging

import asyncio
from lxml import etree

import aiohttp
from kyoukai.context import HTTPRequestContext

from owapi import util

B_BASE_URL = "https://playoverwatch.com/en-us/"
B_PAGE_URL = B_BASE_URL + "career/{platform}{region}/{btag}"

logger = logging.getLogger("OWAPI")


async def get_page_body(ctx: HTTPRequestContext, url: str, cache_time=300) -> str:
    """
    Downloads page body from MasterOverwatch and caches it.
    """
    session = aiohttp.ClientSession(headers={"User-Agent": "OWAPI Scraper/1.0.0"})

    async def _real_get_body(_, url: str):
        # Real function.
        logger.info("GET => {}".format(url))
        async with session.get(url) as req:
            assert isinstance(req, aiohttp.ClientResponse)
            if req.status != 200:
                return None
            return (await req.read()).decode()

    result = await util.with_cache(ctx, _real_get_body, url, expires=cache_time)
    session.close()
    return result


def _parse_page(content: str) -> etree._Element:
    """
    Internal function to parse a page and return the data.
    """
    if content:
        data = etree.HTML(content)
        return data


async def get_user_page(ctx: HTTPRequestContext, battletag: str, platform: str="pc", region: str = "us", extra="",
                        cache_time=300) -> etree._Element:
    """
    Downloads the BZ page for a user, and parses it.
    """
    if platform != "pc":
        region = ""
    built_url = B_PAGE_URL.format(
        region=region, btag=battletag.replace("#", "-"), platform=platform) + "{}".format(extra)
    page_body = await get_page_body(ctx, built_url, cache_time=cache_time)

    if not page_body:
        return None

    # parse the page
    parse_partial = functools.partial(_parse_page, page_body)
    loop = asyncio.get_event_loop()
    parsed = await loop.run_in_executor(None, parse_partial)

    return parsed


async def region_helper(ctx: HTTPRequestContext, battletag: str, platform="pc", region=None, extra=""):
    """
    Downloads the correct page for a user in the right region.

    This will return either (etree._Element, region) or (None, None).
    """
    if region is None:
        reg_l = ["/eu", "/us", "/kr"]
    else:
        if not region.startswith("/"):
            # ugh
            region = "/" + region
        reg_l = [region]

    for reg in reg_l:
        # Get the user page.
        page = await get_user_page(ctx, battletag, platform, reg, extra)
        # Check if the page was returned successfully.
        # If it was, return it.
{'fraction_non_alphanumeric': 0.11624883936861653, 'fraction_numerical': 0.004456824512534819, 'mean_word_length': 2.803672316384181, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5361934', 'n_tokens_mistral': 1702, 'n_tokens_neox': 1652, 'n_words': 489}