# Path: federation/utils/network.py
# def fetch_document(url=None, host=None, path="/", timeout=10, raise_ssl_errors=True, extra_headers=None):
#     """Helper method to fetch remote document.
# 
#     Must be given either the ``url`` or ``host``.
#     If ``url`` is given, only that will be tried without falling back to http from https.
#     If ``host`` given, `path` will be added to it. Will fall back to http on non-success status code.
# 
#     :arg url: Full url to fetch, including protocol
#     :arg host: Domain part only without path or protocol
#     :arg path: Path without domain (defaults to "/")
#     :arg timeout: Seconds to wait for response (defaults to 10)
#     :arg raise_ssl_errors: Pass False if you want to try HTTP even for sites with SSL errors (default True)
#     :arg extra_headers: Optional extra headers dictionary to add to requests
#     :returns: Tuple of document (str or None), status code (int or None) and error (an exception class instance or None)
#     :raises ValueError: If neither url nor host are given as parameters
#     """
#     if not url and not host:
#         raise ValueError("Need url or host.")
# 
#     logger.debug("fetch_document: url=%s, host=%s, path=%s, timeout=%s, raise_ssl_errors=%s",
#                  url, host, path, timeout, raise_ssl_errors)
#     headers = {'user-agent': USER_AGENT}
#     if extra_headers:
#         headers.update(extra_headers)
#     if url:
#         # Use url since it was given
#         logger.debug("fetch_document: trying %s", url)
#         try:
#             response = requests.get(url, timeout=timeout, headers=headers)
#             logger.debug("fetch_document: found document, code %s", response.status_code)
#             response.raise_for_status()
#             return response.text, response.status_code, None
#         except RequestException as ex:
#             logger.debug("fetch_document: exception %s", ex)
#             return None, None, ex
#     # Build url with some little sanitizing
#     host_string = host.replace("http://", "").replace("https://", "").strip("/")
#     path_string = path if path.startswith("/") else "/%s" % path
#     url = "https://%s%s" % (host_string, path_string)
#     logger.debug("fetch_document: trying %s", url)
#     try:
#         response = requests.get(url, timeout=timeout, headers=headers)
#         logger.debug("fetch_document: found document, code %s", response.status_code)
#         response.raise_for_status()
#         return response.text, response.status_code, None
#     except (HTTPError, SSLError, ConnectionError) as ex:
#         if isinstance(ex, SSLError) and raise_ssl_errors:
#             logger.debug("fetch_document: exception %s", ex)
#             return None, None, ex
#         # Try http then
#         url = url.replace("https://", "http://")
#         logger.debug("fetch_document: trying %s", url)
#         try:
#             response = requests.get(url, timeout=timeout, headers=headers)
#             logger.debug("fetch_document: found document, code %s", response.status_code)
#             response.raise_for_status()
#             return response.text, response.status_code, None
#         except RequestException as ex:
#             logger.debug("fetch_document: exception %s", ex)
#             return None, None, ex
#     except RequestException as ex:
#         logger.debug("fetch_document: exception %s", ex)
#         return None, None, ex
# 
# def send_document(url, data, timeout=10, method="post", *args, **kwargs):
#     """Helper method to send a document via POST.
# 
#     Additional ``*args`` and ``**kwargs`` will be passed on to ``requests.post``.
# 
#     :arg url: Full url to send to, including protocol
#     :arg data: Dictionary (will be form-encoded), bytes, or file-like object to send in the body
#     :arg timeout: Seconds to wait for response (defaults to 10)
#     :arg method: Method to use, defaults to post
#     :returns: Tuple of status code (int or None) and error (exception class instance or None)
#     """
#     logger.debug("send_document: url=%s, data=%s, timeout=%s, method=%s", url, data, timeout, method)
#     if not method:
#         method = "post"
#     headers = CaseInsensitiveDict({
#         'User-Agent': USER_AGENT,
#     })
#     if "headers" in kwargs:
#         # Update from kwargs
#         headers.update(kwargs.get("headers"))
#     kwargs.update({
#         "data": data, "timeout": timeout, "headers": headers
#     })
#     request_func = getattr(requests, method)
#     try:
#         response = request_func(url, *args, **kwargs)
#         logger.debug("send_document: response status code %s", response.status_code)
#         return response.status_code, None
#     # TODO support rate limit 429 code
#     except RequestException as ex:
#         logger.debug("send_document: exception %s", ex)
#         return None, ex


import json
import re
from copy import deepcopy
from typing import Dict
from federation.utils.network import fetch_document, send_document
        from federation.hostmeta.fetchers import fetch_nodeinfo_document
        from federation.hostmeta.fetchers import fetch_misskey_document
    result['version'] = doc.get('version', '')
    result['features'] = doc.get('features', {})

    if not mastodon_document:
        # Fetch also Mastodon API doc to get some counts...
        api_doc, _status_code, _error = fetch_document(host=host, path='/api/v1/instance')
        if api_doc:
            try:
                mastodon_document = json.loads(api_doc)
            except json.JSONDecodeError:
                pass
    if mastodon_document:
        result['activity']['users']['total'] = int_or_none(mastodon_document.get('stats', {}).get('user_count'))
        result['activity']['local_posts'] = int_or_none(mastodon_document.get('stats', {}).get('status_count'))

        if "contact_account" in mastodon_document and mastodon_document.get('contact_account') is not None:
            contact_account = mastodon_document.get('contact_account', {})
        else:
            contact_account = {}
        result['organization']['account'] = contact_account.get('url', '')

    return result


def parse_nodeinfo_document(doc, host):
    result = deepcopy(defaults)
    nodeinfo_version = doc.get('version', '1.0')
    result['host'] = host
    result['name'] = doc.get('metadata', {}).get('nodeName', host)
    result['version'] = doc.get('software', {}).get('version', '')

# Path: federation/utils/network.py
# def fetch_document(url=None, host=None, path="/", timeout=10, raise_ssl_errors=True, extra_headers=None):
#     """Helper method to fetch remote document.
# 
#     Must be given either the ``url`` or ``host``.
#     If ``url`` is given, only that will be tried without falling back to http from https.
#     If ``host`` given, `path` will be added to it. Will fall back to http on non-success status code.
# 
#     :arg url: Full url to fetch, including protocol
#     :arg host: Domain part only without path or protocol
#     :arg path: Path without domain (defaults to "/")
#     :arg timeout: Seconds to wait for response (defaults to 10)
#     :arg raise_ssl_errors: Pass False if you want to try HTTP even for sites with SSL errors (default True)
#     :arg extra_headers: Optional extra headers dictionary to add to requests
#     :returns: Tuple of document (str or None), status code (int or None) and error (an exception class instance or None)
#     :raises ValueError: If neither url nor host are given as parameters
#     """
#     if not url and not host:
#         raise ValueError("Need url or host.")
# 
#     logger.debug("fetch_document: url=%s, host=%s, path=%s, timeout=%s, raise_ssl_errors=%s",
#                  url, host, path, timeout, raise_ssl_errors)
#     headers = {'user-agent': USER_AGENT}
#     if extra_headers:
#         headers.update(extra_headers)
#     if url:
#         # Use url since it was given
#         logger.debug("fetch_document: trying %s", url)
#         try:
#             response = requests.get(url, timeout=timeout, headers=headers)
#             logger.debug("fetch_document: found document, code %s", response.status_code)
#             response.raise_for_status()
#             return response.text, response.status_code, None
#         except RequestException as ex:
#             logger.debug("fetch_document: exception %s", ex)
#             return None, None, ex
#     # Build url with some little sanitizing
#     host_string = host.replace("http://", "").replace("https://", "").strip("/")
#     path_string = path if path.startswith("/") else "/%s" % path
#     url = "https://%s%s" % (host_string, path_string)
#     logger.debug("fetch_document: trying %s", url)
#     try:
#         response = requests.get(url, timeout=timeout, headers=headers)
#         logger.debug("fetch_document: found document, code %s", response.status_code)
#         response.raise_for_status()
#         return response.text, response.status_code, None
#     except (HTTPError, SSLError, ConnectionError) as ex:
#         if isinstance(ex, SSLError) and raise_ssl_errors:
#             logger.debug("fetch_document: exception %s", ex)
#             return None, None, ex
#         # Try http then
#         url = url.replace("https://", "http://")
#         logger.debug("fetch_document: trying %s", url)
#         try:
#             response = requests.get(url, timeout=timeout, headers=headers)
#             logger.debug("fetch_document: found document, code %s", response.status_code)
#             response.raise_for_status()
#             return response.text, response.status_code, None
#         except RequestException as ex:
#             logger.debug("fetch_document: exception %s", ex)
#             return None, None, ex
#     except RequestException as ex:
#         logger.debug("fetch_document: exception %s", ex)
#         return None, None, ex
# 
# def send_document(url, data, timeout=10, method="post", *args, **kwargs):
#     """Helper method to send a document via POST.
# 
#     Additional ``*args`` and ``**kwargs`` will be passed on to ``requests.post``.
# 
#     :arg url: Full url to send to, including protocol
#     :arg data: Dictionary (will be form-encoded), bytes, or file-like object to send in the body
#     :arg timeout: Seconds to wait for response (defaults to 10)
#     :arg method: Method to use, defaults to post
#     :returns: Tuple of status code (int or None) and error (exception class instance or None)
#     """
#     logger.debug("send_document: url=%s, data=%s, timeout=%s, method=%s", url, data, timeout, method)
#     if not method:
#         method = "post"
#     headers = CaseInsensitiveDict({
#         'User-Agent': USER_AGENT,
#     })
#     if "headers" in kwargs:
#         # Update from kwargs
#         headers.update(kwargs.get("headers"))
#     kwargs.update({
#         "data": data, "timeout": timeout, "headers": headers
#     })
#     request_func = getattr(requests, method)
#     try:
#         response = request_func(url, *args, **kwargs)
#         logger.debug("send_document: response status code %s", response.status_code)
#         return response.status_code, None
#     # TODO support rate limit 429 code
#     except RequestException as ex:
#         logger.debug("send_document: exception %s", ex)
#         return None, ex


# Path: federation/hostmeta/parsers.py
import json
import re
from copy import deepcopy
from typing import Dict
from federation.utils.network import fetch_document, send_document
        from federation.hostmeta.fetchers import fetch_nodeinfo_document
        from federation.hostmeta.fetchers import fetch_misskey_document

    result['version'] = doc.get('version', '')
    result['features'] = doc.get('features', {})

    if not mastodon_document:
        # Fetch also Mastodon API doc to get some counts...
        api_doc, _status_code, _error = fetch_document(host=host, path='/api/v1/instance')
        if api_doc:
            try:
                mastodon_document = json.loads(api_doc)
            except json.JSONDecodeError:
                pass
    if mastodon_document:
        result['activity']['users']['total'] = int_or_none(mastodon_document.get('stats', {}).get('user_count'))
        result['activity']['local_posts'] = int_or_none(mastodon_document.get('stats', {}).get('status_count'))

        if "contact_account" in mastodon_document and mastodon_document.get('contact_account') is not None:
            contact_account = mastodon_document.get('contact_account', {})
        else:
            contact_account = {}
        result['organization']['account'] = contact_account.get('url', '')

    return result


def parse_nodeinfo_document(doc, host):
    result = deepcopy(defaults)
    nodeinfo_version = doc.get('version', '1.0')
    result['host'] = host
    result['name'] = doc.get('metadata', {}).get('nodeName', host)
    result['version'] = doc.get('software', {}).get('version', '')

    result['platform'] = doc.get('software', {}).get('name', 'unknown').lower()