<commit_before>import { Axios } from "axios";
import { URL } from "url";
import { isIP } from "net";
import { promises as dns, SrvRecord } from "dns"
import Logging from "../components/logging";

interface MatrixServerWellKnown {
    "m.server": string;
}

const OneMinute = 1000 * 60;
const OneHour = OneMinute * 60;

export const MinCacheForMs = OneMinute * 5;
export const MaxCacheForMs = OneHour * 48;
export const DefaultCacheForMs = OneHour * 24;
const CacheFailureForMS = MinCacheForMs;
const DefaultMatrixServerPort = 8448;
const MaxPortNumber = 65535;
const WellKnownTimeout = 10000;

const log = Logging.get('MatrixHostResolver');

type CachedResult = {timestamp: number, result: HostResolveResult}|{timestamp: number, error: Error};

export interface HostResolveResult {
    host: string;
    hostname: string;
    port: number;
    cacheFor: number;
}

interface DnsInterface {
    resolveSrv(hostname: string): Promise<SrvRecord[]>;
}

/**
 * Class to lookup the hostname, port and host headers of a given Matrix servername
 * according to the
 * [server discovery section of the spec](https://spec.matrix.org/v1.1/server-server-api/#server-discovery).
 */
export class MatrixHostResolver {
    private axios: Axios;
    private dns: DnsInterface;
    private resultCache = new Map<string, CachedResult>();

    constructor(private readonly opts: {axios?: Axios, dns?: DnsInterface, currentTimeMs?: number} = {}) {
        // To allow for easier mocking.
        this.axios = opts.axios || new Axios({ timeout: WellKnownTimeout });
        this.dns = opts.dns || dns;
    }

    get currentTime(): number {
        return this.opts.currentTimeMs || Date.now();
    }


    private static sortSrvRecords(a: SrvRecord, b: SrvRecord): number {
        // This algorithm is intentionally simple, as we're unlikely
        // to encounter many Matrix servers that actually load balance this way.
        const diffPrio = a.priority - b.priority;
        if (diffPrio != 0) {
            return diffPrio;
        }
        return a.weight - b.weight;
    }

    private static determineHostType(serverName: string): {type: 4|6|"unknown", host: string, port?: number} {
        const hostPortPair = /(.+):(\d+)/.exec(serverName);
        let host = serverName;
        let port = undefined;
        if (hostPortPair) {
            port = parseInt(hostPortPair[2]);
            if (host.startsWith('[') && host.endsWith(']')) {
                host = host.slice(1, host.length - 2);
                // IPv6 square bracket notation
                if (isIP(host) !== 6) {
                    throw Error('Unknown IPv6 notation');
                }
            }
            else if (isIP(serverName) === 6) {
                // Address is IPv6, but it doesn't have a port
                port = undefined;
                host = serverName;
            }
            else {
                host = hostPortPair[1];
            }
        }

        const ipResult = isIP(host) as 4|6|0;
        return {
            type: ipResult === 0 ? "unknown" : ipResult,
            port,
            host,
        }
    }

    private async getWellKnown(serverName: string): Promise<{mServer: string, cacheFor: number}> {
        const url = `https://${serverName}/.well-known/matrix/server`;
        const wellKnown = await this.axios.get<MatrixServerWellKnown>(
            url, {
            validateStatus: null,
        });
        if (wellKnown.status !== 200) {
            throw Error('Well known request returned non-200');
        }
        const mServer = wellKnown.data["m.server"];
        if (typeof mServer !== "string") {
            throw Error("Missing 'm.server' in well-known response");
        }

        const [host, portStr] = mServer.split(':');
        const port = portStr ? parseInt(portStr, 10) : DefaultMatrixServerPort;
        if (!host || (port && port < 1 || port > MaxPortNumber)) {
            throw Error("'m.server' was not in the format of <delegated_hostname>[:<delegated_port>]")
        }

        let cacheFor = DefaultCacheForMs;
        if (wellKnown.headers['Expires']) {<commit_msg>Fix getWellKnown failing to handle non-JSON payloads<commit_after>import { Axios } from "axios";
import { URL } from "url";
import { isIP } from "net";
import { promises as dns, SrvRecord } from "dns"
import Logging from "../components/logging";

interface MatrixServerWellKnown {
    "m.server": string;
}

const OneMinute = 1000 * 60;
const OneHour = OneMinute * 60;

export const MinCacheForMs = OneMinute * 5;
export const MaxCacheForMs = OneHour * 48;
export const DefaultCacheForMs = OneHour * 24;
const CacheFailureForMS = MinCacheForMs;
const DefaultMatrixServerPort = 8448;
const MaxPortNumber = 65535;
const WellKnownTimeout = 10000;

const log = Logging.get('MatrixHostResolver');

type CachedResult = {timestamp: number, result: HostResolveResult}|{timestamp: number, error: Error};

export interface HostResolveResult {
    host: string;
    hostname: string;
    port: number;
    cacheFor: number;
}

interface DnsInterface {
    resolveSrv(hostname: string): Promise<SrvRecord[]>;
}

/**
 * Class to lookup the hostname, port and host headers of a given Matrix servername
 * according to the
 * [server discovery section of the spec](https://spec.matrix.org/v1.1/server-server-api/#server-discovery).
 */
export class MatrixHostResolver {
    private axios: Axios;
    private dns: DnsInterface;
    private resultCache = new Map<string, CachedResult>();

    constructor(private readonly opts: {axios?: Axios, dns?: DnsInterface, currentTimeMs?: number} = {}) {
        // To allow for easier mocking.
        this.axios = opts.axios || new Axios({ timeout: WellKnownTimeout });
        this.dns = opts.dns || dns;
    }

    get currentTime(): number {
        return this.opts.currentTimeMs || Date.now();
    }


    private static sortSrvRecords(a: SrvRecord, b: SrvRecord): number {
        // This algorithm is intentionally simple, as we're unlikely
        // to encounter many Matrix servers that actually load balance this way.
        const diffPrio = a.priority - b.priority;
        if (diffPrio != 0) {
            return diffPrio;
        }
        return a.weight - b.weight;
    }

    private static determineHostType(serverName: string): {type: 4|6|"unknown", host: string, port?: number} {
        const hostPortPair = /(.+):(\d+)/.exec(serverName);
        let host = serverName;
        let port = undefined;
        if (hostPortPair) {
            port = parseInt(hostPortPair[2]);
            if (host.startsWith('[') && host.endsWith(']')) {
                host = host.slice(1, host.length - 2);
                // IPv6 square bracket notation
                if (isIP(host) !== 6) {
                    throw Error('Unknown IPv6 notation');
                }
            }
            else if (isIP(serverName) === 6) {
                // Address is IPv6, but it doesn't have a port
                port = undefined;
                host = serverName;
            }
            else {
                host = hostPortPair[1];
            }
        }

        const ipResult = isIP(host) as 4|6|0;
        return {
            type: ipResult === 0 ? "unknown" : ipResult,
            port,
            host,
        }
    }

    private async getWellKnown(serverName: string): Promise<{mServer: string, cacheFor: number}> {
        const url = `https://${serverName}/.well-known/matrix/server`;
        const wellKnown = await this.axios.get<MatrixServerWellKnown>(
            url, {
            validateStatus: null,
        });
        if (wellKnown.status !== 200) {
            throw Error('Well known request returned non-200');
        }
        let data: MatrixServerWellKnown;
        if (typeof wellKnown.data === "object") {
            data = wellKnown.data;
        }
        else if (typeof wellKnown.data === "string") {
            data = JSON.parse(wellKnown.data);
        }
        else {
            throw Error('Invalid datatype for well-known response');
        }
        const mServer = data["m.server"];
        if (typeof mServer !== "string") {
            throw Error("Missing 'm.server' in well-known response");
        }

        const [host, portStr] = mServer.split(':');
        const port = portStr ? parseInt(portStr, 10) : DefaultMatrixServerPort;
        if (!host || (port && port < 1 || port > MaxPortNumber)) {
            throw Error("'m.server' was not in the format of <delegated_hostname>[:<delegated_port>]")
        }

        let cacheFor = DefaultCacheForMs;
        if (wellKnown.headers['Expires']) {
{'fraction_non_alphanumeric': 0.11851421938479396, 'fraction_numerical': 0.012536273940800929, 'mean_word_length': 2.349922239502333, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7371009', 'n_tokens_mistral': 2566, 'n_tokens_neox': 2432, 'n_words': 707}