<commit_before><?php

namespace wcf\system\message\unfurl;

use BadMethodCallException;
use Exception;
use GuzzleHttp\ClientInterface;
use GuzzleHttp\Exception\BadResponseException;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Psr7\Response;
use GuzzleHttp\RequestOptions;
use Psr\Http\Client\ClientExceptionInterface;
use wcf\system\io\http\RedirectGuard;
use wcf\system\io\HttpFactory;
use wcf\system\message\unfurl\exception\DownloadFailed;
use wcf\system\message\unfurl\exception\ParsingFailed;
use wcf\system\message\unfurl\exception\UrlInaccessible;
use wcf\util\ArrayUtil;
use wcf\util\StringUtil;
use wcf\util\Url;

/**
 * Helper class to unfurl specific urls.
 *
 * @author      <NAME>
 * @copyright   2001-2021 WoltLab GmbH
 * @license     GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package     WoltLabSuite\Core\System\Message\Unfurl
 * @since       5.4
 */
final class UnfurlResponse
{
    /**
     * 10 Mebibyte
     */
    private const MAX_SIZE = (10 * (1 << 20));

    /**
     * 3 Mebibyte
     */
    public const MAX_IMAGE_SIZE = (3 * (1 << 20));

    /**
     * @var ClientInterface
     */
    private static $httpClient;

    /**
     * @var string
     */
    private $url;

    /**
     * @var Response
     */
    private $response;

    /**
     * @var ?string
     */
    private $body;

    /**
     * @var string
     */
    private $responseCharset = "UTF-8";

    /**
     * @var \DOMDocument
     */
    private $domDocument;

    /**
     * @var \DomXPath
     */
    private $domXPath;

    /**
     * Fetches a given Url and returns an UnfurlResponse instance.
     *
     * @throws ParsingFailed If the body cannot be parsed (e.g. the url is an image).
     * @throws DownloadFailed If the url can not be downloaded. This can be a temporary error.
     * @throws UrlInaccessible If the url is inaccessible (e.g. sends status code 403).
     */
    public static function fetch(string $url): self
    {
        if (!Url::is($url)) {
            throw new \InvalidArgumentException('Given URL "' . $url . '" is not a valid URL.');
        }

        try {
            $request = new Request('GET', $url, [
                'accept' => 'text/html',
                'range' => \sprintf('bytes=%d-%d', 0, self::MAX_SIZE - 1),
            ]);
            $response = self::getHttpClient()->send($request);

            return new self($url, $response);
        } catch (BadResponseException $e) {
            $response = $e->getResponse();

            if (self::isUrlInaccessible($response)) {
                $message = "Request failed with status code {$response->getStatusCode()}.";

                throw new UrlInaccessible($message, $response->getStatusCode(), $e);
            } else {
                throw new DownloadFailed("Could not download content.", $response->getStatusCode(), $e);
            }
        } catch (ClientExceptionInterface $e) {
            throw new DownloadFailed("Could not download content.", 0, $e);
        }
    }

    /**
     * @throws ParsingFailed If the body cannot be parsed (e.g. the url is an image).
     * @throws DownloadFailed If the url can not be downloaded. This can be a temporary error.
     */
    private function __construct(string $url, Response $response)
    {
        $this->url = $url;
        $this->response = $response;

        $this->readBody();
        $this->readDomDocument();
    }

    /**
     * Reads the body of the given url and converts the body to utf-8.
     */
    private function readBody(): void
    {
        $this->validateHeaders();

        $this->body = "";
        while (!$this->response->getBody()->eof()) {
            $this->body .= $this->response->getBody()->read(8192);
<commit_msg>Correctly wrap \RuntimeException from body reading in DownloadFailed in UnfurlResponse<commit_after><?php

namespace wcf\system\message\unfurl;

use BadMethodCallException;
use Exception;
use GuzzleHttp\ClientInterface;
use GuzzleHttp\Exception\BadResponseException;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Psr7\Response;
use GuzzleHttp\RequestOptions;
use Psr\Http\Client\ClientExceptionInterface;
use wcf\system\io\http\RedirectGuard;
use wcf\system\io\HttpFactory;
use wcf\system\message\unfurl\exception\DownloadFailed;
use wcf\system\message\unfurl\exception\ParsingFailed;
use wcf\system\message\unfurl\exception\UrlInaccessible;
use wcf\util\ArrayUtil;
use wcf\util\StringUtil;
use wcf\util\Url;

/**
 * Helper class to unfurl specific urls.
 *
 * @author      <NAME>
 * @copyright   2001-2021 WoltLab GmbH
 * @license     GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package     WoltLabSuite\Core\System\Message\Unfurl
 * @since       5.4
 */
final class UnfurlResponse
{
    /**
     * 10 Mebibyte
     */
    private const MAX_SIZE = (10 * (1 << 20));

    /**
     * 3 Mebibyte
     */
    public const MAX_IMAGE_SIZE = (3 * (1 << 20));

    /**
     * @var ClientInterface
     */
    private static $httpClient;

    /**
     * @var string
     */
    private $url;

    /**
     * @var Response
     */
    private $response;

    /**
     * @var ?string
     */
    private $body;

    /**
     * @var string
     */
    private $responseCharset = "UTF-8";

    /**
     * @var \DOMDocument
     */
    private $domDocument;

    /**
     * @var \DomXPath
     */
    private $domXPath;

    /**
     * Fetches a given Url and returns an UnfurlResponse instance.
     *
     * @throws ParsingFailed If the body cannot be parsed (e.g. the url is an image).
     * @throws DownloadFailed If the url can not be downloaded. This can be a temporary error.
     * @throws UrlInaccessible If the url is inaccessible (e.g. sends status code 403).
     */
    public static function fetch(string $url): self
    {
        if (!Url::is($url)) {
            throw new \InvalidArgumentException('Given URL "' . $url . '" is not a valid URL.');
        }

        try {
            $request = new Request('GET', $url, [
                'accept' => 'text/html',
                'range' => \sprintf('bytes=%d-%d', 0, self::MAX_SIZE - 1),
            ]);
            $response = self::getHttpClient()->send($request);

            return new self($url, $response);
        } catch (BadResponseException $e) {
            $response = $e->getResponse();

            if (self::isUrlInaccessible($response)) {
                $message = "Request failed with status code {$response->getStatusCode()}.";

                throw new UrlInaccessible($message, $response->getStatusCode(), $e);
            } else {
                throw new DownloadFailed("Could not download content.", $response->getStatusCode(), $e);
            }
        } catch (ClientExceptionInterface $e) {
            throw new DownloadFailed("Could not download content.", 0, $e);
        }
    }

    /**
     * @throws ParsingFailed If the body cannot be parsed (e.g. the url is an image).
     * @throws DownloadFailed If the url can not be downloaded. This can be a temporary error.
     */
    private function __construct(string $url, Response $response)
    {
        $this->url = $url;
        $this->response = $response;

        $this->readBody();
        $this->readDomDocument();
    }

    /**
     * Reads the body of the given url and converts the body to utf-8.
     */
    private function readBody(): void
    {
        $this->validateHeaders();

        $this->body = "";
        while (!$this->response->getBody()->eof()) {
            try {
                $this->body .= $this->response->getBody()->read(8192);
            } catch (\RuntimeException $e) {
                throw new DownloadFailed(
                    'Failed to read response body.',
                    0,
                    $e
                );
            }

{'fraction_non_alphanumeric': 0.12941176470588237, 'fraction_numerical': 0.009335038363171356, 'mean_word_length': 2.1818551668022783, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5565349', 'n_tokens_mistral': 2389, 'n_tokens_neox': 2271, 'n_words': 619}