self.base_url = self.base_url[:-1]
        if load_all_paths:
            # set web_path to the sitemap if we want to crawl all paths
            web_page = f"{self.base_url}/sitemap.xml"
        super().__init__(web_paths=(web_page,), continue_on_failure=continue_on_failure)
        self.load_all_paths = load_all_paths
        self.content_selector = content_selector
[docs]    def load(self) -> List[Document]:
        """Fetch text from one single GitBook page."""
        if self.load_all_paths:
            soup_info = self.scrape()
            relative_paths = self._get_paths(soup_info)
            urls = [urljoin(self.base_url, path) for path in relative_paths]
            soup_infos = self.scrape_all(urls)
            _documents = [
                self._get_document(soup_info, url)
                for soup_info, url in zip(soup_infos, urls)
            ]
        else:
            soup_info = self.scrape()
            _documents = [self._get_document(soup_info, self.web_path)]
        documents = [d for d in _documents if d]
        return documents
    def _get_document(
        self, soup: Any, custom_url: Optional[str] = None
    ) -> Optional[Document]:
        """Fetch content from page and return Document."""
        page_content_raw = soup.find(self.content_selector)
        if not page_content_raw:
            return None
        content = page_content_raw.get_text(separator="\n").strip()
        title_if_exists = page_content_raw.find("h1")
        title = title_if_exists.text if title_if_exists else ""
        metadata = {"source": custom_url or self.web_path, "title": title}
        return Document(page_content=content, metadata=metadata)