Source code for langchain.document_loaders.generic
from __future__ import annotations
from pathlib import Path
from typing import Iterator, List, Literal, Optional, Sequence, Union
from langchain.document_loaders.base import BaseBlobParser, BaseLoader
from langchain.document_loaders.blob_loaders import BlobLoader, FileSystemBlobLoader
from langchain.document_loaders.parsers.registry import get_parser
from langchain.schema import Document
from langchain.text_splitter import TextSplitter
_PathLike = Union[str, Path]
DEFAULT = Literal["default"]
[docs]class GenericLoader(BaseLoader):
    """Generic Document Loader.
    A generic document loader that allows combining an arbitrary blob loader with
    a blob parser.
    Examples:
       .. code-block:: python
        from langchain.document_loaders import GenericLoader
        from langchain.document_loaders.blob_loaders import FileSystemBlobLoader
        loader = GenericLoader.from_filesystem(
            path="path/to/directory",
            glob="**/[!.]*",
            suffixes=[".pdf"],
            show_progress=True,
        )
        docs = loader.lazy_load()
        next(docs)
        Example instantiations to change which files are loaded:
        .. code-block:: python
            # Recursively load all text files in a directory.
            loader = GenericLoader.from_filesystem("/path/to/dir", glob="**/*.txt")
            # Recursively load all non-hidden files in a directory.
            loader = GenericLoader.from_filesystem("/path/to/dir", glob="**/[!.]*")
            # Load all files in a directory without recursion.
            loader = GenericLoader.from_filesystem("/path/to/dir", glob="*")
        Example instantiations to change which parser is used:
        .. code-block:: python
            from langchain.document_loaders.parsers.pdf import PyPDFParser