Source code for langchain.document_transformers.embeddings_redundant_filter
"""Transform documents"""
from typing import Any, Callable, List, Sequence
import numpy as np
from langchain.pydantic_v1 import BaseModel, Field
from langchain.schema import BaseDocumentTransformer, Document
from langchain.schema.embeddings import Embeddings
from langchain.utils.math import cosine_similarity
class _DocumentWithState(Document):
    """Wrapper for a document that includes arbitrary state."""
    state: dict = Field(default_factory=dict)
    """State associated with the document."""
    def to_document(self) -> Document:
        """Convert the DocumentWithState to a Document."""
        return Document(page_content=self.page_content, metadata=self.metadata)
    @classmethod
    def from_document(cls, doc: Document) -> "_DocumentWithState":
        """Create a DocumentWithState from a Document."""
        if isinstance(doc, cls):
            return doc
        return cls(page_content=doc.page_content, metadata=doc.metadata)
[docs]def get_stateful_documents(
    documents: Sequence[Document],
) -> Sequence[_DocumentWithState]:
    """Convert a list of documents to a list of documents with state.
    Args:
        documents: The documents to convert.
    Returns:
        A list of documents with state.
    """
    return [_DocumentWithState.from_document(doc) for doc in documents]
def _filter_similar_embeddings(
    embedded_documents: List[List[float]], similarity_fn: Callable, threshold: float
) -> List[int]:
    """Filter redundant documents based on the similarity of their embeddings."""
    similarity = np.tril(similarity_fn(embedded_documents, embedded_documents), k=-1)
    redundant = np.where(similarity > threshold)
    redundant_stacked = np.column_stack(redundant)