Toggle Menu
Prev Up Next

# Source code for langchain_community.document_transformers.html2text

    from typing import Any, Sequence

    from langchain_core.documents import BaseDocumentTransformer, Document

    class Html2TextTransformer(BaseDocumentTransformer):
        """Replace occurrences of a particular search pattern with a replacement string

        Arguments:
            ignore_links: Whether links should be ignored; defaults to True.
            ignore_images: Whether images should be ignored; defaults to True.

        Example:
            .. code-block:: python
                from langchain_community.document_transformers import Html2TextTransformer
                html2text = Html2TextTransformer()
                docs_transform = html2text.transform_documents(docs)
        """

        def __init__(self, ignore_links: bool = True, ignore_images: bool = True) -> None:
            self.ignore_links = ignore_links
            self.ignore_images = ignore_images

        def transform_documents(
            self,
            documents: Sequence[Document],
            **kwargs: Any,
        ) -> Sequence[Document]:
            try:
                import html2text
            except ImportError:
                raise ImportError(
                    """html2text package not found, please
                    install it with `pip install html2text`"""
                )

            # Create a html2text.HTML2Text object and override some properties
            h = html2text.HTML2Text()
            h.ignore_links = self.ignore_links
            h.ignore_images = self.ignore_images

            new_documents = []

            for d in documents:
                new_document = Document(
                    page_content=h.handle(d.page_content), metadata={**d.metadata}
                )
                new_documents.append(new_document)
            return new_documents

        async def atransform_documents(
            self,
            documents: Sequence[Document],
            **kwargs: Any,
        ) -> Sequence[Document]:
            raise NotImplementedError