Toggle Menu
Prev Up Next

# Source code for langchain_community.document_loaders.text

    import logging
    from pathlib import Path
    from typing import Iterator, Optional, Union

    from langchain_core.documents import Document

    from langchain_community.document_loaders.base import BaseLoader
    from langchain_community.document_loaders.helpers import detect_file_encodings

    logger = logging.getLogger(__name__)

    class TextLoader(BaseLoader):
        """Load text file.

        Args:
            file_path: Path to the file to load.

            encoding: File encoding to use. If `None`, the file will be loaded
            with the default system encoding.

            autodetect_encoding: Whether to try to autodetect the file encoding
                if the specified encoding fails.
        """

        def __init__(
            self,
            file_path: Union[str, Path],
            encoding: Optional[str] = None,
            autodetect_encoding: bool = False,
        ):
            """Initialize with file path."""
            self.file_path = file_path
            self.encoding = encoding
            self.autodetect_encoding = autodetect_encoding

        def lazy_load(self) -> Iterator[Document]:
            """Load from file path."""
            text = ""
            try:
                with open(self.file_path, encoding=self.encoding) as f:
                    text = f.read()
            except UnicodeDecodeError as e:
                if self.autodetect_encoding:
                    detected_encodings = detect_file_encodings(self.file_path)
                    for encoding in detected_encodings:
                        logger.debug(f"Trying encoding: {encoding.encoding}")
                        try:
                            with open(self.file_path, encoding=encoding.encoding) as f:
                                text = f.read()
                            break
                        except UnicodeDecodeError:
                            continue
                else:
                    raise RuntimeError(f"Error loading {self.file_path}") from e
            except Exception as e:
                raise RuntimeError(f"Error loading {self.file_path}") from e

            metadata = {"source": str(self.file_path)}
            yield Document(page_content=text, metadata=metadata)