diff --git a/libs/langchain/langchain/document_loaders/confluence.py b/libs/langchain/langchain/document_loaders/confluence.py index 9619071332..8615faa7d8 100644 --- a/libs/langchain/langchain/document_loaders/confluence.py +++ b/libs/langchain/langchain/document_loaders/confluence.py @@ -205,6 +205,7 @@ class ConfluenceLoader(BaseLoader): limit: Optional[int] = 50, max_pages: Optional[int] = 1000, ocr_languages: Optional[str] = None, + keep_markdown_format: bool = False, ) -> List[Document]: """ :param space_key: Space key retrieved from a confluence URL, defaults to None @@ -234,6 +235,9 @@ class ConfluenceLoader(BaseLoader): language, you'll first need to install the appropriate Tesseract language pack. :type ocr_languages: str, optional + :param keep_markdown_format: Whether to keep the markdown format, defaults to + False + :type keep_markdown_format: bool :raises ValueError: _description_ :raises ImportError: _description_ :return: _description_ @@ -263,6 +267,7 @@ class ConfluenceLoader(BaseLoader): include_comments, content_format, ocr_languages, + keep_markdown_format, ) if label: @@ -294,6 +299,7 @@ class ConfluenceLoader(BaseLoader): include_comments, content_format, ocr_languages, + keep_markdown_format, ) if page_ids: @@ -319,6 +325,7 @@ class ConfluenceLoader(BaseLoader): include_comments, content_format, ocr_languages, + keep_markdown_format, ) docs.append(doc) @@ -397,6 +404,7 @@ class ConfluenceLoader(BaseLoader): include_comments: bool, content_format: ContentFormat, ocr_languages: Optional[str] = None, + keep_markdown_format: Optional[bool] = False, ) -> List[Document]: """Process a list of pages into a list of documents.""" docs = [] @@ -409,6 +417,7 @@ class ConfluenceLoader(BaseLoader): include_comments, content_format, ocr_languages, + keep_markdown_format, ) docs.append(doc) @@ -421,24 +430,42 @@ class ConfluenceLoader(BaseLoader): include_comments: bool, content_format: ContentFormat, ocr_languages: Optional[str] = None, + keep_markdown_format: Optional[bool] = False, ) -> Document: - try: - from bs4 import BeautifulSoup # type: ignore - except ImportError: - raise ImportError( - "`beautifulsoup4` package not found, please run " - "`pip install beautifulsoup4`" - ) + if keep_markdown_format: + try: + from markdownify import markdownify + except ImportError: + raise ImportError( + "`markdownify` package not found, please run " + "`pip install markdownify`" + ) + else: + try: + from bs4 import BeautifulSoup # type: ignore + except ImportError: + raise ImportError( + "`beautifulsoup4` package not found, please run " + "`pip install beautifulsoup4`" + ) if include_attachments: attachment_texts = self.process_attachment(page["id"], ocr_languages) else: attachment_texts = [] - content = content_format.get_content(page) - text = BeautifulSoup(content, "lxml").get_text(" ", strip=True) + "".join( - attachment_texts - ) + if keep_markdown_format: + # Use markdownify to keep the page Markdown style + text = markdownify( + page["body"]["storage"]["value"], heading_style="ATX" + ) + "".join(attachment_texts) + + else: + content = content_format.get_content(page) + text = BeautifulSoup(content, "lxml").get_text(" ", strip=True) + "".join( + attachment_texts + ) + if include_comments: comments = self.confluence.get_page_comments( page["id"], expand="body.view.value", depth="all"