Harrison/sitemap local (#4704)

Co-authored-by: Lukas Bauer <lukas.bauer@mayflower.de>
This commit is contained in:
Harrison Chase 2023-05-14 22:04:38 -07:00 committed by GitHub
parent 12b4ee1fc7
commit b6e3ac17c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 127 additions and 4 deletions

View File

@ -0,0 +1,35 @@
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:xhtml="http://www.w3.org/1999/xhtml">
<url>
<loc>https://python.langchain.com/en/stable/</loc>
<lastmod>2023-05-04T16:15:31.377584+00:00</lastmod>
<changefreq>weekly</changefreq>
<priority>1</priority>
</url>
<url>
<loc>https://python.langchain.com/en/latest/</loc>
<lastmod>2023-05-05T07:52:19.633878+00:00</lastmod>
<changefreq>daily</changefreq>
<priority>0.9</priority>
</url>
<url>
<loc>https://python.langchain.com/en/harrison-docs-refactor-3-24/</loc>
<lastmod>2023-03-27T02:32:55.132916+00:00</lastmod>
<changefreq>monthly</changefreq>
<priority>0.8</priority>
</url>
</urlset>

View File

@ -108,7 +108,9 @@
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
@ -125,6 +127,34 @@
"documents[0]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Local Sitemap\n",
"\n",
"The sitemap loader can also be used to load local files."
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Fetching pages: 100%|####################################################################################################################################| 3/3 [00:00<00:00, 3.91it/s]\n"
]
}
],
"source": [
"sitemap_loader = SitemapLoader(web_path=\"example_data/sitemap.xml\", is_local=True)\n",
"\n",
"docs = sitemap_loader.load()"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -149,7 +179,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
"version": "3.9.1"
}
},
"nbformat": 4,

View File

@ -32,11 +32,12 @@ class SitemapLoader(WebBaseLoader):
blocksize: Optional[int] = None,
blocknum: int = 0,
meta_function: Optional[Callable] = None,
is_local: bool = False,
):
"""Initialize with webpage path and optional filter URLs.
Args:
web_path: url of the sitemap
web_path: url of the sitemap. can also be a local path
filter_urls: list of strings or regexes that will be applied to filter the
urls that are parsed and loaded
parsing_function: Function to parse bs4.Soup output
@ -45,6 +46,7 @@ class SitemapLoader(WebBaseLoader):
meta_function: Function to parse bs4.Soup output for metadata
remember when setting this method to also copy metadata["loc"]
to metadata["source"] if you are using this field
is_local: whether the sitemap is a local file
"""
if blocksize is not None and blocksize < 1:
@ -67,6 +69,7 @@ class SitemapLoader(WebBaseLoader):
self.meta_function = meta_function or _default_meta_function
self.blocksize = blocksize
self.blocknum = blocknum
self.is_local = is_local
def parse_sitemap(self, soup: Any) -> List[dict]:
"""Parse sitemap xml and load into a list of dicts."""
@ -100,6 +103,16 @@ class SitemapLoader(WebBaseLoader):
def load(self) -> List[Document]:
"""Load sitemap."""
if self.is_local:
try:
import bs4
except ImportError:
raise ValueError(
"bs4 package not found, please install it with " "`pip install bs4`"
)
fp = open(self.web_path)
soup = bs4.BeautifulSoup(fp, "xml")
else:
soup = self.scrape("xml")
els = self.parse_sitemap(soup)

View File

@ -1,3 +1,4 @@
from pathlib import Path
from typing import Any
import pytest
@ -122,3 +123,12 @@ def test_sitemap_metadata_default() -> None:
assert len(documents) > 1
assert "source" in documents[0].metadata
assert "loc" in documents[0].metadata
def test_local_sitemap() -> None:
"""Test sitemap loader."""
file_path = Path(__file__).parent.parent / "examples/sitemap.xml"
loader = SitemapLoader(str(file_path))
documents = loader.load()
assert len(documents) > 1
assert "🦜🔗" in documents[0].page_content

View File

@ -0,0 +1,35 @@
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:xhtml="http://www.w3.org/1999/xhtml">
<url>
<loc>https://python.langchain.com/en/stable/</loc>
<lastmod>2023-05-04T16:15:31.377584+00:00</lastmod>
<changefreq>weekly</changefreq>
<priority>1</priority>
</url>
<url>
<loc>https://python.langchain.com/en/latest/</loc>
<lastmod>2023-05-05T07:52:19.633878+00:00</lastmod>
<changefreq>daily</changefreq>
<priority>0.9</priority>
</url>
<url>
<loc>https://python.langchain.com/en/harrison-docs-refactor-3-24/</loc>
<lastmod>2023-03-27T02:32:55.132916+00:00</lastmod>
<changefreq>monthly</changefreq>
<priority>0.8</priority>
</url>
</urlset>