From 2dcda8a8aca4c427ff5716e6ac37ab0c24a7f2e5 Mon Sep 17 00:00:00 2001 From: Shelby Jenkins <47464908+ShelbyJenkins@users.noreply.github.com> Date: Mon, 5 Jun 2023 18:33:55 -0500 Subject: [PATCH] Strips whitespace and \n from loc before filtering urls from sitemap (#5728) Fixes #5699 #### Who can review? Tag maintainers/contributors who might be interested: @woodworker @LeSphax @johannhartmann --------- Co-authored-by: Harrison Chase --- langchain/document_loaders/sitemap.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/langchain/document_loaders/sitemap.py b/langchain/document_loaders/sitemap.py index 029f12f7..64e3707a 100644 --- a/langchain/document_loaders/sitemap.py +++ b/langchain/document_loaders/sitemap.py @@ -79,8 +79,11 @@ class SitemapLoader(WebBaseLoader): if not loc: continue + # Strip leading and trailing whitespace and newlines + loc_text = loc.text.strip() + if self.filter_urls and not any( - re.match(r, loc.text) for r in self.filter_urls + re.match(r, loc_text) for r in self.filter_urls ): continue