Fix web research retriever for unknown links in results (#9115)

Fixes an issue with web research retriever for unknown links in results.
This is currently making the retrieve crash sometimes.

@rlancemartin
harrison/clean-up-imports
Jacques Arnoux 1 year ago committed by GitHub
parent 31cfc00845
commit 926c64da60
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -179,15 +179,16 @@ class WebResearchRetriever(BaseRetriever):
logger.info(f"Questions for Google Search: {questions}")
# Get urls
logger.info("Searching for relevat urls ...")
logger.info("Searching for relevant urls...")
urls_to_look = []
for query in questions:
# Google search
search_results = self.search_tool(query, self.num_search_results)
logger.info("Searching for relevat urls ...")
logger.info("Searching for relevant urls...")
logger.info(f"Search results: {search_results}")
for res in search_results:
urls_to_look.append(res["link"])
if res.get("link", None):
urls_to_look.append(res["link"])
# Relevant urls
urls = set(urls_to_look)

Loading…
Cancel
Save