Archives/langchain
2
0
Fork 1
mirror of https://github.com/hwchase17/langchain synced 2024-11-06 03:20:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix web research retriever for unknown links in results (#9115)

Browse Source 
Fixes an issue with web research retriever for unknown links in results.
This is currently making the retrieve crash sometimes.

@rlancemartin
This commit is contained in:
Jacques Arnoux 2023-08-11 10:50:37 -07:00 committed by GitHub
parent 31cfc00845
commit 926c64da60
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
libs/langchain/langchain/retrievers/web_research.py
View File

@ -179,15 +179,16 @@ class WebResearchRetriever(BaseRetriever):
logger.info(f"Questions for Google Search: {questions}")
# Get urls
logger.info("Searching for relevat urls ...")
logger.info("Searching for relevant urls...")
urls_to_look = []
for query in questions:
# Google search
search_results = self.search_tool(query, self.num_search_results)
logger.info("Searching for relevat urls ...")
logger.info("Searching for relevant urls...")
logger.info(f"Search results: {search_results}")
for res in search_results:
urls_to_look.append(res["link"])
if res.get("link", None):
urls_to_look.append(res["link"])
# Relevant urls
urls = set(urls_to_look)