mirror of
https://github.com/hwchase17/langchain
synced 2024-11-02 09:40:22 +00:00
community(you): Better support for You.com News API (#22622)
## Description
While `YouRetriever` supports both You.com's Search and News APIs, news
is supported as an afterthought.
More specifically, not all of the News API parameters are exposed for
the user, only those that happen to overlap with the Search API.
This PR:
- improves support for both APIs, exposing the remaining News API
parameters while retaining backward compatibility
- refactor some REST parameter generation logic
- updates the docstring of `YouSearchAPIWrapper`
- add input validation and warnings to ensure parameters are properly
set by user
- 🚨 Breaking: Limit the news results to `k` items
If no one reviews your PR within a few days, please @-mention one of
baskaryan, efriis, eyurtsev, ccurme, vbarda, hwchase17.
This commit is contained in:
Christopher Tee
GitHub
parent
e09c6bb58b
commit
ada03dd273
@ -2,6 +2,8 @@
|
||||
|
||||
In order to set this up, follow instructions at:
|
||||
"""
|
||||
|
||||
import warnings
|
||||
from typing import Any, Dict, List, Literal, Optional
|
||||
|
||||
import aiohttp
|
||||
@ -44,11 +46,11 @@ class YouDocument(BaseModel):
|
||||
|
||||
|
||||
class YouSearchAPIWrapper(BaseModel):
|
||||
"""Wrapper for you.com Search API.
|
||||
"""Wrapper for you.com Search and News API.
|
||||
|
||||
To connect to the You.com api requires an API key which
|
||||
you can get at https://api.you.com.
|
||||
You can check out the docs at https://documentation.you.com.
|
||||
You can check out the docs at https://documentation.you.com/api-reference/.
|
||||
|
||||
You need to set the environment variable `YDC_API_KEY` for retriever to operate.
|
||||
|
||||
@ -56,33 +58,50 @@ class YouSearchAPIWrapper(BaseModel):
|
||||
----------
|
||||
ydc_api_key: str, optional
|
||||
you.com api key, if YDC_API_KEY is not set in the environment
|
||||
endpoint_type: str, optional
|
||||
you.com endpoints: search, news, rag;
|
||||
`web` and `snippet` alias `search`
|
||||
`rag` returns `{'message': 'Forbidden'}`
|
||||
@todo `news` endpoint
|
||||
num_web_results: int, optional
|
||||
The max number of web results to return, must be under 20
|
||||
The max number of web results to return, must be under 20.
|
||||
This is mapped to the `count` query parameter for the News API.
|
||||
safesearch: str, optional
|
||||
Safesearch settings, one of off, moderate, strict, defaults to moderate
|
||||
country: str, optional
|
||||
Country code, ex: 'US' for united states, see api docs for list
|
||||
Country code, ex: 'US' for United States, see api docs for list
|
||||
search_lang: str, optional
|
||||
(News API) Language codes, ex: 'en' for English, see api docs for list
|
||||
ui_lang: str, optional
|
||||
(News API) User interface language for the response, ex: 'en' for English,
|
||||
see api docs for list
|
||||
spellcheck: bool, optional
|
||||
(News API) Whether to spell check query or not, defaults to True
|
||||
k: int, optional
|
||||
max number of Documents to return using `results()`
|
||||
n_hits: int, optional, deprecated
|
||||
Alias for num_web_results
|
||||
n_snippets_per_hit: int, optional
|
||||
limit the number of snippets returned per hit
|
||||
endpoint_type: str, optional
|
||||
you.com endpoints: search, news, rag;
|
||||
`web` and `snippet` alias `search`
|
||||
`rag` returns `{'message': 'Forbidden'}`
|
||||
@todo `news` endpoint
|
||||
"""
|
||||
|
||||
ydc_api_key: Optional[str] = None
|
||||
num_web_results: Optional[int] = None
|
||||
safesearch: Optional[str] = None
|
||||
country: Optional[str] = None
|
||||
k: Optional[int] = None
|
||||
n_snippets_per_hit: Optional[int] = None
|
||||
|
||||
# @todo deprecate `snippet`, not part of API
|
||||
endpoint_type: Literal["search", "news", "rag", "snippet"] = "search"
|
||||
|
||||
# Common fields between Search and News API
|
||||
num_web_results: Optional[int] = None
|
||||
safesearch: Optional[Literal["off", "moderate", "strict"]] = None
|
||||
country: Optional[str] = None
|
||||
|
||||
# News API specific fields
|
||||
search_lang: Optional[str] = None
|
||||
ui_lang: Optional[str] = None
|
||||
spellcheck: Optional[bool] = None
|
||||
|
||||
k: Optional[int] = None
|
||||
n_snippets_per_hit: Optional[int] = None
|
||||
# should deprecate n_hits
|
||||
n_hits: Optional[int] = None
|
||||
|
||||
@ -94,6 +113,74 @@ class YouSearchAPIWrapper(BaseModel):
|
||||
|
||||
return values
|
||||
|
||||
@root_validator
|
||||
def warn_if_set_fields_have_no_effect(cls, values: Dict) -> Dict:
|
||||
if values["endpoint_type"] != "news":
|
||||
news_api_fields = ("search_lang", "ui_lang", "spellcheck")
|
||||
for field in news_api_fields:
|
||||
if values[field]:
|
||||
warnings.warn(
|
||||
(
|
||||
f"News API-specific field '{field}' is set but "
|
||||
f"`endpoint_type=\"{values['endpoint_type']}\"`. "
|
||||
"This will have no effect."
|
||||
),
|
||||
UserWarning,
|
||||
)
|
||||
if values["endpoint_type"] not in ("search", "snippet"):
|
||||
if values["n_snippets_per_hit"]:
|
||||
warnings.warn(
|
||||
(
|
||||
"Field 'n_snippets_per_hit' only has effect on "
|
||||
'`endpoint_type="search"`.'
|
||||
),
|
||||
UserWarning,
|
||||
)
|
||||
return values
|
||||
|
||||
@root_validator
|
||||
def warn_if_deprecated_endpoints_are_used(cls, values: Dict) -> Dict:
|
||||
if values["endpoint_type"] == "snippets":
|
||||
warnings.warn(
|
||||
(
|
||||
f"`endpoint_type=\"{values['endpoint_type']}\"` is deprecated. "
|
||||
'Use `endpoint_type="search"` instead.'
|
||||
),
|
||||
DeprecationWarning,
|
||||
)
|
||||
return values
|
||||
|
||||
def _generate_params(self, query: str, **kwargs: Any) -> Dict:
|
||||
"""
|
||||
Parse parameters required for different You.com APIs.
|
||||
|
||||
Args:
|
||||
query: The query to search for.
|
||||
"""
|
||||
params = {
|
||||
"safesearch": self.safesearch,
|
||||
"country": self.country,
|
||||
**kwargs,
|
||||
}
|
||||
|
||||
# Add endpoint-specific params
|
||||
if self.endpoint_type in ("search", "snippet"):
|
||||
params.update(
|
||||
query=query,
|
||||
num_web_results=self.num_web_results,
|
||||
)
|
||||
elif self.endpoint_type == "news":
|
||||
params.update(
|
||||
q=query,
|
||||
count=self.num_web_results,
|
||||
search_lang=self.search_lang,
|
||||
ui_lang=self.ui_lang,
|
||||
spellcheck=self.spellcheck,
|
||||
)
|
||||
|
||||
params = {k: v for k, v in params.items() if v is not None}
|
||||
return params
|
||||
|
||||
def _parse_results(self, raw_search_results: Dict) -> List[Document]:
|
||||
"""
|
||||
Extracts snippets from each hit and puts them in a Document
|
||||
@ -105,9 +192,12 @@ class YouSearchAPIWrapper(BaseModel):
|
||||
|
||||
# return news results
|
||||
if self.endpoint_type == "news":
|
||||
news_results = raw_search_results["news"]["results"]
|
||||
if self.k is not None:
|
||||
news_results = news_results[: self.k]
|
||||
return [
|
||||
Document(page_content=result["description"], metadata=result)
|
||||
for result in raw_search_results["news"]["results"]
|
||||
for result in news_results
|
||||
]
|
||||
|
||||
docs = []
|
||||
@ -138,26 +228,10 @@ class YouSearchAPIWrapper(BaseModel):
|
||||
|
||||
Args:
|
||||
query: The query to search for.
|
||||
num_web_results: The maximum number of results to return.
|
||||
safesearch: Safesearch settings,
|
||||
one of off, moderate, strict, defaults to moderate
|
||||
country: Country code
|
||||
Returns: YouAPIOutput
|
||||
"""
|
||||
headers = {"X-API-Key": self.ydc_api_key or ""}
|
||||
params = {
|
||||
"query": query,
|
||||
"num_web_results": self.num_web_results,
|
||||
"safesearch": self.safesearch,
|
||||
"country": self.country,
|
||||
**kwargs,
|
||||
}
|
||||
|
||||
params = {k: v for k, v in params.items() if v is not None}
|
||||
# news endpoint expects `q` instead of `query`
|
||||
if self.endpoint_type == "news":
|
||||
params["q"] = params["query"]
|
||||
del params["query"]
|
||||
params = self._generate_params(query, **kwargs)
|
||||
|
||||
# @todo deprecate `snippet`, not part of API
|
||||
if self.endpoint_type == "snippet":
|
||||
@ -192,18 +266,7 @@ class YouSearchAPIWrapper(BaseModel):
|
||||
"""Get results from the you.com Search API asynchronously."""
|
||||
|
||||
headers = {"X-API-Key": self.ydc_api_key or ""}
|
||||
params = {
|
||||
"query": query,
|
||||
"num_web_results": self.num_web_results,
|
||||
"safesearch": self.safesearch,
|
||||
"country": self.country,
|
||||
**kwargs,
|
||||
}
|
||||
params = {k: v for k, v in params.items() if v is not None}
|
||||
# news endpoint expects `q` instead of `query`
|
||||
if self.endpoint_type == "news":
|
||||
params["q"] = params["query"]
|
||||
del params["query"]
|
||||
params = self._generate_params(query, **kwargs)
|
||||
|
||||
# @todo deprecate `snippet`, not part of API
|
||||
if self.endpoint_type == "snippet":
|
||||
|
||||
Loading…
Reference in New Issue
Block a user