From ba5a2f06b939bef04e79c67555aa8d58a12cf0b3 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Mon, 6 Feb 2023 18:14:25 -0800 Subject: [PATCH] Harrison/inference endpoint (#861) Co-authored-by: Eno Reyes --- langchain/llms/huggingface_endpoint.py | 140 ++++++++++++++++++ .../llms/test_huggingface_endpoint.py | 50 +++++++ 2 files changed, 190 insertions(+) create mode 100644 langchain/llms/huggingface_endpoint.py create mode 100644 tests/integration_tests/llms/test_huggingface_endpoint.py diff --git a/langchain/llms/huggingface_endpoint.py b/langchain/llms/huggingface_endpoint.py new file mode 100644 index 0000000000..dcc5e76155 --- /dev/null +++ b/langchain/llms/huggingface_endpoint.py @@ -0,0 +1,140 @@ +"""Wrapper around HuggingFace APIs.""" +from typing import Any, Dict, List, Mapping, Optional + +import requests +from pydantic import BaseModel, Extra, root_validator + +from langchain.llms.base import LLM +from langchain.llms.utils import enforce_stop_tokens +from langchain.utils import get_from_dict_or_env + +VALID_TASKS = ("text2text-generation", "text-generation") + + +class HuggingFaceEndpoint(LLM, BaseModel): + """Wrapper around HuggingFaceHub Inference Endpoints. + + To use, you should have the ``huggingface_hub`` python package installed, and the + environment variable ``HUGGINGFACEHUB_API_TOKEN`` set with your API token, or pass + it as a named parameter to the constructor. + + Only supports `text-generation` and `text2text-generation` for now. + + Example: + .. code-block:: python + + from langchain import HuggingFaceEndpoint + endpoint_url = ( + "https://abcdefghijklmnop.us-east-1.aws.endpoints.huggingface.cloud" + ) + hf = HuggingFaceEndpoint( + endpoint_url=endpoint_url, + huggingfacehub_api_token="my-api-key" + ) + """ + + endpoint_url: str = "" + """Endpoint URL to use.""" + task: Optional[str] = None + """Task to call the model with. Should be a task that returns `generated_text`.""" + model_kwargs: Optional[dict] = None + """Key word arguments to pass to the model.""" + + huggingfacehub_api_token: Optional[str] = None + + class Config: + """Configuration for this pydantic object.""" + + extra = Extra.forbid + + @root_validator() + def validate_environment(cls, values: Dict) -> Dict: + """Validate that api key and python package exists in environment.""" + huggingfacehub_api_token = get_from_dict_or_env( + values, "huggingfacehub_api_token", "HUGGINGFACEHUB_API_TOKEN" + ) + try: + from huggingface_hub.hf_api import HfApi + + try: + HfApi( + endpoint="https://huggingface.co", # Can be a Private Hub endpoint. + token=huggingfacehub_api_token, + ).whoami() + except Exception as e: + raise ValueError( + "Could not authenticate with huggingface_hub. " + "Please check your API token." + ) from e + + except ImportError: + raise ValueError( + "Could not import huggingface_hub python package. " + "Please it install it with `pip install huggingface_hub`." + ) + return values + + @property + def _identifying_params(self) -> Mapping[str, Any]: + """Get the identifying parameters.""" + _model_kwargs = self.model_kwargs or {} + return { + **{"endpoint_url": self.endpoint_url, "task": self.task}, + **{"model_kwargs": _model_kwargs}, + } + + @property + def _llm_type(self) -> str: + """Return type of llm.""" + return "huggingface_endpoint" + + def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str: + """Call out to HuggingFace Hub's inference endpoint. + + Args: + prompt: The prompt to pass into the model. + stop: Optional list of stop words to use when generating. + + Returns: + The string generated by the model. + + Example: + .. code-block:: python + + response = hf("Tell me a joke.") + """ + _model_kwargs = self.model_kwargs or {} + + # payload samples + parameter_payload = {"inputs": prompt, "parameters": _model_kwargs} + + # HTTP headers for authorization + headers = { + "Authorization": f"Bearer {self.huggingfacehub_api_token}", + "Content-Type": "application/json", + } + + # send request + try: + response = requests.post( + self.endpoint_url, headers=headers, json=parameter_payload + ) + except requests.exceptions.RequestException as e: # This is the correct syntax + raise ValueError(f"Error raised by inference endpoint: {e}") + if self.task == "text-generation": + # Text generation return includes the starter text. + generated_text = response.json() + text = generated_text[0]["generated_text"][len(prompt) :] + elif self.task == "text2text-generation": + generated_text = response.json() + text = generated_text[0]["generated_text"] + else: + raise ValueError( + f"Got invalid task {self.task}, " + f"currently only {VALID_TASKS} are supported" + ) + if stop is not None: + # This is a bit hacky, but I can't figure out a better way to enforce + # stop tokens when making calls to huggingface_hub. + text = enforce_stop_tokens(text, stop) + return text diff --git a/tests/integration_tests/llms/test_huggingface_endpoint.py b/tests/integration_tests/llms/test_huggingface_endpoint.py new file mode 100644 index 0000000000..61639669d3 --- /dev/null +++ b/tests/integration_tests/llms/test_huggingface_endpoint.py @@ -0,0 +1,50 @@ +"""Test HuggingFace API wrapper.""" + +import unittest +from pathlib import Path + +import pytest + +from langchain.llms.huggingface_endpoint import HuggingFaceEndpoint +from langchain.llms.loading import load_llm +from tests.integration_tests.llms.utils import assert_llm_equality + + +@unittest.skip( + "This test requires an inference endpoint. Tested with Hugging Face endpoints" +) +def test_huggingface_endpoint_text_generation() -> None: + """Test valid call to HuggingFace text generation model.""" + llm = HuggingFaceEndpoint( + endpoint_url="", task="text-generation", model_kwargs={"max_new_tokens": 10} + ) + output = llm("Say foo:") + print(output) + assert isinstance(output, str) + + +@unittest.skip( + "This test requires an inference endpoint. Tested with Hugging Face endpoints" +) +def test_huggingface_endpoint_text2text_generation() -> None: + """Test valid call to HuggingFace text2text model.""" + llm = HuggingFaceEndpoint(endpoint_url="", task="text2text-generation") + output = llm("The capital of New York is") + assert output == "Albany" + + +def test_huggingface_endpoint_call_error() -> None: + """Test valid call to HuggingFace that errors.""" + llm = HuggingFaceEndpoint(model_kwargs={"max_new_tokens": -1}) + with pytest.raises(ValueError): + llm("Say foo:") + + +def test_saving_loading_endpoint_llm(tmp_path: Path) -> None: + """Test saving/loading an HuggingFaceHub LLM.""" + llm = HuggingFaceEndpoint( + endpoint_url="", task="text-generation", model_kwargs={"max_new_tokens": 10} + ) + llm.save(file_path=tmp_path / "hf.yaml") + loaded_llm = load_llm(tmp_path / "hf.yaml") + assert_llm_equality(llm, loaded_llm)