From 0d51a1f12bb7ac06b59ed130b0b1a36577e4f478 Mon Sep 17 00:00:00 2001 From: kYLe Date: Thu, 11 May 2023 02:39:59 -0500 Subject: [PATCH] Add LLMs support for Anyscale Service (#4350) Add Anyscale service integration under LLM Co-authored-by: Dev 2049 --- docs/ecosystem/anyscale.md | 17 ++ .../models/llms/integrations/anyscale.ipynb | 171 ++++++++++++++++++ langchain/llms/anyscale.py | 119 ++++++++++++ tests/integration_tests/llms/test_anyscale.py | 10 + 4 files changed, 317 insertions(+) create mode 100644 docs/ecosystem/anyscale.md create mode 100644 docs/modules/models/llms/integrations/anyscale.ipynb create mode 100644 langchain/llms/anyscale.py create mode 100644 tests/integration_tests/llms/test_anyscale.py diff --git a/docs/ecosystem/anyscale.md b/docs/ecosystem/anyscale.md new file mode 100644 index 00000000..4d98dd31 --- /dev/null +++ b/docs/ecosystem/anyscale.md @@ -0,0 +1,17 @@ +# Anyscale + +This page covers how to use the Anyscale ecosystem within LangChain. +It is broken into two parts: installation and setup, and then references to specific Anyscale wrappers. + +## Installation and Setup +- Get an Anyscale Service URL, route and API key and set them as environment variables (`ANYSCALE_SERVICE_URL`,`ANYSCALE_SERVICE_ROUTE`, `ANYSCALE_SERVICE_TOKEN`). +- Please see [the Anyscale docs](https://docs.anyscale.com/productionize/services-v2/get-started) for more details. + +## Wrappers + +### LLM + +There exists an Anyscale LLM wrapper, which you can access with +```python +from langchain.llms import Anyscale +``` diff --git a/docs/modules/models/llms/integrations/anyscale.ipynb b/docs/modules/models/llms/integrations/anyscale.ipynb new file mode 100644 index 00000000..73e6dfaa --- /dev/null +++ b/docs/modules/models/llms/integrations/anyscale.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9597802c", + "metadata": {}, + "source": [ + "# Anysacle\n", + "\n", + "[Anyscale](https://www.anyscale.com/) is a fully-managed [Ray](https://www.ray.io/) platform, on which you can build, deploy, and manage scalable AI and Python applications\n", + "\n", + "This example goes over how to use LangChain to interact with `Anyscale` [service](https://docs.anyscale.com/productionize/services-v2/get-started)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5472a7cd-af26-48ca-ae9b-5f6ae73c74d2", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ[\"ANYSCALE_SERVICE_URL\"] = ANYSCALE_SERVICE_URL\n", + "os.environ[\"ANYSCALE_SERVICE_ROUTE\"] = ANYSCALE_SERVICE_ROUTE\n", + "os.environ[\"ANYSCALE_SERVICE_TOKEN\"] = ANYSCALE_SERVICE_TOKEN" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6fb585dd", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from langchain.llms import Anyscale\n", + "from langchain import PromptTemplate, LLMChain" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "035dea0f", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "template = \"\"\"Question: {question}\n", + "\n", + "Answer: Let's think step by step.\"\"\"\n", + "\n", + "prompt = PromptTemplate(template=template, input_variables=[\"question\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3f3458d9", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "llm = Anyscale()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a641dbd9", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "llm_chain = LLMChain(prompt=prompt, llm=llm)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f844993", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "question = \"When was George Washington president?\"\n", + "\n", + "llm_chain.run(question)" + ] + }, + { + "cell_type": "markdown", + "id": "42f05b34-1a44-4cbd-8342-35c1572b6765", + "metadata": {}, + "source": [ + "With Ray, we can distribute the queries without asyncrhonized implementation. This not only applies to Anyscale LLM model, but to any other Langchain LLM models which do not have `_acall` or `_agenerate` implemented" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "08b23adc-2b29-4c38-b538-47b3c3d840a6", + "metadata": {}, + "outputs": [], + "source": [ + "prompt_list = [\n", + " \"When was George Washington president?\",\n", + " \"Explain to me the difference between nuclear fission and fusion.\",\n", + " \"Give me a list of 5 science fiction books I should read next.\",\n", + " \"Explain the difference between Spark and Ray.\",\n", + " \"Suggest some fun holiday ideas.\",\n", + " \"Tell a joke.\",\n", + " \"What is 2+2?\",\n", + " \"Explain what is machine learning like I am five years old.\",\n", + " \"Explain what is artifical intelligence.\",\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b45abb9-b764-497d-af99-0df1d4e335e0", + "metadata": {}, + "outputs": [], + "source": [ + "import ray\n", + "\n", + "@ray.remote\n", + "def send_query(llm, prompt):\n", + " resp = llm(prompt)\n", + " return resp\n", + "\n", + "futures = [send_query.remote(llm, prompt) for prompt in prompt_list]\n", + "results = ray.get(futures)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.8" + }, + "vscode": { + "interpreter": { + "hash": "a0a0263b650d907a3bfe41c0f8d6a63a071b884df3cfdc1579f00cdc1aed6b03" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/langchain/llms/anyscale.py b/langchain/llms/anyscale.py new file mode 100644 index 00000000..972957d7 --- /dev/null +++ b/langchain/llms/anyscale.py @@ -0,0 +1,119 @@ +"""Wrapper around Anyscale""" +from typing import Any, Dict, List, Mapping, Optional + +import requests +from pydantic import Extra, root_validator + +from langchain.callbacks.manager import CallbackManagerForLLMRun +from langchain.llms.base import LLM +from langchain.llms.utils import enforce_stop_tokens +from langchain.utils import get_from_dict_or_env + + +class Anyscale(LLM): + """Wrapper around Anyscale Services. + To use, you should have the environment variable ``ANYSCALE_SERVICE_URL``, + ``ANYSCALE_SERVICE_ROUTE`` and ``ANYSCALE_SERVICE_TOKEN`` set with your Anyscale + Service, or pass it as a named parameter to the constructor. + + Example: + .. code-block:: python + from langchain.llms import Anyscale + anyscale = Anyscale(anyscale_service_url="SERVICE_URL", + anyscale_service_route="SERVICE_ROUTE", + anyscale_service_token="SERVICE_TOKEN") + + # Use Ray for distributed processing + import ray + prompt_list=[] + @ray.remote + def send_query(llm, prompt): + resp = llm(prompt) + return resp + futures = [send_query.remote(anyscale, prompt) for prompt in prompt_list] + results = ray.get(futures) + """ + + model_kwargs: Optional[dict] = None + """Key word arguments to pass to the model. Reserved for future use""" + + anyscale_service_url: Optional[str] = None + anyscale_service_route: Optional[str] = None + anyscale_service_token: Optional[str] = None + + class Config: + """Configuration for this pydantic object.""" + + extra = Extra.forbid + + @root_validator() + def validate_environment(cls, values: Dict) -> Dict: + """Validate that api key and python package exists in environment.""" + anyscale_service_url = get_from_dict_or_env( + values, "anyscale_service_url", "ANYSCALE_SERVICE_URL" + ) + anyscale_service_route = get_from_dict_or_env( + values, "anyscale_service_route", "ANYSCALE_SERVICE_ROUTE" + ) + anyscale_service_token = get_from_dict_or_env( + values, "anyscale_service_token", "ANYSCALE_SERVICE_TOKEN" + ) + try: + anyscale_service_endpoint = f"{anyscale_service_url}/-/route" + headers = {"Authorization": f"Bearer {anyscale_service_token}"} + requests.get(anyscale_service_endpoint, headers=headers) + except requests.exceptions.RequestException as e: + raise ValueError(e) + values["anyscale_service_url"] = anyscale_service_url + values["anyscale_service_route"] = anyscale_service_route + values["anyscale_service_token"] = anyscale_service_token + return values + + @property + def _identifying_params(self) -> Mapping[str, Any]: + """Get the identifying parameters.""" + return { + "anyscale_service_url": self.anyscale_service_url, + "anyscale_service_route": self.anyscale_service_route, + } + + @property + def _llm_type(self) -> str: + """Return type of llm.""" + return "anyscale" + + def _call( + self, + prompt: str, + stop: Optional[List[str]] = None, + run_manager: Optional[CallbackManagerForLLMRun] = None, + ) -> str: + """Call out to Anyscale Service endpoint. + Args: + prompt: The prompt to pass into the model. + stop: Optional list of stop words to use when generating. + Returns: + The string generated by the model. + Example: + .. code-block:: python + response = anyscale("Tell me a joke.") + """ + + anyscale_service_endpoint = ( + f"{self.anyscale_service_url}/{self.anyscale_service_route}" + ) + headers = {"Authorization": f"Bearer {self.anyscale_service_token}"} + body = {"prompt": prompt} + resp = requests.post(anyscale_service_endpoint, headers=headers, json=body) + + if resp.status_code != 200: + raise ValueError( + f"Error returned by service, status code {resp.status_code}" + ) + text = resp.text + + if stop is not None: + # This is a bit hacky, but I can't figure out a better way to enforce + # stop tokens when making calls to huggingface_hub. + text = enforce_stop_tokens(text, stop) + return text diff --git a/tests/integration_tests/llms/test_anyscale.py b/tests/integration_tests/llms/test_anyscale.py new file mode 100644 index 00000000..605ff67b --- /dev/null +++ b/tests/integration_tests/llms/test_anyscale.py @@ -0,0 +1,10 @@ +"""Test Anyscale API wrapper.""" + +from langchain.llms.anyscale import Anyscale + + +def test_anyscale_call() -> None: + """Test valid call to Anyscale.""" + llm = Anyscale() + output = llm("Say foo:") + assert isinstance(output, str)