From 9aaa0fdce084028632c305cadae6390e4e0d2ed6 Mon Sep 17 00:00:00 2001 From: Predrag Gruevski Date: Mon, 28 Aug 2023 14:20:48 +0000 Subject: [PATCH 1/5] Use unified Python setup steps for release workflow. --- .github/workflows/_release.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/_release.yml b/.github/workflows/_release.yml index 5ddf79ca7c..1d7668978b 100644 --- a/.github/workflows/_release.yml +++ b/.github/workflows/_release.yml @@ -31,13 +31,15 @@ jobs: working-directory: ${{ inputs.working-directory }} steps: - uses: actions/checkout@v3 - - name: Install poetry - run: pipx install "poetry==$POETRY_VERSION" - - name: Set up Python 3.10 - uses: actions/setup-python@v4 + + - name: Set up Python + Poetry ${{ env.POETRY_VERSION }} + uses: "./.github/actions/poetry_setup" with: python-version: "3.10" - cache: "poetry" + poetry-version: ${{ env.POETRY_VERSION }} + working-directory: ${{ inputs.working-directory }} + cache-key: release + - name: Build project for distribution run: poetry build - name: Check Version From 97741d41c5fc75a2bedb5706f94d89d87ea74e1e Mon Sep 17 00:00:00 2001 From: hughcrt Date: Mon, 28 Aug 2023 19:24:50 +0200 Subject: [PATCH 2/5] Add LLMonitorCallbackHandler --- .../integrations/callbacks/llmonitor.md | 63 ++++ .../langchain/langchain/callbacks/__init__.py | 2 + .../langchain/callbacks/llmonitor_callback.py | 319 ++++++++++++++++++ 3 files changed, 384 insertions(+) create mode 100644 docs/extras/integrations/callbacks/llmonitor.md create mode 100644 libs/langchain/langchain/callbacks/llmonitor_callback.py diff --git a/docs/extras/integrations/callbacks/llmonitor.md b/docs/extras/integrations/callbacks/llmonitor.md new file mode 100644 index 0000000000..57b1ec7c95 --- /dev/null +++ b/docs/extras/integrations/callbacks/llmonitor.md @@ -0,0 +1,63 @@ +# LLMonitor + +[LLMonitor](https://llmonitor.com) is an open-source observability platform that provides cost tracking, user tracking and powerful agent tracing. + + + +## Setup +Create an account on [llmonitor.com](https://llmonitor.com), create an `App`, and then copy the associated `tracking id`. +Once you have it, set it as an environment variable by running: +```bash +export LLMONITOR_APP_ID="..." +``` + +If you'd prefer not to set an environment variable, you can pass the key directly when initializing the callback handler: +```python +from langchain.callbacks import LLMonitorCallbackHandler + +handler = LLMonitorCallbackHandler(app_id="...") +``` + +## Usage with LLM/Chat models +```python +from langchain.llms import OpenAI +from langchain.chat_models import ChatOpenAI +from langchain.callbacks import LLMonitorCallbackHandler + +handler = LLMonitorCallbackHandler(app_id="...") + +llm = OpenAI( + callbacks=[handler], +) + +chat = ChatOpenAI( + callbacks=[handler], + metadata={"userId": "123"}, # you can assign user ids to models in the metadata +) +``` + + +## Usage with agents +```python +from langchain.agents import load_tools, initialize_agent, AgentType +from langchain.llms import OpenAI +from langchain.callbacks import LLMonitorCallbackHandler + +handler = LLMonitorCallbackHandler(app_id="...") + +llm = OpenAI(temperature=0) +tools = load_tools(["serpapi", "llm-math"], llm=llm) +agent = initialize_agent(tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION) +agent.run( + "Who is Leo DiCaprio's girlfriend? What is her current age raised to the 0.43 power?", + callbacks=[handler], + metadata={ + "agentName": "Leo DiCaprio's girlfriend", # you can assign a custom agent in the metadata + }, +) +``` + +## Support +For any question or issue with integration you can reach out to the LLMonitor team on [Discord](http://discord.com/invite/8PafSG58kK) or via [email](mailto:vince@llmonitor.com). diff --git a/libs/langchain/langchain/callbacks/__init__.py b/libs/langchain/langchain/callbacks/__init__.py index 8398741be3..12e18d52c7 100644 --- a/libs/langchain/langchain/callbacks/__init__.py +++ b/libs/langchain/langchain/callbacks/__init__.py @@ -19,6 +19,7 @@ from langchain.callbacks.flyte_callback import FlyteCallbackHandler from langchain.callbacks.human import HumanApprovalCallbackHandler from langchain.callbacks.infino_callback import InfinoCallbackHandler from langchain.callbacks.labelstudio_callback import LabelStudioCallbackHandler +from langchain.callbacks.llmonitor_callback import LLMonitorCallbackHandler from langchain.callbacks.manager import ( get_openai_callback, tracing_enabled, @@ -53,6 +54,7 @@ __all__ = [ "HumanApprovalCallbackHandler", "InfinoCallbackHandler", "MlflowCallbackHandler", + "LLMonitorCallbackHandler", "OpenAICallbackHandler", "StdOutCallbackHandler", "AsyncIteratorCallbackHandler", diff --git a/libs/langchain/langchain/callbacks/llmonitor_callback.py b/libs/langchain/langchain/callbacks/llmonitor_callback.py new file mode 100644 index 0000000000..99ca1e9290 --- /dev/null +++ b/libs/langchain/langchain/callbacks/llmonitor_callback.py @@ -0,0 +1,319 @@ +import os +import traceback +from datetime import datetime +from typing import Any, Dict, List, Literal, Optional, Union +from uuid import UUID + +import requests + +from langchain.callbacks.base import BaseCallbackHandler +from langchain.schema.agent import AgentAction, AgentFinish +from langchain.schema.messages import BaseMessage +from langchain.schema.output import LLMResult + +DEFAULT_API_URL = "https://app.llmonitor.com" + + +def _parse_lc_role(role: str) -> Literal["user", "ai", "system", "function"] | None: + if role == "human": + return "user" + elif role == "ai": + return "ai" + elif role == "system": + return "system" + elif role == "function": + return "function" + else: + return None + + +def _serialize_lc_message(message: BaseMessage) -> Dict[str, Any]: + return {"text": message.content, "role": _parse_lc_role(message.type)} + + +class LLMonitorCallbackHandler(BaseCallbackHandler): + """Initializes the `LLMonitorCallbackHandler`. + #### Parameters: + - `app_id`: The app id of the app you want to report to. Defaults to `None`, which means that `LLMONITOR_APP_ID` will be used. + - `api_url`: The url of the LLMonitor API. Defaults to `None`, which means that either `LLMONITOR_API_URL` environment variable or `https://app.llmonitor.com` will be used. + + #### Raises: + - `ValueError`: if `app_id` is not provided either as an argument or as an environment variable. + - `ConnectionError`: if the connection to the API fails. + + + #### Example: + ```python + from langchain.llms import OpenAI + from langchain.callbacks import LLMonitorCallbackHandler + + llmonitor_callback = LLMonitorCallbackHandler() + llm = OpenAI(callbacks=[llmonitor_callback], metadata={"userId": "user-123"}) + llm.predict("Hello, how are you?") + ``` + """ + + __api_url: str + __app_id: str + + def __init__(self, app_id: str | None = None, api_url: str | None = None) -> None: + super().__init__() + + self.__api_url = api_url or os.getenv("LLMONITOR_API_URL") or DEFAULT_API_URL + + _app_id = app_id or os.getenv("LLMONITOR_APP_ID") + if _app_id is None: + raise ValueError( + "app_id must be provided either as an argument or as an environment variable" + ) + self.__app_id = _app_id + + try: + res = requests.get(f"{self.__api_url}/api/app/{self.__app_id}") + if not res.ok: + raise ConnectionError() + except Exception as e: + raise ConnectionError( + f"Could not connect to the LLMonitor API at {self.__api_url}" + ) from e + + def __send_event(self, event: Dict[str, Any]) -> None: + headers = {"Content-Type": "application/json"} + event = {**event, "app": self.__app_id, "timestamp": str(datetime.utcnow())} + data = {"events": event} + requests.post(headers=headers, url=f"{self.__api_url}/api/report", json=data) + + def on_llm_start( + self, + serialized: Dict[str, Any], + prompts: List[str], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: List[str] | None = None, + metadata: Dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + event = { + "event": "start", + "type": "llm", + "userId": (metadata or {}).get("userId"), + "runId": str(run_id), + "parentRunId": str(parent_run_id) if parent_run_id else None, + "input": prompts[0], + "name": kwargs.get("invocation_params", {}).get("model_name"), + "tags": tags, + "metadata": metadata, + } + self.__send_event(event) + + def on_chat_model_start( + self, + serialized: Dict[str, Any], + messages: List[List[BaseMessage]], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: List[str] | None = None, + metadata: Dict[str, Any] | None = None, + **kwargs: Any, + ) -> Any: + event = { + "event": "start", + "type": "llm", + "userId": (metadata or {}).get("userId"), + "runId": str(run_id), + "parentRunId": str(parent_run_id) if parent_run_id else None, + "input": [_serialize_lc_message(message[0]) for message in messages], + "name": kwargs.get("invocation_params", {}).get("model_name"), + "tags": tags, + "metadata": metadata, + } + self.__send_event(event) + + def on_llm_end( + self, + response: LLMResult, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> None: + token_usage = (response.llm_output or {}).get("token_usage", {}) + + event = { + "event": "end", + "type": "llm", + "runId": str(run_id), + "parent_run_id": str(parent_run_id) if parent_run_id else None, + "output": {"text": response.generations[0][0].text, "role": "ai"}, + "tokensUsage": { + "prompt": token_usage.get("prompt_tokens", 0), + "completion": token_usage.get("completion_tokens", 0), + }, + } + self.__send_event(event) + + def on_llm_error( + self, + error: Exception | KeyboardInterrupt, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> Any: + event = { + "event": "error", + "type": "llm", + "runId": str(run_id), + "parent_run_id": str(parent_run_id) if parent_run_id else None, + "error": {"message": str(error), "stack": traceback.format_exc()}, + } + self.__send_event(event) + + def on_tool_start( + self, + serialized: Dict[str, Any], + input_str: str, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: List[str] | None = None, + metadata: Dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + event = { + "event": "start", + "type": "tool", + "userId": (metadata or {}).get("userId"), + "runId": str(run_id), + "parentRunId": str(parent_run_id) if parent_run_id else None, + "name": serialized.get("name"), + "input": input_str, + "tags": tags, + "metadata": metadata, + } + self.__send_event(event) + + def on_tool_end( + self, + output: str, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: List[str] | None = None, + **kwargs: Any, + ) -> None: + event = { + "event": "end", + "type": "tool", + "runId": str(run_id), + "parent_run_id": str(parent_run_id) if parent_run_id else None, + "output": output, + } + self.__send_event(event) + + def on_chain_start( + self, + serialized: Dict[str, Any], + inputs: Dict[str, Any], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: List[str] | None = None, + metadata: Dict[str, Any] | None = None, + **kwargs: Any, + ) -> Any: + name = serialized.get("id", [None, None, None, None])[3] + type = "chain" + + agentName = (metadata or {}).get("agentName") + if agentName is not None: + type = "agent" + name = agentName + if name == "AgentExecutor" or name == "PlanAndExecute": + type = "agent" + event = { + "event": "start", + "type": type, + "userId": (metadata or {}).get("userId"), + "runId": str(run_id), + "parentRunId": str(parent_run_id) if parent_run_id else None, + "input": inputs.get("input", inputs), + "tags": tags, + "metadata": metadata, + "name": serialized.get("id", [None, None, None, None])[3], + } + + self.__send_event(event) + + def on_chain_end( + self, + outputs: Dict[str, Any], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> Any: + event = { + "event": "end", + "type": "chain", + "runId": str(run_id), + "output": outputs.get("output", outputs), + } + self.__send_event(event) + + def on_chain_error( + self, + error: Exception | KeyboardInterrupt, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> Any: + event = { + "event": "error", + "type": "chain", + "runId": str(run_id), + "parent_run_id": str(parent_run_id) if parent_run_id else None, + "error": {"message": str(error), "stack": traceback.format_exc()}, + } + self.__send_event(event) + + def on_agent_action( + self, + action: AgentAction, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> Any: + event = { + "event": "start", + "type": "tool", + "runId": str(run_id), + "parentRunId": str(parent_run_id) if parent_run_id else None, + "name": action.tool, + "input": action.tool_input, + } + self.__send_event(event) + + def on_agent_finish( + self, + finish: AgentFinish, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> Any: + event = { + "event": "end", + "type": "agent", + "runId": str(run_id), + "parentRunId": str(parent_run_id) if parent_run_id else None, + "output": finish.return_values, + } + self.__send_event(event) + + +__all__ = ["LLMonitorCallbackHandler"] From 3a4d4c940c71d67affecfd21503943648f75175f Mon Sep 17 00:00:00 2001 From: hughcrt Date: Mon, 28 Aug 2023 19:26:33 +0200 Subject: [PATCH 3/5] Change video width --- docs/extras/integrations/callbacks/llmonitor.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/extras/integrations/callbacks/llmonitor.md b/docs/extras/integrations/callbacks/llmonitor.md index 57b1ec7c95..daec3dad81 100644 --- a/docs/extras/integrations/callbacks/llmonitor.md +++ b/docs/extras/integrations/callbacks/llmonitor.md @@ -2,7 +2,7 @@ [LLMonitor](https://llmonitor.com) is an open-source observability platform that provides cost tracking, user tracking and powerful agent tracing. -