diff --git a/docs/extras/modules/agents/toolkits/multion.ipynb b/docs/extras/modules/agents/toolkits/multion.ipynb new file mode 100644 index 0000000000..4758a0fa9c --- /dev/null +++ b/docs/extras/modules/agents/toolkits/multion.ipynb @@ -0,0 +1,129 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Multion Toolkit\n", + "\n", + "This notebook walks you through connecting LangChain to the MultiOn Client in your browser\n", + "\n", + "To use this toolkit, you will need to add MultiOn Extension to your browser as explained in the [MultiOn for Chrome](https://multion.notion.site/Download-MultiOn-ddddcfe719f94ab182107ca2612c07a5)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install --upgrade multion > /dev/null" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## MultiOn Setup\n", + "\n", + "Login to establish connection with your extension." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Authorize connection to your Browser extention\n", + "import multion \n", + "multion.login()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Use Multion Toolkit within an Agent" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from langchain.agents.agent_toolkits import create_multion_agent\n", + "from langchain.tools.multion.tool import MultionClientTool\n", + "from langchain.agents.agent_types import AgentType\n", + "from langchain.chat_models import ChatOpenAI" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "agent_executor = create_multion_agent(\n", + " llm=ChatOpenAI(temperature=0),\n", + " tool=MultionClientTool(),\n", + " agent_type=AgentType.OPENAI_FUNCTIONS,\n", + " verbose=True\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "agent.run(\"show me the weather today\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "agent.run(\n", + " \"Tweet about Elon Musk\"\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/libs/langchain/langchain/agents/agent_toolkits/multion/__init__.py b/libs/langchain/langchain/agents/agent_toolkits/multion/__init__.py new file mode 100644 index 0000000000..56c7215199 --- /dev/null +++ b/libs/langchain/langchain/agents/agent_toolkits/multion/__init__.py @@ -0,0 +1 @@ +"""MultiOn Toolkit.""" diff --git a/libs/langchain/langchain/agents/agent_toolkits/multion/base.py b/libs/langchain/langchain/agents/agent_toolkits/multion/base.py new file mode 100644 index 0000000000..e3de7ee5eb --- /dev/null +++ b/libs/langchain/langchain/agents/agent_toolkits/multion/base.py @@ -0,0 +1,58 @@ +"""MultiOn agent.""" + +from typing import Any, Dict, Optional + +from langchain.agents.agent import AgentExecutor, BaseSingleActionAgent +from langchain.agents.agent_toolkits.python.prompt import PREFIX +from langchain.agents.mrkl.base import ZeroShotAgent +from langchain.agents.openai_functions_agent.base import OpenAIFunctionsAgent +from langchain.agents.types import AgentType +from langchain.base_language import BaseLanguageModel +from langchain.callbacks.base import BaseCallbackManager +from langchain.chains.llm import LLMChain +from langchain.schema import SystemMessage +from langchain.tools.multion.tool import MultionClientTool + + +def create_multion_agent( + llm: BaseLanguageModel, + tool: MultionClientTool, + agent_type: AgentType = AgentType.ZERO_SHOT_REACT_DESCRIPTION, + callback_manager: Optional[BaseCallbackManager] = None, + verbose: bool = False, + prefix: str = PREFIX, + agent_executor_kwargs: Optional[Dict[str, Any]] = None, + **kwargs: Dict[str, Any], +) -> AgentExecutor: + """Construct a multion agent from an LLM and tool.""" + tools = [tool] + agent: BaseSingleActionAgent + + if agent_type == AgentType.ZERO_SHOT_REACT_DESCRIPTION: + prompt = ZeroShotAgent.create_prompt(tools, prefix=prefix) + llm_chain = LLMChain( + llm=llm, + prompt=prompt, + callback_manager=callback_manager, + ) + tool_names = [tool.name for tool in tools] + agent = ZeroShotAgent(llm_chain=llm_chain, allowed_tools=tool_names, **kwargs) + elif agent_type == AgentType.OPENAI_FUNCTIONS: + system_message = SystemMessage(content=prefix) + _prompt = OpenAIFunctionsAgent.create_prompt(system_message=system_message) + agent = OpenAIFunctionsAgent( + llm=llm, + prompt=_prompt, + tools=[tool], + callback_manager=callback_manager, + **kwargs, + ) + else: + raise ValueError(f"Agent type {agent_type} not supported at the moment.") + return AgentExecutor.from_agent_and_tools( + agent=agent, + tools=tools, + callback_manager=callback_manager, + verbose=verbose, + **(agent_executor_kwargs or {}), + ) diff --git a/libs/langchain/langchain/tools/multion/__init__.py b/libs/langchain/langchain/tools/multion/__init__.py new file mode 100644 index 0000000000..92c08b48a9 --- /dev/null +++ b/libs/langchain/langchain/tools/multion/__init__.py @@ -0,0 +1 @@ +"""MutliOn Client API toolkit.""" diff --git a/libs/langchain/langchain/tools/multion/tool.py b/libs/langchain/langchain/tools/multion/tool.py new file mode 100644 index 0000000000..9c44764d82 --- /dev/null +++ b/libs/langchain/langchain/tools/multion/tool.py @@ -0,0 +1,50 @@ +"""Tool for MultiOn Extension API""" +from typing import Any, Optional + +from pydantic import Field + +from langchain.callbacks.manager import ( + AsyncCallbackManagerForToolRun, + CallbackManagerForToolRun, +) +from langchain.tools.base import BaseTool +from langchain.utilities.multion import MultionClientAPIWrapper + + +def _get_default_multion_client() -> MultionClientAPIWrapper: + return MultionClientAPIWrapper() + + +class MultionClientTool(BaseTool): + """Simulates a Browser interacting agent.""" + + name = "Multion_Client" + description = ( + "A api to communicate with browser extension multion " + "Useful for automating tasks and actions in the browser " + "Input should be a task and a url." + "The result is text form of action that was executed in the given url." + ) + api_wrapper: MultionClientAPIWrapper = Field( + default_factory=_get_default_multion_client + ) + + def _run( + self, + task: str, + url: str = "https://www.google.com/", + tabId: Optional[Any] = None, + run_manager: Optional[CallbackManagerForToolRun] = None, + ) -> str: + """Use the tool.""" + return self.api_wrapper.run(task, url, tabId) + + async def _arun( + self, + task: str, + url: str, + tabId: Optional[Any] = None, + run_manager: Optional[AsyncCallbackManagerForToolRun] = None, + ) -> str: + """Use the tool asynchronously.""" + raise NotImplementedError("Multion Client does not support async yet") diff --git a/libs/langchain/langchain/utilities/multion.py b/libs/langchain/langchain/utilities/multion.py new file mode 100644 index 0000000000..20276d3c3e --- /dev/null +++ b/libs/langchain/langchain/utilities/multion.py @@ -0,0 +1,75 @@ +"""Util that calls MultiOn Client. + +In order to set this up, follow instructions at: +https://multion.notion.site/Download-MultiOn-ddddcfe719f94ab182107ca2612c07a5 +""" +from typing import Any, Optional + +from pydantic import BaseModel + + +class MultionAPI: + def __init__(self) -> None: + self.tabId = None + self.new_session_count = 0 + + def create_session(self, query: str, url: str) -> str: + """Always the first step to run any activities that can be done using browser. + + Args: + 'query': the query that you need to perform in the given url. + If there is no 'query' set it as open. + 'url': the base url of a site. + """ + import multion + + # Only create new session once and continue using update session + if self.new_session_count < 2: + response = multion.new_session({"input": query, "url": url}) + self.new_session_count += 1 + self.tabId = response["tabId"] + return response["message"] + else: + return "Continue using update session" + + def update_session(self, query: str, url: str) -> str: + """Updates the existing browser session. + + Updates with given action and url, used consequently to handle browser + activities after creating one session of browser. + + Args: + 'query': the query that you need to perform in the given url. + If there is no 'query' set it as open. + 'url': the base url of a site. + """ + import multion + + response = multion.update_session(self.tabId, {"input": query, "url": url}) + return response["message"] + + +class MultionClientAPIWrapper(BaseModel): + """Wrapper for Multion Client API. + + In order to set this up, follow instructions at: + NEED TO ADD + """ + + client: Any = MultionAPI() + + def run(self, task: str, url: str, tabId: Optional[Any]) -> str: + """Run body through Multion Client and respond with action. + + Args: + task: + url: + tabId: + """ + if self.client.tabId is None or tabId is None: + self.client = MultionAPI() + message = self.client.create_session(task, url) + else: + message = self.client.update_session(task, url) + + return message