diff --git a/docs/extras/integrations/toolkits/multion.ipynb b/docs/extras/integrations/toolkits/multion.ipynb index 4758a0fa9c..3382af6210 100644 --- a/docs/extras/integrations/toolkits/multion.ipynb +++ b/docs/extras/integrations/toolkits/multion.ipynb @@ -5,7 +5,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Multion Toolkit\n", + "# MultiOn Toolkit\n", "\n", "This notebook walks you through connecting LangChain to the MultiOn Client in your browser\n", "\n", @@ -18,17 +18,22 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install --upgrade multion > /dev/null" + "!pip install --upgrade multion langchain -q" ] }, { - "attachments": {}, - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "## MultiOn Setup\n", + "from langchain.agents.agent_toolkits import MultionToolkit\n", + "import os\n", "\n", - "Login to establish connection with your extension." + "\n", + "toolkit = MultionToolkit()\n", + "\n", + "toolkit" ] }, { @@ -37,47 +42,37 @@ "metadata": {}, "outputs": [], "source": [ - "# Authorize connection to your Browser extention\n", - "import multion \n", - "multion.login()\n" + "tools = toolkit.get_tools()\n", + "tools" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "## Use Multion Toolkit within an Agent" + "## MultiOn Setup\n", + "\n", + "Login to establish connection with your extension." ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ - "from langchain.agents.agent_toolkits import create_multion_agent\n", - "from langchain.tools.multion.tool import MultionClientTool\n", - "from langchain.agents.agent_types import AgentType\n", - "from langchain.chat_models import ChatOpenAI" + "# Authorize connection to your Browser extention\n", + "import multion\n", + "multion.login()\n", + "\n" ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], + "cell_type": "markdown", + "metadata": {}, "source": [ - "\n", - "agent_executor = create_multion_agent(\n", - " llm=ChatOpenAI(temperature=0),\n", - " tool=MultionClientTool(),\n", - " agent_type=AgentType.OPENAI_FUNCTIONS,\n", - " verbose=True\n", - ")\n" + "## Use Multion Toolkit within an Agent" ] }, { @@ -88,7 +83,18 @@ }, "outputs": [], "source": [ - "agent.run(\"show me the weather today\")" + "from langchain import OpenAI\n", + "from langchain.agents import initialize_agent, AgentType\n", + "llm = OpenAI(temperature=0)\n", + "from langchain.agents.agent_toolkits import MultionToolkit\n", + "toolkit = MultionToolkit()\n", + "tools=toolkit.get_tools()\n", + "agent = initialize_agent(\n", + " tools=toolkit.get_tools(),\n", + " llm=llm,\n", + " agent=AgentType.STRUCTURED_CHAT_ZERO_SHOT_REACT_DESCRIPTION,\n", + " verbose = True\n", + ")" ] }, { @@ -100,7 +106,7 @@ "outputs": [], "source": [ "agent.run(\n", - " \"Tweet about Elon Musk\"\n", + " \"Tweet 'Hi from MultiOn'\"\n", ")" ] } diff --git a/libs/langchain/langchain/agents/agent_toolkits/__init__.py b/libs/langchain/langchain/agents/agent_toolkits/__init__.py index a5baba21b0..f914d0c1e2 100644 --- a/libs/langchain/langchain/agents/agent_toolkits/__init__.py +++ b/libs/langchain/langchain/agents/agent_toolkits/__init__.py @@ -17,7 +17,7 @@ from langchain.agents.agent_toolkits.gmail.toolkit import GmailToolkit from langchain.agents.agent_toolkits.jira.toolkit import JiraToolkit from langchain.agents.agent_toolkits.json.base import create_json_agent from langchain.agents.agent_toolkits.json.toolkit import JsonToolkit -from langchain.agents.agent_toolkits.multion.base import create_multion_agent +from langchain.agents.agent_toolkits.multion.toolkit import MultionToolkit from langchain.agents.agent_toolkits.nla.toolkit import NLAToolkit from langchain.agents.agent_toolkits.office365.toolkit import O365Toolkit from langchain.agents.agent_toolkits.openapi.base import create_openapi_agent @@ -52,6 +52,7 @@ __all__ = [ "GmailToolkit", "JiraToolkit", "JsonToolkit", + "MultionToolkit", "NLAToolkit", "O365Toolkit", "OpenAPIToolkit", @@ -65,7 +66,6 @@ __all__ = [ "ZapierToolkit", "create_csv_agent", "create_json_agent", - "create_multion_agent", "create_openapi_agent", "create_pandas_dataframe_agent", "create_pbi_agent", diff --git a/libs/langchain/langchain/agents/agent_toolkits/multion/base.py b/libs/langchain/langchain/agents/agent_toolkits/multion/base.py deleted file mode 100644 index e3de7ee5eb..0000000000 --- a/libs/langchain/langchain/agents/agent_toolkits/multion/base.py +++ /dev/null @@ -1,58 +0,0 @@ -"""MultiOn agent.""" - -from typing import Any, Dict, Optional - -from langchain.agents.agent import AgentExecutor, BaseSingleActionAgent -from langchain.agents.agent_toolkits.python.prompt import PREFIX -from langchain.agents.mrkl.base import ZeroShotAgent -from langchain.agents.openai_functions_agent.base import OpenAIFunctionsAgent -from langchain.agents.types import AgentType -from langchain.base_language import BaseLanguageModel -from langchain.callbacks.base import BaseCallbackManager -from langchain.chains.llm import LLMChain -from langchain.schema import SystemMessage -from langchain.tools.multion.tool import MultionClientTool - - -def create_multion_agent( - llm: BaseLanguageModel, - tool: MultionClientTool, - agent_type: AgentType = AgentType.ZERO_SHOT_REACT_DESCRIPTION, - callback_manager: Optional[BaseCallbackManager] = None, - verbose: bool = False, - prefix: str = PREFIX, - agent_executor_kwargs: Optional[Dict[str, Any]] = None, - **kwargs: Dict[str, Any], -) -> AgentExecutor: - """Construct a multion agent from an LLM and tool.""" - tools = [tool] - agent: BaseSingleActionAgent - - if agent_type == AgentType.ZERO_SHOT_REACT_DESCRIPTION: - prompt = ZeroShotAgent.create_prompt(tools, prefix=prefix) - llm_chain = LLMChain( - llm=llm, - prompt=prompt, - callback_manager=callback_manager, - ) - tool_names = [tool.name for tool in tools] - agent = ZeroShotAgent(llm_chain=llm_chain, allowed_tools=tool_names, **kwargs) - elif agent_type == AgentType.OPENAI_FUNCTIONS: - system_message = SystemMessage(content=prefix) - _prompt = OpenAIFunctionsAgent.create_prompt(system_message=system_message) - agent = OpenAIFunctionsAgent( - llm=llm, - prompt=_prompt, - tools=[tool], - callback_manager=callback_manager, - **kwargs, - ) - else: - raise ValueError(f"Agent type {agent_type} not supported at the moment.") - return AgentExecutor.from_agent_and_tools( - agent=agent, - tools=tools, - callback_manager=callback_manager, - verbose=verbose, - **(agent_executor_kwargs or {}), - ) diff --git a/libs/langchain/langchain/agents/agent_toolkits/multion/toolkit.py b/libs/langchain/langchain/agents/agent_toolkits/multion/toolkit.py new file mode 100644 index 0000000000..76f374a21c --- /dev/null +++ b/libs/langchain/langchain/agents/agent_toolkits/multion/toolkit.py @@ -0,0 +1,22 @@ +"""MultiOn agent.""" +from __future__ import annotations + +from typing import List + +from langchain.agents.agent_toolkits.base import BaseToolkit +from langchain.tools import BaseTool +from langchain.tools.multion.create_session import MultionCreateSession +from langchain.tools.multion.update_session import MultionUpdateSession + + +class MultionToolkit(BaseToolkit): + """Toolkit for interacting with the Browser Agent""" + + class Config: + """Pydantic config.""" + + arbitrary_types_allowed = True + + def get_tools(self) -> List[BaseTool]: + """Get the tools in the toolkit.""" + return [MultionCreateSession(), MultionUpdateSession()] diff --git a/libs/langchain/langchain/tools/multion/__init__.py b/libs/langchain/langchain/tools/multion/__init__.py index 92c08b48a9..bf8068935c 100644 --- a/libs/langchain/langchain/tools/multion/__init__.py +++ b/libs/langchain/langchain/tools/multion/__init__.py @@ -1 +1,6 @@ -"""MutliOn Client API toolkit.""" +"""MutliOn Client API tools.""" + +from langchain.tools.multion.create_session import MultionCreateSession +from langchain.tools.multion.update_session import MultionUpdateSession + +__all__ = ["MultionCreateSession", "MultionUpdateSession"] diff --git a/libs/langchain/langchain/tools/multion/create_session.py b/libs/langchain/langchain/tools/multion/create_session.py new file mode 100644 index 0000000000..9ae2332cf3 --- /dev/null +++ b/libs/langchain/langchain/tools/multion/create_session.py @@ -0,0 +1,50 @@ +from typing import TYPE_CHECKING, Optional, Type + +from pydantic import BaseModel, Field + +from langchain.callbacks.manager import CallbackManagerForToolRun +from langchain.tools.base import BaseTool + +if TYPE_CHECKING: + # This is for linting and IDE typehints + import multion +else: + try: + # We do this so pydantic can resolve the types when instantiating + import multion + except ImportError: + pass + + +class CreateSessionSchema(BaseModel): + """Input for CreateSessionTool.""" + + query: str = Field( + ..., + description="The query to run in multion agent.", + ) + url: str = Field( + "https://www.google.com/", + description="""The Url to run the agent at. Note: accepts only secure \ + links having https://""", + ) + + +class MultionCreateSession(BaseTool): + name: str = "create_multion_session" + description: str = """Use this tool to create a new Multion Browser Window \ + with provided fields.Always the first step to run \ + any activities that can be done using browser.""" + args_schema: Type[CreateSessionSchema] = CreateSessionSchema + + def _run( + self, + query: str, + url: Optional[str] = "https://www.google.com/", + run_manager: Optional[CallbackManagerForToolRun] = None, + ) -> dict: + try: + response = multion.new_session({"input": query, "url": url}) + return {"tabId": response["tabId"], "Response": response["message"]} + except Exception as e: + raise Exception(f"An error occurred: {e}") diff --git a/libs/langchain/langchain/tools/multion/tool.py b/libs/langchain/langchain/tools/multion/tool.py deleted file mode 100644 index be2ac31010..0000000000 --- a/libs/langchain/langchain/tools/multion/tool.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Tool for MultiOn Extension API""" -from typing import Any, Optional - -from pydantic import Field - -from langchain.callbacks.manager import CallbackManagerForToolRun -from langchain.tools.base import BaseTool -from langchain.utilities.multion import MultionClientAPIWrapper - - -def _get_default_multion_client() -> MultionClientAPIWrapper: - return MultionClientAPIWrapper() - - -class MultionClientTool(BaseTool): - """Simulates a Browser interacting agent.""" - - name = "Multion_Client" - description = ( - "A api to communicate with browser extension multion " - "Useful for automating tasks and actions in the browser " - "Input should be a task and a url." - "The result is text form of action that was executed in the given url." - ) - api_wrapper: MultionClientAPIWrapper = Field( - default_factory=_get_default_multion_client - ) - - def _run( - self, - task: str, - url: str = "https://www.google.com/", - tabId: Optional[Any] = None, - run_manager: Optional[CallbackManagerForToolRun] = None, - ) -> str: - """Use the tool.""" - return self.api_wrapper.run(task, url, tabId) diff --git a/libs/langchain/langchain/tools/multion/update_session.py b/libs/langchain/langchain/tools/multion/update_session.py new file mode 100644 index 0000000000..0e724726e9 --- /dev/null +++ b/libs/langchain/langchain/tools/multion/update_session.py @@ -0,0 +1,63 @@ +from typing import TYPE_CHECKING, Optional, Type + +from pydantic import BaseModel, Field + +from langchain.callbacks.manager import CallbackManagerForToolRun +from langchain.tools.base import BaseTool + +if TYPE_CHECKING: + # This is for linting and IDE typehints + import multion +else: + try: + # We do this so pydantic can resolve the types when instantiating + import multion + except ImportError: + pass + + +class UpdateSessionSchema(BaseModel): + """Input for UpdateSessionTool.""" + + tabId: str = Field( + ..., description="The tabID, received from one of the createSessions run before" + ) + query: str = Field( + ..., + description="The query to run in multion agent.", + ) + url: str = Field( + "https://www.google.com/", + description="""The Url to run the agent at. \ + Note: accepts only secure links having https://""", + ) + + +class MultionUpdateSession(BaseTool): + name: str = "update_multion_session" + description: str = """Use this tool to update \ + a existing corresponding \ + Multion Browser Window with provided fields. \ + Note:TabId is got from one of the previous Browser window creation.""" + args_schema: Type[UpdateSessionSchema] = UpdateSessionSchema + + def _run( + self, + tabId: str, + query: str, + url: Optional[str] = "https://www.google.com/", + run_manager: Optional[CallbackManagerForToolRun] = None, + ) -> dict: + try: + try: + response = multion.update_session(tabId, {"input": query, "url": url}) + content = {"tabId": tabId, "Response": response["message"]} + self.tabId = tabId + return content + except Exception as e: + print(f"{e}, creating a new session") + response = multion.new_session({"input": query, "url": url}) + self.tabID = response["tabId"] + return {"tabId": response["tabId"], "Response": response["message"]} + except Exception as e: + raise Exception(f"An error occurred: {e}") diff --git a/libs/langchain/langchain/utilities/multion.py b/libs/langchain/langchain/utilities/multion.py deleted file mode 100644 index 20276d3c3e..0000000000 --- a/libs/langchain/langchain/utilities/multion.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Util that calls MultiOn Client. - -In order to set this up, follow instructions at: -https://multion.notion.site/Download-MultiOn-ddddcfe719f94ab182107ca2612c07a5 -""" -from typing import Any, Optional - -from pydantic import BaseModel - - -class MultionAPI: - def __init__(self) -> None: - self.tabId = None - self.new_session_count = 0 - - def create_session(self, query: str, url: str) -> str: - """Always the first step to run any activities that can be done using browser. - - Args: - 'query': the query that you need to perform in the given url. - If there is no 'query' set it as open. - 'url': the base url of a site. - """ - import multion - - # Only create new session once and continue using update session - if self.new_session_count < 2: - response = multion.new_session({"input": query, "url": url}) - self.new_session_count += 1 - self.tabId = response["tabId"] - return response["message"] - else: - return "Continue using update session" - - def update_session(self, query: str, url: str) -> str: - """Updates the existing browser session. - - Updates with given action and url, used consequently to handle browser - activities after creating one session of browser. - - Args: - 'query': the query that you need to perform in the given url. - If there is no 'query' set it as open. - 'url': the base url of a site. - """ - import multion - - response = multion.update_session(self.tabId, {"input": query, "url": url}) - return response["message"] - - -class MultionClientAPIWrapper(BaseModel): - """Wrapper for Multion Client API. - - In order to set this up, follow instructions at: - NEED TO ADD - """ - - client: Any = MultionAPI() - - def run(self, task: str, url: str, tabId: Optional[Any]) -> str: - """Run body through Multion Client and respond with action. - - Args: - task: - url: - tabId: - """ - if self.client.tabId is None or tabId is None: - self.client = MultionAPI() - message = self.client.create_session(task, url) - else: - message = self.client.update_session(task, url) - - return message