langchain/docs/extras/use_cases/multi_modal/image_agent.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cd835d40",
   "metadata": {},
   "source": [
    "## Multi-modal outputs: Image & Text"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa88e03a",
   "metadata": {},
   "source": [
    "This notebook shows how non-text producing tools can be used to create multi-modal agents.\n",
    "\n",
    "This example is limited to text and image outputs and uses UUIDs to transfer content across tools and agents. \n",
    "\n",
    "This example uses Steamship to generate and store generated images. Generated are auth protected by default. \n",
    "\n",
    "You can get your Steamship api key here: https://steamship.com/account/api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0653da01",
   "metadata": {},
   "outputs": [],
   "source": [
    "from steamship import Block, Steamship\n",
    "import re\n",
    "from IPython.display import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f6933033",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain import OpenAI\n",
    "from langchain.agents import initialize_agent\n",
    "from langchain.agents import AgentType\n",
    "from langchain.tools import SteamshipImageGenerationTool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "71e51e53",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = OpenAI(temperature=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9fc769d",
   "metadata": {},
   "source": [
    "## Dall-E "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cd177dfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "tools = [SteamshipImageGenerationTool(model_name=\"dall-e\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c71b1e46",
   "metadata": {},
   "outputs": [],
   "source": [
    "mrkl = initialize_agent(\n",
    "    tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "603aeb9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m I need to generate an image of a parrot playing soccer.\n",
      "Action: GenerateImage\n",
      "Action Input: A parrot wearing a soccer uniform, kicking a soccer ball.\u001b[0m\n",
      "Observation: \u001b[36;1m\u001b[1;3mE28BE7C7-D105-41E0-8A5B-2CE21424DFEC\u001b[0m\n",
      "Thought:\u001b[32;1m\u001b[1;3m I now have the UUID of the generated image.\n",
      "Final Answer: The UUID of the generated image is E28BE7C7-D105-41E0-8A5B-2CE21424DFEC.\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "output = mrkl.run(\"How would you visualize a parot playing soccer?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "25eb4efe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_output(output):\n",
    "    \"\"\"Display the multi-modal output from the agent.\"\"\"\n",
    "    UUID_PATTERN = re.compile(\n",
    "        r\"([0-9A-Za-z]{8}-[0-9A-Za-z]{4}-[0-9A-Za-z]{4}-[0-9A-Za-z]{4}-[0-9A-Za-z]{12})\"\n",
    "    )\n",
    "\n",
    "    outputs = UUID_PATTERN.split(output)\n",
    "    outputs = [\n",
    "        re.sub(r\"^\\W+\", \"\", el) for el in outputs\n",
    "    ]  # Clean trailing and leading non-word characters\n",
    "\n",
    "    for output in outputs:\n",
    "        maybe_block_id = UUID_PATTERN.search(output)\n",
    "        if maybe_block_id:\n",
    "            display(Image(Block.get(Steamship(), _id=maybe_block_id.group()).raw()))\n",
    "        else:\n",
    "            print(output, end=\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "082792a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The UUID of the generated image is \n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAAAaGVYSWZNTQAqAAAACAACknwAAgAAACkAAAAmkoYAAgAAABgAAABQAAAAAE9wZW5BSS0tYjcxM2Y4N2E3MGY4M2NhOGI3MzQwZDlkNWZiZjljOTAAAE1hZGUgd2l0aCBPcGVuQUkgREFMTC1FAAkzXE4AAQAASURBVHgBABaE6XsB/v3+/wAAAQEBAAD/AQAA/wAAAAAAAf8AAAH//v8AAQABAQEA/wD//wABAQAAAP8A/wEAAQD//wAAAQECAP8AAAH/////AQAAAAABAAAA/wABAQD/AP8AAAEB///+AAEBAAAAAP8AAAEAAAD/AQAB//8AAQEB/wD/AAAAAQAA/wAAAf//AAIB//4AAAD/AQEC/wH+AP8BAAABAQAA///+AQEAAAAA/wABAQAA/wAAAf8AAAH/AAAA/wABAP8AAAEAAf8B/wD/AQAAAAEA/wAAAP//AAAAAQIB//4AAAEAAf8AAAEA//8A/wAAAAAAAgAA/wAAAAIAAP4AAAD/AAAAAAEAAP8BAQAAAQIB//7/AQEA/wAAAP8AAAEA/wAAAQAB/wD/AP//AQEAAP8BAAEAAAAA////AQEA//8BAAD/AAABAQEAAAAAAP8A/wD/AAEBAAAAAAAAAAD/AQAA/wEAAP8BAAD/Af8C/wD/AAAAAQAA/wH+AAABAAAAAAEBAP8AAP//AAABAAAAAQEA////AAEBAAAAAQAA/wAAAP//AAABAAH/AQAA/wABAAD/AP8BAQD//wEBAAD+AP8CAAAAAAAAAAEAAP8A/wEAAQAAAf8A/wEAAAAAAf8AAQIA//8AAAD//wABAQD///8BAQEB/wD/AQAAAAD//wABAQD/AP8B/wD/AAEBAQD/AAEBAP8AAQH///8CAAD/////AAEBAAAAAP8AAAEBAQAA/wD/Af8B/wEAAP/+AQAB/wEBAP8AAAEAAP8AAAD+AAEBAQAA//8AAAL/Af8B//8AAAEAAQAAAP///wEBAAAAAAH/AP8BAAAAAf8A/wH/Af8BAAEA/wAAAAD/AQAB/wAAAAAAAP8AAAEAAAAAAP8AAAABAQD//wEAAAD/AQAB/wAAAP8AAAIAAf4AAAIAAP8B/wH+AQABAP//AP8BAAAA/wIAAf8BAAD/AAAA/wAAAQAA/wD/AAABAf8A/wH/AAABAP8AAAEAAv8A/wEAAAAA/wAAAAD/AAAAAAABAAH/AP8BAAH/AP8AAAEBAv8B/gD+/wABAQAAAQAAAP//AAIAAP8B//8AAQIA/wD/Af8B/wD/AAABAf8B/wH+AAAAAAABAAAAAQEA//8AAAEAAAAAAQAAAAAA//8AAAAAAAABAQD/AP8A/wEAAQEB//7/AQEAAAAA////AAEBAAEAAP8BAAAAAAAAAP8AAAH/AAAAAAAAAQAA/wAAAAEAAP4AAAEAAAAAAAAA//8AAgIA//4AAAH/AP8BAQAAAAAAAAEBAAD/AP8A/wEAAgAA/wAAAAAAAAAAAAABAAD///8BAAEAAP8AAQH//wD/AQABAAAAAAAAAAAA/wAAAAEAAP8AAAAAAP8AAAEAAQABAAAAAAEA//8AAQD//wD/Af8BAAH//gABAgAAAP8AAAL///8BAAAAAQAB////AQEB/wAAAP//AQEAAAAAAAAA//8A/wEAAgEA//8A//8AAQEAAQAB/wEAAP7/AQIAAP///wACAQD///8BAQEAAAAA/wAAAAH/Af8AAP8A/wEAAf8A/wEAAAEAAP8AAf8AAAEAAP8A/wEBAQD//wABAAD/AQAB/wAAAQD/AAH///8AAf8B/wEAAQEBAP///wAAAQEAAP8AAAAAAQAA/wEA//4AAAEBAAD/AQAAAAAA/wABAQD/AAEAAP8AAAAAAAAAAAEAAAAAAAAAAf8A/wAAAAEAAQAB/wAAAP8AAAAAAAH/AP4BAAEAAAD/AAABAQAAAAEAAP8AAAD/AAEAAP4B/wD+x8vMQ0RF+PTzAAABAAABAAEAfkQEf0n+Af/+AAMBAP//AAAAAAH/AfIK/coS8u/+Nsrp3fz8//8CAQAAdXVzh4qIAP8AAAEAAv4A/wEAAAEA/wAAAQD/AP8B/wEAAf//AAAAAP8BAQEB/gD/AQAAAAEA//8AAAAAAQAA/wD/AQAB//8AAQEB/wD/AQAAAP8BAAD//wEBAAD/AQAAAAAB/wAAAQD/AAABAAH/AP8AAAAA/wAAAQAAAAAA/wEAAgAAAAEBAP8A/gEAAQABAAEA/wEAAP8BAP//AQEBAQAC//8BAAD/AAEA/wAAAP8BAAABAAH/AQAAAP4A/wAA/wD/AQAAAQAAAf////8AAQD//gAA//8AAAEA/gEAAQEAAAD/AQEB/wAA/wEAAf8AAAEAAQD//wD/AQEAAAAAAAEAAAABAgD/AAEAAAABAQAA/wD/AAEAAAEA//8A/wEBAf8BAf8BAAEAAP8BAAEAAAABAAAAAf//AQAAAAEAAAD/AQAB/wEBAf8A//8AAAD/AQEBAQEAAQD/AAABAAD/Af8A/wD/AAEBAQIA/wEBAP8AAAABAAD/AAAAAAABAQIBAQEAAQABAAEAAP/+/v8A////AAAAAP8B//8A/wH/AQD//wABAAABAf//AQAB/////wD/AAABAQD/AQAAAAEBAQEBAAD//wAAAP8BAAABAQD//wD//wD/AAH/AAD//wD+AAAA/wD/AAEA/wABAAECAP8AAAABAgEAAQAAAAH//gD/AAEBAP8B/wEBAP8AAAABAAD//wD/AP//AAEAAQEBAgECAP8AAP8AAAD/AAD/AAAA//8BAAIBAQAAAgEBAAEB/wEBAAD/AAD//wH//wD/AAAA/wH//wD///8BAP7+/wD///8AAAAA////AAEB/wD//gD/AP8A////AAAAAP8AAf8AAAEBAQEAAAAAAQD//wD/AP7/Af8A////AAEBAQEAAAH/AAD/AP///wD//wH/AP8B/wEB/wAAAf8BAAAA/wEA/wH//wAB/wEBAQEA//8AAAABAAD/AQAAAAABAAD/AP8AAP8BAQAA/wEAAAD+AAEAAAAAAf8AAAD/AQAAAP///wAA/wAAAQD/AAD//wAAAAAAAAD//wD+AAABAAEB//8BAAAAAAAAAAABAAABAAAAAAD+AP8BAP///////gAAAP//AQAAAQABAP8AAAEAAAAA/wAB//8AAAH/AAAAAAEAAQIAAP8B/wEAAAEA/wABAAEA//4BAAD/AP///wABAQAAAgEBAQEAAAAAAQEAAAH/AAEAAAD/AP///wEAAQIAAAEAAAEBAAEBAAD/AQD/AAECAP///wEAAQAA/wEAAAEAAAAB////AQAAAAEAAAEAAP8AAAABAAAAAAD/AAAAAAD//wAB/wEAAAEB//8BAAACAQD/AP///gABAf8AAAAA/wEBAAH/AAH/AAAB/wAAAAAAAAAC/wEAAAEAAAEBAP8A/wAAAAEAAAEB/wEAAAEBAAEBAAIAAAD/AAECAAEBAgABAAABAP8CAAAB/wAAAP8BAQD/AP8AAP8AAAD/AAAAAAD//gAAAAAA/wEA/wEB/gAAAAH/AQAAAP//AAAAAQEAAAEAAAH/AQEAAAIAAAAAAAEAAQAAAAAA/wD+/wD//gAAAAAAAQACAf8BAQIA/wAA/wAAAP8AAQD/AAD/Af//AAAAAP8A/wEAAAEBAf8AAAAAAAAAAQIBAAIAAP///wEAAAIBAAEAAAAA/wH//wAB/wD+AP8AAAEAAP//AAAA//////8A/wD//gD/AP4AAP8BAAEBAAABAAAAAAD/AAAAAP8BAP8AAAAAAQEA//8AAAAAAAH/AQAAAAAA/wABAAAAAgAAAQEB/wH/AAD//wAAAAEBAP//AAAAAAAAAP8AAAD/AQEBAQEBAAEBAAEA/wABAQEAAAABAQAAAP8AAQAAAAAAAAAAAQEAAQEAAf8BAAEAAAD/Af8AAAD/AAAA/wABAAAAAQAC/wEAAAEAAQAA//8BAAAAAP8BAAABAP//AQABAAABAbKztPj19QA
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_output(output)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e247b2c4",
   "metadata": {},
   "source": [
    "## StableDiffusion "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "315025e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "tools = [SteamshipImageGenerationTool(model_name=\"stable-diffusion\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7930064a",
   "metadata": {},
   "outputs": [],
   "source": [
    "mrkl = initialize_agent(\n",
    "    tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "611a833d",
   "metadata": {},
   "outputs": [],
   "source": [
    "output = mrkl.run(\"How would you visualize a parot playing soccer?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d7a3edaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "show_output(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffdf9c53",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}