manifest/examples/manifest_connection_pool.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "OPENAI_KEY1 = \"sk-XXX\"\n",
    "OPENAI_KEY2 = \"sk-XX\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use OpenAI\n",
    "\n",
    "Set you `OPENAI_API_KEY` environment variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from manifest import Manifest\n",
    "from manifest.connections.client_pool import ClientConnection\n",
    "\n",
    "openai_ada = ClientConnection(\n",
    "    client_name=\"openai\",\n",
    "    client_connection=OPENAI_KEY1,\n",
    "    engine=\"text-ada-001\"\n",
    ")\n",
    "\n",
    "openai_curie = ClientConnection(\n",
    "    client_name=\"openai\",\n",
    "    client_connection=OPENAI_KEY2,\n",
    "    engine=\"text-curie-001\"\n",
    ")\n",
    "\n",
    "manifest = Manifest(client_pool=[openai_ada, openai_curie], client_pool_schedule=\"round_robin\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "I am a model.\n",
      "1\n",
      "I am a MacBook Pro with a retina\n"
     ]
    }
   ],
   "source": [
    "res = manifest.run(\"What model are you?\", temperature=0.0)\n",
    "print(manifest.client_pool.current_client_id)\n",
    "print(res)\n",
    "res = manifest.run(\"What model are you?\", temperature=0.0)\n",
    "print(manifest.client_pool.current_client_id)\n",
    "print(res)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## With Async"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nest_asyncio\n",
    "# This is required for asyncio.run(...) to work in Jupyter notebooks.\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from manifest import Manifest\n",
    "from manifest.connections.client_pool import ClientConnection\n",
    "\n",
    "openai_ada = ClientConnection(\n",
    "    client_name=\"openai\",\n",
    "    client_connection=OPENAI_KEY1,\n",
    "    engine=\"text-ada-001\"\n",
    ")\n",
    "\n",
    "openai_babbage = ClientConnection(\n",
    "    client_name=\"openai\",\n",
    "    client_connection=OPENAI_KEY2,\n",
    "    engine=\"text-babbage-001\"\n",
    ")\n",
    "\n",
    "openai_curie = ClientConnection(\n",
    "    client_name=\"openai\",\n",
    "    client_connection=OPENAI_KEY2,\n",
    "    engine=\"text-curie-001\"\n",
    ")\n",
    "\n",
    "manifest = Manifest(client_pool=[openai_ada, openai_babbage, openai_curie], client_pool_schedule=\"round_robin\")\n",
    "manifest_single_client = Manifest(client_pool=[openai_babbage])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For loop: 128.68\n",
      "Running with async single client\n",
      "Running 1 tasks across all clients.\n",
      "Async loop: 4.02\n",
      "Running with async two clients but not chunking\n",
      "Running 1 tasks across all clients.\n",
      "Async loop: 3.92\n",
      "Running with async two clients and chunk size\n",
      "Running 20 tasks across all clients.\n",
      "Async loop: 1.44\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import asyncio\n",
    "\n",
    "prompts = [f\"Tell me something interesting about {i}\" for i in range(400)]\n",
    "st = time.time()\n",
    "for pmt in prompts:\n",
    "    _ = manifest_single_client.run(pmt, max_tokens=30)\n",
    "print(f\"For loop: {time.time() - st :.2f}\")\n",
    "\n",
    "print(\"Running with async single client\")\n",
    "st = time.time()\n",
    "_ = asyncio.run(manifest_single_client.arun_batch(prompts, max_tokens=30, chunk_size=-1))\n",
    "print(f\"Async loop: {time.time() - st :.2f}\")\n",
    "\n",
    "print(\"Running with async two clients but not chunking\")\n",
    "st = time.time()\n",
    "_ = asyncio.run(manifest.arun_batch(prompts, max_tokens=30, chunk_size=-1))\n",
    "print(f\"Async loop: {time.time() - st :.2f}\")\n",
    "\n",
    "print(\"Running with async two clients and chunk size\")\n",
    "st = time.time()\n",
    "_ = asyncio.run(manifest.arun_batch(prompts, max_tokens=30, chunk_size=20))\n",
    "print(f\"Async loop: {time.time() - st :.2f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "manifest",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "fddffe4ac3b9f00470127629076101c1b5f38ecb1e7358b567d19305425e9491"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}