From c02eb199b6587aeeb50fbb083693572bd2f030cc Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Sat, 19 Nov 2022 20:32:45 -0800 Subject: [PATCH 1/2] add few shot example (#148) --- README.md | 4 +- docs/examples/demos/map reduce.ipynb | 6 +- docs/examples/demos/simple_prompts.ipynb | 6 +- docs/examples/integrations/embeddings.ipynb | 2 +- .../integrations/huggingface_hub.ipynb | 6 +- docs/examples/integrations/manifest.ipynb | 6 +- docs/examples/model_laboratory.ipynb | 6 +- docs/examples/prompts/example_prompt.json | 4 + docs/examples/prompts/examples.json | 4 + docs/examples/prompts/few shot examples.ipynb | 306 ++++++++++ docs/examples/prompts/few_shot_prompt.json | 11 + docs/examples/prompts/few_shot_prompt.yaml | 14 + .../few_shot_prompt_example_prompt.json | 8 + .../prompts/few_shot_prompt_examples_in.json | 14 + docs/examples/prompts/generate_examples.ipynb | 74 +-- docs/examples/prompts/prompt_management.ipynb | 540 ++++++++++++++++++ .../prompts/prompt_optimization.ipynb | 179 ------ .../prompts/prompt_serialization.ipynb | 538 +++++++++++++++++ docs/examples/prompts/simple_prompt.json | 4 + docs/examples/prompts/simple_prompt.yaml | 4 + .../simple_prompt_with_template_file.json | 4 + docs/examples/prompts/simple_template.txt | 1 + docs/{ => explanation}/core_concepts.md | 6 +- docs/{ => explanation}/glossary.md | 0 docs/explanation/prompts.md | 138 +++++ docs/getting_started/chains.md | 9 +- docs/index.rst | 6 +- docs/modules/example_selector.rst | 5 + langchain/__init__.py | 6 +- langchain/chains/llm.py | 8 +- langchain/chains/llm_math/prompt.py | 4 +- langchain/chains/mapreduce.py | 4 +- langchain/chains/mrkl/base.py | 8 +- langchain/chains/natbot/prompt.py | 4 +- langchain/chains/react/prompt.py | 4 +- .../chains/self_ask_with_search/prompt.py | 4 +- langchain/chains/sql_database/prompt.py | 4 +- langchain/chains/vector_db_qa/prompt.py | 6 +- langchain/example_generator.py | 19 +- langchain/model_laboratory.py | 6 +- langchain/prompts/__init__.py | 14 +- langchain/prompts/base.py | 20 +- langchain/prompts/dynamic.py | 112 ---- .../prompts/example_selector/__init__.py | 7 + langchain/prompts/example_selector/base.py | 11 + .../prompts/example_selector/length_based.py | 54 ++ .../example_selector/semantic_similarity.py | 75 +++ langchain/prompts/few_shot.py | 110 ++++ langchain/prompts/loading.py | 103 ++++ langchain/prompts/optimized.py | 166 ------ langchain/prompts/prompt.py | 40 +- langchain/vectorstores/base.py | 8 +- .../vectorstores/elastic_vector_search.py | 16 +- langchain/vectorstores/faiss.py | 13 +- setup.py | 2 +- test_requirements.txt | 1 + .../vectorstores/__init__.py | 1 + .../vectorstores/test_elasticsearch.py | 42 ++ .../{ => vectorstores}/test_faiss.py | 17 + tests/unit_tests/chains/test_llm.py | 4 +- tests/unit_tests/chains/test_mrkl.py | 4 +- tests/unit_tests/chains/test_react.py | 4 +- tests/unit_tests/prompts/__init__.py | 1 + tests/unit_tests/prompts/test_few_shot.py | 87 +++ .../test_length_based_example_selector.py | 48 ++ tests/unit_tests/prompts/test_loading.py | 134 +++++ tests/unit_tests/{ => prompts}/test_prompt.py | 20 +- tests/unit_tests/test_dynamic_prompt.py | 111 ---- 68 files changed, 2494 insertions(+), 713 deletions(-) create mode 100644 docs/examples/prompts/example_prompt.json create mode 100644 docs/examples/prompts/examples.json create mode 100644 docs/examples/prompts/few shot examples.ipynb create mode 100644 docs/examples/prompts/few_shot_prompt.json create mode 100644 docs/examples/prompts/few_shot_prompt.yaml create mode 100644 docs/examples/prompts/few_shot_prompt_example_prompt.json create mode 100644 docs/examples/prompts/few_shot_prompt_examples_in.json create mode 100644 docs/examples/prompts/prompt_management.ipynb delete mode 100644 docs/examples/prompts/prompt_optimization.ipynb create mode 100644 docs/examples/prompts/prompt_serialization.ipynb create mode 100644 docs/examples/prompts/simple_prompt.json create mode 100644 docs/examples/prompts/simple_prompt.yaml create mode 100644 docs/examples/prompts/simple_prompt_with_template_file.json create mode 100644 docs/examples/prompts/simple_template.txt rename docs/{ => explanation}/core_concepts.md (85%) rename docs/{ => explanation}/glossary.md (100%) create mode 100644 docs/explanation/prompts.md create mode 100644 docs/modules/example_selector.rst delete mode 100644 langchain/prompts/dynamic.py create mode 100644 langchain/prompts/example_selector/__init__.py create mode 100644 langchain/prompts/example_selector/base.py create mode 100644 langchain/prompts/example_selector/length_based.py create mode 100644 langchain/prompts/example_selector/semantic_similarity.py create mode 100644 langchain/prompts/few_shot.py create mode 100644 langchain/prompts/loading.py delete mode 100644 langchain/prompts/optimized.py create mode 100644 tests/integration_tests/vectorstores/__init__.py create mode 100644 tests/integration_tests/vectorstores/test_elasticsearch.py rename tests/integration_tests/{ => vectorstores}/test_faiss.py (68%) create mode 100644 tests/unit_tests/prompts/__init__.py create mode 100644 tests/unit_tests/prompts/test_few_shot.py create mode 100644 tests/unit_tests/prompts/test_length_based_example_selector.py create mode 100644 tests/unit_tests/prompts/test_loading.py rename tests/unit_tests/{ => prompts}/test_prompt.py (79%) delete mode 100644 tests/unit_tests/test_dynamic_prompt.py diff --git a/README.md b/README.md index a7bded55..7490e29f 100644 --- a/README.md +++ b/README.md @@ -68,12 +68,12 @@ llm_math.run("How many of the integers between 0 and 99 inclusive are divisible You can also use this for simple prompting pipelines, as in the below example and this [example notebook](https://github.com/hwchase17/langchain/blob/master/docs/examples/demos/simple_prompts.ipynb). ```python -from langchain import Prompt, OpenAI, LLMChain +from langchain import PromptTemplate, OpenAI, LLMChain template = """Question: {question} Answer: Let's think step by step.""" -prompt = Prompt(template=template, input_variables=["question"]) +prompt = PromptTemplate(template=template, input_variables=["question"]) llm = OpenAI(temperature=0) llm_chain = LLMChain(prompt=prompt, llm=llm) diff --git a/docs/examples/demos/map reduce.ipynb b/docs/examples/demos/map reduce.ipynb index 18dcd617..2862b1a7 100644 --- a/docs/examples/demos/map reduce.ipynb +++ b/docs/examples/demos/map reduce.ipynb @@ -17,7 +17,7 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain import OpenAI, Prompt, LLMChain\n", + "from langchain import OpenAI, PromptTemplate, LLMChain\n", "from langchain.text_splitter import CharacterTextSplitter\n", "from langchain.chains.mapreduce import MapReduceChain\n", "\n", @@ -30,7 +30,7 @@ "\n", "\n", "CONCISE SUMMARY:\"\"\"\n", - "prompt = Prompt(template=_prompt, input_variables=[\"text\"])\n", + "prompt = PromptTemplate(template=_prompt, input_variables=[\"text\"])\n", "\n", "text_splitter = CharacterTextSplitter()\n", "\n", @@ -85,7 +85,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.8.7" } }, "nbformat": 4, diff --git a/docs/examples/demos/simple_prompts.ipynb b/docs/examples/demos/simple_prompts.ipynb index 4fb65a18..4299261d 100644 --- a/docs/examples/demos/simple_prompts.ipynb +++ b/docs/examples/demos/simple_prompts.ipynb @@ -28,12 +28,12 @@ } ], "source": [ - "from langchain import Prompt, OpenAI, LLMChain\n", + "from langchain import PromptTemplate, OpenAI, LLMChain\n", "\n", "template = \"\"\"Question: {question}\n", "\n", "Answer: Let's think step by step.\"\"\"\n", - "prompt = Prompt(template=template, input_variables=[\"question\"])\n", + "prompt = PromptTemplate(template=template, input_variables=[\"question\"])\n", "llm_chain = LLMChain(prompt=prompt, llm=OpenAI(temperature=0))\n", "\n", "question = \"What NFL team won the Super Bowl in the year Justin Beiber was born?\"\n", @@ -66,7 +66,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.8.7" } }, "nbformat": 4, diff --git a/docs/examples/integrations/embeddings.ipynb b/docs/examples/integrations/embeddings.ipynb index e80a675f..632f2d84 100644 --- a/docs/examples/integrations/embeddings.ipynb +++ b/docs/examples/integrations/embeddings.ipynb @@ -169,7 +169,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.8.7" } }, "nbformat": 4, diff --git a/docs/examples/integrations/huggingface_hub.ipynb b/docs/examples/integrations/huggingface_hub.ipynb index b2e59376..7f15cc1e 100644 --- a/docs/examples/integrations/huggingface_hub.ipynb +++ b/docs/examples/integrations/huggingface_hub.ipynb @@ -25,12 +25,12 @@ } ], "source": [ - "from langchain import Prompt, HuggingFaceHub, LLMChain\n", + "from langchain import PromptTemplate, HuggingFaceHub, LLMChain\n", "\n", "template = \"\"\"Question: {question}\n", "\n", "Answer: Let's think step by step.\"\"\"\n", - "prompt = Prompt(template=template, input_variables=[\"question\"])\n", + "prompt = PromptTemplate(template=template, input_variables=[\"question\"])\n", "llm_chain = LLMChain(prompt=prompt, llm=HuggingFaceHub(repo_id=\"google/flan-t5-xl\", model_kwargs={\"temperature\":1e-10}))\n", "\n", "question = \"What NFL team won the Super Bowl in the year Justin Beiber was born?\"\n", @@ -63,7 +63,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.8.7" } }, "nbformat": 4, diff --git a/docs/examples/integrations/manifest.ipynb b/docs/examples/integrations/manifest.ipynb index 2c809487..40e6b8c0 100644 --- a/docs/examples/integrations/manifest.ipynb +++ b/docs/examples/integrations/manifest.ipynb @@ -69,7 +69,7 @@ "outputs": [], "source": [ "# Map reduce example\n", - "from langchain import Prompt\n", + "from langchain import PromptTemplate\n", "from langchain.text_splitter import CharacterTextSplitter\n", "from langchain.chains.mapreduce import MapReduceChain\n", "\n", @@ -81,7 +81,7 @@ "\n", "\n", "CONCISE SUMMARY:\"\"\"\n", - "prompt = Prompt(template=_prompt, input_variables=[\"text\"])\n", + "prompt = PromptTemplate(template=_prompt, input_variables=[\"text\"])\n", "\n", "text_splitter = CharacterTextSplitter()\n", "\n", @@ -202,7 +202,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.8.7" }, "vscode": { "interpreter": { diff --git a/docs/examples/model_laboratory.ipynb b/docs/examples/model_laboratory.ipynb index 8c5af92f..13892a73 100644 --- a/docs/examples/model_laboratory.ipynb +++ b/docs/examples/model_laboratory.ipynb @@ -17,7 +17,7 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain import LLMChain, OpenAI, Cohere, HuggingFaceHub, Prompt\n", + "from langchain import LLMChain, OpenAI, Cohere, HuggingFaceHub, PromptTemplate\n", "from langchain.model_laboratory import ModelLaboratory" ] }, @@ -88,7 +88,7 @@ "metadata": {}, "outputs": [], "source": [ - "prompt = Prompt(template=\"What is the capital of {state}?\", input_variables=[\"state\"])\n", + "prompt = PromptTemplate(template=\"What is the capital of {state}?\", input_variables=[\"state\"])\n", "model_lab_with_prompt = ModelLaboratory.from_llms(llms, prompt=prompt)" ] }, @@ -246,7 +246,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.8.7" } }, "nbformat": 4, diff --git a/docs/examples/prompts/example_prompt.json b/docs/examples/prompts/example_prompt.json new file mode 100644 index 00000000..05098c4a --- /dev/null +++ b/docs/examples/prompts/example_prompt.json @@ -0,0 +1,4 @@ +{ + "input_variables": ["input", "output"], + "template": "Input: {input}\nOutput: {output}" +} diff --git a/docs/examples/prompts/examples.json b/docs/examples/prompts/examples.json new file mode 100644 index 00000000..70defee8 --- /dev/null +++ b/docs/examples/prompts/examples.json @@ -0,0 +1,4 @@ +[ + {"input": "happy", "output": "sad"}, + {"input": "tall", "output": "short"} +] diff --git a/docs/examples/prompts/few shot examples.ipynb b/docs/examples/prompts/few shot examples.ipynb new file mode 100644 index 00000000..5a325532 --- /dev/null +++ b/docs/examples/prompts/few shot examples.ipynb @@ -0,0 +1,306 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f8b01b97", + "metadata": {}, + "source": [ + "# Few Shot Prompt examples\n", + "Notebook showing off how canonical prompts in LangChain can be recreated as FewShotPrompts" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "18c67cc9", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.prompts.few_shot import FewShotPromptTemplate\n", + "from langchain.prompts.prompt import PromptTemplate" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2a729c9f", + "metadata": {}, + "outputs": [], + "source": [ + "# Self Ask with Search\n", + "\n", + "examples = [\n", + " {\n", + " \"question\": \"Who lived longer, Muhammad Ali or Alan Turing?\",\n", + " \"answer\": \"Are follow up questions needed here: Yes.\\nFollow up: How old was Muhammad Ali when he died?\\nIntermediate answer: Muhammad Ali was 74 years old when he died.\\nFollow up: How old was Alan Turing when he died?\\nIntermediate answer: Alan Turing was 41 years old when he died.\\nSo the final answer is: Muhammad Ali\"\n", + " },\n", + " {\n", + " \"question\": \"When was the founder of craigslist born?\",\n", + " \"answer\": \"Are follow up questions needed here: Yes.\\nFollow up: Who was the founder of craigslist?\\nIntermediate answer: Craigslist was founded by Craig Newmark.\\nFollow up: When was Craig Newmark born?\\nIntermediate answer: Craig Newmark was born on December 6, 1952.\\nSo the final answer is: December 6, 1952\"\n", + " },\n", + " {\n", + " \"question\": \"Who was the maternal grandfather of George Washington?\",\n", + " \"answer\": \"Are follow up questions needed here: Yes.\\nFollow up: Who was the mother of George Washington?\\nIntermediate answer: The mother of George Washington was Mary Ball Washington.\\nFollow up: Who was the father of Mary Ball Washington?\\nIntermediate answer: The father of Mary Ball Washington was Joseph Ball.\\nSo the final answer is: Joseph Ball\"\n", + " },\n", + " {\n", + " \"question\": \"Are both the directors of Jaws and Casino Royale from the same country?\",\n", + " \"answer\": \"Are follow up questions needed here: Yes.\\nFollow up: Who is the director of Jaws?\\nIntermediate Answer: The director of Jaws is Steven Spielberg.\\nFollow up: Where is Steven Spielberg from?\\nIntermediate Answer: The United States.\\nFollow up: Who is the director of Casino Royale?\\nIntermediate Answer: The director of Casino Royale is Martin Campbell.\\nFollow up: Where is Martin Campbell from?\\nIntermediate Answer: New Zealand.\\nSo the final answer is: No\"\n", + " }\n", + "]\n", + "example_prompt = PromptTemplate(input_variables=[\"question\", \"answer\"], template=\"Question: {question}\\n{answer}\")\n", + "\n", + "prompt = FewShotPromptTemplate(\n", + " examples=examples, \n", + " example_prompt=example_prompt, \n", + " suffix=\"Question: {input}\", \n", + " input_variables=[\"input\"]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "95fc0059", + "metadata": {}, + "outputs": [], + "source": [ + "# ReAct\n", + "\n", + "examples = [\n", + " {\n", + " \"question\": \"What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into?\",\n", + " \"answer\": \"Thought 1: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado orogeny extends into, then find the elevation range of that area.\\nAction 1: Search[Colorado orogeny]\\nObservation 1: The Colorado orogeny was an episode of mountain building (an orogeny) in Colorado and surrounding areas.\\nThought 2: It does not mention the eastern sector. So I need to look up eastern sector.\\nAction 2: Lookup[eastern sector]\\nObservation 2: (Result 1 / 1) The eastern sector extends into the High Plains and is called the Central Plains orogeny.\\nThought 3: The eastern sector of Colorado orogeny extends into the High Plains. So I need to search High Plains and find its elevation range.\\nAction 3: Search[High Plains]\\nObservation 3: High Plains refers to one of two distinct land regions\\nThought 4: I need to instead search High Plains (United States).\\nAction 4: Search[High Plains (United States)]\\nObservation 4: The High Plains are a subregion of the Great Plains. From east to west, the High Plains rise in elevation from around 1,800 to 7,000 ft (550 to 2,130 m).[3]\\nThought 5: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft.\\nAction 5: Finish[1,800 to 7,000 ft]\"\n", + " },\n", + " {\n", + " \"question\": \"Musician and satirist Allie Goertz wrote a song about the \\\"The Simpsons\\\" character Milhouse, who Matt Groening named after who?\",\n", + " \"answer\": \"Thought 1: The question simplifies to \\\"The Simpsons\\\" character Milhouse is named after who. I only need to search Milhouse and find who it is named after.\\nAction 1: Search[Milhouse]\\nObservation 1: Milhouse Mussolini Van Houten is a recurring character in the Fox animated television series The Simpsons voiced by Pamela Hayden and created by Matt Groening.\\nThought 2: The paragraph does not tell who Milhouse is named after, maybe I can look up \\\"named after\\\".\\nAction 2: Lookup[named after]\\nObservation 2: (Result 1 / 1) Milhouse was named after U.S. president Richard Nixon, whose middle name was Milhous.\\nThought 3: Milhouse was named after U.S. president Richard Nixon, so the answer is Richard Nixon.\\nAction 3: Finish[Richard Nixon]\"\n", + " },\n", + " {\n", + " \"question\": \"Which documentary is about Finnish rock groups, Adam Clayton Powell or The Saimaa Gesture?\",\n", + " \"answer\": \"Thought 1: I need to search Adam Clayton Powell and The Saimaa Gesture, and find which documentary is about Finnish rock groups.\\nAction 1: Search[Adam Clayton Powell]\\nObservation 1 Could not find [Adam Clayton Powell]. Similar: [’Adam Clayton Powell III’, ’Seventh Avenue (Manhattan)’, ’Adam Clayton Powell Jr. State Office Building’, ’Isabel Washington Powell’, ’Adam Powell’, ’Adam Clayton Powell (film)’, ’Giancarlo Esposito’].\\nThought 2: To find the documentary, I can search Adam Clayton Powell (film).\\nAction 2: Search[Adam Clayton Powell (film)]\\nObservation 2: Adam Clayton Powell is a 1989 American documentary film directed by Richard Kilberg. The film is about the rise and fall of influential African-American politician Adam Clayton Powell Jr.[3][4] It was later aired as part of the PBS series The American Experience.\\nThought 3: Adam Clayton Powell (film) is a documentary about an African-American politician, not Finnish rock groups. So the documentary about Finnish rock groups must instead be The Saimaa Gesture.\\nAction 3: Finish[The Saimaa Gesture]\"\n", + " },\n", + " {\n", + " \"question\": \"What profession does Nicholas Ray and Elia Kazan have in common?\",\n", + " \"answer\": \"Thought 1: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common.\\nAction 1: Search[Nicholas Ray]\\nObservation 1: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 - June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause.\\nThought 2: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions.\\nAction 2: Search[Elia Kazan]\\nObservation 2: Elia Kazan was an American film and theatre director, producer, screenwriter and actor.\\nThought 3: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor.\\nAction 3: Finish[director, screenwriter, actor]\"\n", + " },\n", + " {\n", + " \"question\": \"Which magazine was started first Arthur’s Magazine or First for Women?\",\n", + " \"answer\": \"Thought 1: I need to search Arthur’s Magazine and First for Women, and find which was started first.\\nAction 1: Search[Arthur’s Magazine]\\nObservation 1: Arthur’s Magazine (1844-1846) was an American literary periodical published in Philadelphia in the 19th century.\\nThought 2: Arthur’s Magazine was started in 1844. I need to search First for Women next.\\nAction 2: Search[First for Women]\\nObservation 2: First for Women is a woman’s magazine published by Bauer Media Group in the USA.[1] The magazine was started in 1989.\\nThought 3: First for Women was started in 1989. 1844 (Arthur’s Magazine) < 1989 (First for Women), so Arthur’s Magazine was started first.\\nAction 3: Finish[Arthur’s Magazine]\"\n", + " },\n", + " {\n", + " \"question\": \"Were Pavel Urysohn and Leonid Levin known for the same type of work?\",\n", + " \"answer\": \"Thought 1: I need to search Pavel Urysohn and Leonid Levin, find their types of work, then find if they are the same.\\nAction 1: Search[Pavel Urysohn]\\nObservation 1: Pavel Samuilovich Urysohn (February 3, 1898 - August 17, 1924) was a Soviet mathematician who is best known for his contributions in dimension theory.\\nThought 2: Pavel Urysohn is a mathematician. I need to search Leonid Levin next and find its type of work.\\nAction 2: Search[Leonid Levin]\\nObservation 2: Leonid Anatolievich Levin is a Soviet-American mathematician and computer scientist.\\nThought 3: Leonid Levin is a mathematician and computer scientist. So Pavel Urysohn and Leonid Levin have the same type of work.\\nAction 3: Finish[yes]\"\n", + " }\n", + "]\n", + "example_prompt = PromptTemplate(input_variables=[\"question\", \"answer\"], template=\"Question: {question}\\n{answer}\")\n", + "\n", + "prompt = FewShotPrompt(\n", + " examples=examples, \n", + " example_prompt=example_prompt, \n", + " suffix=\"Question: {input}\", \n", + " input_variables=[\"input\"]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "897d4e08", + "metadata": {}, + "outputs": [], + "source": [ + "# LLM Math\n", + "examples = [\n", + " {\n", + " \"question\": \"What is 37593 * 67?\",\n", + " \"answer\": \"```python\\nprint(37593 * 67)\\n```\\n```output\\n2518731\\n```\\nAnswer: 2518731\"\n", + " }\n", + "]\n", + "example_prompt = PromptTemplate(input_variables=[\"question\", \"answer\"], template=\"Question: {question}\\n\\n{answer}\")\n", + "\n", + "prompt = FewShotPromptTemplate(\n", + " examples=examples, \n", + " example_prompt=example_prompt, \n", + " suffix=\"Question: {input}\", \n", + " input_variables=[\"input\"]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7ab7379f", + "metadata": {}, + "outputs": [], + "source": [ + "# NatBot\n", + "example_seperator = \"==================================================\"\n", + "content_1 = \"\"\"About\n", + "Store\n", + "Gmail\n", + "Images\n", + "(Google apps)\n", + "Sign in\n", + "\"(Google)\"/\n", + "\n", + "\n", + "\n", + "\n", + "Advertising\n", + "Business\n", + "How Search works\n", + "Carbon neutral since 2007\n", + "Privacy\n", + "Terms\n", + "Settings\"\"\"\n", + "content_2 = \"\"\"About\n", + "Store\n", + "Gmail\n", + "Images\n", + "(Google apps)\n", + "Sign in\n", + "\"(Google)\"/\n", + "\n", + "\n", + "\n", + "\n", + "Advertising\n", + "Business\n", + "How Search works\n", + "Carbon neutral since 2007\n", + "Privacy\n", + "Terms\n", + "Settings\"\"\"\n", + "content_3 = \"\"\"\n", + "\n", + "\n", + "\n", + "OpenTable logo\n", + "\n", + "Find your table for any occasion\n", + "\n", + "Sep 28, 2022\n", + "7:00 PM\n", + "2 people\n", + "\n", + "\n", + "It looks like you're in Peninsula. Not correct?\n", + "\n", + "\"\"\"\n", + "examples = [\n", + " {\n", + " \"i\": 1,\n", + " \"content\": content_1,\n", + " \"objective\": \"Find a 2 bedroom house for sale in Anchorage AK for under $750k\",\n", + " \"current_url\": \"https://www.google.com/\",\n", + " \"command\": 'TYPESUBMIT 8 \"anchorage redfin\"'\n", + " },\n", + " {\n", + " \"i\": 2,\n", + " \"content\": content_2,\n", + " \"objective\": \"Make a reservation for 4 at Dorsia at 8pm\",\n", + " \"current_url\": \"https://www.google.com/\",\n", + " \"command\": 'TYPESUBMIT 8 \"dorsia nyc opentable\"'\n", + " },\n", + " {\n", + " \"i\": 3,\n", + " \"content\": content_3,\n", + " \"objective\": \"Make a reservation for 4 for dinner at Dorsia in New York City at 8pm\",\n", + " \"current_url\": \"https://www.opentable.com/\",\n", + " \"command\": 'TYPESUBMIT 12 \"dorsia new york city\"'\n", + " },\n", + "]\n", + "example_prompt_template=\"\"\"EXAMPLE {i}:\n", + "==================================================\n", + "CURRENT BROWSER CONTENT:\n", + "------------------\n", + "{content}\n", + "------------------\n", + "OBJECTIVE: {objective}\n", + "CURRENT URL: {current_url}\n", + "YOUR COMMAND:\n", + "{command}\"\"\"\n", + "example_prompt = PromptTemplate(input_variables=[\"i\", \"content\", \"objective\", \"current_url\", \"command\"], template=example_prompt_template)\n", + "\n", + "\n", + "prefix = \"\"\"\n", + "You are an agent controlling a browser. You are given:\n", + "\t(1) an objective that you are trying to achieve\n", + "\t(2) the URL of your current web page\n", + "\t(3) a simplified text description of what's visible in the browser window (more on that below)\n", + "You can issue these commands:\n", + "\tSCROLL UP - scroll up one page\n", + "\tSCROLL DOWN - scroll down one page\n", + "\tCLICK X - click on a given element. You can only click on links, buttons, and inputs!\n", + "\tTYPE X \"TEXT\" - type the specified text into the input with id X\n", + "\tTYPESUBMIT X \"TEXT\" - same as TYPE above, except then it presses ENTER to submit the form\n", + "The format of the browser content is highly simplified; all formatting elements are stripped.\n", + "Interactive elements such as links, inputs, buttons are represented like this:\n", + "\t\ttext\n", + "\t\t\n", + "\t\ttext\n", + "Images are rendered as their alt text like this:\n", + "\t\t\"\"/\n", + "Based on your given objective, issue whatever command you believe will get you closest to achieving your goal.\n", + "You always start on Google; you should submit a search query to Google that will take you to the best page for\n", + "achieving your objective. And then interact with that page to achieve your objective.\n", + "If you find yourself on Google and there are no search results displayed yet, you should probably issue a command\n", + "like \"TYPESUBMIT 7 \"search query\"\" to get to a more useful page.\n", + "Then, if you find yourself on a Google search results page, you might issue the command \"CLICK 24\" to click\n", + "on the first link in the search results. (If your previous command was a TYPESUBMIT your next command should\n", + "probably be a CLICK.)\n", + "Don't try to interact with elements that you can't see.\n", + "Here are some examples:\n", + "\"\"\"\n", + "suffix=\"\"\"\n", + "The current browser content, objective, and current URL follow. Reply with your next command to the browser.\n", + "CURRENT BROWSER CONTENT:\n", + "------------------\n", + "{browser_content}\n", + "------------------\n", + "OBJECTIVE: {objective}\n", + "CURRENT URL: {url}\n", + "PREVIOUS COMMAND: {previous_command}\n", + "YOUR COMMAND:\n", + "\"\"\"\n", + "PROMPT = FewShotPromptTemplate(\n", + " examples = examples,\n", + " example_prompt=example_prompt,\n", + " example_separator=example_seperator,\n", + " input_variables=[\"browser_content\", \"url\", \"previous_command\", \"objective\"],\n", + " prefix=prefix,\n", + " suffix=suffix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce5927c6", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/examples/prompts/few_shot_prompt.json b/docs/examples/prompts/few_shot_prompt.json new file mode 100644 index 00000000..b6ed30ad --- /dev/null +++ b/docs/examples/prompts/few_shot_prompt.json @@ -0,0 +1,11 @@ +{ + "_type": "few_shot", + "input_variables": ["adjective"], + "prefix": "Write antonyms for the following words.", + "example_prompt": { + "input_variables": ["input", "output"], + "template": "Input: {input}\nOutput: {output}" + }, + "examples": "examples.json", + "suffix": "Input: {adjective}\nOutput:" +} diff --git a/docs/examples/prompts/few_shot_prompt.yaml b/docs/examples/prompts/few_shot_prompt.yaml new file mode 100644 index 00000000..b42d494f --- /dev/null +++ b/docs/examples/prompts/few_shot_prompt.yaml @@ -0,0 +1,14 @@ +_type: few_shot +input_variables: + ["adjective"] +prefix: + Write antonyms for the following words. +example_prompt: + input_variables: + ["input", "output"] + template: + "Input: {input}\nOutput: {output}" +examples: + examples.json +suffix: + "Input: {adjective}\nOutput:" diff --git a/docs/examples/prompts/few_shot_prompt_example_prompt.json b/docs/examples/prompts/few_shot_prompt_example_prompt.json new file mode 100644 index 00000000..35765240 --- /dev/null +++ b/docs/examples/prompts/few_shot_prompt_example_prompt.json @@ -0,0 +1,8 @@ +{ + "_type": "few_shot", + "input_variables": ["adjective"], + "prefix": "Write antonyms for the following words.", + "example_prompt_path": "example_prompt.json", + "examples": "examples.json", + "suffix": "Input: {adjective}\nOutput:" +} diff --git a/docs/examples/prompts/few_shot_prompt_examples_in.json b/docs/examples/prompts/few_shot_prompt_examples_in.json new file mode 100644 index 00000000..8b95f63c --- /dev/null +++ b/docs/examples/prompts/few_shot_prompt_examples_in.json @@ -0,0 +1,14 @@ +{ + "_type": "few_shot", + "input_variables": ["adjective"], + "prefix": "Write antonyms for the following words.", + "example_prompt": { + "input_variables": ["input", "output"], + "template": "Input: {input}\nOutput: {output}" + }, + "examples": [ + {"input": "happy", "output": "sad"}, + {"input": "tall", "output": "short"} + ], + "suffix": "Input: {adjective}\nOutput:" +} diff --git a/docs/examples/prompts/generate_examples.ipynb b/docs/examples/prompts/generate_examples.ipynb index db28cdec..0ef43eb7 100644 --- a/docs/examples/prompts/generate_examples.ipynb +++ b/docs/examples/prompts/generate_examples.ipynb @@ -25,9 +25,9 @@ }, "outputs": [], "source": [ - "from langchain.chains.react.prompt import EXAMPLES\n", "from langchain.llms.openai import OpenAI\n", - "from langchain.example_generator import generate_example, generate_example_from_dynamic_prompt" + "from langchain.example_generator import generate_example\n", + "from langchain.prompts import PromptTemplate" ] }, { @@ -39,26 +39,41 @@ "name": "#%%\n" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "'Question: What is the elevation range for the area that the eastern sector of the\\nColorado orogeny extends into?\\nThought 1: I need to search Colorado orogeny, find the area that the eastern sector\\nof the Colorado orogeny extends into, then find the elevation range of the\\narea.\\nAction 1: Search[Colorado orogeny]\\nObservation 1: The Colorado orogeny was an episode of mountain building (an orogeny) in\\nColorado and surrounding areas.\\nThought 2: It does not mention the eastern sector. So I need to look up eastern\\nsector.\\nAction 2: Lookup[eastern sector]\\nObservation 2: (Result 1 / 1) The eastern sector extends into the High Plains and is called\\nthe Central Plains orogeny.\\nThought 3: The eastern sector of Colorado orogeny extends into the High Plains. So I\\nneed to search High Plains and find its elevation range.\\nAction 3: Search[High Plains]\\nObservation 3: High Plains refers to one of two distinct land regions\\nThought 4: I need to instead search High Plains (United States).\\nAction 4: Search[High Plains (United States)]\\nObservation 4: The High Plains are a subregion of the Great Plains. From east to west, the\\nHigh Plains rise in elevation from around 1,800 to 7,000 ft (550 to 2,130\\nm).[3]\\nThought 5: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer\\nis 1,800 to 7,000 ft.\\nAction 5: Finish[1,800 to 7,000 ft]'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "# print initial example for visibility\n", - "EXAMPLES[0]" + "# Use examples from ReAct\n", + "examples = [\n", + " {\n", + " \"question\": \"What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into?\",\n", + " \"answer\": \"Thought 1: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado orogeny extends into, then find the elevation range of that area.\\nAction 1: Search[Colorado orogeny]\\nObservation 1: The Colorado orogeny was an episode of mountain building (an orogeny) in Colorado and surrounding areas.\\nThought 2: It does not mention the eastern sector. So I need to look up eastern sector.\\nAction 2: Lookup[eastern sector]\\nObservation 2: (Result 1 / 1) The eastern sector extends into the High Plains and is called the Central Plains orogeny.\\nThought 3: The eastern sector of Colorado orogeny extends into the High Plains. So I need to search High Plains and find its elevation range.\\nAction 3: Search[High Plains]\\nObservation 3: High Plains refers to one of two distinct land regions\\nThought 4: I need to instead search High Plains (United States).\\nAction 4: Search[High Plains (United States)]\\nObservation 4: The High Plains are a subregion of the Great Plains. From east to west, the High Plains rise in elevation from around 1,800 to 7,000 ft (550 to 2,130 m).[3]\\nThought 5: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft.\\nAction 5: Finish[1,800 to 7,000 ft]\"\n", + " },\n", + " {\n", + " \"question\": \"Musician and satirist Allie Goertz wrote a song about the \\\"The Simpsons\\\" character Milhouse, who Matt Groening named after who?\",\n", + " \"answer\": \"Thought 1: The question simplifies to \\\"The Simpsons\\\" character Milhouse is named after who. I only need to search Milhouse and find who it is named after.\\nAction 1: Search[Milhouse]\\nObservation 1: Milhouse Mussolini Van Houten is a recurring character in the Fox animated television series The Simpsons voiced by Pamela Hayden and created by Matt Groening.\\nThought 2: The paragraph does not tell who Milhouse is named after, maybe I can look up \\\"named after\\\".\\nAction 2: Lookup[named after]\\nObservation 2: (Result 1 / 1) Milhouse was named after U.S. president Richard Nixon, whose middle name was Milhous.\\nThought 3: Milhouse was named after U.S. president Richard Nixon, so the answer is Richard Nixon.\\nAction 3: Finish[Richard Nixon]\"\n", + " },\n", + " {\n", + " \"question\": \"Which documentary is about Finnish rock groups, Adam Clayton Powell or The Saimaa Gesture?\",\n", + " \"answer\": \"Thought 1: I need to search Adam Clayton Powell and The Saimaa Gesture, and find which documentary is about Finnish rock groups.\\nAction 1: Search[Adam Clayton Powell]\\nObservation 1 Could not find [Adam Clayton Powell]. Similar: [’Adam Clayton Powell III’, ’Seventh Avenue (Manhattan)’, ’Adam Clayton Powell Jr. State Office Building’, ’Isabel Washington Powell’, ’Adam Powell’, ’Adam Clayton Powell (film)’, ’Giancarlo Esposito’].\\nThought 2: To find the documentary, I can search Adam Clayton Powell (film).\\nAction 2: Search[Adam Clayton Powell (film)]\\nObservation 2: Adam Clayton Powell is a 1989 American documentary film directed by Richard Kilberg. The film is about the rise and fall of influential African-American politician Adam Clayton Powell Jr.[3][4] It was later aired as part of the PBS series The American Experience.\\nThought 3: Adam Clayton Powell (film) is a documentary about an African-American politician, not Finnish rock groups. So the documentary about Finnish rock groups must instead be The Saimaa Gesture.\\nAction 3: Finish[The Saimaa Gesture]\"\n", + " },\n", + " {\n", + " \"question\": \"What profession does Nicholas Ray and Elia Kazan have in common?\",\n", + " \"answer\": \"Thought 1: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common.\\nAction 1: Search[Nicholas Ray]\\nObservation 1: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 - June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause.\\nThought 2: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions.\\nAction 2: Search[Elia Kazan]\\nObservation 2: Elia Kazan was an American film and theatre director, producer, screenwriter and actor.\\nThought 3: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor.\\nAction 3: Finish[director, screenwriter, actor]\"\n", + " },\n", + " {\n", + " \"question\": \"Which magazine was started first Arthur’s Magazine or First for Women?\",\n", + " \"answer\": \"Thought 1: I need to search Arthur’s Magazine and First for Women, and find which was started first.\\nAction 1: Search[Arthur’s Magazine]\\nObservation 1: Arthur’s Magazine (1844-1846) was an American literary periodical published in Philadelphia in the 19th century.\\nThought 2: Arthur’s Magazine was started in 1844. I need to search First for Women next.\\nAction 2: Search[First for Women]\\nObservation 2: First for Women is a woman’s magazine published by Bauer Media Group in the USA.[1] The magazine was started in 1989.\\nThought 3: First for Women was started in 1989. 1844 (Arthur’s Magazine) < 1989 (First for Women), so Arthur’s Magazine was started first.\\nAction 3: Finish[Arthur’s Magazine]\"\n", + " },\n", + " {\n", + " \"question\": \"Were Pavel Urysohn and Leonid Levin known for the same type of work?\",\n", + " \"answer\": \"Thought 1: I need to search Pavel Urysohn and Leonid Levin, find their types of work, then find if they are the same.\\nAction 1: Search[Pavel Urysohn]\\nObservation 1: Pavel Samuilovich Urysohn (February 3, 1898 - August 17, 1924) was a Soviet mathematician who is best known for his contributions in dimension theory.\\nThought 2: Pavel Urysohn is a mathematician. I need to search Leonid Levin next and find its type of work.\\nAction 2: Search[Leonid Levin]\\nObservation 2: Leonid Anatolievich Levin is a Soviet-American mathematician and computer scientist.\\nThought 3: Leonid Levin is a mathematician and computer scientist. So Pavel Urysohn and Leonid Levin have the same type of work.\\nAction 3: Finish[yes]\"\n", + " }\n", + "]\n", + "example_template = PromptTemplate(template=\"Question: {question}\\n{answer}\", input_variables=[\"question\", \"answer\"])" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 5, "id": "a7bd36bc", "metadata": { "pycharm": { @@ -67,12 +82,12 @@ }, "outputs": [], "source": [ - "new_example = generate_example(EXAMPLES, OpenAI())" + "new_example = generate_example(examples, OpenAI(), example_template)" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "id": "e1efb008", "metadata": { "pycharm": { @@ -85,22 +100,15 @@ "text/plain": [ "['',\n", " '',\n", - " 'Question: Which ocean is the world’s smallest?',\n", " '',\n", - " 'Thought 1: I need to search for oceans and find which one is the world’s smallest.',\n", - " '',\n", - " 'Action 1: Search[oceans]',\n", - " '',\n", - " 'Observation 1: There are five oceans: the Pacific, Atlantic, Indian, Southern, and Arctic.',\n", - " '',\n", - " 'Thought 2: I need to compare the sizes of the oceans and find which one is the smallest.',\n", - " '',\n", - " 'Action 2: Compare[Pacific, Atlantic, Indian, Southern, Arctic]',\n", - " '',\n", - " 'Observation 2: The Arctic is the smallest ocean.']" + " 'Question: Is the film \"The Omen\" based on a book?',\n", + " 'Thought 1: I need to search \"The Omen\" and find if it is based on a book.',\n", + " 'Action 1: Search[\"The Omen\"]',\n", + " 'Observation 1: The Omen is a 1976 American supernatural horror film directed by Richard Donner and written by David Seltzer.',\n", + " 'Thought 2: The Omen is not based on a book.']" ] }, - "execution_count": 4, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -134,7 +142,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.8.7" } }, "nbformat": 4, diff --git a/docs/examples/prompts/prompt_management.ipynb b/docs/examples/prompts/prompt_management.ipynb new file mode 100644 index 00000000..6e23448e --- /dev/null +++ b/docs/examples/prompts/prompt_management.ipynb @@ -0,0 +1,540 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "43fb16cb", + "metadata": {}, + "source": [ + "# Prompt Management\n", + "\n", + "Managing your prompts is annoying and tedious, with everyone writing their own slightly different variants of the same ideas. But it shouldn't be this way. \n", + "\n", + "LangChain provides a standard and flexible way for specifying and managing all your prompts, as well as clear and specific terminology around them. This notebook goes through the core components of working with prompts, showing how to use them as well as explaining what they do.\n", + "\n", + "This notebook covers how to work with prompts in Python. If you are interested in how to work with serialized versions of prompts and load them from disk, see [this notebook](prompt_serialization.ipynb)." + ] + }, + { + "cell_type": "markdown", + "id": "890aad4d", + "metadata": {}, + "source": [ + "### The BasePromptTemplate Interface\n", + "\n", + "A prompt template is a mechanism for constructing a prompt to pass to the language model given some user input. Below is the interface that all different types of prompt templates should expose.\n", + "\n", + "```python\n", + "class BasePromptTemplate(ABC):\n", + "\n", + " input_variables: List[str]\n", + " \"\"\"A list of the names of the variables the prompt template expects.\"\"\"\n", + "\n", + " @abstractmethod\n", + " def format(self, **kwargs: Any) -> str:\n", + " \"\"\"Format the prompt with the inputs.\n", + "\n", + " Args:\n", + " kwargs: Any arguments to be passed to the prompt template.\n", + "\n", + " Returns:\n", + " A formatted string.\n", + "\n", + " Example:\n", + "\n", + " .. code-block:: python\n", + "\n", + " prompt.format(variable1=\"foo\")\n", + " \"\"\"\n", + "```\n", + "\n", + "The only two things that define a prompt are:\n", + "\n", + "1. `input_variables`: The user inputted variables that are needed to format the prompt.\n", + "2. `format`: A method which takes in keyword arguments are returns a formatted prompt. The keys are expected to be the input variables\n", + " \n", + "The rest of the logic of how the prompt is constructed is left up to different implementations. Let's take a look at some below." + ] + }, + { + "cell_type": "markdown", + "id": "cddb465e", + "metadata": {}, + "source": [ + "### PromptTemplate\n", + "\n", + "This is the most simple type of prompt - a string template that takes any number of input variables. The template should be formatted as a Python f-string, although we will support other formats (Jinja, Mako, etc) in the future. \n", + "\n", + "If you just want to use a hardcoded prompt template, you should use this implementation.\n", + "\n", + "Let's walk through a few examples." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "094229f4", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.prompts import PromptTemplate" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ab46bd2a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Tell me a joke.'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# An example prompt with no input variables\n", + "no_input_prompt = PromptTemplate(input_variables=[], template=\"Tell me a joke.\")\n", + "no_input_prompt.format()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "c3ad0fa8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Tell me a funny joke.'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# An example prompt with one input variable\n", + "one_input_prompt = PromptTemplate(input_variables=[\"adjective\"], template=\"Tell me a {adjective} joke.\")\n", + "one_input_prompt.format(adjective=\"funny\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "ba577dcf", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Tell me a funny joke about chickens.'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# An example prompt with multiple input variables\n", + "multiple_input_prompt = PromptTemplate(\n", + " input_variables=[\"adjective\", \"content\"], \n", + " template=\"Tell me a {adjective} joke about {content}.\"\n", + ")\n", + "multiple_input_prompt.format(adjective=\"funny\", content=\"chickens\")" + ] + }, + { + "cell_type": "markdown", + "id": "1492b49d", + "metadata": {}, + "source": [ + "### Few Shot Prompts\n", + "\n", + "A FewShotPromptTemplate is a prompt template that includes some examples. If you have collected some examples of how the task should be done, you can insert them into prompt using this class.\n", + "\n", + "Examples are datapoints that can be included in the prompt in order to give the model more context what to do. Examples are represented as a dictionary of key-value pairs, with the key being the input (or label) name, and the value being the input (or label) value. \n", + "\n", + "In addition to the example, we also need to specify how the example should be formatted when it's inserted in the prompt. We can do this using the above `PromptTemplate`!" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "3eb36972", + "metadata": {}, + "outputs": [], + "source": [ + "# These are some examples of a pretend task of creating antonyms.\n", + "examples = [\n", + " {\"input\": \"happy\", \"output\": \"sad\"},\n", + " {\"input\": \"tall\", \"output\": \"short\"},\n", + "]\n", + "# This how we specify how the example should be formatted.\n", + "example_prompt = PromptTemplate(\n", + " input_variables=[\"input\",\"output\"],\n", + " template=\"Input: {input}\\nOutput: {output}\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "80a91d96", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.prompts.few_shot import FewShotPromptTemplate" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "7931e5f2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Give the antonym of every input\n", + "\n", + "Input: happy\n", + "Output: sad\n", + "\n", + "Input: tall\n", + "Output: short\n", + "\n", + "Input: big\n", + "Output:\n" + ] + } + ], + "source": [ + "prompt_from_string_examples = FewShotPromptTemplate(\n", + " # These are the examples we want to insert into the prompt.\n", + " examples=examples,\n", + " # This is how we want to format the examples when we insert them into the prompt.\n", + " example_prompt=example_prompt,\n", + " # The prefix is some text that goes before the examples in the prompt.\n", + " # Usually, this consists of intructions.\n", + " prefix=\"Give the antonym of every input\",\n", + " # The suffix is some text that goes after the examples in the prompt.\n", + " # Usually, this is where the user input will go\n", + " suffix=\"Input: {adjective}\\nOutput:\", \n", + " # The input variables are the variables that the overall prompt expects.\n", + " input_variables=[\"adjective\"],\n", + " # The example_separator is the string we will use to join the prefix, examples, and suffix together with.\n", + " example_separator=\"\\n\\n\"\n", + " \n", + ")\n", + "print(prompt_from_string_examples.format(adjective=\"big\"))" + ] + }, + { + "cell_type": "markdown", + "id": "bf038596", + "metadata": {}, + "source": [ + "### ExampleSelector\n", + "If you have a large number of examples, you may need to select which ones to include in the prompt. The ExampleSelector is the class responsible for doing so. The base interface is defined as below.\n", + "\n", + "```python\n", + "class BaseExampleSelector(ABC):\n", + " \"\"\"Interface for selecting examples to include in prompts.\"\"\"\n", + "\n", + " @abstractmethod\n", + " def select_examples(self, input_variables: Dict[str, str]) -> List[dict]:\n", + " \"\"\"Select which examples to use based on the inputs.\"\"\"\n", + "\n", + "```\n", + "\n", + "The only method it needs to expose is a `select_examples` method. This takes in the input variables and then returns a list of examples. It is up to each specific implementation as to how those examples are selected. Let's take a look at some below." + ] + }, + { + "cell_type": "markdown", + "id": "861a4d1f", + "metadata": {}, + "source": [ + "### LengthBased ExampleSelector\n", + "\n", + "This ExampleSelector selects which examples to use based on length. This is useful when you are worried about constructing a prompt that will go over the length of the context window. For longer inputs, it will select fewer examples to include, while for shorter inputs it will select more.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "7c469c95", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.prompts.example_selector.length_based import LengthBasedExampleSelector" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "0ec6d950", + "metadata": {}, + "outputs": [], + "source": [ + "# These are a lot of examples of a pretend task of creating antonyms.\n", + "examples = [\n", + " {\"input\": \"happy\", \"output\": \"sad\"},\n", + " {\"input\": \"tall\", \"output\": \"short\"},\n", + " {\"input\": \"energetic\", \"output\": \"lethargic\"},\n", + " {\"input\": \"sunny\", \"output\": \"gloomy\"},\n", + " {\"input\": \"windy\", \"output\": \"calm\"},\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "207e55f7", + "metadata": {}, + "outputs": [], + "source": [ + "example_selector = LengthBasedExampleSelector(\n", + " # These are the examples is has available to choose from.\n", + " examples=examples, \n", + " # This is the PromptTemplate being used to format the examples.\n", + " example_prompt=example_prompt, \n", + " # This is the maximum length that the formatted examples should be.\n", + " # Length is measured by the get_text_length function below.\n", + " max_length=18,\n", + " # This is the function used to get the length of a string, which is used\n", + " # to determine which examples to include. It is commented out because\n", + " # it is provided as a default value if none is specified.\n", + " # get_text_length: Callable[[str], int] = lambda x: len(re.split(\"\\n| \", x))\n", + ")\n", + "dynamic_prompt = FewShotPromptTemplate(\n", + " # We provide an ExampleSelector instead of examples.\n", + " example_selector=example_selector,\n", + " example_prompt=example_prompt,\n", + " prefix=\"Give the antonym of every input\",\n", + " suffix=\"Input: {adjective}\\nOutput:\", \n", + " input_variables=[\"adjective\"],\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "d00b4385", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Give the antonym of every input\n", + "\n", + "Input: happy\n", + "Output: sad\n", + "\n", + "Input: tall\n", + "Output: short\n", + "\n", + "Input: energetic\n", + "Output: lethargic\n", + "\n", + "Input: sunny\n", + "Output: gloomy\n", + "\n", + "Input: windy\n", + "Output: calm\n", + "\n", + "Input: big\n", + "Output:\n" + ] + } + ], + "source": [ + "# An example with small input, so it selects all examples.\n", + "print(dynamic_prompt.format(adjective=\"big\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "878bcde9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Give the antonym of every input\n", + "\n", + "Input: happy\n", + "Output: sad\n", + "\n", + "Input: big and huge and massive and large and gigantic and tall and bigger than everything else\n", + "Output:\n" + ] + } + ], + "source": [ + "# An example with long input, so it selects only one example.\n", + "long_string = \"big and huge and massive and large and gigantic and tall and bigger than everything else\"\n", + "print(dynamic_prompt.format(adjective=long_string))" + ] + }, + { + "cell_type": "markdown", + "id": "2d007b0a", + "metadata": {}, + "source": [ + "### Similarity ExampleSelector\n", + "\n", + "The SemanticSimilarityExampleSelector selects examples based on which examples are most similar to the inputs. It does this by finding the examples with the embeddings that have the greatest cosine similarity with the inputs.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "241bfe80", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.prompts.example_selector.semantic_similarity import SemanticSimilarityExampleSelector\n", + "from langchain.vectorstores import FAISS\n", + "from langchain.embeddings import OpenAIEmbeddings" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "50d0a701", + "metadata": {}, + "outputs": [], + "source": [ + "example_selector = SemanticSimilarityExampleSelector.from_examples(\n", + " # This is the list of examples available to select from.\n", + " examples, \n", + " # This is the embedding class used to produce embeddings which are used to measure semantic similarity.\n", + " OpenAIEmbeddings(), \n", + " # This is the VectorStore class that is used to store the embeddings and do a similarity search over.\n", + " FAISS, \n", + " # This is the number of examples to produce.\n", + " k=1\n", + ")\n", + "similar_prompt = FewShotPromptTemplate(\n", + " # We provide an ExampleSelector instead of examples.\n", + " example_selector=example_selector,\n", + " example_prompt=example_prompt,\n", + " prefix=\"Give the antonym of every input\",\n", + " suffix=\"Input: {adjective}\\nOutput:\", \n", + " input_variables=[\"adjective\"],\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "4c8fdf45", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Give the antonym of every input\n", + "\n", + "Input: happy\n", + "Output: sad\n", + "\n", + "Input: worried\n", + "Output:\n" + ] + } + ], + "source": [ + "# Input is a feeling, so should select the happy/sad example\n", + "print(similar_prompt.format(adjective=\"worried\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "829af21a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Give the antonym of every input\n", + "\n", + "Input: tall\n", + "Output: short\n", + "\n", + "Input: fat\n", + "Output:\n" + ] + } + ], + "source": [ + "# Input is a measurment, so should select the tall/short example\n", + "print(similar_prompt.format(adjective=\"fat\"))" + ] + }, + { + "cell_type": "markdown", + "id": "dbc32551", + "metadata": {}, + "source": [ + "### Serialization\n", + "\n", + "PromptTemplates and examples can be serialized and loaded from disk, making it easy to share and store prompts. For a detailed walkthrough on how to do that, see [this notebook](prompt_serialization.ipynb)." + ] + }, + { + "cell_type": "markdown", + "id": "1e1e13c6", + "metadata": {}, + "source": [ + "### Customizability\n", + "The above covers all the ways currently supported in LangChain to represent prompts and example selectors. However, due to the simple interface that the base classes (`BasePromptTemplate`, `BaseExampleSelector`) expose, it should be easy to subclass them and write your own implementation in your own codebase. And of course, if you'd like to contribute that back to LangChain, we'd love that :)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c746d6f4", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/examples/prompts/prompt_optimization.ipynb b/docs/examples/prompts/prompt_optimization.ipynb deleted file mode 100644 index c06d3a98..00000000 --- a/docs/examples/prompts/prompt_optimization.ipynb +++ /dev/null @@ -1,179 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "d7467b67", - "metadata": {}, - "source": [ - "# Optimized Prompts\n", - "\n", - "This example showcases how using the OptimizedPrompt class enables selection of the most relevant examples to include as few-shot examples in the prompt." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "e9e2b50b", - "metadata": {}, - "outputs": [], - "source": [ - "from langchain.chains.react.prompt import EXAMPLES, SUFFIX\n", - "from langchain.embeddings.openai import OpenAIEmbeddings\n", - "from langchain.example_generator import generate_example, generate_example_from_dynamic_prompt\n", - "from langchain.llms.openai import OpenAI\n", - "from langchain.prompts.optimized import OptimizedPrompt\n", - "from langchain.vectorstores.elastic_vector_search import ElasticVectorSearch\n", - "from langchain.vectorstores.faiss_search import FAISS" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "cb069606", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Question: What is the elevation range for the area that the eastern sector of the\\nColorado orogeny extends into?\\nThought 1: I need to search Colorado orogeny, find the area that the eastern sector\\nof the Colorado orogeny extends into, then find the elevation range of the\\narea.\\nAction 1: Search[Colorado orogeny]\\nObservation 1: The Colorado orogeny was an episode of mountain building (an orogeny) in\\nColorado and surrounding areas.\\nThought 2: It does not mention the eastern sector. So I need to look up eastern\\nsector.\\nAction 2: Lookup[eastern sector]\\nObservation 2: (Result 1 / 1) The eastern sector extends into the High Plains and is called\\nthe Central Plains orogeny.\\nThought 3: The eastern sector of Colorado orogeny extends into the High Plains. So I\\nneed to search High Plains and find its elevation range.\\nAction 3: Search[High Plains]\\nObservation 3: High Plains refers to one of two distinct land regions\\nThought 4: I need to instead search High Plains (United States).\\nAction 4: Search[High Plains (United States)]\\nObservation 4: The High Plains are a subregion of the Great Plains. From east to west, the\\nHigh Plains rise in elevation from around 1,800 to 7,000 ft (550 to 2,130\\nm).[3]\\nThought 5: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer\\nis 1,800 to 7,000 ft.\\nAction 5: Finish[1,800 to 7,000 ft]'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "EXAMPLES[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "5fda75a4", - "metadata": {}, - "outputs": [], - "source": [ - "prompt = OptimizedPrompt.from_examples(\n", - " examples=EXAMPLES, \n", - " suffix=SUFFIX, \n", - " input_variables=[\"input\"],\n", - " embeddings=OpenAIEmbeddings(),\n", - " vectorstore_cls=FAISS\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "7a601df8", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "\n", - "Question: What is the elevation range for the area that the eastern sector of the\n", - "Colorado orogeny extends into?\n", - "Thought 1: I need to search Colorado orogeny, find the area that the eastern sector\n", - "of the Colorado orogeny extends into, then find the elevation range of the\n", - "area.\n", - "Action 1: Search[Colorado orogeny]\n", - "Observation 1: The Colorado orogeny was an episode of mountain building (an orogeny) in\n", - "Colorado and surrounding areas.\n", - "Thought 2: It does not mention the eastern sector. So I need to look up eastern\n", - "sector.\n", - "Action 2: Lookup[eastern sector]\n", - "Observation 2: (Result 1 / 1) The eastern sector extends into the High Plains and is called\n", - "the Central Plains orogeny.\n", - "Thought 3: The eastern sector of Colorado orogeny extends into the High Plains. So I\n", - "need to search High Plains and find its elevation range.\n", - "Action 3: Search[High Plains]\n", - "Observation 3: High Plains refers to one of two distinct land regions\n", - "Thought 4: I need to instead search High Plains (United States).\n", - "Action 4: Search[High Plains (United States)]\n", - "Observation 4: The High Plains are a subregion of the Great Plains. From east to west, the\n", - "High Plains rise in elevation from around 1,800 to 7,000 ft (550 to 2,130\n", - "m).[3]\n", - "Thought 5: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer\n", - "is 1,800 to 7,000 ft.\n", - "Action 5: Finish[1,800 to 7,000 ft]\n", - "\n", - "\n", - "\n", - "Question: What is the highest mountain peak in Asia?\n" - ] - } - ], - "source": [ - "print(prompt.format(k=1, input=\"What is the highest mountain peak in Asia?\"))" - ] - }, - { - "cell_type": "markdown", - "id": "a5dc3525", - "metadata": {}, - "source": [ - "## Requires having ElasticSearch setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bbd92d08", - "metadata": {}, - "outputs": [], - "source": [ - "prompt = OptimizedPrompt.from_examples(\n", - " examples=EXAMPLES, \n", - " suffix=SUFFIX, \n", - " input_variables=[\"input\"],\n", - " embeddings=OpenAIEmbeddings(),\n", - " vectorstore_cls=ElasticVectorSearch,\n", - " elasticsearch_url=\"http://localhost:9200\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bd91f408", - "metadata": {}, - "outputs": [], - "source": [ - "print(prompt.format(k=1, input=\"What is the highest mountain peak in Asia?\"))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "716165c2", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/examples/prompts/prompt_serialization.ipynb b/docs/examples/prompts/prompt_serialization.ipynb new file mode 100644 index 00000000..366511c5 --- /dev/null +++ b/docs/examples/prompts/prompt_serialization.ipynb @@ -0,0 +1,538 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "43fb16cb", + "metadata": {}, + "source": [ + "# Prompt Serialization\n", + "\n", + "It is often preferrable to store prompts not as python code but as files. This can make it easy to share, store, and version prompts. This notebook covers how to do that in LangChain, walking through all the different types of prompts and the different serialization options.\n", + "\n", + "At a high level, the following design principles are applied to serialization:\n", + "1. Both JSON and YAML are supported. We want to support serialization methods are human readable on disk, and YAML and JSON are two of the most popular methods for that. Note that this rule applies to prompts. For other assets, like Examples, different serialization methods may be supported.\n", + "2. We support specifying everything in one file, or storing different components (templates, examples, etc) in different files and referencing them. For some cases, storing everything in file makes the most sense, but for others it is preferrable to split up some of the assets (long templates, large examples, reusable components). LangChain supports both." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "2c8d7587", + "metadata": {}, + "outputs": [], + "source": [ + "# All prompts are loading through the `load_prompt` function.\n", + "from langchain.prompts.loading import load_prompt" + ] + }, + { + "cell_type": "markdown", + "id": "cddb465e", + "metadata": {}, + "source": [ + "## PromptTemplate\n", + "\n", + "This section covers examples for loading a PromptTemplate." + ] + }, + { + "cell_type": "markdown", + "id": "4d4b40f2", + "metadata": {}, + "source": [ + "### Loading from YAML\n", + "This shows an example of loading a PromptTemplate from YAML." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "2d6e5117", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "input_variables:\r\n", + " [\"adjective\", \"content\"]\r\n", + "template: \r\n", + " Tell me a {adjective} joke about {content}.\r\n" + ] + } + ], + "source": [ + "!cat simple_prompt.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "4f4ca686", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tell me a funny joke about chickens.\n" + ] + } + ], + "source": [ + "prompt = load_prompt(\"simple_prompt.yaml\")\n", + "print(prompt.format(adjective=\"funny\", content=\"chickens\"))" + ] + }, + { + "cell_type": "markdown", + "id": "362eadb2", + "metadata": {}, + "source": [ + "### Loading from JSON\n", + "This shows an example of loading a PromptTemplate from JSON." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "510def23", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\r\n", + " \"input_variables\": [\"adjective\", \"content\"],\r\n", + " \"template\": \"Tell me a {adjective} joke about {content}.\"\r\n", + "}\r\n" + ] + } + ], + "source": [ + "!cat simple_prompt.json" + ] + }, + { + "cell_type": "markdown", + "id": "d788a83c", + "metadata": {}, + "source": [ + "### Loading Template from a File\n", + "This shows an example of storing the template in a separate file and then referencing it in the config. Notice that the key changes from `template` to `template_path`." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "5547760d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tell me a {adjective} joke about {content}." + ] + } + ], + "source": [ + "!cat simple_template.txt" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "9cb13ac5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\r\n", + " \"input_variables\": [\"adjective\", \"content\"],\r\n", + " \"template_path\": \"simple_template.txt\"\r\n", + "}\r\n" + ] + } + ], + "source": [ + "!cat simple_prompt_with_template_file.json" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "762cb4bf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tell me a funny joke about chickens.\n" + ] + } + ], + "source": [ + "prompt = load_prompt(\"simple_prompt_with_template_file.json\")\n", + "print(prompt.format(adjective=\"funny\", content=\"chickens\"))" + ] + }, + { + "cell_type": "markdown", + "id": "2ae191cc", + "metadata": {}, + "source": [ + "## FewShotPromptTemplate\n", + "\n", + "This section covers examples for loading few shot prompt templates." + ] + }, + { + "cell_type": "markdown", + "id": "9828f94c", + "metadata": {}, + "source": [ + "### Examples\n", + "This shows an example of what examples stored as json might look like." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "b21f5b95", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[\r\n", + " {\"input\": \"happy\", \"output\": \"sad\"},\r\n", + " {\"input\": \"tall\", \"output\": \"short\"}\r\n", + "]\r\n" + ] + } + ], + "source": [ + "!cat examples.json" + ] + }, + { + "cell_type": "markdown", + "id": "8e300335", + "metadata": {}, + "source": [ + "### Loading from YAML\n", + "This shows an example of loading a few shot example from YAML." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "e2bec0fc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "_type: few_shot\r\n", + "input_variables:\r\n", + " [\"adjective\"]\r\n", + "prefix: \r\n", + " Write antonyms for the following words.\r\n", + "example_prompt:\r\n", + " input_variables:\r\n", + " [\"input\", \"output\"]\r\n", + " template:\r\n", + " \"Input: {input}\\nOutput: {output}\"\r\n", + "examples:\r\n", + " examples.json\r\n", + "suffix:\r\n", + " \"Input: {adjective}\\nOutput:\"\r\n" + ] + } + ], + "source": [ + "!cat few_shot_prompt.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "98c8f356", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Write antonyms for the following words.\n", + "\n", + "Input: happy\n", + "Output: sad\n", + "\n", + "Input: tall\n", + "Output: short\n", + "\n", + "Input: funny\n", + "Output:\n" + ] + } + ], + "source": [ + "prompt = load_prompt(\"few_shot_prompt.yaml\")\n", + "print(prompt.format(adjective=\"funny\"))" + ] + }, + { + "cell_type": "markdown", + "id": "4870aa9d", + "metadata": {}, + "source": [ + "### Loading from JSON\n", + "This shows an example of loading a few shot example from JSON." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "9d996a86", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\r\n", + " \"_type\": \"few_shot\",\r\n", + " \"input_variables\": [\"adjective\"],\r\n", + " \"prefix\": \"Write antonyms for the following words.\",\r\n", + " \"example_prompt\": {\r\n", + " \"input_variables\": [\"input\", \"output\"],\r\n", + " \"template\": \"Input: {input}\\nOutput: {output}\"\r\n", + " },\r\n", + " \"examples\": \"examples.json\",\r\n", + " \"suffix\": \"Input: {adjective}\\nOutput:\"\r\n", + "} \r\n" + ] + } + ], + "source": [ + "!cat few_shot_prompt.json" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "dd2c10bb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Write antonyms for the following words.\n", + "\n", + "Input: happy\n", + "Output: sad\n", + "\n", + "Input: tall\n", + "Output: short\n", + "\n", + "Input: funny\n", + "Output:\n" + ] + } + ], + "source": [ + "prompt = load_prompt(\"few_shot_prompt.json\")\n", + "print(prompt.format(adjective=\"funny\"))" + ] + }, + { + "cell_type": "markdown", + "id": "9d23faf4", + "metadata": {}, + "source": [ + "### Examples in the Config\n", + "This shows an example of referencing the examples directly in the config." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "6cd781ef", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\r\n", + " \"_type\": \"few_shot\",\r\n", + " \"input_variables\": [\"adjective\"],\r\n", + " \"prefix\": \"Write antonyms for the following words.\",\r\n", + " \"example_prompt\": {\r\n", + " \"input_variables\": [\"input\", \"output\"],\r\n", + " \"template\": \"Input: {input}\\nOutput: {output}\"\r\n", + " },\r\n", + " \"examples\": [\r\n", + " {\"input\": \"happy\", \"output\": \"sad\"},\r\n", + " {\"input\": \"tall\", \"output\": \"short\"}\r\n", + " ],\r\n", + " \"suffix\": \"Input: {adjective}\\nOutput:\"\r\n", + "} \r\n" + ] + } + ], + "source": [ + "!cat few_shot_prompt_examples_in.json" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "533ab8a7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Write antonyms for the following words.\n", + "\n", + "Input: happy\n", + "Output: sad\n", + "\n", + "Input: tall\n", + "Output: short\n", + "\n", + "Input: funny\n", + "Output:\n" + ] + } + ], + "source": [ + "prompt = load_prompt(\"few_shot_prompt_examples_in.json\")\n", + "print(prompt.format(adjective=\"funny\"))" + ] + }, + { + "cell_type": "markdown", + "id": "2e86139e", + "metadata": {}, + "source": [ + "### Example Prompt from a File\n", + "This shows an example of loading the PromptTemplate that is used to format the examples from a separate file. Note that the key changes from `example_prompt` to `example_prompt_path`." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "0b6dd7b8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\r\n", + " \"input_variables\": [\"input\", \"output\"],\r\n", + " \"template\": \"Input: {input}\\nOutput: {output}\" \r\n", + "}\r\n" + ] + } + ], + "source": [ + "!cat example_prompt.json" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "76a1065d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\r\n", + " \"_type\": \"few_shot\",\r\n", + " \"input_variables\": [\"adjective\"],\r\n", + " \"prefix\": \"Write antonyms for the following words.\",\r\n", + " \"example_prompt_path\": \"example_prompt.json\",\r\n", + " \"examples\": \"examples.json\",\r\n", + " \"suffix\": \"Input: {adjective}\\nOutput:\"\r\n", + "} \r\n" + ] + } + ], + "source": [ + "!cat few_shot_prompt_example_prompt.json " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "744d275d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Write antonyms for the following words.\n", + "\n", + "Input: happy\n", + "Output: sad\n", + "\n", + "Input: tall\n", + "Output: short\n", + "\n", + "Input: funny\n", + "Output:\n" + ] + } + ], + "source": [ + "prompt = load_prompt(\"few_shot_prompt_example_prompt.json\")\n", + "print(prompt.format(adjective=\"funny\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dcfc7176", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/examples/prompts/simple_prompt.json b/docs/examples/prompts/simple_prompt.json new file mode 100644 index 00000000..05a3731a --- /dev/null +++ b/docs/examples/prompts/simple_prompt.json @@ -0,0 +1,4 @@ +{ + "input_variables": ["adjective", "content"], + "template": "Tell me a {adjective} joke about {content}." +} diff --git a/docs/examples/prompts/simple_prompt.yaml b/docs/examples/prompts/simple_prompt.yaml new file mode 100644 index 00000000..4642b08d --- /dev/null +++ b/docs/examples/prompts/simple_prompt.yaml @@ -0,0 +1,4 @@ +input_variables: + ["adjective", "content"] +template: + Tell me a {adjective} joke about {content}. diff --git a/docs/examples/prompts/simple_prompt_with_template_file.json b/docs/examples/prompts/simple_prompt_with_template_file.json new file mode 100644 index 00000000..d3acffe3 --- /dev/null +++ b/docs/examples/prompts/simple_prompt_with_template_file.json @@ -0,0 +1,4 @@ +{ + "input_variables": ["adjective", "content"], + "template_path": "simple_template.txt" +} diff --git a/docs/examples/prompts/simple_template.txt b/docs/examples/prompts/simple_template.txt new file mode 100644 index 00000000..3e1ab1df --- /dev/null +++ b/docs/examples/prompts/simple_template.txt @@ -0,0 +1 @@ +Tell me a {adjective} joke about {content}. \ No newline at end of file diff --git a/docs/core_concepts.md b/docs/explanation/core_concepts.md similarity index 85% rename from docs/core_concepts.md rename to docs/explanation/core_concepts.md index e309cea0..3ef3af83 100644 --- a/docs/core_concepts.md +++ b/docs/explanation/core_concepts.md @@ -3,11 +3,13 @@ This section goes over the core concepts of LangChain. Understanding these will go a long way in helping you understand the codebase and how to construct chains. -## Prompts -Prompts generically have a `format` method that takes in variables and returns a formatted string. +## PromptTemplates +PromptTemplates generically have a `format` method that takes in variables and returns a formatted string. The most simple implementation of this is to have a template string with some variables in it, and then format it with the incoming variables. More complex iterations dynamically construct the template string from few shot examples, etc. +For a more detailed explanation of how LangChain approaches prompts and prompt templates, see [here](prompts.md). + ## LLMs Wrappers around Large Language Models (in particular, the `generate` ability of large language models) are some of the core functionality of LangChain. These wrappers are classes that are callable: they take in an input string, and return the generated output string. diff --git a/docs/glossary.md b/docs/explanation/glossary.md similarity index 100% rename from docs/glossary.md rename to docs/explanation/glossary.md diff --git a/docs/explanation/prompts.md b/docs/explanation/prompts.md new file mode 100644 index 00000000..86842c46 --- /dev/null +++ b/docs/explanation/prompts.md @@ -0,0 +1,138 @@ +# Prompts + +Prompts and all the tooling around them are integral to working with language models, and therefor +really important to get right, from both and interface and naming perspective. This is a "design doc" +of sorts explaining how we think about prompts and the related concepts, and why the interfaces +for working with are the way they are in LangChain. + +For a more code-based walkthrough of all these concept, checkout our example [here](/examples/prompts/prompt_management) + +## Prompt + +### Concept +A prompt is the final string that gets fed into the language model. + +### LangChain Implementation +In LangChain a prompt is represented as just a string. + +## Input Variables + +### Concept +Input variables are parts of a prompt that are not known until runtime, eg could be user provided. + +### LangChain Implementation +In LangChain input variables are just represented as a dictionary of key-value pairs, with the key +being the variable name and the value being the variable value. + +## Examples + +### Concept +Examples are basically datapoints that can be used to teach the model what to do. These can be included +in prompts to better instruct the model on what to do. + +### LangChain Implementation +In LangChain examples are represented as a dictionary of key-value pairs, with the key being the feature +(or label) name, and the value being the feature (or label) value. + +## Example Selector + +### Concept +If you have a large number of examples, you may need to select which ones to include in the prompt. The +Example Selector is the class responsible for doing so. + +### LangChain Implementation + +#### BaseExampleSelector +In LangChain there is a BaseExampleSelector that exposes the following interface + +```python +class BaseExampleSelector: + + def select_examples(self, input_variables: dict): +``` + +Notice that it does not take in examples at runtime when it's selecting them - those are assumed to have been provided ahead of time. + +#### LengthExampleSelector +The LengthExampleSelector selects examples based on the length of the input variables. +This is useful when you are worried about constructing a prompt that will go over the length +of the context window. For longer inputs, it will select fewer examples to include, while for +shorter inputs it will select more. + +#### SemanticSimilarityExampleSelector +The SemanticSimilarityExampleSelector selects examples based on which examples are most similar +to the inputs. It does this by finding the examples with the embeddings that have the greatest +cosine similarity with the inputs. + + +## Prompt Template + +### Concept +The prompts that get fed into the language model are nearly always not hardcoded, but rather a combination +of parts, including Examples and Input Variables. A prompt template is responsible +for taking those parts and constructing a prompt. + +### LangChain Implementation + +#### BasePromptTemplate +In LangChain there is a BasePromptTemplate that exposes the following interface + +```python +class BasePromptTemplate: + + @property + def input_variables(self) -> List[str]: + + def format(self, **kwargs) -> str: +``` +The input variables property is used to provide introspection of the PromptTemplate and know +what inputs it expects. The format method takes in input variables and returns the prompt. + +#### PromptTemplate +The PromptTemplate implementation is the most simple form of a prompt template. It consists of three parts: +- input variables: which variables this prompt template expects +- template: the template into which these variables will be formatted +- template format: the format of the template (eg mustache, python f-strings, etc) + +For example, if I was making an application that took a user inputted concept and asked a language model +to make a joke about that concept, I might use this specification for the PromptTemplate +- input variables = `["thing"]` +- template = `"Tell me a joke about {thing}"` +- template format = `"f-string"` + +#### FewShotPromptTemplate +A FewShotPromptTemplate is a Prompt Template that includes some examples. It consists of: +- examples OR example selector: a list of examples to use, or an Example Selector to select which examples to use +- example prompt template: a Prompt Template responsible for taking an individual example (a dictionary) and turning it into a string to be used in the prompt. +- prefix: the template put in the prompt before listing any examples +- suffix: the template put in the prompt after listing any examples +- example separator: a string separator which is used to join the prefix, the examples, and the suffix together + + +For example, if I wanted to turn the above example into a few shot prompt, this is what it would +look like: + +First I would collect some examples, like +```python +examples = [ + {"concept": "chicken", "joke": "Why did the chicken cross the road?"}, + ... +] +``` + +I would then make sure to define a prompt template for how each example should be formatted +when inserted into the prompt: +```python +prompt_template = PromptTemplate( + input_variables=["concept", "joke"], + template="Tell me a joke about {concept}\n{joke}" +) +``` + +Then, I would define the components as: +- examples: The above examples +- example_prompt: The above example prompt +- prefix = `"You are a comedian telling jokes on demand."` +- suffix = `"Tell me a joke about {concept}"` +- input variables = `["concept"]` +- template format = `"f-string"` diff --git a/docs/getting_started/chains.md b/docs/getting_started/chains.md index 68013b6c..86d57a41 100644 --- a/docs/getting_started/chains.md +++ b/docs/getting_started/chains.md @@ -12,9 +12,10 @@ This is easy to do with LangChain! First lets define the prompt: ```python -from langchain.prompts import Prompt -prompt = Prompt( - input_variables=["product"], +from langchain.prompts import PromptTemplate + +prompt = PromptTemplate( + input_variables=["product"], template="What is a good name for a company that makes {product}?", ) ``` @@ -35,4 +36,4 @@ chain.run("colorful socks") There we go! There's the first chain. That is it for the Getting Started example. -As a next step, we would suggest checking out the more complex chains in the [Demos section](/examples/demos.rst) +As a next step, we would suggest checking out the more complex chains in the [Demos section](/examples/demos) diff --git a/docs/index.rst b/docs/index.rst index 016b72a7..6d1f28e8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -56,6 +56,7 @@ common tasks or cool demos. installation.md integrations.md modules/prompt + modules/example_selector modules/llms modules/embeddings modules/text_splitter @@ -72,8 +73,9 @@ see detailed information about the various classes, methods, and APIs. :caption: Resources :name: resources - core_concepts.md - glossary.md + explanation/core_concepts.md + explanation/prompts.md + explanation/glossary.md Discord Higher level, conceptual explanations of the LangChain components. diff --git a/docs/modules/example_selector.rst b/docs/modules/example_selector.rst new file mode 100644 index 00000000..261d356d --- /dev/null +++ b/docs/modules/example_selector.rst @@ -0,0 +1,5 @@ +:mod:`langchain.prompts.example_selector` +========================================= + +.. automodule:: langchain.prompts.example_selector + :members: diff --git a/langchain/__init__.py b/langchain/__init__.py index 4ccd7dc5..1591eb8f 100644 --- a/langchain/__init__.py +++ b/langchain/__init__.py @@ -18,7 +18,7 @@ from langchain.chains import ( ) from langchain.docstore import Wikipedia from langchain.llms import Cohere, HuggingFaceHub, OpenAI -from langchain.prompts import BasePrompt, DynamicPrompt, Prompt +from langchain.prompts import BasePromptTemplate, PromptTemplate from langchain.sql_database import SQLDatabase from langchain.vectorstores import FAISS, ElasticVectorSearch @@ -30,9 +30,9 @@ __all__ = [ "SerpAPIChain", "Cohere", "OpenAI", - "BasePrompt", + "BasePromptTemplate", "DynamicPrompt", - "Prompt", + "PromptTemplate", "ReActChain", "Wikipedia", "HuggingFaceHub", diff --git a/langchain/chains/llm.py b/langchain/chains/llm.py index fe173a38..5133129e 100644 --- a/langchain/chains/llm.py +++ b/langchain/chains/llm.py @@ -5,7 +5,7 @@ from pydantic import BaseModel, Extra from langchain.chains.base import Chain from langchain.llms.base import LLM -from langchain.prompts.base import BasePrompt +from langchain.prompts.base import BasePromptTemplate class LLMChain(Chain, BaseModel): @@ -16,11 +16,13 @@ class LLMChain(Chain, BaseModel): from langchain import LLMChain, OpenAI, Prompt prompt_template = "Tell me a {adjective} joke" - prompt = Prompt(input_variables=["adjective"], template=prompt_template) + prompt = PromptTemplate( + input_variables=["adjective"], template=prompt_template + ) llm = LLMChain(llm=OpenAI(), prompt=prompt) """ - prompt: BasePrompt + prompt: BasePromptTemplate """Prompt object to use.""" llm: LLM """LLM wrapper to use.""" diff --git a/langchain/chains/llm_math/prompt.py b/langchain/chains/llm_math/prompt.py index b389e917..252ec493 100644 --- a/langchain/chains/llm_math/prompt.py +++ b/langchain/chains/llm_math/prompt.py @@ -1,5 +1,5 @@ # flake8: noqa -from langchain.prompts.prompt import Prompt +from langchain.prompts.prompt import PromptTemplate _PROMPT_TEMPLATE = """You are GPT-3, and you can't do math. @@ -35,4 +35,4 @@ Answer: 2518731 Question: {question}""" -PROMPT = Prompt(input_variables=["question"], template=_PROMPT_TEMPLATE) +PROMPT = PromptTemplate(input_variables=["question"], template=_PROMPT_TEMPLATE) diff --git a/langchain/chains/mapreduce.py b/langchain/chains/mapreduce.py index 623b95cd..8ec5a0b4 100644 --- a/langchain/chains/mapreduce.py +++ b/langchain/chains/mapreduce.py @@ -11,7 +11,7 @@ from pydantic import BaseModel, Extra from langchain.chains.base import Chain from langchain.chains.llm import LLMChain from langchain.llms.base import LLM -from langchain.prompts.base import BasePrompt +from langchain.prompts.base import BasePromptTemplate from langchain.text_splitter import TextSplitter @@ -29,7 +29,7 @@ class MapReduceChain(Chain, BaseModel): @classmethod def from_params( - cls, llm: LLM, prompt: BasePrompt, text_splitter: TextSplitter + cls, llm: LLM, prompt: BasePromptTemplate, text_splitter: TextSplitter ) -> "MapReduceChain": """Construct a map-reduce chain that uses the chain for map and reduce.""" llm_chain = LLMChain(llm=llm, prompt=prompt) diff --git a/langchain/chains/mrkl/base.py b/langchain/chains/mrkl/base.py index 38d04048..b7c26e63 100644 --- a/langchain/chains/mrkl/base.py +++ b/langchain/chains/mrkl/base.py @@ -8,7 +8,7 @@ from langchain.chains.llm import LLMChain from langchain.chains.mrkl.prompt import BASE_TEMPLATE from langchain.input import ChainedInput, get_color_mapping from langchain.llms.base import LLM -from langchain.prompts import BasePrompt, Prompt +from langchain.prompts import BasePromptTemplate, PromptTemplate FINAL_ANSWER_ACTION = "Final Answer: " @@ -57,7 +57,7 @@ class MRKLChain(Chain, BaseModel): from langchain import OpenAI, Prompt, MRKLChain from langchain.chains.mrkl.base import ChainConfig llm = OpenAI(temperature=0) - prompt = Prompt(...) + prompt = PromptTemplate(...) action_to_chain_map = {...} mrkl = MRKLChain( llm=llm, @@ -68,7 +68,7 @@ class MRKLChain(Chain, BaseModel): llm: LLM """LLM wrapper to use as router.""" - prompt: BasePrompt + prompt: BasePromptTemplate """Prompt to use as router.""" action_to_chain_map: Dict[str, Callable] """Mapping from action name to chain to execute.""" @@ -119,7 +119,7 @@ class MRKLChain(Chain, BaseModel): ) tool_names = ", ".join([chain.action_name for chain in chains]) template = BASE_TEMPLATE.format(tools=tools, tool_names=tool_names) - prompt = Prompt(template=template, input_variables=["input"]) + prompt = PromptTemplate(template=template, input_variables=["input"]) action_to_chain_map = {chain.action_name: chain.action for chain in chains} return cls( llm=llm, prompt=prompt, action_to_chain_map=action_to_chain_map, **kwargs diff --git a/langchain/chains/natbot/prompt.py b/langchain/chains/natbot/prompt.py index f67775b0..700d93d6 100644 --- a/langchain/chains/natbot/prompt.py +++ b/langchain/chains/natbot/prompt.py @@ -1,5 +1,5 @@ # flake8: noqa -from langchain.prompts.prompt import Prompt +from langchain.prompts.prompt import PromptTemplate _PROMPT_TEMPLATE = """ You are an agent controlling a browser. You are given: @@ -138,7 +138,7 @@ CURRENT URL: {url} PREVIOUS COMMAND: {previous_command} YOUR COMMAND: """ -PROMPT = Prompt( +PROMPT = PromptTemplate( input_variables=["browser_content", "url", "previous_command", "objective"], template=_PROMPT_TEMPLATE, ) diff --git a/langchain/chains/react/prompt.py b/langchain/chains/react/prompt.py index 8a3b2cfe..33fbada6 100644 --- a/langchain/chains/react/prompt.py +++ b/langchain/chains/react/prompt.py @@ -1,5 +1,5 @@ # flake8: noqa -from langchain.prompts.prompt import Prompt +from langchain.prompts.prompt import PromptTemplate EXAMPLES = [ """Question: What is the elevation range for the area that the eastern sector of the @@ -109,4 +109,4 @@ Action 3: Finish[yes]""", ] SUFFIX = """\n\nQuestion: {input}""" -PROMPT = Prompt.from_examples(EXAMPLES, SUFFIX, ["input"]) +PROMPT = PromptTemplate.from_examples(EXAMPLES, SUFFIX, ["input"]) diff --git a/langchain/chains/self_ask_with_search/prompt.py b/langchain/chains/self_ask_with_search/prompt.py index 02f7ab3f..f9d0c5b6 100644 --- a/langchain/chains/self_ask_with_search/prompt.py +++ b/langchain/chains/self_ask_with_search/prompt.py @@ -1,5 +1,5 @@ # flake8: noqa -from langchain.prompts.prompt import Prompt +from langchain.prompts.prompt import PromptTemplate _DEFAULT_TEMPLATE = """Question: Who lived longer, Muhammad Ali or Alan Turing? Are follow up questions needed here: Yes. @@ -38,4 +38,4 @@ Intermediate Answer: New Zealand. So the final answer is: No Question: {input}""" -PROMPT = Prompt(input_variables=["input"], template=_DEFAULT_TEMPLATE) +PROMPT = PromptTemplate(input_variables=["input"], template=_DEFAULT_TEMPLATE) diff --git a/langchain/chains/sql_database/prompt.py b/langchain/chains/sql_database/prompt.py index 43bb3fcf..1bc9ffd9 100644 --- a/langchain/chains/sql_database/prompt.py +++ b/langchain/chains/sql_database/prompt.py @@ -1,5 +1,5 @@ # flake8: noqa -from langchain.prompts.prompt import Prompt +from langchain.prompts.prompt import PromptTemplate _DEFAULT_TEMPLATE = """Given an input question, first create a syntactically correct {dialect} query to run, then look at the results of the query and return the answer. Use the following format: @@ -14,6 +14,6 @@ Only use the following tables: {table_info} Question: {input}""" -PROMPT = Prompt( +PROMPT = PromptTemplate( input_variables=["input", "table_info", "dialect"], template=_DEFAULT_TEMPLATE ) diff --git a/langchain/chains/vector_db_qa/prompt.py b/langchain/chains/vector_db_qa/prompt.py index 54c4d7f6..734223a3 100644 --- a/langchain/chains/vector_db_qa/prompt.py +++ b/langchain/chains/vector_db_qa/prompt.py @@ -1,5 +1,5 @@ # flake8: noqa -from langchain.prompts import Prompt +from langchain.prompts import PromptTemplate prompt_template = """Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer. @@ -7,4 +7,6 @@ prompt_template = """Use the following pieces of context to answer the question Question: {question} Helpful Answer:""" -prompt = Prompt(template=prompt_template, input_variables=["context", "question"]) +prompt = PromptTemplate( + template=prompt_template, input_variables=["context", "question"] +) diff --git a/langchain/example_generator.py b/langchain/example_generator.py index 818a848a..58816e56 100644 --- a/langchain/example_generator.py +++ b/langchain/example_generator.py @@ -3,18 +3,21 @@ from typing import List from langchain.chains.llm import LLMChain from langchain.llms.base import LLM -from langchain.prompts.dynamic import DynamicPrompt +from langchain.prompts.few_shot import FewShotPromptTemplate +from langchain.prompts.prompt import PromptTemplate TEST_GEN_TEMPLATE_SUFFIX = "Add another example." -def generate_example(examples: List[str], llm: LLM) -> str: +def generate_example( + examples: List[dict], llm: LLM, prompt_template: PromptTemplate +) -> str: """Return another example given a list of examples for a prompt.""" - prompt = DynamicPrompt(examples=examples, suffix=TEST_GEN_TEMPLATE_SUFFIX) + prompt = FewShotPromptTemplate( + examples=examples, + suffix=TEST_GEN_TEMPLATE_SUFFIX, + input_variables=[], + example_prompt=prompt_template, + ) chain = LLMChain(llm=llm, prompt=prompt) return chain.predict() - - -def generate_example_from_dynamic_prompt(prompt: DynamicPrompt, llm: LLM) -> str: - """Return another example given a DynamicPrompt object.""" - return generate_example(prompt.examples, llm) diff --git a/langchain/model_laboratory.py b/langchain/model_laboratory.py index d61265c0..2bf30904 100644 --- a/langchain/model_laboratory.py +++ b/langchain/model_laboratory.py @@ -5,7 +5,7 @@ from langchain.chains.base import Chain from langchain.chains.llm import LLMChain from langchain.input import get_color_mapping, print_text from langchain.llms.base import LLM -from langchain.prompts.prompt import Prompt +from langchain.prompts.prompt import PromptTemplate class ModelLaboratory: @@ -44,7 +44,7 @@ class ModelLaboratory: @classmethod def from_llms( - cls, llms: List[LLM], prompt: Optional[Prompt] = None + cls, llms: List[LLM], prompt: Optional[PromptTemplate] = None ) -> "ModelLaboratory": """Initialize with LLMs to experiment with and optional prompt. @@ -54,7 +54,7 @@ class ModelLaboratory: If a prompt was provided, it should only have one input variable. """ if prompt is None: - prompt = Prompt(input_variables=["_input"], template="{_input}") + prompt = PromptTemplate(input_variables=["_input"], template="{_input}") chains = [LLMChain(llm=llm, prompt=prompt) for llm in llms] names = [str(llm) for llm in llms] return cls(chains, names=names) diff --git a/langchain/prompts/__init__.py b/langchain/prompts/__init__.py index 177aa155..019317ec 100644 --- a/langchain/prompts/__init__.py +++ b/langchain/prompts/__init__.py @@ -1,6 +1,12 @@ """Prompt template classes.""" -from langchain.prompts.base import BasePrompt -from langchain.prompts.dynamic import DynamicPrompt -from langchain.prompts.prompt import Prompt +from langchain.prompts.base import BasePromptTemplate +from langchain.prompts.few_shot import FewShotPromptTemplate +from langchain.prompts.loading import load_prompt +from langchain.prompts.prompt import PromptTemplate -__all__ = ["BasePrompt", "Prompt", "DynamicPrompt"] +__all__ = [ + "BasePromptTemplate", + "load_prompt", + "PromptTemplate", + "FewShotPromptTemplate", +] diff --git a/langchain/prompts/base.py b/langchain/prompts/base.py index d99d9400..44cdad9f 100644 --- a/langchain/prompts/base.py +++ b/langchain/prompts/base.py @@ -9,7 +9,25 @@ DEFAULT_FORMATTER_MAPPING = { } -class BasePrompt(ABC): +def check_valid_template( + template: str, template_format: str, input_variables: List[str] +) -> None: + """Check that template string is valid.""" + if template_format not in DEFAULT_FORMATTER_MAPPING: + valid_formats = list(DEFAULT_FORMATTER_MAPPING) + raise ValueError( + f"Invalid template format. Got `{template_format}`;" + f" should be one of {valid_formats}" + ) + dummy_inputs = {input_variable: "foo" for input_variable in input_variables} + try: + formatter_func = DEFAULT_FORMATTER_MAPPING[template_format] + formatter_func(template, **dummy_inputs) + except KeyError: + raise ValueError("Invalid prompt schema.") + + +class BasePromptTemplate(ABC): """Base prompt should expose the format method, returning a prompt.""" input_variables: List[str] diff --git a/langchain/prompts/dynamic.py b/langchain/prompts/dynamic.py deleted file mode 100644 index fbf0c351..00000000 --- a/langchain/prompts/dynamic.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Dynamic prompt schema definition.""" -import re -from typing import Any, Callable, Dict, List - -from pydantic import BaseModel, Extra, root_validator - -from langchain.prompts.base import DEFAULT_FORMATTER_MAPPING, BasePrompt - - -class DynamicPrompt(BaseModel, BasePrompt): - r"""Schema to represent a dynamic prompt for an LLM. - - Example: - .. code-block:: python - - from langchain import DynamicPrompt - dynamic_prompt = DynamicPrompt( - examples=["Say hi. Hi", "Say ho. Ho"], - example_separator="\n\n", - prefix="", - suffix="\n\nSay {foo}" - input_variables=["foo"], - max_length=200, - get_text_length=word_count - ) - """ - - examples: List[str] - """A list of the examples that the prompt template expects.""" - - example_separator: str = "\n\n" - """Example separator, e.g. \n\n, for the dynamic prompt creation.""" - - input_variables: List[str] = [] - """A list of the names of the variables the prompt template expects.""" - - prefix: str = "" - """Prefix for the prompt.""" - - suffix: str = "" - """Suffix for the prompt.""" - - template_format: str = "f-string" - """The format of the prompt template. Options are: 'f-string'.""" - - get_text_length: Callable[[str], int] = lambda x: len(re.split("\n| ", x)) - """Function to measure prompt length. Defaults to word count.""" - - max_length: int = 2048 - """Max length for the prompt, beyond which examples are cut.""" - - class Config: - """Configuration for this pydantic object.""" - - extra = Extra.forbid - - def template(self, example_list: List[str], **kwargs: Any) -> str: - """Return template given example list.""" - template = self.example_separator.join( - [self.prefix, *example_list, self.suffix] - ) - return DEFAULT_FORMATTER_MAPPING[self.template_format](template, **kwargs) - - def format(self, **kwargs: Any) -> str: - """Dynamically format the prompt with the inputs. - - Args: - kwargs: Any arguments to be passed to the prompt template. - - Returns: - A formatted string. - - Example: - - .. code-block:: python - - prompt.format(variable1="foo") - """ - curr_examples = self.examples - template = self.template(curr_examples, **kwargs) - while self.get_text_length(template) > self.max_length and curr_examples: - curr_examples = curr_examples[:-1] - template = self.template(curr_examples, **kwargs) - return template - - @root_validator() - def template_is_valid(cls, values: Dict) -> Dict: - """Check that prefix, suffix and input variables are consistent.""" - input_variables = values["input_variables"] - prefix = values["prefix"] - suffix = values["suffix"] - template_format = values["template_format"] - if template_format not in DEFAULT_FORMATTER_MAPPING: - valid_formats = list(DEFAULT_FORMATTER_MAPPING) - raise ValueError( - f"Invalid template format. Got `{template_format}`;" - f" should be one of {valid_formats}" - ) - try: - result = values["get_text_length"]("foo") - assert isinstance(result, int) - except AssertionError: - raise ValueError( - "Invalid text length callable, must take string & return int;" - ) - dummy_inputs = {input_variable: "foo" for input_variable in input_variables} - try: - formatter_func = DEFAULT_FORMATTER_MAPPING[template_format] - formatter_func(prefix + suffix, **dummy_inputs) - except KeyError: - raise ValueError("Invalid prompt schema.") - return values diff --git a/langchain/prompts/example_selector/__init__.py b/langchain/prompts/example_selector/__init__.py new file mode 100644 index 00000000..e58b5352 --- /dev/null +++ b/langchain/prompts/example_selector/__init__.py @@ -0,0 +1,7 @@ +"""Logic for selecting examples to include in prompts.""" +from langchain.prompts.example_selector.length_based import LengthBasedExampleSelector +from langchain.prompts.example_selector.semantic_similarity import ( + SemanticSimilarityExampleSelector, +) + +__all__ = ["LengthBasedExampleSelector", "SemanticSimilarityExampleSelector"] diff --git a/langchain/prompts/example_selector/base.py b/langchain/prompts/example_selector/base.py new file mode 100644 index 00000000..9af9e307 --- /dev/null +++ b/langchain/prompts/example_selector/base.py @@ -0,0 +1,11 @@ +"""Interface for selecting examples to include in prompts.""" +from abc import ABC, abstractmethod +from typing import Dict, List + + +class BaseExampleSelector(ABC): + """Interface for selecting examples to include in prompts.""" + + @abstractmethod + def select_examples(self, input_variables: Dict[str, str]) -> List[dict]: + """Select which examples to use based on the inputs.""" diff --git a/langchain/prompts/example_selector/length_based.py b/langchain/prompts/example_selector/length_based.py new file mode 100644 index 00000000..ae13e884 --- /dev/null +++ b/langchain/prompts/example_selector/length_based.py @@ -0,0 +1,54 @@ +"""Select examples based on length.""" +import re +from typing import Callable, Dict, List + +from pydantic import BaseModel, validator + +from langchain.prompts.example_selector.base import BaseExampleSelector +from langchain.prompts.prompt import PromptTemplate + + +class LengthBasedExampleSelector(BaseExampleSelector, BaseModel): + """Select examples based on length.""" + + examples: List[dict] + """A list of the examples that the prompt template expects.""" + + example_prompt: PromptTemplate + """Prompt template used to format the examples.""" + + get_text_length: Callable[[str], int] = lambda x: len(re.split("\n| ", x)) + """Function to measure prompt length. Defaults to word count.""" + + max_length: int = 2048 + """Max length for the prompt, beyond which examples are cut.""" + + example_text_lengths: List[int] = [] #: :meta private: + + @validator("example_text_lengths", always=True) + def calculate_example_text_lengths(cls, v: List[int], values: Dict) -> List[int]: + """Calculate text lengths if they don't exist.""" + # Check if text lengths were passed in + if v: + return v + # If they were not, calculate them + example_prompt = values["example_prompt"] + get_text_length = values["get_text_length"] + string_examples = [example_prompt.format(**eg) for eg in values["examples"]] + return [get_text_length(eg) for eg in string_examples] + + def select_examples(self, input_variables: Dict[str, str]) -> List[dict]: + """Select which examples to use based on the input lengths.""" + inputs = " ".join(input_variables.values()) + remaining_length = self.max_length - self.get_text_length(inputs) + i = 0 + examples = [] + while remaining_length > 0 and i < len(self.examples): + new_length = remaining_length - self.example_text_lengths[i] + if i < 0: + break + else: + examples.append(self.examples[i]) + remaining_length = new_length + i += 1 + return examples diff --git a/langchain/prompts/example_selector/semantic_similarity.py b/langchain/prompts/example_selector/semantic_similarity.py new file mode 100644 index 00000000..82c087df --- /dev/null +++ b/langchain/prompts/example_selector/semantic_similarity.py @@ -0,0 +1,75 @@ +"""Example selector that selects examples based on SemanticSimilarity.""" +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Extra + +from langchain.embeddings.base import Embeddings +from langchain.prompts.example_selector.base import BaseExampleSelector +from langchain.vectorstores.base import VectorStore + + +class SemanticSimilarityExampleSelector(BaseExampleSelector, BaseModel): + """Example selector that selects examples based on SemanticSimilarity.""" + + vectorstore: VectorStore + """VectorStore than contains information about examples.""" + k: int = 4 + """Number of examples to select.""" + example_keys: Optional[List[str]] = None + """Optional keys to filter examples to.""" + + class Config: + """Configuration for this pydantic object.""" + + extra = Extra.forbid + arbitrary_types_allowed = True + + def select_examples(self, input_variables: Dict[str, str]) -> List[dict]: + """Select which examples to use based on semantic similarity.""" + # Get the docs with the highest similarity. + query = " ".join(input_variables.values()) + example_docs = self.vectorstore.similarity_search(query, k=self.k) + # Get the examples from the metadata. + # This assumes that examples are stored in metadata. + examples = [dict(e.metadata) for e in example_docs] + # If example keys are provided, filter examples to those keys. + if self.example_keys: + examples = [{k: eg[k] for k in self.example_keys} for eg in examples] + return examples + + @classmethod + def from_examples( + cls, + examples: List[dict], + embeddings: Embeddings, + vectorstore_cls: VectorStore, + k: int = 4, + **vectorstore_cls_kwargs: Any, + ) -> "SemanticSimilarityExampleSelector": + """Create k-shot example selector using example list and embeddings. + + Reshuffles examples dynamically based on query similarity. + + Args: + examples: List of examples to use in the prompt. + suffix: String to go after the list of examples. Should generally + set up the user's input. + input_variables: A list of variable names the final prompt template + will expect. + embeddings: An iniialized embedding API interface, e.g. OpenAIEmbeddings(). + vectorstore_cls: A vector store DB interface class, e.g. FAISS. + example_separator: The seperator to use in between examples. Defaults + to two new line characters. + prefix: String that should go before any examples. Generally includes + examples. Default to an empty string. + k: Number of examples to select + vectorstore_cls_kwargs: optional kwargs containing url for vector store + + Returns: + The ExampleSelector instantiated, backed by a vector store. + """ + string_examples = [" ".join(eg.values()) for eg in examples] + vectorstore = vectorstore_cls.from_texts( + string_examples, embeddings, metadatas=examples, **vectorstore_cls_kwargs + ) + return cls(vectorstore=vectorstore, k=k) diff --git a/langchain/prompts/few_shot.py b/langchain/prompts/few_shot.py new file mode 100644 index 00000000..a73c1256 --- /dev/null +++ b/langchain/prompts/few_shot.py @@ -0,0 +1,110 @@ +"""Prompt template that contains few shot examples.""" +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Extra, root_validator + +from langchain.prompts.base import ( + DEFAULT_FORMATTER_MAPPING, + BasePromptTemplate, + check_valid_template, +) +from langchain.prompts.example_selector.base import BaseExampleSelector +from langchain.prompts.prompt import PromptTemplate + + +class FewShotPromptTemplate(BasePromptTemplate, BaseModel): + """Prompt template that contains few shot examples.""" + + examples: Optional[List[dict]] = None + """Examples to format into the prompt. + Either this or example_selector should be provided.""" + + example_selector: Optional[BaseExampleSelector] = None + """ExampleSelector to choose the examples to format into the prompt. + Either this or examples should be provided.""" + + example_prompt: PromptTemplate + """PromptTemplate used to format an individual example.""" + + suffix: str + """A prompt template string to put after the examples.""" + + input_variables: List[str] + """A list of the names of the variables the prompt template expects.""" + + example_separator: str = "\n\n" + """String separator used to join the prefix, the examples, and suffix.""" + + prefix: str = "" + """A prompt template string to put before the examples.""" + + template_format: str = "f-string" + """The format of the prompt template. Options are: 'f-string'.""" + + @root_validator(pre=True) + def check_examples_and_selector(cls, values: Dict) -> Dict: + """Check that one and only one of examples/example_selector are provided.""" + examples = values.get("examples", None) + example_selector = values.get("example_selector", None) + if examples and example_selector: + raise ValueError( + "Only one of 'examples' and 'example_selector' should be provided" + ) + + if examples is None and example_selector is None: + raise ValueError( + "One of 'examples' and 'example_selector' should be provided" + ) + + return values + + @root_validator() + def template_is_valid(cls, values: Dict) -> Dict: + """Check that prefix, suffix and input variables are consistent.""" + check_valid_template( + values["prefix"] + values["suffix"], + values["template_format"], + values["input_variables"], + ) + return values + + class Config: + """Configuration for this pydantic object.""" + + extra = Extra.forbid + arbitrary_types_allowed = True + + def _get_examples(self, **kwargs: Any) -> List[dict]: + if self.examples is not None: + return self.examples + elif self.example_selector is not None: + return self.example_selector.select_examples(kwargs) + else: + raise ValueError + + def format(self, **kwargs: Any) -> str: + """Format the prompt with the inputs. + + Args: + kwargs: Any arguments to be passed to the prompt template. + + Returns: + A formatted string. + + Example: + + .. code-block:: python + + prompt.format(variable1="foo") + """ + # Get the examples to use. + examples = self._get_examples(**kwargs) + # Format the examples. + example_strings = [ + self.example_prompt.format(**example) for example in examples + ] + # Create the overall template. + pieces = [self.prefix, *example_strings, self.suffix] + template = self.example_separator.join([piece for piece in pieces if piece]) + # Format the template with the input variables. + return DEFAULT_FORMATTER_MAPPING[self.template_format](template, **kwargs) diff --git a/langchain/prompts/loading.py b/langchain/prompts/loading.py new file mode 100644 index 00000000..d99e51ef --- /dev/null +++ b/langchain/prompts/loading.py @@ -0,0 +1,103 @@ +"""Load prompts from disk.""" +import json +from pathlib import Path +from typing import Union + +import yaml + +from langchain.prompts.base import BasePromptTemplate +from langchain.prompts.few_shot import FewShotPromptTemplate +from langchain.prompts.prompt import PromptTemplate + + +def load_prompt_from_config(config: dict) -> BasePromptTemplate: + """Get the right type from the config and load it accordingly.""" + prompt_type = config.pop("_type", "prompt") + if prompt_type == "prompt": + return _load_prompt(config) + elif prompt_type == "few_shot": + return _load_few_shot_prompt(config) + else: + raise ValueError + + +def _load_template(var_name: str, config: dict) -> dict: + """Load template from disk if applicable.""" + # Check if template_path exists in config. + if f"{var_name}_path" in config: + # If it does, make sure template variable doesn't also exist. + if var_name in config: + raise ValueError( + f"Both `{var_name}_path` and `{var_name}` cannot be provided." + ) + # Pop the template path from the config. + template_path = Path(config.pop(f"{var_name}_path")) + # Load the template. + if template_path.suffix == ".txt": + with open(template_path) as f: + template = f.read() + else: + raise ValueError + # Set the template variable to the extracted variable. + config[var_name] = template + return config + + +def _load_examples(config: dict) -> dict: + """Load examples if necessary.""" + if isinstance(config["examples"], list): + pass + elif isinstance(config["examples"], str): + with open(config["examples"]) as f: + examples = json.load(f) + config["examples"] = examples + else: + raise ValueError + return config + + +def _load_few_shot_prompt(config: dict) -> FewShotPromptTemplate: + """Load the few shot prompt from the config.""" + # Load the suffix and prefix templates. + config = _load_template("suffix", config) + config = _load_template("prefix", config) + # Load the example prompt. + if "example_prompt_path" in config: + if "example_prompt" in config: + raise ValueError( + "Only one of example_prompt and example_prompt_path should " + "be specified." + ) + config["example_prompt"] = load_prompt(config.pop("example_prompt_path")) + else: + config["example_prompt"] = _load_prompt(config["example_prompt"]) + # Load the examples. + config = _load_examples(config) + return FewShotPromptTemplate(**config) + + +def _load_prompt(config: dict) -> PromptTemplate: + """Load the prompt template from config.""" + # Load the template from disk if necessary. + config = _load_template("template", config) + return PromptTemplate(**config) + + +def load_prompt(file: Union[str, Path]) -> BasePromptTemplate: + """Load prompt from file.""" + # Convert file to Path object. + if isinstance(file, str): + file_path = Path(file) + else: + file_path = file + # Load from either json or yaml. + if file_path.suffix == ".json": + with open(file_path) as f: + config = json.load(f) + elif file_path.suffix == ".yaml": + with open(file_path, "r") as f: + config = yaml.safe_load(f) + else: + raise ValueError + # Load the prompt from the config now. + return load_prompt_from_config(config) diff --git a/langchain/prompts/optimized.py b/langchain/prompts/optimized.py deleted file mode 100644 index c8240671..00000000 --- a/langchain/prompts/optimized.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Optimized prompt schema definition.""" -import re -from typing import Any, Callable, Dict, List - -from pydantic import BaseModel, Extra, root_validator - -from langchain.embeddings.base import Embeddings -from langchain.prompts.base import DEFAULT_FORMATTER_MAPPING -from langchain.vectorstores.base import VectorStore - - -class OptimizedPrompt(BaseModel): - r"""Schema to represent an optimized prompt for an LLM. - - Example: - .. code-block:: python - - from langchain import DynamicPrompt - vectorstore = FAISS.from_texts(examples, OpenAIEmbeddings() - optimized_prompt = OptimizedPrompt( - example_separator="\n\n", - prefix="", - suffix="\n\nSay {foo}" - input_variables=["foo"], - max_length=200, - get_text_length=word_count, - vectorstore=vectorstore) - ) - """ - - example_separator: str = "\n\n" - """Example separator, e.g. \n\n, for the dynamic prompt creation.""" - - input_variables: List[str] = [] - """A list of the names of the variables the prompt template expects.""" - - prefix: str = "" - """Prefix for the prompt.""" - - suffix: str = "" - """Suffix for the prompt.""" - - template_format: str = "f-string" - """The format of the prompt template. Options are: 'f-string'.""" - - get_text_length: Callable[[str], int] = lambda x: len(re.split("\n| ", x)) - """Function to measure prompt length. Defaults to word count.""" - - max_length: int = 2048 - """Max length for the prompt, beyond which examples are cut.""" - - vectorstore: VectorStore - """Vectorstore to use for storing the embeddings.""" - - class Config: - """Configuration for this pydantic object.""" - - arbitrary_types_allowed = True - - extra = Extra.forbid - - def template(self, example_list: List[str], **kwargs: Any) -> str: - """Return template given full example list.""" - template = self.example_separator.join( - [self.prefix, *example_list, self.suffix] - ) - return DEFAULT_FORMATTER_MAPPING[self.template_format](template, **kwargs) - - def format(self, k: int = 4, **kwargs: Any) -> str: - """Optimize the examples in the prompt for the given inputs. - - Args: - k: Number of examples to aim for (may be trimmed by optimizer afterwards) - kwargs: Any arguments to be passed to the prompt template. - - Returns: - A formatted string. - - Example: - - .. code-block:: python - - prompt.format(variable1="foo") - """ - query = " ".join([v for k, v in kwargs.items()]) - example_docs = self.vectorstore.similarity_search(query, k=k) - curr_examples = [str(e.page_content) for e in example_docs] - template = self.template(curr_examples, **kwargs) - while self.get_text_length(template) > self.max_length and curr_examples: - curr_examples = curr_examples[:-1] - template = self.template(curr_examples, **kwargs) - return template - - @root_validator() - def template_is_valid(cls, values: Dict) -> Dict: - """Check that prefix, suffix and input variables are consistent.""" - input_variables = values["input_variables"] - if len(input_variables) > 1: - raise ValueError("Only one input variable allowed for optimized prompt;") - prefix = values["prefix"] - suffix = values["suffix"] - template_format = values["template_format"] - if template_format not in DEFAULT_FORMATTER_MAPPING: - valid_formats = list(DEFAULT_FORMATTER_MAPPING) - raise ValueError( - f"Invalid template format. Got `{template_format}`;" - f" should be one of {valid_formats}" - ) - try: - result = values["get_text_length"]("foo") - assert isinstance(result, int) - except AssertionError: - raise ValueError( - "Invalid text length callable, must take string & return int;" - ) - dummy_inputs = {input_variable: "foo" for input_variable in input_variables} - try: - formatter_func = DEFAULT_FORMATTER_MAPPING[template_format] - formatter_func(prefix + suffix, **dummy_inputs) - except KeyError: - raise ValueError("Invalid prompt schema.") - return values - - @classmethod - def from_examples( - cls, - examples: List[str], - suffix: str, - input_variables: List[str], - embeddings: Embeddings, - vectorstore_cls: VectorStore, - example_separator: str = "\n\n", - prefix: str = "", - **vectorstore_cls_kwargs: Any, - ) -> "OptimizedPrompt": - """Create k-shot prompt optimizer using example list and embeddings. - - Reshuffles examples for the prompt dynamically based on query similarity. - - Args: - examples: List of examples to use in the prompt. - suffix: String to go after the list of examples. Should generally - set up the user's input. - input_variables: A list of variable names the final prompt template - will expect. - embeddings: An iniialized embedding API interface, e.g. OpenAIEmbeddings(). - vectorstore_cls: A vector store DB interface class, e.g. FAISS. - example_separator: The seperator to use in between examples. Defaults - to two new line characters. - prefix: String that should go before any examples. Generally includes - examples. Default to an empty string. - vectorstore_cls_kwargs: optional kwargs containing url for vector store - - Returns: - The OptimizedPrompt instantiated, backed by a vector store. - """ - vectorstore = vectorstore_cls.from_texts( - examples, embeddings, **vectorstore_cls_kwargs - ) - return cls( - suffix=suffix, - input_variables=input_variables, - example_separator=example_separator, - prefix=prefix, - vectorstore=vectorstore, - ) diff --git a/langchain/prompts/prompt.py b/langchain/prompts/prompt.py index f27c678f..6eccaaa3 100644 --- a/langchain/prompts/prompt.py +++ b/langchain/prompts/prompt.py @@ -3,17 +3,21 @@ from typing import Any, Dict, List from pydantic import BaseModel, Extra, root_validator -from langchain.prompts.base import DEFAULT_FORMATTER_MAPPING, BasePrompt +from langchain.prompts.base import ( + DEFAULT_FORMATTER_MAPPING, + BasePromptTemplate, + check_valid_template, +) -class Prompt(BaseModel, BasePrompt): +class PromptTemplate(BaseModel, BasePromptTemplate): """Schema to represent a prompt for an LLM. Example: .. code-block:: python - from langchain import Prompt - prompt = Prompt(input_variables=["foo"], template="Say {foo}") + from langchain import PromptTemplate + prompt = PromptTemplate(input_variables=["foo"], template="Say {foo}") """ input_variables: List[str] @@ -50,21 +54,9 @@ class Prompt(BaseModel, BasePrompt): @root_validator() def template_is_valid(cls, values: Dict) -> Dict: """Check that template and input variables are consistent.""" - input_variables = values["input_variables"] - template = values["template"] - template_format = values["template_format"] - if template_format not in DEFAULT_FORMATTER_MAPPING: - valid_formats = list(DEFAULT_FORMATTER_MAPPING) - raise ValueError( - f"Invalid template format. Got `{template_format}`;" - f" should be one of {valid_formats}" - ) - dummy_inputs = {input_variable: "foo" for input_variable in input_variables} - try: - formatter_func = DEFAULT_FORMATTER_MAPPING[template_format] - formatter_func(template, **dummy_inputs) - except KeyError: - raise ValueError("Invalid prompt schema.") + check_valid_template( + values["template"], values["template_format"], values["input_variables"] + ) return values @classmethod @@ -75,7 +67,7 @@ class Prompt(BaseModel, BasePrompt): input_variables: List[str], example_separator: str = "\n\n", prefix: str = "", - ) -> "Prompt": + ) -> "PromptTemplate": """Take examples in list format with prefix and suffix to create a prompt. Intended be used as a way to dynamically create a prompt from examples. @@ -98,7 +90,9 @@ class Prompt(BaseModel, BasePrompt): return cls(input_variables=input_variables, template=template) @classmethod - def from_file(cls, template_file: str, input_variables: List[str]) -> "Prompt": + def from_file( + cls, template_file: str, input_variables: List[str] + ) -> "PromptTemplate": """Load a prompt from a file. Args: @@ -111,3 +105,7 @@ class Prompt(BaseModel, BasePrompt): with open(template_file, "r") as f: template = f.read() return cls(input_variables=input_variables, template=template) + + +# For backwards compatibility. +Prompt = PromptTemplate diff --git a/langchain/vectorstores/base.py b/langchain/vectorstores/base.py index db4f7196..a7097893 100644 --- a/langchain/vectorstores/base.py +++ b/langchain/vectorstores/base.py @@ -1,6 +1,6 @@ """Interface for vector stores.""" from abc import ABC, abstractmethod -from typing import Any, List +from typing import Any, List, Optional from langchain.docstore.document import Document from langchain.embeddings.base import Embeddings @@ -16,6 +16,10 @@ class VectorStore(ABC): @classmethod @abstractmethod def from_texts( - cls, texts: List[str], embedding: Embeddings, **kwargs: Any + cls, + texts: List[str], + embedding: Embeddings, + metadatas: Optional[List[dict]] = None, + **kwargs: Any ) -> "VectorStore": """Return VectorStore initialized from texts and embeddings.""" diff --git a/langchain/vectorstores/elastic_vector_search.py b/langchain/vectorstores/elastic_vector_search.py index 549277b3..b186cfff 100644 --- a/langchain/vectorstores/elastic_vector_search.py +++ b/langchain/vectorstores/elastic_vector_search.py @@ -1,6 +1,6 @@ """Wrapper around Elasticsearch vector database.""" import uuid -from typing import Any, Callable, Dict, List +from typing import Any, Callable, Dict, List, Optional from langchain.docstore.document import Document from langchain.embeddings.base import Embeddings @@ -78,13 +78,19 @@ class ElasticVectorSearch(VectorStore): embedding = self.embedding_function(query) script_query = _default_script_query(embedding) response = self.client.search(index=self.index_name, query=script_query) - texts = [hit["_source"]["text"] for hit in response["hits"]["hits"][:k]] - documents = [Document(page_content=text) for text in texts] + hits = [hit["_source"] for hit in response["hits"]["hits"][:k]] + documents = [ + Document(page_content=hit["text"], metadata=hit["metadata"]) for hit in hits + ] return documents @classmethod def from_texts( - cls, texts: List[str], embedding: Embeddings, **kwargs: Any + cls, + texts: List[str], + embedding: Embeddings, + metadatas: Optional[List[dict]] = None, + **kwargs: Any, ) -> "ElasticVectorSearch": """Construct ElasticVectorSearch wrapper from raw documents. @@ -133,11 +139,13 @@ class ElasticVectorSearch(VectorStore): client.indices.create(index=index_name, mappings=mapping) requests = [] for i, text in enumerate(texts): + metadata = metadatas[i] if metadatas else {} request = { "_op_type": "index", "_index": index_name, "vector": embeddings[i], "text": text, + "metadata": metadata, } requests.append(request) bulk(client, requests) diff --git a/langchain/vectorstores/faiss.py b/langchain/vectorstores/faiss.py index 937ad80e..8ae2e3f0 100644 --- a/langchain/vectorstores/faiss.py +++ b/langchain/vectorstores/faiss.py @@ -1,5 +1,5 @@ """Wrapper around FAISS vector database.""" -from typing import Any, Callable, List +from typing import Any, Callable, List, Optional import numpy as np @@ -54,7 +54,11 @@ class FAISS(VectorStore): @classmethod def from_texts( - cls, texts: List[str], embedding: Embeddings, **kwargs: Any + cls, + texts: List[str], + embedding: Embeddings, + metadatas: Optional[List[dict]] = None, + **kwargs: Any, ) -> "FAISS": """Construct FAISS wrapper from raw documents. @@ -84,6 +88,9 @@ class FAISS(VectorStore): embeddings = embedding.embed_documents(texts) index = faiss.IndexFlatL2(len(embeddings[0])) index.add(np.array(embeddings, dtype=np.float32)) - documents = [Document(page_content=text) for text in texts] + documents = [] + for i, text in enumerate(texts): + metadata = metadatas[i] if metadatas else {} + documents.append(Document(page_content=text, metadata=metadata)) docstore = InMemoryDocstore({str(i): doc for i, doc in enumerate(documents)}) return cls(embedding.embed_query, index, docstore) diff --git a/setup.py b/setup.py index 6936dee6..fbc129da 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup( version=__version__, packages=find_packages(), description="Building applications with LLMs through composability", - install_requires=["pydantic", "sqlalchemy", "numpy", "requests"], + install_requires=["pydantic", "sqlalchemy", "numpy", "requests", "pyyaml"], long_description=long_description, license="MIT", url="https://github.com/hwchase17/langchain", diff --git a/test_requirements.txt b/test_requirements.txt index 4318012c..b8f7af92 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -9,3 +9,4 @@ mypy flake8 flake8-docstrings types-requests +types-PyYAML diff --git a/tests/integration_tests/vectorstores/__init__.py b/tests/integration_tests/vectorstores/__init__.py new file mode 100644 index 00000000..4396bb23 --- /dev/null +++ b/tests/integration_tests/vectorstores/__init__.py @@ -0,0 +1 @@ +"""Test vectorstores.""" diff --git a/tests/integration_tests/vectorstores/test_elasticsearch.py b/tests/integration_tests/vectorstores/test_elasticsearch.py new file mode 100644 index 00000000..d3fd801f --- /dev/null +++ b/tests/integration_tests/vectorstores/test_elasticsearch.py @@ -0,0 +1,42 @@ +"""Test ElasticSearch functionality.""" +from typing import List + +from langchain.docstore.document import Document +from langchain.embeddings.base import Embeddings +from langchain.vectorstores.elastic_vector_search import ElasticVectorSearch + + +class FakeEmbeddings(Embeddings): + """Fake embeddings functionality for testing.""" + + def embed_documents(self, texts: List[str]) -> List[List[float]]: + """Return simple embeddings.""" + return [[1.0] * 9 + [i] for i in range(len(texts))] + + def embed_query(self, text: str) -> List[float]: + """Return simple embeddings.""" + return [1.0] * 9 + [0.0] + + +def test_elasticsearch() -> None: + """Test end to end construction and search.""" + texts = ["foo", "bar", "baz"] + docsearch = ElasticVectorSearch.from_texts( + texts, FakeEmbeddings(), elasticsearch_url="http://localhost:9200" + ) + output = docsearch.similarity_search("foo", k=1) + assert output == [Document(page_content="foo")] + + +def test_elasticsearch_with_metadatas() -> None: + """Test end to end construction and search.""" + texts = ["foo", "bar", "baz"] + metadatas = [{"page": i} for i in range(len(texts))] + docsearch = ElasticVectorSearch.from_texts( + texts, + FakeEmbeddings(), + metadatas=metadatas, + elasticsearch_url="http://localhost:9200", + ) + output = docsearch.similarity_search("foo", k=1) + assert output == [Document(page_content="foo", metadata={"page": 0})] diff --git a/tests/integration_tests/test_faiss.py b/tests/integration_tests/vectorstores/test_faiss.py similarity index 68% rename from tests/integration_tests/test_faiss.py rename to tests/integration_tests/vectorstores/test_faiss.py index 8df51989..2b3cbd1d 100644 --- a/tests/integration_tests/test_faiss.py +++ b/tests/integration_tests/vectorstores/test_faiss.py @@ -37,6 +37,23 @@ def test_faiss() -> None: assert output == [Document(page_content="foo")] +def test_faiss_with_metadatas() -> None: + """Test end to end construction and search.""" + texts = ["foo", "bar", "baz"] + metadatas = [{"page": i} for i in range(len(texts))] + docsearch = FAISS.from_texts(texts, FakeEmbeddings(), metadatas=metadatas) + expected_docstore = InMemoryDocstore( + { + "0": Document(page_content="foo", metadata={"page": 0}), + "1": Document(page_content="bar", metadata={"page": 1}), + "2": Document(page_content="baz", metadata={"page": 2}), + } + ) + assert docsearch.docstore.__dict__ == expected_docstore.__dict__ + output = docsearch.similarity_search("foo", k=1) + assert output == [Document(page_content="foo", metadata={"page": 0})] + + def test_faiss_search_not_found() -> None: """Test what happens when document is not found.""" texts = ["foo", "bar", "baz"] diff --git a/tests/unit_tests/chains/test_llm.py b/tests/unit_tests/chains/test_llm.py index 0077df86..425713ff 100644 --- a/tests/unit_tests/chains/test_llm.py +++ b/tests/unit_tests/chains/test_llm.py @@ -2,14 +2,14 @@ import pytest from langchain.chains.llm import LLMChain -from langchain.prompts.prompt import Prompt +from langchain.prompts.prompt import PromptTemplate from tests.unit_tests.llms.fake_llm import FakeLLM @pytest.fixture def fake_llm_chain() -> LLMChain: """Fake LLM chain for testing purposes.""" - prompt = Prompt(input_variables=["bar"], template="This is a {bar}:") + prompt = PromptTemplate(input_variables=["bar"], template="This is a {bar}:") return LLMChain(prompt=prompt, llm=FakeLLM(), output_key="text1") diff --git a/tests/unit_tests/chains/test_mrkl.py b/tests/unit_tests/chains/test_mrkl.py index d881426a..7783361a 100644 --- a/tests/unit_tests/chains/test_mrkl.py +++ b/tests/unit_tests/chains/test_mrkl.py @@ -4,7 +4,7 @@ import pytest from langchain.chains.mrkl.base import ChainConfig, MRKLChain, get_action_and_input from langchain.chains.mrkl.prompt import BASE_TEMPLATE -from langchain.prompts import Prompt +from langchain.prompts import PromptTemplate from tests.unit_tests.llms.fake_llm import FakeLLM @@ -66,5 +66,5 @@ def test_from_chains() -> None: tools=expected_tools_prompt, tool_names=expected_tool_names ) prompt = mrkl_chain.prompt - assert isinstance(prompt, Prompt) + assert isinstance(prompt, PromptTemplate) assert prompt.template == expected_template diff --git a/tests/unit_tests/chains/test_react.py b/tests/unit_tests/chains/test_react.py index f7be9c54..b16c4e1d 100644 --- a/tests/unit_tests/chains/test_react.py +++ b/tests/unit_tests/chains/test_react.py @@ -9,7 +9,7 @@ from langchain.chains.react.base import ReActChain, predict_until_observation from langchain.docstore.base import Docstore from langchain.docstore.document import Document from langchain.llms.base import LLM -from langchain.prompts.prompt import Prompt +from langchain.prompts.prompt import PromptTemplate _PAGE_CONTENT = """This is a page about LangChain. @@ -19,7 +19,7 @@ What isn't there to love about langchain? Made in 2022.""" -_FAKE_PROMPT = Prompt(input_variables=["input"], template="{input}") +_FAKE_PROMPT = PromptTemplate(input_variables=["input"], template="{input}") class FakeListLLM(LLM): diff --git a/tests/unit_tests/prompts/__init__.py b/tests/unit_tests/prompts/__init__.py new file mode 100644 index 00000000..dc72afe0 --- /dev/null +++ b/tests/unit_tests/prompts/__init__.py @@ -0,0 +1 @@ +"""Test prompt functionality.""" diff --git a/tests/unit_tests/prompts/test_few_shot.py b/tests/unit_tests/prompts/test_few_shot.py new file mode 100644 index 00000000..59a0ac7a --- /dev/null +++ b/tests/unit_tests/prompts/test_few_shot.py @@ -0,0 +1,87 @@ +"""Test few shot prompt template.""" +import pytest + +from langchain.prompts.few_shot import FewShotPromptTemplate +from langchain.prompts.prompt import PromptTemplate + +EXAMPLE_PROMPT = PromptTemplate( + input_variables=["question", "answer"], template="{question}: {answer}" +) + + +def test_suffix_only() -> None: + """Test prompt works with just a suffix.""" + suffix = "This is a {foo} test." + input_variables = ["foo"] + prompt = FewShotPromptTemplate( + input_variables=input_variables, + suffix=suffix, + examples=[], + example_prompt=EXAMPLE_PROMPT, + ) + output = prompt.format(foo="bar") + expected_output = "This is a bar test." + assert output == expected_output + + +def test_prompt_missing_input_variables() -> None: + """Test error is raised when input variables are not provided.""" + # Test when missing in suffix + template = "This is a {foo} test." + with pytest.raises(ValueError): + FewShotPromptTemplate( + input_variables=[], + suffix=template, + examples=[], + example_prompt=EXAMPLE_PROMPT, + ) + + # Test when missing in prefix + template = "This is a {foo} test." + with pytest.raises(ValueError): + FewShotPromptTemplate( + input_variables=[], + suffix="foo", + examples=[], + prefix=template, + example_prompt=EXAMPLE_PROMPT, + ) + + +def test_prompt_extra_input_variables() -> None: + """Test error is raised when there are too many input variables.""" + template = "This is a {foo} test." + input_variables = ["foo", "bar"] + with pytest.raises(ValueError): + FewShotPromptTemplate( + input_variables=input_variables, + suffix=template, + examples=[], + example_prompt=EXAMPLE_PROMPT, + ) + + +def test_few_shot_functionality() -> None: + """Test that few shot works with examples.""" + prefix = "This is a test about {content}." + suffix = "Now you try to talk about {new_content}." + examples = [ + {"question": "foo", "answer": "bar"}, + {"question": "baz", "answer": "foo"}, + ] + prompt = FewShotPromptTemplate( + suffix=suffix, + prefix=prefix, + input_variables=["content", "new_content"], + examples=examples, + example_prompt=EXAMPLE_PROMPT, + example_separator="\n", + ) + output = prompt.format(content="animals", new_content="party") + expected_output = ( + "This is a test about animals.\n" + "foo: bar\n" + "baz: foo\n" + "Now you try to talk about party." + ) + assert output == expected_output diff --git a/tests/unit_tests/prompts/test_length_based_example_selector.py b/tests/unit_tests/prompts/test_length_based_example_selector.py new file mode 100644 index 00000000..7f9ca3cd --- /dev/null +++ b/tests/unit_tests/prompts/test_length_based_example_selector.py @@ -0,0 +1,48 @@ +"""Test functionality related to dynamic prompts.""" +import pytest + +from langchain.prompts.example_selector.length_based import LengthBasedExampleSelector +from langchain.prompts.prompt import PromptTemplate + +EXAMPLES = [ + {"question": "Question: who are you?\nAnswer: foo"}, + {"question": "Question: who are you?\nAnswer: foo"}, +] + + +@pytest.fixture +def selector() -> LengthBasedExampleSelector: + """Get length based selector to use in tests.""" + prompts = PromptTemplate(input_variables=["question"], template="{question}") + selector = LengthBasedExampleSelector( + examples=EXAMPLES, + example_prompt=prompts, + max_length=25, + ) + return selector + + +def test_dynamic_prompt_valid(selector: LengthBasedExampleSelector) -> None: + """Test dynamic prompt can be successfully constructed from examples.""" + short_question = "Short question?" + output = selector.select_examples({"question": short_question}) + assert output == EXAMPLES + + +def test_dynamic_prompt_trims_one_example(selector: LengthBasedExampleSelector) -> None: + """Test dynamic prompt can trim one example.""" + long_question = """I am writing a really long question, + this probably is going to affect the example right?""" + output = selector.select_examples({"question": long_question}) + assert output == EXAMPLES[:1] + + +def test_dynamic_prompt_trims_all_examples( + selector: LengthBasedExampleSelector, +) -> None: + """Test dynamic prompt can trim all examples.""" + longest_question = """This question is super super super, + super super super super super super super super super super super, + super super super super long, this will affect the example right?""" + output = selector.select_examples({"question": longest_question}) + assert output == [] diff --git a/tests/unit_tests/prompts/test_loading.py b/tests/unit_tests/prompts/test_loading.py new file mode 100644 index 00000000..6af7f24c --- /dev/null +++ b/tests/unit_tests/prompts/test_loading.py @@ -0,0 +1,134 @@ +"""Test loading functionality.""" + +import os +from contextlib import contextmanager +from pathlib import Path +from typing import Iterator + +from langchain.prompts.few_shot import FewShotPromptTemplate +from langchain.prompts.loading import load_prompt +from langchain.prompts.prompt import PromptTemplate + + +@contextmanager +def change_directory() -> Iterator: + """Change the working directory to the right folder.""" + origin = Path().absolute() + try: + os.chdir("docs/examples/prompts") + yield + finally: + os.chdir(origin) + + +def test_loading_from_YAML() -> None: + """Test loading from yaml file.""" + with change_directory(): + prompt = load_prompt("simple_prompt.yaml") + expected_prompt = PromptTemplate( + input_variables=["adjective", "content"], + template="Tell me a {adjective} joke about {content}.", + ) + assert prompt == expected_prompt + + +def test_loading_from_JSON() -> None: + """Test loading from json file.""" + with change_directory(): + prompt = load_prompt("simple_prompt.json") + expected_prompt = PromptTemplate( + input_variables=["adjective", "content"], + template="Tell me a {adjective} joke about {content}.", + ) + assert prompt == expected_prompt + + +def test_loading_with_template_as_file() -> None: + """Test loading when the template is a file.""" + with change_directory(): + prompt = load_prompt("simple_prompt_with_template_file.json") + expected_prompt = PromptTemplate( + input_variables=["adjective", "content"], + template="Tell me a {adjective} joke about {content}.", + ) + assert prompt == expected_prompt + + +def test_loading_few_shot_prompt_from_yaml() -> None: + """Test loading few shot prompt from yaml.""" + with change_directory(): + prompt = load_prompt("few_shot_prompt.yaml") + expected_prompt = FewShotPromptTemplate( + input_variables=["adjective"], + prefix="Write antonyms for the following words.", + example_prompt=PromptTemplate( + input_variables=["input", "output"], + template="Input: {input}\nOutput: {output}", + ), + examples=[ + {"input": "happy", "output": "sad"}, + {"input": "tall", "output": "short"}, + ], + suffix="Input: {adjective}\nOutput:", + ) + assert prompt == expected_prompt + + +def test_loading_few_shot_prompt_from_json() -> None: + """Test loading few shot prompt from json.""" + with change_directory(): + prompt = load_prompt("few_shot_prompt.json") + expected_prompt = FewShotPromptTemplate( + input_variables=["adjective"], + prefix="Write antonyms for the following words.", + example_prompt=PromptTemplate( + input_variables=["input", "output"], + template="Input: {input}\nOutput: {output}", + ), + examples=[ + {"input": "happy", "output": "sad"}, + {"input": "tall", "output": "short"}, + ], + suffix="Input: {adjective}\nOutput:", + ) + assert prompt == expected_prompt + + +def test_loading_few_shot_prompt_when_examples_in_config() -> None: + """Test loading few shot prompt when the examples are in the config.""" + with change_directory(): + prompt = load_prompt("few_shot_prompt_examples_in.json") + expected_prompt = FewShotPromptTemplate( + input_variables=["adjective"], + prefix="Write antonyms for the following words.", + example_prompt=PromptTemplate( + input_variables=["input", "output"], + template="Input: {input}\nOutput: {output}", + ), + examples=[ + {"input": "happy", "output": "sad"}, + {"input": "tall", "output": "short"}, + ], + suffix="Input: {adjective}\nOutput:", + ) + assert prompt == expected_prompt + + +def test_loading_few_shot_prompt_example_prompt() -> None: + """Test loading few shot when the example prompt is in its own file.""" + with change_directory(): + prompt = load_prompt("few_shot_prompt_example_prompt.json") + expected_prompt = FewShotPromptTemplate( + input_variables=["adjective"], + prefix="Write antonyms for the following words.", + example_prompt=PromptTemplate( + input_variables=["input", "output"], + template="Input: {input}\nOutput: {output}", + ), + examples=[ + {"input": "happy", "output": "sad"}, + {"input": "tall", "output": "short"}, + ], + suffix="Input: {adjective}\nOutput:", + ) + assert prompt == expected_prompt diff --git a/tests/unit_tests/test_prompt.py b/tests/unit_tests/prompts/test_prompt.py similarity index 79% rename from tests/unit_tests/test_prompt.py rename to tests/unit_tests/prompts/test_prompt.py index 7265cae3..cec597e0 100644 --- a/tests/unit_tests/test_prompt.py +++ b/tests/unit_tests/prompts/test_prompt.py @@ -1,14 +1,14 @@ """Test functionality related to prompts.""" import pytest -from langchain.prompts.prompt import Prompt +from langchain.prompts.prompt import PromptTemplate def test_prompt_valid() -> None: """Test prompts can be constructed.""" template = "This is a {foo} test." input_variables = ["foo"] - prompt = Prompt(input_variables=input_variables, template=template) + prompt = PromptTemplate(input_variables=input_variables, template=template) assert prompt.template == template assert prompt.input_variables == input_variables @@ -18,7 +18,7 @@ def test_prompt_missing_input_variables() -> None: template = "This is a {foo} test." input_variables: list = [] with pytest.raises(ValueError): - Prompt(input_variables=input_variables, template=template) + PromptTemplate(input_variables=input_variables, template=template) def test_prompt_extra_input_variables() -> None: @@ -26,7 +26,7 @@ def test_prompt_extra_input_variables() -> None: template = "This is a {foo} test." input_variables = ["foo", "bar"] with pytest.raises(ValueError): - Prompt(input_variables=input_variables, template=template) + PromptTemplate(input_variables=input_variables, template=template) def test_prompt_wrong_input_variables() -> None: @@ -34,7 +34,7 @@ def test_prompt_wrong_input_variables() -> None: template = "This is a {foo} test." input_variables = ["bar"] with pytest.raises(ValueError): - Prompt(input_variables=input_variables, template=template) + PromptTemplate(input_variables=input_variables, template=template) def test_prompt_from_examples_valid() -> None: @@ -57,14 +57,16 @@ Answer:""" """Question: who are you?\nAnswer: foo""", """Question: what are you?\nAnswer: bar""", ] - prompt_from_examples = Prompt.from_examples( + prompt_from_examples = PromptTemplate.from_examples( examples, suffix, input_variables, example_separator=example_separator, prefix=prefix, ) - prompt_from_template = Prompt(input_variables=input_variables, template=template) + prompt_from_template = PromptTemplate( + input_variables=input_variables, template=template + ) assert prompt_from_examples.template == prompt_from_template.template assert prompt_from_examples.input_variables == prompt_from_template.input_variables @@ -74,7 +76,7 @@ def test_prompt_invalid_template_format() -> None: template = "This is a {foo} test." input_variables = ["foo"] with pytest.raises(ValueError): - Prompt( + PromptTemplate( input_variables=input_variables, template=template, template_format="bar" ) @@ -83,5 +85,5 @@ def test_prompt_from_file() -> None: """Test prompt can be successfully constructed from a file.""" template_file = "tests/unit_tests/data/prompt_file.txt" input_variables = ["question"] - prompt = Prompt.from_file(template_file, input_variables) + prompt = PromptTemplate.from_file(template_file, input_variables) assert prompt.template == "Question: {question}\nAnswer:" diff --git a/tests/unit_tests/test_dynamic_prompt.py b/tests/unit_tests/test_dynamic_prompt.py deleted file mode 100644 index 72f56eea..00000000 --- a/tests/unit_tests/test_dynamic_prompt.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Test functionality related to dynamic prompts.""" -from langchain.prompts.dynamic import DynamicPrompt -from langchain.prompts.prompt import Prompt - -# FULL TEMPLATES -LONGER_TEMPLATE = """Test Prompt: - -Question: who are you? -Answer: foo - -Question: what are you? -Answer: bar - -Question: {question} -Answer:""" -SHORTER_TEMPLATE = """Test Prompt: - -Question: who are you? -Answer: foo - -Question: {question} -Answer:""" -SHORTEST_TEMPLATE = """Test Prompt: - -Question: {question} -Answer:""" - -# DYNAMIC PROMPT COMPONENTS -PREFIX = """Test Prompt:""" -SUFFIX = """Question: {question}\nAnswer:""" -EXAMPLES = [ - """Question: who are you?\nAnswer: foo""", - """Question: what are you?\nAnswer: bar""", -] - -# INPUTS -TEST_LONG_QUESTION = """I am writing a really long question, -this probably is going to affect the example right?""" -TEST_LONGEST_QUESTION = """This question is super super super, -super super super super super super super super super super super, -super super super super long, this will affect the example right?""" -TEST_SHORT_QUESTION = "Short question?" - - -def test_dynamic_prompt_valid() -> None: - """Test dynamic prompt can be successfully constructed from examples.""" - input_variables = ["question"] - example_separator = "\n\n" - dynamic_prompt_cls = DynamicPrompt( - examples=EXAMPLES, - suffix=SUFFIX, - input_variables=input_variables, - example_separator=example_separator, - prefix=PREFIX, - ) - prompt_cls = Prompt(input_variables=input_variables, template=LONGER_TEMPLATE) - dynamic_prompt_template = dynamic_prompt_cls.format(question="foo?") - prompt_template = prompt_cls.format(question="foo?") - assert dynamic_prompt_template == prompt_template - assert dynamic_prompt_cls.input_variables == prompt_cls.input_variables - - -def test_dynamic_prompt_trims_one_example() -> None: - """Test dynamic prompt can trim one example.""" - input_variables = ["question"] - example_separator = "\n\n" - dynamic_prompt_cls = DynamicPrompt( - examples=EXAMPLES, - suffix=SUFFIX, - input_variables=input_variables, - example_separator=example_separator, - prefix=PREFIX, - max_length=30, - ) - dynamic_prompt = dynamic_prompt_cls.format(question=TEST_LONG_QUESTION) - shorter_prompt = SHORTER_TEMPLATE.format(question=TEST_LONG_QUESTION) - assert dynamic_prompt == shorter_prompt - - -def test_dynamic_prompt_trims_no_examples() -> None: - """Test dynamic prompt can trim no examples.""" - input_variables = ["question"] - example_separator = "\n\n" - dynamic_prompt_cls = DynamicPrompt( - examples=EXAMPLES, - suffix=SUFFIX, - input_variables=input_variables, - example_separator=example_separator, - prefix=PREFIX, - max_length=30, - ) - dynamic_prompt = dynamic_prompt_cls.format(question=TEST_SHORT_QUESTION) - full_prompt = LONGER_TEMPLATE.format(question=TEST_SHORT_QUESTION) - assert dynamic_prompt == full_prompt - - -def test_dynamic_prompt_trims_all_examples() -> None: - """Test dynamic prompt can trim all examples.""" - input_variables = ["question"] - example_separator = "\n\n" - dynamic_prompt_cls = DynamicPrompt( - examples=EXAMPLES, - suffix=SUFFIX, - input_variables=input_variables, - example_separator=example_separator, - prefix=PREFIX, - max_length=30, - ) - dynamic_prompt = dynamic_prompt_cls.format(question=TEST_LONGEST_QUESTION) - full_prompt = SHORTEST_TEMPLATE.format(question=TEST_LONGEST_QUESTION) - assert dynamic_prompt == full_prompt From a19ad935b3329ff6c745a8a51343eb3ece48d949 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Sat, 19 Nov 2022 20:39:35 -0800 Subject: [PATCH 2/2] Harrison/verbose prompt (#159) Add printing of prompt to LLMChain --- docs/examples/demos/simple_prompts.ipynb | 21 ++++++++++++++++++--- langchain/chains/llm.py | 5 ++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/examples/demos/simple_prompts.ipynb b/docs/examples/demos/simple_prompts.ipynb index 4299261d..208b2e48 100644 --- a/docs/examples/demos/simple_prompts.ipynb +++ b/docs/examples/demos/simple_prompts.ipynb @@ -12,17 +12,32 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "51a54c4d", "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mQuestion: What NFL team won the Super Bowl in the year Justin Beiber was born?\n", + "\n", + "Answer: Let's think step by step.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n" + ] + }, { "data": { "text/plain": [ "' The year Justin Beiber was born was 1994. In 1994, the Dallas Cowboys won the Super Bowl.'" ] }, - "execution_count": 2, + "execution_count": 1, "metadata": {}, "output_type": "execute_result" } @@ -34,7 +49,7 @@ "\n", "Answer: Let's think step by step.\"\"\"\n", "prompt = PromptTemplate(template=template, input_variables=[\"question\"])\n", - "llm_chain = LLMChain(prompt=prompt, llm=OpenAI(temperature=0))\n", + "llm_chain = LLMChain(prompt=prompt, llm=OpenAI(temperature=0), verbose=True)\n", "\n", "question = \"What NFL team won the Super Bowl in the year Justin Beiber was born?\"\n", "\n", diff --git a/langchain/chains/llm.py b/langchain/chains/llm.py index 5133129e..93ac27cb 100644 --- a/langchain/chains/llm.py +++ b/langchain/chains/llm.py @@ -4,6 +4,7 @@ from typing import Any, Dict, List from pydantic import BaseModel, Extra from langchain.chains.base import Chain +from langchain.input import print_text from langchain.llms.base import LLM from langchain.prompts.base import BasePromptTemplate @@ -53,7 +54,9 @@ class LLMChain(Chain, BaseModel): def _call(self, inputs: Dict[str, Any]) -> Dict[str, str]: selected_inputs = {k: inputs[k] for k in self.prompt.input_variables} prompt = self.prompt.format(**selected_inputs) - + if self.verbose: + print("Prompt after formatting:") + print_text(prompt, color="green", end="\n") kwargs = {} if "stop" in inputs: kwargs["stop"] = inputs["stop"]