diff --git a/docs/modules/chains/examples/llm_summarization_checker.ipynb b/docs/modules/chains/examples/llm_summarization_checker.ipynb new file mode 100644 index 0000000000..7448f84f5f --- /dev/null +++ b/docs/modules/chains/examples/llm_summarization_checker.ipynb @@ -0,0 +1,1124 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LLMSummarizationCheckerChain\n", + "This notebook shows some examples of LLMSummarizationCheckerChain in use with different types of texts. It has a few distinct differences from the `LLMCheckerChain`, in that it doesn't have any assumtions to the format of the input text (or summary).\n", + "Additionally, as the LLMs like to hallucinate when fact checking or get confused by context, it is sometimes beneficial to run the checker multiple times. It does this by feeding the rewritten \"True\" result back on itself, and checking the \"facts\" for truth. As you can see from the examples below, this can be very effective in arriving at a generally true body of text.\n", + "\n", + "You can control the number of times the checker runs by setting the `max_checks` parameter. The default is 2, but you can set it to 1 if you don't want any double-checking." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new LLMSummarizationCheckerChain chain...\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new SequentialChain chain...\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mGiven some text, extract a list of facts from the text.\n", + "\n", + "Format your output as a bulleted list.\n", + "\n", + "Text:\n", + "\"\"\"\n", + "\n", + "Your 9-year old might like these recent discoveries made by The James Webb Space Telescope (JWST):\n", + "• In 2023, The JWST spotted a number of galaxies nicknamed \"green peas.\" They were given this name because they are small, round, and green, like peas.\n", + "• The telescope captured images of galaxies that are over 13 billion years old. This means that the light from these galaxies has been traveling for over 13 billion years to reach us.\n", + "• JWST took the very first pictures of a planet outside of our own solar system. These distant worlds are called \"exoplanets.\" Exo means \"from outside.\"\n", + "These discoveries can spark a child's imagination about the infinite wonders of the universe.\n", + "\"\"\"\n", + "\n", + "Facts:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mYou are an expert fact checker. You have been hired by a major news organization to fact check a very important story.\n", + "\n", + "Here is a bullet point list of facts:\n", + "\"\"\"\n", + "\n", + "• The James Webb Space Telescope (JWST) spotted a number of galaxies nicknamed \"green peas.\"\n", + "• The telescope captured images of galaxies that are over 13 billion years old.\n", + "• JWST took the very first pictures of a planet outside of our own solar system.\n", + "• These distant worlds are called \"exoplanets.\"\n", + "\"\"\"\n", + "\n", + "For each fact, determine whether it is true or false about the subject. If you are unable to determine whether the fact is true or false, output \"Undetermined\".\n", + "If the fact is false, explain why.\n", + "\n", + "\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mBelow are some assertions that have been fact checked and are labeled as true of false. If the answer is false, a suggestion is given for a correction.\n", + "\n", + "Checked Assertions:\n", + "\"\"\"\n", + "• The James Webb Space Telescope (JWST) spotted a number of galaxies nicknamed \"green peas.\" - True \n", + "\n", + "• The telescope captured images of galaxies that are over 13 billion years old. - True \n", + "\n", + "• JWST took the very first pictures of a planet outside of our own solar system. - False. The first exoplanet was discovered in 1992, before the JWST was launched. \n", + "\n", + "• These distant worlds are called \"exoplanets.\" - True\n", + "\"\"\"\n", + "\n", + "Original Summary:\n", + "\"\"\"\n", + "\n", + "Your 9-year old might like these recent discoveries made by The James Webb Space Telescope (JWST):\n", + "• In 2023, The JWST spotted a number of galaxies nicknamed \"green peas.\" They were given this name because they are small, round, and green, like peas.\n", + "• The telescope captured images of galaxies that are over 13 billion years old. This means that the light from these galaxies has been traveling for over 13 billion years to reach us.\n", + "• JWST took the very first pictures of a planet outside of our own solar system. These distant worlds are called \"exoplanets.\" Exo means \"from outside.\"\n", + "These discoveries can spark a child's imagination about the infinite wonders of the universe.\n", + "\"\"\"\n", + "\n", + "Using these checked assertions, rewrite the original summary to be completely true.\n", + "\n", + "The output should have the same structure and formatting as the original summary.\n", + "\n", + "Summary:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mBelow are some assertions that have been fact checked and are labeled as true or false.\n", + "\n", + "If all of the assertions are true, return \"True\". If any of the assertions are false, return \"False\".\n", + "\n", + "Here are some examples:\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is red: False\n", + "- Water is made of lava: False\n", + "- The sun is a star: True\n", + "\"\"\"\n", + "Result: False\n", + "\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is blue: True\n", + "- Water is wet: True\n", + "- The sun is a star: True\n", + "\"\"\"\n", + "Result: True\n", + "\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is blue - True\n", + "- Water is made of lava- False\n", + "- The sun is a star - True\n", + "\"\"\"\n", + "Result: False\n", + "\n", + "===\n", + "\n", + "Checked Assertions:\"\"\"\n", + "• The James Webb Space Telescope (JWST) spotted a number of galaxies nicknamed \"green peas.\" - True \n", + "\n", + "• The telescope captured images of galaxies that are over 13 billion years old. - True \n", + "\n", + "• JWST took the very first pictures of a planet outside of our own solar system. - False. The first exoplanet was discovered in 1992, before the JWST was launched. \n", + "\n", + "• These distant worlds are called \"exoplanets.\" - True\n", + "\"\"\"\n", + "Result:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "Your 9-year old might like these recent discoveries made by The James Webb Space Telescope (JWST):\n", + "• In 2023, The JWST spotted a number of galaxies nicknamed \"green peas.\" They were given this name because they are small, round, and green, like peas.\n", + "• The telescope captured images of galaxies that are over 13 billion years old. This means that the light from these galaxies has been traveling for over 13 billion years to reach us.\n", + "• JWST has provided us with the first images of exoplanets, which are planets outside of our own solar system. These distant worlds were first discovered in 1992, and the JWST has allowed us to see them in greater detail.\n", + "These discoveries can spark a child's imagination about the infinite wonders of the universe.\n", + "\n", + "\n", + "\u001b[1m> Entering new SequentialChain chain...\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mGiven some text, extract a list of facts from the text.\n", + "\n", + "Format your output as a bulleted list.\n", + "\n", + "Text:\n", + "\"\"\"\n", + "\n", + "\n", + "Your 9-year old might like these recent discoveries made by The James Webb Space Telescope (JWST):\n", + "• In 2023, The JWST spotted a number of galaxies nicknamed \"green peas.\" They were given this name because they are small, round, and green, like peas.\n", + "• The telescope captured images of galaxies that are over 13 billion years old. This means that the light from these galaxies has been traveling for over 13 billion years to reach us.\n", + "• JWST has provided us with the first images of exoplanets, which are planets outside of our own solar system. These distant worlds were first discovered in 1992, and the JWST has allowed us to see them in greater detail.\n", + "These discoveries can spark a child's imagination about the infinite wonders of the universe.\n", + "\"\"\"\n", + "\n", + "Facts:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mYou are an expert fact checker. You have been hired by a major news organization to fact check a very important story.\n", + "\n", + "Here is a bullet point list of facts:\n", + "\"\"\"\n", + "\n", + "• The James Webb Space Telescope (JWST) spotted a number of galaxies nicknamed \"green peas.\"\n", + "• The light from these galaxies has been traveling for over 13 billion years to reach us.\n", + "• JWST has provided us with the first images of exoplanets, which are planets outside of our own solar system.\n", + "• Exoplanets were first discovered in 1992.\n", + "• The JWST has allowed us to see exoplanets in greater detail.\n", + "\"\"\"\n", + "\n", + "For each fact, determine whether it is true or false about the subject. If you are unable to determine whether the fact is true or false, output \"Undetermined\".\n", + "If the fact is false, explain why.\n", + "\n", + "\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mBelow are some assertions that have been fact checked and are labeled as true of false. If the answer is false, a suggestion is given for a correction.\n", + "\n", + "Checked Assertions:\n", + "\"\"\"\n", + "\n", + "• The James Webb Space Telescope (JWST) spotted a number of galaxies nicknamed \"green peas.\" - True \n", + "\n", + "• The light from these galaxies has been traveling for over 13 billion years to reach us. - True \n", + "\n", + "• JWST has provided us with the first images of exoplanets, which are planets outside of our own solar system. - False. The first exoplanet was discovered in 1992, but the first images of exoplanets were taken by the Hubble Space Telescope in 1995. \n", + "\n", + "• Exoplanets were first discovered in 1992. - True \n", + "\n", + "• The JWST has allowed us to see exoplanets in greater detail. - Undetermined. It is too early to tell as the JWST has not been launched yet.\n", + "\"\"\"\n", + "\n", + "Original Summary:\n", + "\"\"\"\n", + "\n", + "\n", + "Your 9-year old might like these recent discoveries made by The James Webb Space Telescope (JWST):\n", + "• In 2023, The JWST spotted a number of galaxies nicknamed \"green peas.\" They were given this name because they are small, round, and green, like peas.\n", + "• The telescope captured images of galaxies that are over 13 billion years old. This means that the light from these galaxies has been traveling for over 13 billion years to reach us.\n", + "• JWST has provided us with the first images of exoplanets, which are planets outside of our own solar system. These distant worlds were first discovered in 1992, and the JWST has allowed us to see them in greater detail.\n", + "These discoveries can spark a child's imagination about the infinite wonders of the universe.\n", + "\"\"\"\n", + "\n", + "Using these checked assertions, rewrite the original summary to be completely true.\n", + "\n", + "The output should have the same structure and formatting as the original summary.\n", + "\n", + "Summary:\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mBelow are some assertions that have been fact checked and are labeled as true or false.\n", + "\n", + "If all of the assertions are true, return \"True\". If any of the assertions are false, return \"False\".\n", + "\n", + "Here are some examples:\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is red: False\n", + "- Water is made of lava: False\n", + "- The sun is a star: True\n", + "\"\"\"\n", + "Result: False\n", + "\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is blue: True\n", + "- Water is wet: True\n", + "- The sun is a star: True\n", + "\"\"\"\n", + "Result: True\n", + "\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is blue - True\n", + "- Water is made of lava- False\n", + "- The sun is a star - True\n", + "\"\"\"\n", + "Result: False\n", + "\n", + "===\n", + "\n", + "Checked Assertions:\"\"\"\n", + "\n", + "• The James Webb Space Telescope (JWST) spotted a number of galaxies nicknamed \"green peas.\" - True \n", + "\n", + "• The light from these galaxies has been traveling for over 13 billion years to reach us. - True \n", + "\n", + "• JWST has provided us with the first images of exoplanets, which are planets outside of our own solar system. - False. The first exoplanet was discovered in 1992, but the first images of exoplanets were taken by the Hubble Space Telescope in 1995. \n", + "\n", + "• Exoplanets were first discovered in 1992. - True \n", + "\n", + "• The JWST has allowed us to see exoplanets in greater detail. - Undetermined. It is too early to tell as the JWST has not been launched yet.\n", + "\"\"\"\n", + "Result:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "Your 9-year old might like these recent discoveries made by The James Webb Space Telescope (JWST):\n", + "• In 2023, The JWST will spot a number of galaxies nicknamed \"green peas.\" They were given this name because they are small, round, and green, like peas.\n", + "• The telescope will capture images of galaxies that are over 13 billion years old. This means that the light from these galaxies has been traveling for over 13 billion years to reach us.\n", + "• Exoplanets, which are planets outside of our own solar system, were first discovered in 1992. The JWST will allow us to see them in greater detail than ever before.\n", + "These discoveries can spark a child's imagination about the infinite wonders of the universe.\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n" + ] + }, + { + "data": { + "text/plain": [ + "'Your 9-year old might like these recent discoveries made by The James Webb Space Telescope (JWST):\\n• In 2023, The JWST will spot a number of galaxies nicknamed \"green peas.\" They were given this name because they are small, round, and green, like peas.\\n• The telescope will capture images of galaxies that are over 13 billion years old. This means that the light from these galaxies has been traveling for over 13 billion years to reach us.\\n• Exoplanets, which are planets outside of our own solar system, were first discovered in 1992. The JWST will allow us to see them in greater detail than ever before.\\nThese discoveries can spark a child\\'s imagination about the infinite wonders of the universe.'" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from langchain.chains import LLMSummarizationCheckerChain\n", + "from langchain.llms import OpenAI\n", + "\n", + "llm = OpenAI(temperature=0)\n", + "checker_chain = LLMSummarizationCheckerChain(llm=llm, verbose=True, max_checks=2)\n", + "text = \"\"\"\n", + "Your 9-year old might like these recent discoveries made by The James Webb Space Telescope (JWST):\n", + "• In 2023, The JWST spotted a number of galaxies nicknamed \"green peas.\" They were given this name because they are small, round, and green, like peas.\n", + "• The telescope captured images of galaxies that are over 13 billion years old. This means that the light from these galaxies has been traveling for over 13 billion years to reach us.\n", + "• JWST took the very first pictures of a planet outside of our own solar system. These distant worlds are called \"exoplanets.\" Exo means \"from outside.\"\n", + "These discoveries can spark a child's imagination about the infinite wonders of the universe.\"\"\"\n", + "checker_chain.run(text)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new LLMSummarizationCheckerChain chain...\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new SequentialChain chain...\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mGiven some text, extract a list of facts from the text.\n", + "\n", + "Format your output as a bulleted list.\n", + "\n", + "Text:\n", + "\"\"\"\n", + "The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. It has an area of 465,000 square miles and is one of five oceans in the world, alongside the Pacific Ocean, Atlantic Ocean, Indian Ocean, and the Southern Ocean. It is the smallest of the five oceans and is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. The sea is named after the island of Greenland, and is the Arctic Ocean's main outlet to the Atlantic. It is often frozen over so navigation is limited, and is considered the northern branch of the Norwegian Sea.\n", + "\"\"\"\n", + "\n", + "Facts:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mYou are an expert fact checker. You have been hired by a major news organization to fact check a very important story.\n", + "\n", + "Here is a bullet point list of facts:\n", + "\"\"\"\n", + "\n", + "- The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland.\n", + "- It has an area of 465,000 square miles.\n", + "- It is one of five oceans in the world, alongside the Pacific Ocean, Atlantic Ocean, Indian Ocean, and the Southern Ocean.\n", + "- It is the smallest of the five oceans.\n", + "- It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs.\n", + "- The sea is named after the island of Greenland.\n", + "- It is the Arctic Ocean's main outlet to the Atlantic.\n", + "- It is often frozen over so navigation is limited.\n", + "- It is considered the northern branch of the Norwegian Sea.\n", + "\"\"\"\n", + "\n", + "For each fact, determine whether it is true or false about the subject. If you are unable to determine whether the fact is true or false, output \"Undetermined\".\n", + "If the fact is false, explain why.\n", + "\n", + "\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mBelow are some assertions that have been fact checked and are labeled as true of false. If the answer is false, a suggestion is given for a correction.\n", + "\n", + "Checked Assertions:\"\"\"\n", + "\n", + "- The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. True\n", + "\n", + "- It has an area of 465,000 square miles. True\n", + "\n", + "- It is one of five oceans in the world, alongside the Pacific Ocean, Atlantic Ocean, Indian Ocean, and the Southern Ocean. False - The Greenland Sea is not an ocean, it is an arm of the Arctic Ocean.\n", + "\n", + "- It is the smallest of the five oceans. False - The Greenland Sea is not an ocean, it is an arm of the Arctic Ocean.\n", + "\n", + "- It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. True\n", + "\n", + "- The sea is named after the island of Greenland. True\n", + "\n", + "- It is the Arctic Ocean's main outlet to the Atlantic. True\n", + "\n", + "- It is often frozen over so navigation is limited. True\n", + "\n", + "- It is considered the northern branch of the Norwegian Sea. True\n", + "\"\"\"\n", + "\n", + "Original Summary:\"\"\"\n", + "The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. It has an area of 465,000 square miles and is one of five oceans in the world, alongside the Pacific Ocean, Atlantic Ocean, Indian Ocean, and the Southern Ocean. It is the smallest of the five oceans and is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. The sea is named after the island of Greenland, and is the Arctic Ocean's main outlet to the Atlantic. It is often frozen over so navigation is limited, and is considered the northern branch of the Norwegian Sea.\n", + "\"\"\"\n", + "\n", + "Using these checked assertions, rewrite the original summary to be completely true.\n", + "\n", + "The output should have the same structure and formatting as the original summary.\n", + "\n", + "Summary:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mBelow are some assertions that have been fact checked and are labeled as true of false.\n", + "\n", + "If all of the assertions are true, return \"True\". If any of the assertions are false, return \"False\".\n", + "\n", + "Here are some examples:\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is red: False\n", + "- Water is made of lava: False\n", + "- The sun is a star: True\n", + "\"\"\"\n", + "Result: False\n", + "\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is blue: True\n", + "- Water is wet: True\n", + "- The sun is a star: True\n", + "\"\"\"\n", + "Result: True\n", + "\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is blue - True\n", + "- Water is made of lava- False\n", + "- The sun is a star - True\n", + "\"\"\"\n", + "Result: False\n", + "\n", + "===\n", + "\n", + "Checked Assertions:\"\"\"\n", + "\n", + "- The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. True\n", + "\n", + "- It has an area of 465,000 square miles. True\n", + "\n", + "- It is one of five oceans in the world, alongside the Pacific Ocean, Atlantic Ocean, Indian Ocean, and the Southern Ocean. False - The Greenland Sea is not an ocean, it is an arm of the Arctic Ocean.\n", + "\n", + "- It is the smallest of the five oceans. False - The Greenland Sea is not an ocean, it is an arm of the Arctic Ocean.\n", + "\n", + "- It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. True\n", + "\n", + "- The sea is named after the island of Greenland. True\n", + "\n", + "- It is the Arctic Ocean's main outlet to the Atlantic. True\n", + "\n", + "- It is often frozen over so navigation is limited. True\n", + "\n", + "- It is considered the northern branch of the Norwegian Sea. True\n", + "\"\"\"\n", + "Result:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. It has an area of 465,000 square miles and is an arm of the Arctic Ocean. It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. The sea is named after the island of Greenland, and is the Arctic Ocean's main outlet to the Atlantic. It is often frozen over so navigation is limited, and is considered the northern branch of the Norwegian Sea.\n", + "\n", + "\n", + "\u001b[1m> Entering new SequentialChain chain...\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mGiven some text, extract a list of facts from the text.\n", + "\n", + "Format your output as a bulleted list.\n", + "\n", + "Text:\n", + "\"\"\"\n", + "\n", + "The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. It has an area of 465,000 square miles and is an arm of the Arctic Ocean. It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. The sea is named after the island of Greenland, and is the Arctic Ocean's main outlet to the Atlantic. It is often frozen over so navigation is limited, and is considered the northern branch of the Norwegian Sea.\n", + "\"\"\"\n", + "\n", + "Facts:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mYou are an expert fact checker. You have been hired by a major news organization to fact check a very important story.\n", + "\n", + "Here is a bullet point list of facts:\n", + "\"\"\"\n", + "\n", + "- The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland.\n", + "- It has an area of 465,000 square miles.\n", + "- It is an arm of the Arctic Ocean.\n", + "- It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs.\n", + "- It is named after the island of Greenland.\n", + "- It is the Arctic Ocean's main outlet to the Atlantic.\n", + "- It is often frozen over so navigation is limited.\n", + "- It is considered the northern branch of the Norwegian Sea.\n", + "\"\"\"\n", + "\n", + "For each fact, determine whether it is true or false about the subject. If you are unable to determine whether the fact is true or false, output \"Undetermined\".\n", + "If the fact is false, explain why.\n", + "\n", + "\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mBelow are some assertions that have been fact checked and are labeled as true of false. If the answer is false, a suggestion is given for a correction.\n", + "\n", + "Checked Assertions:\"\"\"\n", + "\n", + "- The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. True\n", + "\n", + "- It has an area of 465,000 square miles. True\n", + "\n", + "- It is an arm of the Arctic Ocean. True\n", + "\n", + "- It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. True\n", + "\n", + "- It is named after the island of Greenland. False - It is named after the country of Greenland.\n", + "\n", + "- It is the Arctic Ocean's main outlet to the Atlantic. True\n", + "\n", + "- It is often frozen over so navigation is limited. True\n", + "\n", + "- It is considered the northern branch of the Norwegian Sea. False - It is considered the northern branch of the Atlantic Ocean.\n", + "\"\"\"\n", + "\n", + "Original Summary:\"\"\"\n", + "\n", + "The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. It has an area of 465,000 square miles and is an arm of the Arctic Ocean. It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. The sea is named after the island of Greenland, and is the Arctic Ocean's main outlet to the Atlantic. It is often frozen over so navigation is limited, and is considered the northern branch of the Norwegian Sea.\n", + "\"\"\"\n", + "\n", + "Using these checked assertions, rewrite the original summary to be completely true.\n", + "\n", + "The output should have the same structure and formatting as the original summary.\n", + "\n", + "Summary:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mBelow are some assertions that have been fact checked and are labeled as true of false.\n", + "\n", + "If all of the assertions are true, return \"True\". If any of the assertions are false, return \"False\".\n", + "\n", + "Here are some examples:\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is red: False\n", + "- Water is made of lava: False\n", + "- The sun is a star: True\n", + "\"\"\"\n", + "Result: False\n", + "\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is blue: True\n", + "- Water is wet: True\n", + "- The sun is a star: True\n", + "\"\"\"\n", + "Result: True\n", + "\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is blue - True\n", + "- Water is made of lava- False\n", + "- The sun is a star - True\n", + "\"\"\"\n", + "Result: False\n", + "\n", + "===\n", + "\n", + "Checked Assertions:\"\"\"\n", + "\n", + "- The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. True\n", + "\n", + "- It has an area of 465,000 square miles. True\n", + "\n", + "- It is an arm of the Arctic Ocean. True\n", + "\n", + "- It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. True\n", + "\n", + "- It is named after the island of Greenland. False - It is named after the country of Greenland.\n", + "\n", + "- It is the Arctic Ocean's main outlet to the Atlantic. True\n", + "\n", + "- It is often frozen over so navigation is limited. True\n", + "\n", + "- It is considered the northern branch of the Norwegian Sea. False - It is considered the northern branch of the Atlantic Ocean.\n", + "\"\"\"\n", + "Result:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. It has an area of 465,000 square miles and is an arm of the Arctic Ocean. It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. The sea is named after the country of Greenland, and is the Arctic Ocean's main outlet to the Atlantic. It is often frozen over so navigation is limited, and is considered the northern branch of the Atlantic Ocean.\n", + "\n", + "\n", + "\u001b[1m> Entering new SequentialChain chain...\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mGiven some text, extract a list of facts from the text.\n", + "\n", + "Format your output as a bulleted list.\n", + "\n", + "Text:\n", + "\"\"\"\n", + "\n", + "\n", + "The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. It has an area of 465,000 square miles and is an arm of the Arctic Ocean. It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. The sea is named after the country of Greenland, and is the Arctic Ocean's main outlet to the Atlantic. It is often frozen over so navigation is limited, and is considered the northern branch of the Atlantic Ocean.\n", + "\"\"\"\n", + "\n", + "Facts:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mYou are an expert fact checker. You have been hired by a major news organization to fact check a very important story.\n", + "\n", + "Here is a bullet point list of facts:\n", + "\"\"\"\n", + "\n", + "- The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland.\n", + "- It has an area of 465,000 square miles.\n", + "- It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs.\n", + "- The sea is named after the country of Greenland.\n", + "- It is the Arctic Ocean's main outlet to the Atlantic.\n", + "- It is often frozen over so navigation is limited.\n", + "- It is considered the northern branch of the Atlantic Ocean.\n", + "\"\"\"\n", + "\n", + "For each fact, determine whether it is true or false about the subject. If you are unable to determine whether the fact is true or false, output \"Undetermined\".\n", + "If the fact is false, explain why.\n", + "\n", + "\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mBelow are some assertions that have been fact checked and are labeled as true of false. If the answer is false, a suggestion is given for a correction.\n", + "\n", + "Checked Assertions:\"\"\"\n", + "\n", + "- The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. True\n", + "\n", + "- It has an area of 465,000 square miles. True\n", + "\n", + "- It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. True\n", + "\n", + "- The sea is named after the country of Greenland. True\n", + "\n", + "- It is the Arctic Ocean's main outlet to the Atlantic. False - The Arctic Ocean's main outlet to the Atlantic is the Barents Sea.\n", + "\n", + "- It is often frozen over so navigation is limited. True\n", + "\n", + "- It is considered the northern branch of the Atlantic Ocean. False - The Greenland Sea is considered part of the Arctic Ocean, not the Atlantic Ocean.\n", + "\"\"\"\n", + "\n", + "Original Summary:\"\"\"\n", + "\n", + "\n", + "The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. It has an area of 465,000 square miles and is an arm of the Arctic Ocean. It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. The sea is named after the country of Greenland, and is the Arctic Ocean's main outlet to the Atlantic. It is often frozen over so navigation is limited, and is considered the northern branch of the Atlantic Ocean.\n", + "\"\"\"\n", + "\n", + "Using these checked assertions, rewrite the original summary to be completely true.\n", + "\n", + "The output should have the same structure and formatting as the original summary.\n", + "\n", + "Summary:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mBelow are some assertions that have been fact checked and are labeled as true of false.\n", + "\n", + "If all of the assertions are true, return \"True\". If any of the assertions are false, return \"False\".\n", + "\n", + "Here are some examples:\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is red: False\n", + "- Water is made of lava: False\n", + "- The sun is a star: True\n", + "\"\"\"\n", + "Result: False\n", + "\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is blue: True\n", + "- Water is wet: True\n", + "- The sun is a star: True\n", + "\"\"\"\n", + "Result: True\n", + "\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is blue - True\n", + "- Water is made of lava- False\n", + "- The sun is a star - True\n", + "\"\"\"\n", + "Result: False\n", + "\n", + "===\n", + "\n", + "Checked Assertions:\"\"\"\n", + "\n", + "- The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. True\n", + "\n", + "- It has an area of 465,000 square miles. True\n", + "\n", + "- It is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. True\n", + "\n", + "- The sea is named after the country of Greenland. True\n", + "\n", + "- It is the Arctic Ocean's main outlet to the Atlantic. False - The Arctic Ocean's main outlet to the Atlantic is the Barents Sea.\n", + "\n", + "- It is often frozen over so navigation is limited. True\n", + "\n", + "- It is considered the northern branch of the Atlantic Ocean. False - The Greenland Sea is considered part of the Arctic Ocean, not the Atlantic Ocean.\n", + "\"\"\"\n", + "Result:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. It has an area of 465,000 square miles and is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. The sea is named after the country of Greenland, and is the Arctic Ocean's main outlet to the Barents Sea. It is often frozen over so navigation is limited, and is considered part of the Arctic Ocean.\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n" + ] + }, + { + "data": { + "text/plain": [ + "\"The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. It has an area of 465,000 square miles and is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. The sea is named after the country of Greenland, and is the Arctic Ocean's main outlet to the Barents Sea. It is often frozen over so navigation is limited, and is considered part of the Arctic Ocean.\"" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from langchain.chains import LLMSummarizationCheckerChain\n", + "from langchain.llms import OpenAI\n", + "\n", + "llm = OpenAI(temperature=0)\n", + "checker_chain = LLMSummarizationCheckerChain(llm=llm, verbose=True, max_checks=3)\n", + "text = \"The Greenland Sea is an outlying portion of the Arctic Ocean located between Iceland, Norway, the Svalbard archipelago and Greenland. It has an area of 465,000 square miles and is one of five oceans in the world, alongside the Pacific Ocean, Atlantic Ocean, Indian Ocean, and the Southern Ocean. It is the smallest of the five oceans and is covered almost entirely by water, some of which is frozen in the form of glaciers and icebergs. The sea is named after the island of Greenland, and is the Arctic Ocean's main outlet to the Atlantic. It is often frozen over so navigation is limited, and is considered the northern branch of the Norwegian Sea.\"\n", + "checker_chain.run(text)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new LLMSummarizationCheckerChain chain...\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new SequentialChain chain...\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mGiven some text, extract a list of facts from the text.\n", + "\n", + "Format your output as a bulleted list.\n", + "\n", + "Text:\n", + "\"\"\"\n", + "Mammals can lay eggs, birds can lay eggs, therefore birds are mammals.\n", + "\"\"\"\n", + "\n", + "Facts:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mYou are an expert fact checker. You have been hired by a major news organization to fact check a very important story.\n", + "\n", + "Here is a bullet point list of facts:\n", + "\"\"\"\n", + "\n", + "- Mammals can lay eggs\n", + "- Birds can lay eggs\n", + "- Birds are mammals\n", + "\"\"\"\n", + "\n", + "For each fact, determine whether it is true or false about the subject. If you are unable to determine whether the fact is true or false, output \"Undetermined\".\n", + "If the fact is false, explain why.\n", + "\n", + "\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mBelow are some assertions that have been fact checked and are labeled as true of false. If the answer is false, a suggestion is given for a correction.\n", + "\n", + "Checked Assertions:\n", + "\"\"\"\n", + "\n", + "- Mammals can lay eggs: False. Mammals are not capable of laying eggs, as they give birth to live young.\n", + "\n", + "- Birds can lay eggs: True. Birds are capable of laying eggs.\n", + "\n", + "- Birds are mammals: False. Birds are not mammals, they are a class of their own.\n", + "\"\"\"\n", + "\n", + "Original Summary:\n", + "\"\"\"\n", + "Mammals can lay eggs, birds can lay eggs, therefore birds are mammals.\n", + "\"\"\"\n", + "\n", + "Using these checked assertions, rewrite the original summary to be completely true.\n", + "\n", + "The output should have the same structure and formatting as the original summary.\n", + "\n", + "Summary:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mBelow are some assertions that have been fact checked and are labeled as true or false.\n", + "\n", + "If all of the assertions are true, return \"True\". If any of the assertions are false, return \"False\".\n", + "\n", + "Here are some examples:\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is red: False\n", + "- Water is made of lava: False\n", + "- The sun is a star: True\n", + "\"\"\"\n", + "Result: False\n", + "\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is blue: True\n", + "- Water is wet: True\n", + "- The sun is a star: True\n", + "\"\"\"\n", + "Result: True\n", + "\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is blue - True\n", + "- Water is made of lava- False\n", + "- The sun is a star - True\n", + "\"\"\"\n", + "Result: False\n", + "\n", + "===\n", + "\n", + "Checked Assertions:\"\"\"\n", + "\n", + "- Mammals can lay eggs: False. Mammals are not capable of laying eggs, as they give birth to live young.\n", + "\n", + "- Birds can lay eggs: True. Birds are capable of laying eggs.\n", + "\n", + "- Birds are mammals: False. Birds are not mammals, they are a class of their own.\n", + "\"\"\"\n", + "Result:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + " Birds and mammals are both capable of laying eggs, however birds are not mammals, they are a class of their own.\n", + "\n", + "\n", + "\u001b[1m> Entering new SequentialChain chain...\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mGiven some text, extract a list of facts from the text.\n", + "\n", + "Format your output as a bulleted list.\n", + "\n", + "Text:\n", + "\"\"\"\n", + " Birds and mammals are both capable of laying eggs, however birds are not mammals, they are a class of their own.\n", + "\"\"\"\n", + "\n", + "Facts:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mYou are an expert fact checker. You have been hired by a major news organization to fact check a very important story.\n", + "\n", + "Here is a bullet point list of facts:\n", + "\"\"\"\n", + "\n", + "- Birds and mammals are both capable of laying eggs.\n", + "- Birds are not mammals.\n", + "- Birds are a class of their own.\n", + "\"\"\"\n", + "\n", + "For each fact, determine whether it is true or false about the subject. If you are unable to determine whether the fact is true or false, output \"Undetermined\".\n", + "If the fact is false, explain why.\n", + "\n", + "\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mBelow are some assertions that have been fact checked and are labeled as true of false. If the answer is false, a suggestion is given for a correction.\n", + "\n", + "Checked Assertions:\n", + "\"\"\"\n", + "\n", + "- Birds and mammals are both capable of laying eggs: False. Mammals give birth to live young, while birds lay eggs.\n", + "\n", + "- Birds are not mammals: True. Birds are a class of their own, separate from mammals.\n", + "\n", + "- Birds are a class of their own: True. Birds are a class of their own, separate from mammals.\n", + "\"\"\"\n", + "\n", + "Original Summary:\n", + "\"\"\"\n", + " Birds and mammals are both capable of laying eggs, however birds are not mammals, they are a class of their own.\n", + "\"\"\"\n", + "\n", + "Using these checked assertions, rewrite the original summary to be completely true.\n", + "\n", + "The output should have the same structure and formatting as the original summary.\n", + "\n", + "Summary:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\n", + "\u001b[1m> Entering new LLMChain chain...\u001b[0m\n", + "Prompt after formatting:\n", + "\u001b[32;1m\u001b[1;3mBelow are some assertions that have been fact checked and are labeled as true or false.\n", + "\n", + "If all of the assertions are true, return \"True\". If any of the assertions are false, return \"False\".\n", + "\n", + "Here are some examples:\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is red: False\n", + "- Water is made of lava: False\n", + "- The sun is a star: True\n", + "\"\"\"\n", + "Result: False\n", + "\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is blue: True\n", + "- Water is wet: True\n", + "- The sun is a star: True\n", + "\"\"\"\n", + "Result: True\n", + "\n", + "===\n", + "\n", + "Checked Assertions: \"\"\"\n", + "- The sky is blue - True\n", + "- Water is made of lava- False\n", + "- The sun is a star - True\n", + "\"\"\"\n", + "Result: False\n", + "\n", + "===\n", + "\n", + "Checked Assertions:\"\"\"\n", + "\n", + "- Birds and mammals are both capable of laying eggs: False. Mammals give birth to live young, while birds lay eggs.\n", + "\n", + "- Birds are not mammals: True. Birds are a class of their own, separate from mammals.\n", + "\n", + "- Birds are a class of their own: True. Birds are a class of their own, separate from mammals.\n", + "\"\"\"\n", + "Result:\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n" + ] + }, + { + "data": { + "text/plain": [ + "'Birds are not mammals, but they are a class of their own. They lay eggs, unlike mammals which give birth to live young.'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from langchain.chains import LLMSummarizationCheckerChain\n", + "from langchain.llms import OpenAI\n", + "\n", + "llm = OpenAI(temperature=0)\n", + "checker_chain = LLMSummarizationCheckerChain(llm=llm, max_checks=3, verbose=True)\n", + "text = \"Mammals can lay eggs, birds can lay eggs, therefore birds are mammals.\"\n", + "checker_chain.run(text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/langchain/chains/__init__.py b/langchain/chains/__init__.py index 09563968e5..1072ccb5f9 100644 --- a/langchain/chains/__init__.py +++ b/langchain/chains/__init__.py @@ -11,6 +11,7 @@ from langchain.chains.llm_bash.base import LLMBashChain from langchain.chains.llm_checker.base import LLMCheckerChain from langchain.chains.llm_math.base import LLMMathChain from langchain.chains.llm_requests import LLMRequestsChain +from langchain.chains.llm_summarization_checker.base import LLMSummarizationCheckerChain from langchain.chains.loading import load_chain from langchain.chains.mapreduce import MapReduceChain from langchain.chains.moderation import OpenAIModerationChain @@ -30,6 +31,7 @@ __all__ = [ "LLMChain", "LLMBashChain", "LLMCheckerChain", + "LLMSummarizationCheckerChain", "LLMMathChain", "PALChain", "QAWithSourcesChain", diff --git a/langchain/chains/llm_summarization_checker/__init__.py b/langchain/chains/llm_summarization_checker/__init__.py new file mode 100644 index 0000000000..3859908161 --- /dev/null +++ b/langchain/chains/llm_summarization_checker/__init__.py @@ -0,0 +1,7 @@ +"""Summarization checker chain for verifying accuracy of text generation. + +Chain that tries to verify the accuracy of text generation by splitting it into a +list of facts, then checking if those facts are true or not, and rewriting +the text to make it more truth-ful. It will repeat this loop until it hits `max_tries` +or gets to a "true" output. +""" diff --git a/langchain/chains/llm_summarization_checker/base.py b/langchain/chains/llm_summarization_checker/base.py new file mode 100644 index 0000000000..3fd1b943b7 --- /dev/null +++ b/langchain/chains/llm_summarization_checker/base.py @@ -0,0 +1,133 @@ +"""Chain for summarization with self-verification.""" + +from pathlib import Path +from typing import Dict, List + +from pydantic import BaseModel, Extra + +from langchain.chains.base import Chain +from langchain.chains.llm import LLMChain +from langchain.chains.sequential import SequentialChain +from langchain.llms.base import BaseLLM +from langchain.prompts.prompt import PromptTemplate + +PROMPTS_DIR = Path(__file__).parent / "prompts" + +CREATE_ASSERTIONS_PROMPT = PromptTemplate.from_file( + PROMPTS_DIR / "create_facts.txt", ["summary"] +) +CHECK_ASSERTIONS_PROMPT = PromptTemplate.from_file( + PROMPTS_DIR / "check_facts.txt", ["assertions"] +) +REVISED_SUMMARY_PROMPT = PromptTemplate.from_file( + PROMPTS_DIR / "revise_summary.txt", ["checked_assertions", "summary"] +) +ARE_ALL_TRUE_PROMPT = PromptTemplate.from_file( + PROMPTS_DIR / "are_all_true_prompt.txt", ["checked_assertions"] +) + + +class LLMSummarizationCheckerChain(Chain, BaseModel): + """Chain for question-answering with self-verification. + + Example: + .. code-block:: python + + from langchain import OpenAI, LLMSummarizationCheckerChain + llm = OpenAI(temperature=0.0) + checker_chain = LLMSummarizationCheckerChain(llm=llm) + """ + + llm: BaseLLM + """LLM wrapper to use.""" + + create_assertions_prompt: PromptTemplate = CREATE_ASSERTIONS_PROMPT + check_assertions_prompt: PromptTemplate = CHECK_ASSERTIONS_PROMPT + revised_summary_prompt: PromptTemplate = REVISED_SUMMARY_PROMPT + are_all_true_prompt: PromptTemplate = ARE_ALL_TRUE_PROMPT + + input_key: str = "query" #: :meta private: + output_key: str = "result" #: :meta private: + max_checks: int = 2 + """Maximum number of times to check the assertions. Default to double-checking.""" + + class Config: + """Configuration for this pydantic object.""" + + extra = Extra.forbid + arbitrary_types_allowed = True + + @property + def input_keys(self) -> List[str]: + """Return the singular input key. + + :meta private: + """ + return [self.input_key] + + @property + def output_keys(self) -> List[str]: + """Return the singular output key. + + :meta private: + """ + return [self.output_key] + + def _call(self, inputs: Dict[str, str]) -> Dict[str, str]: + all_true = False + count = 0 + output = None + original_input = inputs[self.input_key] + chain_input = original_input + + while not all_true and count < self.max_checks: + chain = SequentialChain( + chains=[ + LLMChain( + llm=self.llm, + prompt=self.create_assertions_prompt, + output_key="assertions", + verbose=self.verbose, + ), + LLMChain( + llm=self.llm, + prompt=self.check_assertions_prompt, + output_key="checked_assertions", + verbose=self.verbose, + ), + LLMChain( + llm=self.llm, + prompt=self.revised_summary_prompt, + output_key="revised_summary", + verbose=self.verbose, + ), + LLMChain( + llm=self.llm, + output_key="all_true", + prompt=self.are_all_true_prompt, + verbose=self.verbose, + ), + ], + input_variables=["summary"], + output_variables=["all_true", "revised_summary"], + verbose=True, + ) + output = chain({"summary": chain_input}) + count += 1 + + if output["all_true"].strip() == "True": + break + + if self.verbose: + print(output["revised_summary"]) + + chain_input = output["revised_summary"] + + if not output: + raise ValueError("No output from chain") + + return {self.output_key: output["revised_summary"].strip()} + + @property + def _chain_type(self) -> str: + return "llm_summarization_checker_chain" diff --git a/langchain/chains/llm_summarization_checker/prompts/are_all_true_prompt.txt b/langchain/chains/llm_summarization_checker/prompts/are_all_true_prompt.txt new file mode 100644 index 0000000000..cb1bedabfd --- /dev/null +++ b/langchain/chains/llm_summarization_checker/prompts/are_all_true_prompt.txt @@ -0,0 +1,38 @@ +Below are some assertions that have been fact checked and are labeled as true or false. + +If all of the assertions are true, return "True". If any of the assertions are false, return "False". + +Here are some examples: +=== + +Checked Assertions: """ +- The sky is red: False +- Water is made of lava: False +- The sun is a star: True +""" +Result: False + +=== + +Checked Assertions: """ +- The sky is blue: True +- Water is wet: True +- The sun is a star: True +""" +Result: True + +=== + +Checked Assertions: """ +- The sky is blue - True +- Water is made of lava- False +- The sun is a star - True +""" +Result: False + +=== + +Checked Assertions:""" +{checked_assertions} +""" +Result: \ No newline at end of file diff --git a/langchain/chains/llm_summarization_checker/prompts/check_facts.txt b/langchain/chains/llm_summarization_checker/prompts/check_facts.txt new file mode 100644 index 0000000000..b675d31823 --- /dev/null +++ b/langchain/chains/llm_summarization_checker/prompts/check_facts.txt @@ -0,0 +1,10 @@ +You are an expert fact checker. You have been hired by a major news organization to fact check a very important story. + +Here is a bullet point list of facts: +""" +{assertions} +""" + +For each fact, determine whether it is true or false about the subject. If you are unable to determine whether the fact is true or false, output "Undetermined". +If the fact is false, explain why. + diff --git a/langchain/chains/llm_summarization_checker/prompts/create_facts.txt b/langchain/chains/llm_summarization_checker/prompts/create_facts.txt new file mode 100644 index 0000000000..e85079a1d0 --- /dev/null +++ b/langchain/chains/llm_summarization_checker/prompts/create_facts.txt @@ -0,0 +1,10 @@ +Given some text, extract a list of facts from the text. + +Format your output as a bulleted list. + +Text: +""" +{summary} +""" + +Facts: \ No newline at end of file diff --git a/langchain/chains/llm_summarization_checker/prompts/revise_summary.txt b/langchain/chains/llm_summarization_checker/prompts/revise_summary.txt new file mode 100644 index 0000000000..70f0d2fe10 --- /dev/null +++ b/langchain/chains/llm_summarization_checker/prompts/revise_summary.txt @@ -0,0 +1,17 @@ +Below are some assertions that have been fact checked and are labeled as true of false. If the answer is false, a suggestion is given for a correction. + +Checked Assertions: +""" +{checked_assertions} +""" + +Original Summary: +""" +{summary} +""" + +Using these checked assertions, rewrite the original summary to be completely true. + +The output should have the same structure and formatting as the original summary. + +Summary: \ No newline at end of file diff --git a/langchain/prompts/base.py b/langchain/prompts/base.py index a22e2ef6a7..8244f2f3d7 100644 --- a/langchain/prompts/base.py +++ b/langchain/prompts/base.py @@ -46,8 +46,11 @@ def check_valid_template( try: formatter_func = DEFAULT_FORMATTER_MAPPING[template_format] formatter_func(template, **dummy_inputs) - except KeyError: - raise ValueError("Invalid prompt schema.") + except KeyError as e: + raise ValueError( + "Invalid prompt schema; check for mismatched or missing input parameters. " + + str(e) + ) class BaseOutputParser(BaseModel, ABC): diff --git a/langchain/prompts/prompt.py b/langchain/prompts/prompt.py index 777c1cbe93..2762e959e3 100644 --- a/langchain/prompts/prompt.py +++ b/langchain/prompts/prompt.py @@ -1,8 +1,9 @@ """Prompt schema definition.""" from __future__ import annotations +from pathlib import Path from string import Formatter -from typing import Any, Dict, List +from typing import Any, Dict, List, Union from pydantic import BaseModel, Extra, root_validator @@ -105,7 +106,7 @@ class PromptTemplate(BasePromptTemplate, BaseModel): @classmethod def from_file( - cls, template_file: str, input_variables: List[str] + cls, template_file: Union[str, Path], input_variables: List[str] ) -> PromptTemplate: """Load a prompt from a file. @@ -116,7 +117,7 @@ class PromptTemplate(BasePromptTemplate, BaseModel): Returns: The prompt loaded from the file. """ - with open(template_file, "r") as f: + with open(str(template_file), "r") as f: template = f.read() return cls(input_variables=input_variables, template=template) diff --git a/tests/unit_tests/chains/test_llm_summarization_checker.py b/tests/unit_tests/chains/test_llm_summarization_checker.py new file mode 100644 index 0000000000..81e4a8fa1c --- /dev/null +++ b/tests/unit_tests/chains/test_llm_summarization_checker.py @@ -0,0 +1,44 @@ +# flake8: noqa E501 + +"""Test LLMSummarization functionality.""" + +import pytest + +from langchain.chains.llm_summarization_checker.base import ( + ARE_ALL_TRUE_PROMPT, + CHECK_ASSERTIONS_PROMPT, + CREATE_ASSERTIONS_PROMPT, + REVISED_SUMMARY_PROMPT, + LLMSummarizationCheckerChain, +) +from tests.unit_tests.llms.fake_llm import FakeLLM + + +@pytest.fixture +def fake_llm_summarization_checker_chain() -> LLMSummarizationCheckerChain: + """Fake LLMCheckerChain for testing.""" + queries = { + CREATE_ASSERTIONS_PROMPT.format( + summary="a", + ): "b", + CHECK_ASSERTIONS_PROMPT.format( + assertions="b", + ): "- b - True", + REVISED_SUMMARY_PROMPT.format( + checked_assertions="- b - True", summary="a" + ): "b", + ARE_ALL_TRUE_PROMPT.format( + checked_assertions="- b - True", + ): "True", + } + fake_llm = FakeLLM(queries=queries) + return LLMSummarizationCheckerChain(llm=fake_llm, input_key="q", output_key="a") + + +def test_simple_text( + fake_llm_summarization_checker_chain: LLMSummarizationCheckerChain, +) -> None: + """Test simple question that should not need python.""" + question = "a" + output = fake_llm_summarization_checker_chain.run(question) + assert output == "b"