diff --git a/docs/modules/chains/examples/sqlite.ipynb b/docs/modules/chains/examples/sqlite.ipynb index 472ac99e..b8b4c03b 100644 --- a/docs/modules/chains/examples/sqlite.ipynb +++ b/docs/modules/chains/examples/sqlite.ipynb @@ -1,5 +1,16 @@ { "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "ca883d49", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, { "cell_type": "markdown", "id": "0ed6aab1", @@ -29,9 +40,15 @@ "To set it up, follow the instructions on https://database.guide/2-sample-databases-sqlite/, placing the `.db` file in a notebooks folder at the root of this repository." ] }, + { + "cell_type": "markdown", + "id": "6e287fa3", + "metadata": {}, + "source": [] + }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "d0e27d88", "metadata": { "pycharm": { @@ -45,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "72ede462", "metadata": { "pycharm": { @@ -55,7 +72,7 @@ "outputs": [], "source": [ "db = SQLDatabase.from_uri(\"sqlite:///../../../../notebooks/Chinook.db\")\n", - "llm = OpenAI(temperature=0)" + "llm = OpenAI(temperature=0, verbose=True)" ] }, { @@ -68,7 +85,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "a8fc8f23", "metadata": {}, "outputs": [], @@ -78,7 +95,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "15ff81df", "metadata": { "pycharm": { @@ -93,7 +110,7 @@ "\n", "\n", "\u001b[1m> Entering new SQLDatabaseChain chain...\u001b[0m\n", - "How many employees are there? \n", + "How many employees are there?\n", "SQLQuery:" ] }, @@ -101,7 +118,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/harrisonchase/workplace/langchain/langchain/sql_database.py:120: SAWarning: Dialect sqlite+pysqlite does *not* support Decimal objects natively, and SQLAlchemy must convert from floating point - rounding errors and other issues may occur. Please consider storing Decimal numbers as strings or integers on this platform for lossless storage.\n", + "/workspace/langchain/langchain/sql_database.py:191: SAWarning: Dialect sqlite+pysqlite does *not* support Decimal objects natively, and SQLAlchemy must convert from floating point - rounding errors and other issues may occur. Please consider storing Decimal numbers as strings or integers on this platform for lossless storage.\n", " sample_rows = connection.execute(command)\n" ] }, @@ -109,19 +126,19 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32;1m\u001b[1;3m SELECT COUNT(*) FROM Employee;\u001b[0m\n", + "\u001b[32;1m\u001b[1;3mSELECT COUNT(*) FROM \"Employee\";\u001b[0m\n", "SQLResult: \u001b[33;1m\u001b[1;3m[(8,)]\u001b[0m\n", - "Answer:\u001b[32;1m\u001b[1;3m There are 8 employees.\u001b[0m\n", + "Answer:\u001b[32;1m\u001b[1;3mThere are 8 employees.\u001b[0m\n", "\u001b[1m> Finished chain.\u001b[0m\n" ] }, { "data": { "text/plain": [ - "' There are 8 employees.'" + "'There are 8 employees.'" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -130,6 +147,61 @@ "db_chain.run(\"How many employees are there?\")" ] }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "bf8a5248", + "metadata": {}, + "source": [ + "## Use Query Checker\n", + "Sometimes the Language Model generates invalid SQL with small mistakes that can be self-corrected using the same technique used by the SQL Database Agent to try and fix the SQL using the LLM. You can simply specify this option when creating the chain:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "97e395db", + "metadata": {}, + "outputs": [], + "source": [ + "db_chain = SQLDatabaseChain.from_llm(llm, db, verbose=True, use_query_checker=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "4afc37db", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new SQLDatabaseChain chain...\u001b[0m\n", + "How many albums by Aerosmith?\n", + "SQLQuery:\u001b[32;1m\u001b[1;3mSELECT COUNT(*) FROM Album WHERE ArtistId = 3;\u001b[0m\n", + "SQLResult: \u001b[33;1m\u001b[1;3m[(1,)]\u001b[0m\n", + "Answer:\u001b[32;1m\u001b[1;3mThere is 1 album by Aerosmith.\u001b[0m\n", + "\u001b[1m> Finished chain.\u001b[0m\n" + ] + }, + { + "data": { + "text/plain": [ + "'There is 1 album by Aerosmith.'" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db_chain.run(\"How many albums by Aerosmith?\")" + ] + }, { "cell_type": "markdown", "id": "aad2cba6", @@ -141,7 +213,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 9, "id": "8ca7bafb", "metadata": {}, "outputs": [], @@ -170,7 +242,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 10, "id": "ec47a2bf", "metadata": {}, "outputs": [], @@ -180,7 +252,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 11, "id": "ebb0674e", "metadata": {}, "outputs": [ @@ -191,20 +263,20 @@ "\n", "\n", "\u001b[1m> Entering new SQLDatabaseChain chain...\u001b[0m\n", - "How many employees are there in the foobar table? \n", - "SQLQuery:\u001b[32;1m\u001b[1;3m SELECT COUNT(*) FROM Employee;\u001b[0m\n", + "How many employees are there in the foobar table?\n", + "SQLQuery:\u001b[32;1m\u001b[1;3mSELECT COUNT(*) FROM Employee;\u001b[0m\n", "SQLResult: \u001b[33;1m\u001b[1;3m[(8,)]\u001b[0m\n", - "Answer:\u001b[32;1m\u001b[1;3m There are 8 employees in the foobar table.\u001b[0m\n", + "Answer:\u001b[32;1m\u001b[1;3mThere are 8 employees in the foobar table.\u001b[0m\n", "\u001b[1m> Finished chain.\u001b[0m\n" ] }, { "data": { "text/plain": [ - "' There are 8 employees in the foobar table.'" + "'There are 8 employees in the foobar table.'" ] }, - "execution_count": 7, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -225,17 +297,17 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 13, "id": "38559487", "metadata": {}, "outputs": [], "source": [ - "db_chain = SQLDatabaseChain.from_llm(llm, db, prompt=PROMPT, verbose=True, return_intermediate_steps=True)" + "db_chain = SQLDatabaseChain.from_llm(llm, db, prompt=PROMPT, verbose=True, use_query_checker=True, return_intermediate_steps=True)" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 14, "id": "78b6af4d", "metadata": {}, "outputs": [ @@ -246,20 +318,28 @@ "\n", "\n", "\u001b[1m> Entering new SQLDatabaseChain chain...\u001b[0m\n", - "How many employees are there in the foobar table? \n", - "SQLQuery:\u001b[32;1m\u001b[1;3m SELECT COUNT(*) FROM Employee;\u001b[0m\n", + "How many employees are there in the foobar table?\n", + "SQLQuery:\u001b[32;1m\u001b[1;3mSELECT COUNT(*) FROM Employee;\u001b[0m\n", "SQLResult: \u001b[33;1m\u001b[1;3m[(8,)]\u001b[0m\n", - "Answer:\u001b[32;1m\u001b[1;3m There are 8 employees in the foobar table.\u001b[0m\n", + "Answer:\u001b[32;1m\u001b[1;3mThere are 8 employees in the foobar table.\u001b[0m\n", "\u001b[1m> Finished chain.\u001b[0m\n" ] }, { "data": { "text/plain": [ - "[' SELECT COUNT(*) FROM Employee;', '[(8,)]']" + "[{'input': 'How many employees are there in the foobar table?\\nSQLQuery:SELECT COUNT(*) FROM Employee;\\nSQLResult: [(8,)]\\nAnswer:',\n", + " 'top_k': '5',\n", + " 'dialect': 'sqlite',\n", + " 'table_info': '\\nCREATE TABLE \"Artist\" (\\n\\t\"ArtistId\" INTEGER NOT NULL, \\n\\t\"Name\" NVARCHAR(120), \\n\\tPRIMARY KEY (\"ArtistId\")\\n)\\n\\n/*\\n3 rows from Artist table:\\nArtistId\\tName\\n1\\tAC/DC\\n2\\tAccept\\n3\\tAerosmith\\n*/\\n\\n\\nCREATE TABLE \"Employee\" (\\n\\t\"EmployeeId\" INTEGER NOT NULL, \\n\\t\"LastName\" NVARCHAR(20) NOT NULL, \\n\\t\"FirstName\" NVARCHAR(20) NOT NULL, \\n\\t\"Title\" NVARCHAR(30), \\n\\t\"ReportsTo\" INTEGER, \\n\\t\"BirthDate\" DATETIME, \\n\\t\"HireDate\" DATETIME, \\n\\t\"Address\" NVARCHAR(70), \\n\\t\"City\" NVARCHAR(40), \\n\\t\"State\" NVARCHAR(40), \\n\\t\"Country\" NVARCHAR(40), \\n\\t\"PostalCode\" NVARCHAR(10), \\n\\t\"Phone\" NVARCHAR(24), \\n\\t\"Fax\" NVARCHAR(24), \\n\\t\"Email\" NVARCHAR(60), \\n\\tPRIMARY KEY (\"EmployeeId\"), \\n\\tFOREIGN KEY(\"ReportsTo\") REFERENCES \"Employee\" (\"EmployeeId\")\\n)\\n\\n/*\\n3 rows from Employee table:\\nEmployeeId\\tLastName\\tFirstName\\tTitle\\tReportsTo\\tBirthDate\\tHireDate\\tAddress\\tCity\\tState\\tCountry\\tPostalCode\\tPhone\\tFax\\tEmail\\n1\\tAdams\\tAndrew\\tGeneral Manager\\tNone\\t1962-02-18 00:00:00\\t2002-08-14 00:00:00\\t11120 Jasper Ave NW\\tEdmonton\\tAB\\tCanada\\tT5K 2N1\\t+1 (780) 428-9482\\t+1 (780) 428-3457\\tandrew@chinookcorp.com\\n2\\tEdwards\\tNancy\\tSales Manager\\t1\\t1958-12-08 00:00:00\\t2002-05-01 00:00:00\\t825 8 Ave SW\\tCalgary\\tAB\\tCanada\\tT2P 2T3\\t+1 (403) 262-3443\\t+1 (403) 262-3322\\tnancy@chinookcorp.com\\n3\\tPeacock\\tJane\\tSales Support Agent\\t2\\t1973-08-29 00:00:00\\t2002-04-01 00:00:00\\t1111 6 Ave SW\\tCalgary\\tAB\\tCanada\\tT2P 5M5\\t+1 (403) 262-3443\\t+1 (403) 262-6712\\tjane@chinookcorp.com\\n*/\\n\\n\\nCREATE TABLE \"Genre\" (\\n\\t\"GenreId\" INTEGER NOT NULL, \\n\\t\"Name\" NVARCHAR(120), \\n\\tPRIMARY KEY (\"GenreId\")\\n)\\n\\n/*\\n3 rows from Genre table:\\nGenreId\\tName\\n1\\tRock\\n2\\tJazz\\n3\\tMetal\\n*/\\n\\n\\nCREATE TABLE \"MediaType\" (\\n\\t\"MediaTypeId\" INTEGER NOT NULL, \\n\\t\"Name\" NVARCHAR(120), \\n\\tPRIMARY KEY (\"MediaTypeId\")\\n)\\n\\n/*\\n3 rows from MediaType table:\\nMediaTypeId\\tName\\n1\\tMPEG audio file\\n2\\tProtected AAC audio file\\n3\\tProtected MPEG-4 video file\\n*/\\n\\n\\nCREATE TABLE \"Playlist\" (\\n\\t\"PlaylistId\" INTEGER NOT NULL, \\n\\t\"Name\" NVARCHAR(120), \\n\\tPRIMARY KEY (\"PlaylistId\")\\n)\\n\\n/*\\n3 rows from Playlist table:\\nPlaylistId\\tName\\n1\\tMusic\\n2\\tMovies\\n3\\tTV Shows\\n*/\\n\\n\\nCREATE TABLE \"Album\" (\\n\\t\"AlbumId\" INTEGER NOT NULL, \\n\\t\"Title\" NVARCHAR(160) NOT NULL, \\n\\t\"ArtistId\" INTEGER NOT NULL, \\n\\tPRIMARY KEY (\"AlbumId\"), \\n\\tFOREIGN KEY(\"ArtistId\") REFERENCES \"Artist\" (\"ArtistId\")\\n)\\n\\n/*\\n3 rows from Album table:\\nAlbumId\\tTitle\\tArtistId\\n1\\tFor Those About To Rock We Salute You\\t1\\n2\\tBalls to the Wall\\t2\\n3\\tRestless and Wild\\t2\\n*/\\n\\n\\nCREATE TABLE \"Customer\" (\\n\\t\"CustomerId\" INTEGER NOT NULL, \\n\\t\"FirstName\" NVARCHAR(40) NOT NULL, \\n\\t\"LastName\" NVARCHAR(20) NOT NULL, \\n\\t\"Company\" NVARCHAR(80), \\n\\t\"Address\" NVARCHAR(70), \\n\\t\"City\" NVARCHAR(40), \\n\\t\"State\" NVARCHAR(40), \\n\\t\"Country\" NVARCHAR(40), \\n\\t\"PostalCode\" NVARCHAR(10), \\n\\t\"Phone\" NVARCHAR(24), \\n\\t\"Fax\" NVARCHAR(24), \\n\\t\"Email\" NVARCHAR(60) NOT NULL, \\n\\t\"SupportRepId\" INTEGER, \\n\\tPRIMARY KEY (\"CustomerId\"), \\n\\tFOREIGN KEY(\"SupportRepId\") REFERENCES \"Employee\" (\"EmployeeId\")\\n)\\n\\n/*\\n3 rows from Customer table:\\nCustomerId\\tFirstName\\tLastName\\tCompany\\tAddress\\tCity\\tState\\tCountry\\tPostalCode\\tPhone\\tFax\\tEmail\\tSupportRepId\\n1\\tLuís\\tGonçalves\\tEmbraer - Empresa Brasileira de Aeronáutica S.A.\\tAv. Brigadeiro Faria Lima, 2170\\tSão José dos Campos\\tSP\\tBrazil\\t12227-000\\t+55 (12) 3923-5555\\t+55 (12) 3923-5566\\tluisg@embraer.com.br\\t3\\n2\\tLeonie\\tKöhler\\tNone\\tTheodor-Heuss-Straße 34\\tStuttgart\\tNone\\tGermany\\t70174\\t+49 0711 2842222\\tNone\\tleonekohler@surfeu.de\\t5\\n3\\tFrançois\\tTremblay\\tNone\\t1498 rue Bélanger\\tMontréal\\tQC\\tCanada\\tH2G 1A7\\t+1 (514) 721-4711\\tNone\\tftremblay@gmail.com\\t3\\n*/\\n\\n\\nCREATE TABLE \"Invoice\" (\\n\\t\"InvoiceId\" INTEGER NOT NULL, \\n\\t\"CustomerId\" INTEGER NOT NULL, \\n\\t\"InvoiceDate\" DATETIME NOT NULL, \\n\\t\"BillingAddress\" NVARCHAR(70), \\n\\t\"BillingCity\" NVARCHAR(40), \\n\\t\"BillingState\" NVARCHAR(40), \\n\\t\"BillingCountry\" NVARCHAR(40), \\n\\t\"BillingPostalCode\" NVARCHAR(10), \\n\\t\"Total\" NUMERIC(10, 2) NOT NULL, \\n\\tPRIMARY KEY (\"InvoiceId\"), \\n\\tFOREIGN KEY(\"CustomerId\") REFERENCES \"Customer\" (\"CustomerId\")\\n)\\n\\n/*\\n3 rows from Invoice table:\\nInvoiceId\\tCustomerId\\tInvoiceDate\\tBillingAddress\\tBillingCity\\tBillingState\\tBillingCountry\\tBillingPostalCode\\tTotal\\n1\\t2\\t2009-01-01 00:00:00\\tTheodor-Heuss-Straße 34\\tStuttgart\\tNone\\tGermany\\t70174\\t1.98\\n2\\t4\\t2009-01-02 00:00:00\\tUllevålsveien 14\\tOslo\\tNone\\tNorway\\t0171\\t3.96\\n3\\t8\\t2009-01-03 00:00:00\\tGrétrystraat 63\\tBrussels\\tNone\\tBelgium\\t1000\\t5.94\\n*/\\n\\n\\nCREATE TABLE \"Track\" (\\n\\t\"TrackId\" INTEGER NOT NULL, \\n\\t\"Name\" NVARCHAR(200) NOT NULL, \\n\\t\"AlbumId\" INTEGER, \\n\\t\"MediaTypeId\" INTEGER NOT NULL, \\n\\t\"GenreId\" INTEGER, \\n\\t\"Composer\" NVARCHAR(220), \\n\\t\"Milliseconds\" INTEGER NOT NULL, \\n\\t\"Bytes\" INTEGER, \\n\\t\"UnitPrice\" NUMERIC(10, 2) NOT NULL, \\n\\tPRIMARY KEY (\"TrackId\"), \\n\\tFOREIGN KEY(\"MediaTypeId\") REFERENCES \"MediaType\" (\"MediaTypeId\"), \\n\\tFOREIGN KEY(\"GenreId\") REFERENCES \"Genre\" (\"GenreId\"), \\n\\tFOREIGN KEY(\"AlbumId\") REFERENCES \"Album\" (\"AlbumId\")\\n)\\n\\n/*\\n3 rows from Track table:\\nTrackId\\tName\\tAlbumId\\tMediaTypeId\\tGenreId\\tComposer\\tMilliseconds\\tBytes\\tUnitPrice\\n1\\tFor Those About To Rock (We Salute You)\\t1\\t1\\t1\\tAngus Young, Malcolm Young, Brian Johnson\\t343719\\t11170334\\t0.99\\n2\\tBalls to the Wall\\t2\\t2\\t1\\tNone\\t342562\\t5510424\\t0.99\\n3\\tFast As a Shark\\t3\\t2\\t1\\tF. Baltes, S. Kaufman, U. Dirkscneider & W. Hoffman\\t230619\\t3990994\\t0.99\\n*/\\n\\n\\nCREATE TABLE \"InvoiceLine\" (\\n\\t\"InvoiceLineId\" INTEGER NOT NULL, \\n\\t\"InvoiceId\" INTEGER NOT NULL, \\n\\t\"TrackId\" INTEGER NOT NULL, \\n\\t\"UnitPrice\" NUMERIC(10, 2) NOT NULL, \\n\\t\"Quantity\" INTEGER NOT NULL, \\n\\tPRIMARY KEY (\"InvoiceLineId\"), \\n\\tFOREIGN KEY(\"TrackId\") REFERENCES \"Track\" (\"TrackId\"), \\n\\tFOREIGN KEY(\"InvoiceId\") REFERENCES \"Invoice\" (\"InvoiceId\")\\n)\\n\\n/*\\n3 rows from InvoiceLine table:\\nInvoiceLineId\\tInvoiceId\\tTrackId\\tUnitPrice\\tQuantity\\n1\\t1\\t2\\t0.99\\t1\\n2\\t1\\t4\\t0.99\\t1\\n3\\t2\\t6\\t0.99\\t1\\n*/\\n\\n\\nCREATE TABLE \"PlaylistTrack\" (\\n\\t\"PlaylistId\" INTEGER NOT NULL, \\n\\t\"TrackId\" INTEGER NOT NULL, \\n\\tPRIMARY KEY (\"PlaylistId\", \"TrackId\"), \\n\\tFOREIGN KEY(\"TrackId\") REFERENCES \"Track\" (\"TrackId\"), \\n\\tFOREIGN KEY(\"PlaylistId\") REFERENCES \"Playlist\" (\"PlaylistId\")\\n)\\n\\n/*\\n3 rows from PlaylistTrack table:\\nPlaylistId\\tTrackId\\n1\\t3402\\n1\\t3389\\n1\\t3390\\n*/',\n", + " 'stop': ['\\nSQLResult:']},\n", + " 'SELECT COUNT(*) FROM Employee;',\n", + " {'query': 'SELECT COUNT(*) FROM Employee;', 'dialect': 'sqlite'},\n", + " 'SELECT COUNT(*) FROM Employee;',\n", + " '[(8,)]']" ] }, - "execution_count": 9, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -280,17 +360,17 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 15, "id": "6adaa799", "metadata": {}, "outputs": [], "source": [ - "db_chain = SQLDatabaseChain.from_llm(llm, db, verbose=True, top_k=3)" + "db_chain = SQLDatabaseChain.from_llm(llm, db, verbose=True, use_query_checker=True, top_k=3)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 16, "id": "edfc8a8e", "metadata": {}, "outputs": [ @@ -301,20 +381,20 @@ "\n", "\n", "\u001b[1m> Entering new SQLDatabaseChain chain...\u001b[0m\n", - "What are some example tracks by composer Johann Sebastian Bach? \n", - "SQLQuery:\u001b[32;1m\u001b[1;3m SELECT Name, Composer FROM Track WHERE Composer LIKE '%Johann Sebastian Bach%' LIMIT 3;\u001b[0m\n", - "SQLResult: \u001b[33;1m\u001b[1;3m[('Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace', 'Johann Sebastian Bach'), ('Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria', 'Johann Sebastian Bach'), ('Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude', 'Johann Sebastian Bach')]\u001b[0m\n", - "Answer:\u001b[32;1m\u001b[1;3m Some example tracks by composer Johann Sebastian Bach are 'Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace', 'Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria', and 'Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude'.\u001b[0m\n", + "What are some example tracks by composer Johann Sebastian Bach?\n", + "SQLQuery:\u001b[32;1m\u001b[1;3mSELECT Name FROM Track WHERE Composer = 'Johann Sebastian Bach' LIMIT 3\u001b[0m\n", + "SQLResult: \u001b[33;1m\u001b[1;3m[('Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace',), ('Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria',), ('Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude',)]\u001b[0m\n", + "Answer:\u001b[32;1m\u001b[1;3mExamples of tracks by Johann Sebastian Bach are Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace, Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria, and Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude.\u001b[0m\n", "\u001b[1m> Finished chain.\u001b[0m\n" ] }, { "data": { "text/plain": [ - "' Some example tracks by composer Johann Sebastian Bach are \\'Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace\\', \\'Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria\\', and \\'Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude\\'.'" + "'Examples of tracks by Johann Sebastian Bach are Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace, Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria, and Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude.'" ] }, - "execution_count": 11, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -334,7 +414,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 17, "id": "9a22ee47", "metadata": {}, "outputs": [], @@ -355,7 +435,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 18, "id": "9de86267", "metadata": {}, "outputs": [ @@ -379,6 +459,7 @@ "\tFOREIGN KEY(\"GenreId\") REFERENCES \"Genre\" (\"GenreId\"), \n", "\tFOREIGN KEY(\"AlbumId\") REFERENCES \"Album\" (\"AlbumId\")\n", ")\n", + "\n", "/*\n", "2 rows from Track table:\n", "TrackId\tName\tAlbumId\tMediaTypeId\tGenreId\tComposer\tMilliseconds\tBytes\tUnitPrice\n", @@ -386,14 +467,6 @@ "2\tBalls to the Wall\t2\t2\t1\tNone\t342562\t5510424\t0.99\n", "*/\n" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/jon/projects/langchain/langchain/sql_database.py:135: SAWarning: Dialect sqlite+pysqlite does *not* support Decimal objects natively, and SQLAlchemy must convert from floating point - rounding errors and other issues may occur. Please consider storing Decimal numbers as strings or integers on this platform for lossless storage.\n", - " sample_rows = connection.execute(command)\n" - ] } ], "source": [ @@ -402,17 +475,17 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 19, "id": "bcb7a489", "metadata": {}, "outputs": [], "source": [ - "db_chain = SQLDatabaseChain.from_llm(llm, db, verbose=True)" + "db_chain = SQLDatabaseChain.from_llm(llm, db, use_query_checker=True, verbose=True)" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 20, "id": "81e05d82", "metadata": {}, "outputs": [ @@ -423,20 +496,20 @@ "\n", "\n", "\u001b[1m> Entering new SQLDatabaseChain chain...\u001b[0m\n", - "What are some example tracks by Bach? \n", - "SQLQuery:\u001b[32;1m\u001b[1;3m SELECT Name FROM Track WHERE Composer LIKE '%Bach%' LIMIT 5;\u001b[0m\n", - "SQLResult: \u001b[33;1m\u001b[1;3m[('American Woman',), ('Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace',), ('Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria',), ('Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude',), ('Toccata and Fugue in D Minor, BWV 565: I. Toccata',)]\u001b[0m\n", - "Answer:\u001b[32;1m\u001b[1;3m Some example tracks by Bach are 'American Woman', 'Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace', 'Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria', 'Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude', and 'Toccata and Fugue in D Minor, BWV 565: I. Toccata'.\u001b[0m\n", + "What are some example tracks by Bach?\n", + "SQLQuery:\u001b[32;1m\u001b[1;3mSELECT \"Name\", \"Composer\" FROM \"Track\" WHERE \"Composer\" LIKE '%Bach%' LIMIT 5\u001b[0m\n", + "SQLResult: \u001b[33;1m\u001b[1;3m[('American Woman', 'B. Cummings/G. Peterson/M.J. Kale/R. Bachman'), ('Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace', 'Johann Sebastian Bach'), ('Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria', 'Johann Sebastian Bach'), ('Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude', 'Johann Sebastian Bach'), ('Toccata and Fugue in D Minor, BWV 565: I. Toccata', 'Johann Sebastian Bach')]\u001b[0m\n", + "Answer:\u001b[32;1m\u001b[1;3mTracks by Bach include 'American Woman', 'Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace', 'Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria', 'Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude', and 'Toccata and Fugue in D Minor, BWV 565: I. Toccata'.\u001b[0m\n", "\u001b[1m> Finished chain.\u001b[0m\n" ] }, { "data": { "text/plain": [ - "' Some example tracks by Bach are \\'American Woman\\', \\'Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace\\', \\'Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria\\', \\'Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude\\', and \\'Toccata and Fugue in D Minor, BWV 565: I. Toccata\\'.'" + "'Tracks by Bach include \\'American Woman\\', \\'Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace\\', \\'Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria\\', \\'Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude\\', and \\'Toccata and Fugue in D Minor, BWV 565: I. Toccata\\'.'" ] }, - "execution_count": 15, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -458,7 +531,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 21, "id": "2ad33ab1", "metadata": {}, "outputs": [], @@ -482,7 +555,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 22, "id": "db144352", "metadata": {}, "outputs": [ @@ -496,6 +569,7 @@ "\t\"Name\" NVARCHAR(120), \n", "\tPRIMARY KEY (\"PlaylistId\")\n", ")\n", + "\n", "/*\n", "2 rows from Playlist table:\n", "PlaylistId\tName\n", @@ -539,7 +613,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 24, "id": "dfbda4e6", "metadata": {}, "outputs": [ @@ -550,20 +624,67 @@ "\n", "\n", "\u001b[1m> Entering new SQLDatabaseChain chain...\u001b[0m\n", - "What are some example tracks by Bach? \n", - "SQLQuery:\u001b[32;1m\u001b[1;3m SELECT Name, Composer FROM Track WHERE Composer LIKE '%Bach%' LIMIT 5;\u001b[0m\n", - "SQLResult: \u001b[33;1m\u001b[1;3m[('American Woman', 'B. Cummings/G. Peterson/M.J. Kale/R. Bachman'), ('Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace', 'Johann Sebastian Bach'), ('Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria', 'Johann Sebastian Bach'), ('Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude', 'Johann Sebastian Bach'), ('Toccata and Fugue in D Minor, BWV 565: I. Toccata', 'Johann Sebastian Bach')]\u001b[0m\n", - "Answer:\u001b[32;1m\u001b[1;3m Some example tracks by Bach are 'American Woman', 'Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace', 'Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria', 'Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude', and 'Toccata and Fugue in D Minor, BWV 565: I. Toccata'.\u001b[0m\n", + "What are some example tracks by Bach?\n", + "SQLQuery:\u001b[32;1m\u001b[1;3mSELECT \"Name\" FROM Track WHERE \"Composer\" LIKE '%Bach%' LIMIT 5;\u001b[0m\n", + "SQLResult: \u001b[33;1m\u001b[1;3m[('American Woman',), ('Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace',), ('Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria',), ('Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude',), ('Toccata and Fugue in D Minor, BWV 565: I. Toccata',)]\u001b[0m\n", + "Answer:text='You are a SQLite expert. Given an input question, first create a syntactically correct SQLite query to run, then look at the results of the query and return the answer to the input question.\\nUnless the user specifies in the question a specific number of examples to obtain, query for at most 5 results using the LIMIT clause as per SQLite. You can order the results to return the most informative data in the database.\\nNever query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in double quotes (\") to denote them as delimited identifiers.\\nPay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table.\\n\\nUse the following format:\\n\\nQuestion: \"Question here\"\\nSQLQuery: \"SQL Query to run\"\\nSQLResult: \"Result of the SQLQuery\"\\nAnswer: \"Final answer here\"\\n\\nOnly use the following tables:\\n\\nCREATE TABLE \"Playlist\" (\\n\\t\"PlaylistId\" INTEGER NOT NULL, \\n\\t\"Name\" NVARCHAR(120), \\n\\tPRIMARY KEY (\"PlaylistId\")\\n)\\n\\n/*\\n2 rows from Playlist table:\\nPlaylistId\\tName\\n1\\tMusic\\n2\\tMovies\\n*/\\n\\nCREATE TABLE Track (\\n\\t\"TrackId\" INTEGER NOT NULL, \\n\\t\"Name\" NVARCHAR(200) NOT NULL,\\n\\t\"Composer\" NVARCHAR(220),\\n\\tPRIMARY KEY (\"TrackId\")\\n)\\n/*\\n3 rows from Track table:\\nTrackId\\tName\\tComposer\\n1\\tFor Those About To Rock (We Salute You)\\tAngus Young, Malcolm Young, Brian Johnson\\n2\\tBalls to the Wall\\tNone\\n3\\tMy favorite song ever\\tThe coolest composer of all time\\n*/\\n\\nQuestion: What are some example tracks by Bach?\\nSQLQuery:SELECT \"Name\" FROM Track WHERE \"Composer\" LIKE \\'%Bach%\\' LIMIT 5;\\nSQLResult: [(\\'American Woman\\',), (\\'Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace\\',), (\\'Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria\\',), (\\'Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude\\',), (\\'Toccata and Fugue in D Minor, BWV 565: I. Toccata\\',)]\\nAnswer:'\n", + "You are a SQLite expert. Given an input question, first create a syntactically correct SQLite query to run, then look at the results of the query and return the answer to the input question.\n", + "Unless the user specifies in the question a specific number of examples to obtain, query for at most 5 results using the LIMIT clause as per SQLite. You can order the results to return the most informative data in the database.\n", + "Never query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in double quotes (\") to denote them as delimited identifiers.\n", + "Pay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table.\n", + "\n", + "Use the following format:\n", + "\n", + "Question: \"Question here\"\n", + "SQLQuery: \"SQL Query to run\"\n", + "SQLResult: \"Result of the SQLQuery\"\n", + "Answer: \"Final answer here\"\n", + "\n", + "Only use the following tables:\n", + "\n", + "CREATE TABLE \"Playlist\" (\n", + "\t\"PlaylistId\" INTEGER NOT NULL, \n", + "\t\"Name\" NVARCHAR(120), \n", + "\tPRIMARY KEY (\"PlaylistId\")\n", + ")\n", + "\n", + "/*\n", + "2 rows from Playlist table:\n", + "PlaylistId\tName\n", + "1\tMusic\n", + "2\tMovies\n", + "*/\n", + "\n", + "CREATE TABLE Track (\n", + "\t\"TrackId\" INTEGER NOT NULL, \n", + "\t\"Name\" NVARCHAR(200) NOT NULL,\n", + "\t\"Composer\" NVARCHAR(220),\n", + "\tPRIMARY KEY (\"TrackId\")\n", + ")\n", + "/*\n", + "3 rows from Track table:\n", + "TrackId\tName\tComposer\n", + "1\tFor Those About To Rock (We Salute You)\tAngus Young, Malcolm Young, Brian Johnson\n", + "2\tBalls to the Wall\tNone\n", + "3\tMy favorite song ever\tThe coolest composer of all time\n", + "*/\n", + "\n", + "Question: What are some example tracks by Bach?\n", + "SQLQuery:SELECT \"Name\" FROM Track WHERE \"Composer\" LIKE '%Bach%' LIMIT 5;\n", + "SQLResult: [('American Woman',), ('Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace',), ('Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria',), ('Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude',), ('Toccata and Fugue in D Minor, BWV 565: I. Toccata',)]\n", + "Answer:\n", + "{'input': 'What are some example tracks by Bach?\\nSQLQuery:SELECT \"Name\" FROM Track WHERE \"Composer\" LIKE \\'%Bach%\\' LIMIT 5;\\nSQLResult: [(\\'American Woman\\',), (\\'Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace\\',), (\\'Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria\\',), (\\'Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude\\',), (\\'Toccata and Fugue in D Minor, BWV 565: I. Toccata\\',)]\\nAnswer:', 'top_k': '5', 'dialect': 'sqlite', 'table_info': '\\nCREATE TABLE \"Playlist\" (\\n\\t\"PlaylistId\" INTEGER NOT NULL, \\n\\t\"Name\" NVARCHAR(120), \\n\\tPRIMARY KEY (\"PlaylistId\")\\n)\\n\\n/*\\n2 rows from Playlist table:\\nPlaylistId\\tName\\n1\\tMusic\\n2\\tMovies\\n*/\\n\\nCREATE TABLE Track (\\n\\t\"TrackId\" INTEGER NOT NULL, \\n\\t\"Name\" NVARCHAR(200) NOT NULL,\\n\\t\"Composer\" NVARCHAR(220),\\n\\tPRIMARY KEY (\"TrackId\")\\n)\\n/*\\n3 rows from Track table:\\nTrackId\\tName\\tComposer\\n1\\tFor Those About To Rock (We Salute You)\\tAngus Young, Malcolm Young, Brian Johnson\\n2\\tBalls to the Wall\\tNone\\n3\\tMy favorite song ever\\tThe coolest composer of all time\\n*/', 'stop': ['\\nSQLResult:']}\n", + "\u001b[32;1m\u001b[1;3mExamples of tracks by Bach include \"American Woman\", \"Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace\", \"Aria Mit 30 Veränderungen, BWV 988 'Goldberg Variations': Aria\", \"Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude\", and \"Toccata and Fugue in D Minor, BWV 565: I. Toccata\".\u001b[0m\n", "\u001b[1m> Finished chain.\u001b[0m\n" ] }, { "data": { "text/plain": [ - "' Some example tracks by Bach are \\'American Woman\\', \\'Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace\\', \\'Aria Mit 30 Veränderungen, BWV 988 \"Goldberg Variations\": Aria\\', \\'Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude\\', and \\'Toccata and Fugue in D Minor, BWV 565: I. Toccata\\'.'" + "'Examples of tracks by Bach include \"American Woman\", \"Concerto for 2 Violins in D Minor, BWV 1043: I. Vivace\", \"Aria Mit 30 Veränderungen, BWV 988 \\'Goldberg Variations\\': Aria\", \"Suite for Solo Cello No. 1 in G Major, BWV 1007: I. Prélude\", and \"Toccata and Fugue in D Minor, BWV 565: I. Toccata\".'" ] }, - "execution_count": 18, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -592,7 +713,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 25, "id": "e59a4740", "metadata": {}, "outputs": [], @@ -603,7 +724,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 26, "id": "58bb49b6", "metadata": {}, "outputs": [], @@ -613,7 +734,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 27, "id": "95017b1a", "metadata": {}, "outputs": [ @@ -625,13 +746,13 @@ "\n", "\u001b[1m> Entering new SQLDatabaseSequentialChain chain...\u001b[0m\n", "Table names to use:\n", - "\u001b[33;1m\u001b[1;3m['Customer', 'Employee']\u001b[0m\n", + "\u001b[33;1m\u001b[1;3m['Employee', 'Customer']\u001b[0m\n", "\n", "\u001b[1m> Entering new SQLDatabaseChain chain...\u001b[0m\n", - "How many employees are also customers? \n", - "SQLQuery:\u001b[32;1m\u001b[1;3m SELECT COUNT(*) FROM Employee INNER JOIN Customer ON Employee.EmployeeId = Customer.SupportRepId;\u001b[0m\n", + "How many employees are also customers?\n", + "SQLQuery:\u001b[32;1m\u001b[1;3mSELECT COUNT(*) FROM Employee e INNER JOIN Customer c ON e.EmployeeId = c.SupportRepId;\u001b[0m\n", "SQLResult: \u001b[33;1m\u001b[1;3m[(59,)]\u001b[0m\n", - "Answer:\u001b[32;1m\u001b[1;3m 59 employees are also customers.\u001b[0m\n", + "Answer:\u001b[32;1m\u001b[1;3m59 employees are also customers.\u001b[0m\n", "\u001b[1m> Finished chain.\u001b[0m\n", "\n", "\u001b[1m> Finished chain.\u001b[0m\n" @@ -640,10 +761,10 @@ { "data": { "text/plain": [ - "' 59 employees are also customers.'" + "'59 employees are also customers.'" ] }, - "execution_count": 22, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -653,12 +774,619 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "attachments": {}, + "cell_type": "markdown", "id": "5eb39db6", "metadata": {}, + "source": [ + "## Using Local Language Models\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "6ab11cb9", + "metadata": {}, + "source": [ + "Sometimes you may not have the luxury of using OpenAI or other service-hosted large language model. You can, ofcourse, try to use the `SQLDatabaseChain` with a local model, but will quickly realize that most models you can run locally even with a large GPU struggle to generate the right output." + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "dd21d9b4", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/workspace/langchain/.venv/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "Loading checkpoint shards: 100%|██████████| 8/8 [00:32<00:00, 4.11s/it]\n" + ] + } + ], + "source": [ + "import logging\n", + "import torch\n", + "from transformers import AutoTokenizer, GPT2TokenizerFast, pipeline, AutoModelForSeq2SeqLM, AutoModelForCausalLM\n", + "from langchain import HuggingFacePipeline\n", + "\n", + "# Note: This model requires a large GPU, e.g. an 80GB A100. See documentation for other ways to run private non-OpenAI models.\n", + "model_id = \"google/flan-ul2\"\n", + "model = AutoModelForSeq2SeqLM.from_pretrained(model_id, temperature=0)\n", + "\n", + "device_id = -1 # default to no-GPU, but use GPU and half precision mode if available\n", + "if torch.cuda.is_available():\n", + " device_id = 0\n", + " try:\n", + " model = model.half()\n", + " except RuntimeError as exc:\n", + " logging.warn(f\"Could not run model in half precision mode: {str(exc)}\")\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(model_id)\n", + "pipe = pipeline(task=\"text2text-generation\", model=model, tokenizer=tokenizer, max_length=1024, device=device_id)\n", + "\n", + "local_llm = HuggingFacePipeline(pipeline=pipe)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "f89fd8d0", + "metadata": {}, "outputs": [], - "source": [] + "source": [ + "from langchain import SQLDatabase, SQLDatabaseChain\n", + "\n", + "db = SQLDatabase.from_uri(\"sqlite:///../../../../notebooks/Chinook.db\", include_tables=['Customer'])\n", + "local_chain = SQLDatabaseChain.from_llm(local_llm, db, verbose=True, return_intermediate_steps=True, use_query_checker=True)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "5a339eed", + "metadata": {}, + "source": [ + "This model should work for very simple SQL queries, as long as you use the query checker as specified above, e.g.:" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "49a43200", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new SQLDatabaseChain chain...\u001b[0m\n", + "How many customers are there?\n", + "SQLQuery:" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/workspace/langchain/.venv/lib/python3.9/site-packages/transformers/pipelines/base.py:1070: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n", + " warnings.warn(\n", + "/workspace/langchain/.venv/lib/python3.9/site-packages/transformers/pipelines/base.py:1070: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32;1m\u001b[1;3mSELECT count(*) FROM Customer\u001b[0m\n", + "SQLResult: \u001b[33;1m\u001b[1;3m[(59,)]\u001b[0m\n", + "Answer:" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/workspace/langchain/.venv/lib/python3.9/site-packages/transformers/pipelines/base.py:1070: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32;1m\u001b[1;3m[59]\u001b[0m\n", + "\u001b[1m> Finished chain.\u001b[0m\n" + ] + }, + { + "data": { + "text/plain": [ + "{'query': 'How many customers are there?',\n", + " 'result': '[59]',\n", + " 'intermediate_steps': [{'input': 'How many customers are there?\\nSQLQuery:SELECT count(*) FROM Customer\\nSQLResult: [(59,)]\\nAnswer:',\n", + " 'top_k': '5',\n", + " 'dialect': 'sqlite',\n", + " 'table_info': '\\nCREATE TABLE \"Customer\" (\\n\\t\"CustomerId\" INTEGER NOT NULL, \\n\\t\"FirstName\" NVARCHAR(40) NOT NULL, \\n\\t\"LastName\" NVARCHAR(20) NOT NULL, \\n\\t\"Company\" NVARCHAR(80), \\n\\t\"Address\" NVARCHAR(70), \\n\\t\"City\" NVARCHAR(40), \\n\\t\"State\" NVARCHAR(40), \\n\\t\"Country\" NVARCHAR(40), \\n\\t\"PostalCode\" NVARCHAR(10), \\n\\t\"Phone\" NVARCHAR(24), \\n\\t\"Fax\" NVARCHAR(24), \\n\\t\"Email\" NVARCHAR(60) NOT NULL, \\n\\t\"SupportRepId\" INTEGER, \\n\\tPRIMARY KEY (\"CustomerId\"), \\n\\tFOREIGN KEY(\"SupportRepId\") REFERENCES \"Employee\" (\"EmployeeId\")\\n)\\n\\n/*\\n3 rows from Customer table:\\nCustomerId\\tFirstName\\tLastName\\tCompany\\tAddress\\tCity\\tState\\tCountry\\tPostalCode\\tPhone\\tFax\\tEmail\\tSupportRepId\\n1\\tLuís\\tGonçalves\\tEmbraer - Empresa Brasileira de Aeronáutica S.A.\\tAv. Brigadeiro Faria Lima, 2170\\tSão José dos Campos\\tSP\\tBrazil\\t12227-000\\t+55 (12) 3923-5555\\t+55 (12) 3923-5566\\tluisg@embraer.com.br\\t3\\n2\\tLeonie\\tKöhler\\tNone\\tTheodor-Heuss-Straße 34\\tStuttgart\\tNone\\tGermany\\t70174\\t+49 0711 2842222\\tNone\\tleonekohler@surfeu.de\\t5\\n3\\tFrançois\\tTremblay\\tNone\\t1498 rue Bélanger\\tMontréal\\tQC\\tCanada\\tH2G 1A7\\t+1 (514) 721-4711\\tNone\\tftremblay@gmail.com\\t3\\n*/',\n", + " 'stop': ['\\nSQLResult:']},\n", + " 'SELECT count(*) FROM Customer',\n", + " {'query': 'SELECT count(*) FROM Customer', 'dialect': 'sqlite'},\n", + " 'SELECT count(*) FROM Customer',\n", + " '[(59,)]']}" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "local_chain(\"How many customers are there?\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "6858f758", + "metadata": {}, + "source": [ + "Even this relatively large model will most likely fail to generate more complicated SQL by itself. However, you can log its inputs and outputs so that you can hand-correct them and use the corrected examples for few shot prompt examples later. In practice, you could log any executions of your chain that raise exceptions (as shown in the example below) or get direct user feedback in cases where the results are incorrect (but did not raise an exception)." + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "1abf2c91", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "11842.36s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: pyyaml in /workspace/langchain/.venv/lib/python3.9/site-packages (6.0)\n", + "Requirement already satisfied: chromadb in /workspace/langchain/.venv/lib/python3.9/site-packages (0.3.21)\n", + "Requirement already satisfied: pandas>=1.3 in /workspace/langchain/.venv/lib/python3.9/site-packages (from chromadb) (2.0.1)\n", + "Requirement already satisfied: requests>=2.28 in /workspace/langchain/.venv/lib/python3.9/site-packages (from chromadb) (2.28.2)\n", + "Requirement already satisfied: pydantic>=1.9 in /workspace/langchain/.venv/lib/python3.9/site-packages (from chromadb) (1.10.7)\n", + "Requirement already satisfied: hnswlib>=0.7 in /workspace/langchain/.venv/lib/python3.9/site-packages (from chromadb) (0.7.0)\n", + "Requirement already satisfied: clickhouse-connect>=0.5.7 in /workspace/langchain/.venv/lib/python3.9/site-packages (from chromadb) (0.5.20)\n", + "Requirement already satisfied: sentence-transformers>=2.2.2 in /workspace/langchain/.venv/lib/python3.9/site-packages (from chromadb) (2.2.2)\n", + "Requirement already satisfied: duckdb>=0.7.1 in /workspace/langchain/.venv/lib/python3.9/site-packages (from chromadb) (0.7.1)\n", + "Requirement already satisfied: fastapi>=0.85.1 in /workspace/langchain/.venv/lib/python3.9/site-packages (from chromadb) (0.95.1)\n", + "Requirement already satisfied: uvicorn[standard]>=0.18.3 in /workspace/langchain/.venv/lib/python3.9/site-packages (from chromadb) (0.21.1)\n", + "Requirement already satisfied: numpy>=1.21.6 in /workspace/langchain/.venv/lib/python3.9/site-packages (from chromadb) (1.24.3)\n", + "Requirement already satisfied: posthog>=2.4.0 in /workspace/langchain/.venv/lib/python3.9/site-packages (from chromadb) (3.0.1)\n", + "Requirement already satisfied: certifi in /workspace/langchain/.venv/lib/python3.9/site-packages (from clickhouse-connect>=0.5.7->chromadb) (2022.12.7)\n", + "Requirement already satisfied: urllib3>=1.26 in /workspace/langchain/.venv/lib/python3.9/site-packages (from clickhouse-connect>=0.5.7->chromadb) (1.26.15)\n", + "Requirement already satisfied: pytz in /workspace/langchain/.venv/lib/python3.9/site-packages (from clickhouse-connect>=0.5.7->chromadb) (2023.3)\n", + "Requirement already satisfied: zstandard in /workspace/langchain/.venv/lib/python3.9/site-packages (from clickhouse-connect>=0.5.7->chromadb) (0.21.0)\n", + "Requirement already satisfied: lz4 in /workspace/langchain/.venv/lib/python3.9/site-packages (from clickhouse-connect>=0.5.7->chromadb) (4.3.2)\n", + "Requirement already satisfied: starlette<0.27.0,>=0.26.1 in /workspace/langchain/.venv/lib/python3.9/site-packages (from fastapi>=0.85.1->chromadb) (0.26.1)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /workspace/langchain/.venv/lib/python3.9/site-packages (from pandas>=1.3->chromadb) (2.8.2)\n", + "Requirement already satisfied: tzdata>=2022.1 in /workspace/langchain/.venv/lib/python3.9/site-packages (from pandas>=1.3->chromadb) (2023.3)\n", + "Requirement already satisfied: six>=1.5 in /workspace/langchain/.venv/lib/python3.9/site-packages (from posthog>=2.4.0->chromadb) (1.16.0)\n", + "Requirement already satisfied: monotonic>=1.5 in /workspace/langchain/.venv/lib/python3.9/site-packages (from posthog>=2.4.0->chromadb) (1.6)\n", + "Requirement already satisfied: backoff>=1.10.0 in /workspace/langchain/.venv/lib/python3.9/site-packages (from posthog>=2.4.0->chromadb) (2.2.1)\n", + "Requirement already satisfied: typing-extensions>=4.2.0 in /workspace/langchain/.venv/lib/python3.9/site-packages (from pydantic>=1.9->chromadb) (4.5.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /workspace/langchain/.venv/lib/python3.9/site-packages (from requests>=2.28->chromadb) (3.1.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /workspace/langchain/.venv/lib/python3.9/site-packages (from requests>=2.28->chromadb) (3.4)\n", + "Requirement already satisfied: transformers<5.0.0,>=4.6.0 in /workspace/langchain/.venv/lib/python3.9/site-packages (from sentence-transformers>=2.2.2->chromadb) (4.28.1)\n", + "Requirement already satisfied: tqdm in /workspace/langchain/.venv/lib/python3.9/site-packages (from sentence-transformers>=2.2.2->chromadb) (4.65.0)\n", + "Requirement already satisfied: torch>=1.6.0 in /workspace/langchain/.venv/lib/python3.9/site-packages (from sentence-transformers>=2.2.2->chromadb) (1.13.1)\n", + "Requirement already satisfied: torchvision in /workspace/langchain/.venv/lib/python3.9/site-packages (from sentence-transformers>=2.2.2->chromadb) (0.14.1)\n", + "Requirement already satisfied: scikit-learn in /workspace/langchain/.venv/lib/python3.9/site-packages (from sentence-transformers>=2.2.2->chromadb) (1.2.2)\n", + "Requirement already satisfied: scipy in /workspace/langchain/.venv/lib/python3.9/site-packages (from sentence-transformers>=2.2.2->chromadb) (1.9.3)\n", + "Requirement already satisfied: nltk in /workspace/langchain/.venv/lib/python3.9/site-packages (from sentence-transformers>=2.2.2->chromadb) (3.8.1)\n", + "Requirement already satisfied: sentencepiece in /workspace/langchain/.venv/lib/python3.9/site-packages (from sentence-transformers>=2.2.2->chromadb) (0.1.98)\n", + "Requirement already satisfied: huggingface-hub>=0.4.0 in /workspace/langchain/.venv/lib/python3.9/site-packages (from sentence-transformers>=2.2.2->chromadb) (0.13.4)\n", + "Requirement already satisfied: click>=7.0 in /workspace/langchain/.venv/lib/python3.9/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (8.1.3)\n", + "Requirement already satisfied: h11>=0.8 in /workspace/langchain/.venv/lib/python3.9/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.14.0)\n", + "Requirement already satisfied: httptools>=0.5.0 in /workspace/langchain/.venv/lib/python3.9/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.5.0)\n", + "Requirement already satisfied: python-dotenv>=0.13 in /workspace/langchain/.venv/lib/python3.9/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.0.0)\n", + "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /workspace/langchain/.venv/lib/python3.9/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.17.0)\n", + "Requirement already satisfied: watchfiles>=0.13 in /workspace/langchain/.venv/lib/python3.9/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.19.0)\n", + "Requirement already satisfied: websockets>=10.4 in /workspace/langchain/.venv/lib/python3.9/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (11.0.2)\n", + "Requirement already satisfied: filelock in /workspace/langchain/.venv/lib/python3.9/site-packages (from huggingface-hub>=0.4.0->sentence-transformers>=2.2.2->chromadb) (3.12.0)\n", + "Requirement already satisfied: packaging>=20.9 in /workspace/langchain/.venv/lib/python3.9/site-packages (from huggingface-hub>=0.4.0->sentence-transformers>=2.2.2->chromadb) (23.1)\n", + "Requirement already satisfied: anyio<5,>=3.4.0 in /workspace/langchain/.venv/lib/python3.9/site-packages (from starlette<0.27.0,>=0.26.1->fastapi>=0.85.1->chromadb) (3.6.2)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /workspace/langchain/.venv/lib/python3.9/site-packages (from torch>=1.6.0->sentence-transformers>=2.2.2->chromadb) (11.7.99)\n", + "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /workspace/langchain/.venv/lib/python3.9/site-packages (from torch>=1.6.0->sentence-transformers>=2.2.2->chromadb) (8.5.0.96)\n", + "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /workspace/langchain/.venv/lib/python3.9/site-packages (from torch>=1.6.0->sentence-transformers>=2.2.2->chromadb) (11.10.3.66)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /workspace/langchain/.venv/lib/python3.9/site-packages (from torch>=1.6.0->sentence-transformers>=2.2.2->chromadb) (11.7.99)\n", + "Requirement already satisfied: setuptools in /workspace/langchain/.venv/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.6.0->sentence-transformers>=2.2.2->chromadb) (67.7.1)\n", + "Requirement already satisfied: wheel in /workspace/langchain/.venv/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.6.0->sentence-transformers>=2.2.2->chromadb) (0.40.0)\n", + "Requirement already satisfied: regex!=2019.12.17 in /workspace/langchain/.venv/lib/python3.9/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers>=2.2.2->chromadb) (2023.3.23)\n", + "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /workspace/langchain/.venv/lib/python3.9/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers>=2.2.2->chromadb) (0.13.3)\n", + "Requirement already satisfied: joblib in /workspace/langchain/.venv/lib/python3.9/site-packages (from nltk->sentence-transformers>=2.2.2->chromadb) (1.2.0)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /workspace/langchain/.venv/lib/python3.9/site-packages (from scikit-learn->sentence-transformers>=2.2.2->chromadb) (3.1.0)\n", + "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /workspace/langchain/.venv/lib/python3.9/site-packages (from torchvision->sentence-transformers>=2.2.2->chromadb) (9.5.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /workspace/langchain/.venv/lib/python3.9/site-packages (from anyio<5,>=3.4.0->starlette<0.27.0,>=0.26.1->fastapi>=0.85.1->chromadb) (1.3.0)\n" + ] + } + ], + "source": [ + "!poetry run pip install pyyaml chromadb\n", + "import yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "id": "0d1633c3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new SQLDatabaseChain chain...\u001b[0m\n", + "List all the customer first names that start with 'a'\n", + "SQLQuery:" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/workspace/langchain/.venv/lib/python3.9/site-packages/transformers/pipelines/base.py:1070: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32;1m\u001b[1;3mSELECT firstname FROM customer WHERE firstname LIKE '%a%'\u001b[0m\n", + "SQLResult: \u001b[33;1m\u001b[1;3m[('François',), ('František',), ('Helena',), ('Astrid',), ('Daan',), ('Kara',), ('Eduardo',), ('Alexandre',), ('Fernanda',), ('Mark',), ('Frank',), ('Jack',), ('Dan',), ('Kathy',), ('Heather',), ('Frank',), ('Richard',), ('Patrick',), ('Julia',), ('Edward',), ('Martha',), ('Aaron',), ('Madalena',), ('Hannah',), ('Niklas',), ('Camille',), ('Marc',), ('Wyatt',), ('Isabelle',), ('Ladislav',), ('Lucas',), ('Johannes',), ('Stanisław',), ('Joakim',), ('Emma',), ('Mark',), ('Manoj',), ('Puja',)]\u001b[0m\n", + "Answer:" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/workspace/langchain/.venv/lib/python3.9/site-packages/transformers/pipelines/base.py:1070: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32;1m\u001b[1;3m[('François', 'Frantiek', 'Helena', 'Astrid', 'Daan', 'Kara', 'Eduardo', 'Alexandre', 'Fernanda', 'Mark', 'Frank', 'Jack', 'Dan', 'Kathy', 'Heather', 'Frank', 'Richard', 'Patrick', 'Julia', 'Edward', 'Martha', 'Aaron', 'Madalena', 'Hannah', 'Niklas', 'Camille', 'Marc', 'Wyatt', 'Isabelle', 'Ladislav', 'Lucas', 'Johannes', 'Stanisaw', 'Joakim', 'Emma', 'Mark', 'Manoj', 'Puja']\u001b[0m\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "*** Query succeeded\n", + "\n", + "answer: '[(''François'', ''Frantiek'', ''Helena'', ''Astrid'', ''Daan'', ''Kara'',\n", + " ''Eduardo'', ''Alexandre'', ''Fernanda'', ''Mark'', ''Frank'', ''Jack'', ''Dan'',\n", + " ''Kathy'', ''Heather'', ''Frank'', ''Richard'', ''Patrick'', ''Julia'', ''Edward'',\n", + " ''Martha'', ''Aaron'', ''Madalena'', ''Hannah'', ''Niklas'', ''Camille'', ''Marc'',\n", + " ''Wyatt'', ''Isabelle'', ''Ladislav'', ''Lucas'', ''Johannes'', ''Stanisaw'', ''Joakim'',\n", + " ''Emma'', ''Mark'', ''Manoj'', ''Puja'']'\n", + "input: List all the customer first names that start with 'a'\n", + "sql_cmd: SELECT firstname FROM customer WHERE firstname LIKE '%a%'\n", + "sql_result: '[(''François'',), (''František'',), (''Helena'',), (''Astrid'',), (''Daan'',),\n", + " (''Kara'',), (''Eduardo'',), (''Alexandre'',), (''Fernanda'',), (''Mark'',), (''Frank'',),\n", + " (''Jack'',), (''Dan'',), (''Kathy'',), (''Heather'',), (''Frank'',), (''Richard'',),\n", + " (''Patrick'',), (''Julia'',), (''Edward'',), (''Martha'',), (''Aaron'',), (''Madalena'',),\n", + " (''Hannah'',), (''Niklas'',), (''Camille'',), (''Marc'',), (''Wyatt'',), (''Isabelle'',),\n", + " (''Ladislav'',), (''Lucas'',), (''Johannes'',), (''Stanisław'',), (''Joakim'',),\n", + " (''Emma'',), (''Mark'',), (''Manoj'',), (''Puja'',)]'\n", + "table_info: \"\\nCREATE TABLE \\\"Customer\\\" (\\n\\t\\\"CustomerId\\\" INTEGER NOT NULL, \\n\\t\\\n", + " \\\"FirstName\\\" NVARCHAR(40) NOT NULL, \\n\\t\\\"LastName\\\" NVARCHAR(20) NOT NULL, \\n\\t\\\n", + " \\\"Company\\\" NVARCHAR(80), \\n\\t\\\"Address\\\" NVARCHAR(70), \\n\\t\\\"City\\\" NVARCHAR(40),\\\n", + " \\ \\n\\t\\\"State\\\" NVARCHAR(40), \\n\\t\\\"Country\\\" NVARCHAR(40), \\n\\t\\\"PostalCode\\\" NVARCHAR(10),\\\n", + " \\ \\n\\t\\\"Phone\\\" NVARCHAR(24), \\n\\t\\\"Fax\\\" NVARCHAR(24), \\n\\t\\\"Email\\\" NVARCHAR(60)\\\n", + " \\ NOT NULL, \\n\\t\\\"SupportRepId\\\" INTEGER, \\n\\tPRIMARY KEY (\\\"CustomerId\\\"), \\n\\t\\\n", + " FOREIGN KEY(\\\"SupportRepId\\\") REFERENCES \\\"Employee\\\" (\\\"EmployeeId\\\")\\n)\\n\\n/*\\n\\\n", + " 3 rows from Customer table:\\nCustomerId\\tFirstName\\tLastName\\tCompany\\tAddress\\t\\\n", + " City\\tState\\tCountry\\tPostalCode\\tPhone\\tFax\\tEmail\\tSupportRepId\\n1\\tLuís\\tGonçalves\\t\\\n", + " Embraer - Empresa Brasileira de Aeronáutica S.A.\\tAv. Brigadeiro Faria Lima, 2170\\t\\\n", + " São José dos Campos\\tSP\\tBrazil\\t12227-000\\t+55 (12) 3923-5555\\t+55 (12) 3923-5566\\t\\\n", + " luisg@embraer.com.br\\t3\\n2\\tLeonie\\tKöhler\\tNone\\tTheodor-Heuss-Straße 34\\tStuttgart\\t\\\n", + " None\\tGermany\\t70174\\t+49 0711 2842222\\tNone\\tleonekohler@surfeu.de\\t5\\n3\\tFrançois\\t\\\n", + " Tremblay\\tNone\\t1498 rue Bélanger\\tMontréal\\tQC\\tCanada\\tH2G 1A7\\t+1 (514) 721-4711\\t\\\n", + " None\\tftremblay@gmail.com\\t3\\n*/\"\n", + "\n" + ] + } + ], + "source": [ + "from typing import Dict\n", + "\n", + "QUERY = \"List all the customer first names that start with 'a'\"\n", + "\n", + "def _parse_example(result: Dict) -> Dict:\n", + " sql_cmd_key = \"sql_cmd\"\n", + " sql_result_key = \"sql_result\"\n", + " table_info_key = \"table_info\"\n", + " input_key = \"input\"\n", + " final_answer_key = \"answer\"\n", + "\n", + " _example = {\n", + " \"input\": result.get(\"query\"),\n", + " }\n", + "\n", + " steps = result.get(\"intermediate_steps\")\n", + " answer_key = sql_cmd_key # the first one\n", + " for step in steps:\n", + " # The steps are in pairs, a dict (input) followed by a string (output).\n", + " # Unfortunately there is no schema but you can look at the input key of the\n", + " # dict to see what the output is supposed to be\n", + " if isinstance(step, dict):\n", + " # Grab the table info from input dicts in the intermediate steps once\n", + " if table_info_key not in _example:\n", + " _example[table_info_key] = step.get(table_info_key)\n", + "\n", + " if input_key in step:\n", + " if step[input_key].endswith(\"SQLQuery:\"):\n", + " answer_key = sql_cmd_key # this is the SQL generation input\n", + " if step[input_key].endswith(\"Answer:\"):\n", + " answer_key = final_answer_key # this is the final answer input\n", + " elif sql_cmd_key in step:\n", + " _example[sql_cmd_key] = step[sql_cmd_key]\n", + " answer_key = sql_result_key # this is SQL execution input\n", + " elif isinstance(step, str):\n", + " # The preceding element should have set the answer_key\n", + " _example[answer_key] = step\n", + " return _example\n", + "\n", + "example: any\n", + "try:\n", + " result = local_chain(QUERY)\n", + " print(\"*** Query succeeded\")\n", + " example = _parse_example(result)\n", + "except Exception as exc:\n", + " print(\"*** Query failed\")\n", + " result = {\n", + " \"query\": QUERY,\n", + " \"intermediate_steps\": exc.intermediate_steps\n", + " }\n", + " example = _parse_example(result)\n", + "\n", + "\n", + "# print for now, in reality you may want to write this out to a YAML file or database for manual fix-ups offline\n", + "yaml_example = yaml.dump(example, allow_unicode=True)\n", + "print(\"\\n\" + yaml_example)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "da618de6", + "metadata": {}, + "source": [ + "Run the snippet above a few times, or log exceptions in your deployed environment, to collect lots of examples of inputs, table_info and sql_cmd generated by your language model. The sql_cmd values will be incorrect and you can manually fix them up to build a collection of examples, e.g. here we are using YAML to keep a neat record of our inputs and corrected SQL output that we can build up over time." + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "id": "b81b9fdd", + "metadata": {}, + "outputs": [], + "source": [ + "YAML_EXAMPLES = \"\"\"\n", + "- input: How many customers are not from Brazil?\n", + " table_info: |\n", + " CREATE TABLE \"Customer\" (\n", + " \"CustomerId\" INTEGER NOT NULL, \n", + " \"FirstName\" NVARCHAR(40) NOT NULL, \n", + " \"LastName\" NVARCHAR(20) NOT NULL, \n", + " \"Company\" NVARCHAR(80), \n", + " \"Address\" NVARCHAR(70), \n", + " \"City\" NVARCHAR(40), \n", + " \"State\" NVARCHAR(40), \n", + " \"Country\" NVARCHAR(40), \n", + " \"PostalCode\" NVARCHAR(10), \n", + " \"Phone\" NVARCHAR(24), \n", + " \"Fax\" NVARCHAR(24), \n", + " \"Email\" NVARCHAR(60) NOT NULL, \n", + " \"SupportRepId\" INTEGER, \n", + " PRIMARY KEY (\"CustomerId\"), \n", + " FOREIGN KEY(\"SupportRepId\") REFERENCES \"Employee\" (\"EmployeeId\")\n", + " )\n", + " sql_cmd: SELECT COUNT(*) FROM \"Customer\" WHERE NOT \"Country\" = \"Brazil\";\n", + " sql_result: \"[(54,)]\"\n", + " answer: 54 customers are not from Brazil.\n", + "- input: list all the genres that start with 'r'\n", + " table_info: |\n", + " CREATE TABLE \"Genre\" (\n", + " \"GenreId\" INTEGER NOT NULL, \n", + " \"Name\" NVARCHAR(120), \n", + " PRIMARY KEY (\"GenreId\")\n", + " )\n", + "\n", + " /*\n", + " 3 rows from Genre table:\n", + " GenreId\tName\n", + " 1\tRock\n", + " 2\tJazz\n", + " 3\tMetal\n", + " */\n", + " sql_cmd: SELECT \"Name\" FROM \"Genre\" WHERE \"Name\" LIKE 'r%';\n", + " sql_result: \"[('Rock',), ('Rock and Roll',), ('Reggae',), ('R&B/Soul',)]\"\n", + " answer: The genres that start with 'r' are Rock, Rock and Roll, Reggae and R&B/Soul. \n", + "\"\"\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "6b13b4d5", + "metadata": {}, + "source": [ + "Now that you have some examples (with manually corrected output SQL), you can do few shot prompt seeding the usual way:" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "id": "0d174f59", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using embedded DuckDB without persistence: data will be transient\n" + ] + } + ], + "source": [ + "from langchain import FewShotPromptTemplate, PromptTemplate\n", + "from langchain.chains.sql_database.prompt import _sqlite_prompt, PROMPT_SUFFIX\n", + "from langchain.embeddings.huggingface import HuggingFaceEmbeddings\n", + "from langchain.prompts.example_selector.semantic_similarity import SemanticSimilarityExampleSelector\n", + "from langchain.vectorstores import Chroma\n", + "\n", + "example_prompt = PromptTemplate(\n", + " input_variables=[\"table_info\", \"input\", \"sql_cmd\", \"sql_result\", \"answer\"],\n", + " template=\"{table_info}\\n\\nQuestion: {input}\\nSQLQuery: {sql_cmd}\\nSQLResult: {sql_result}\\nAnswer: {answer}\",\n", + ")\n", + "\n", + "examples_dict = yaml.safe_load(YAML_EXAMPLES)\n", + "\n", + "local_embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n", + "\n", + "example_selector = SemanticSimilarityExampleSelector.from_examples(\n", + " # This is the list of examples available to select from.\n", + " examples_dict,\n", + " # This is the embedding class used to produce embeddings which are used to measure semantic similarity.\n", + " local_embeddings,\n", + " # This is the VectorStore class that is used to store the embeddings and do a similarity search over.\n", + " Chroma, # type: ignore\n", + " # This is the number of examples to produce and include per prompt\n", + " k=min(3, len(examples_dict)),\n", + " )\n", + "\n", + "few_shot_prompt = FewShotPromptTemplate(\n", + " example_selector=example_selector,\n", + " example_prompt=example_prompt,\n", + " prefix=_sqlite_prompt + \"Here are some examples:\",\n", + " suffix=PROMPT_SUFFIX,\n", + " input_variables=[\"table_info\", \"input\", \"top_k\"],\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "2324e41f", + "metadata": {}, + "source": [ + "The model should do better now with this few shot prompt, especially for inputs similar to the examples you have seeded it with." + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "id": "c4dcf76c", + "metadata": {}, + "outputs": [], + "source": [ + "local_chain = SQLDatabaseChain.from_llm(local_llm, db, prompt=few_shot_prompt, use_query_checker=True, verbose=True, return_intermediate_steps=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "id": "5ee7a26c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new SQLDatabaseChain chain...\u001b[0m\n", + "How many customers are from Brazil?\n", + "SQLQuery:\u001b[32;1m\u001b[1;3mSELECT count(*) FROM Customer WHERE Country = \"Brazil\";\u001b[0m\n", + "SQLResult: \u001b[33;1m\u001b[1;3m[(5,)]\u001b[0m\n", + "Answer:\u001b[32;1m\u001b[1;3m[5]\u001b[0m\n", + "\u001b[1m> Finished chain.\u001b[0m\n" + ] + } + ], + "source": [ + "result = local_chain(\"How many customers are from Brazil?\")" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "id": "ead61709", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new SQLDatabaseChain chain...\u001b[0m\n", + "How many customers are not from Brazil?\n", + "SQLQuery:\u001b[32;1m\u001b[1;3mSELECT count(*) FROM customer WHERE country NOT IN (SELECT country FROM customer WHERE country = 'Brazil')\u001b[0m\n", + "SQLResult: \u001b[33;1m\u001b[1;3m[(54,)]\u001b[0m\n", + "Answer:\u001b[32;1m\u001b[1;3m54 customers are not from Brazil.\u001b[0m\n", + "\u001b[1m> Finished chain.\u001b[0m\n" + ] + } + ], + "source": [ + "result = local_chain(\"How many customers are not from Brazil?\")" + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "id": "7af127d9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new SQLDatabaseChain chain...\u001b[0m\n", + "How many customers are there in total?\n", + "SQLQuery:\u001b[32;1m\u001b[1;3mSELECT count(*) FROM Customer;\u001b[0m\n", + "SQLResult: \u001b[33;1m\u001b[1;3m[(59,)]\u001b[0m\n", + "Answer:\u001b[32;1m\u001b[1;3mThere are 59 customers in total.\u001b[0m\n", + "\u001b[1m> Finished chain.\u001b[0m\n" + ] + } + ], + "source": [ + "result = local_chain(\"How many customers are there in total?\")" + ] } ], "metadata": { @@ -681,7 +1409,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.1" + "version": "3.9.2" } }, "nbformat": 4, diff --git a/langchain/chains/sql_database/base.py b/langchain/chains/sql_database/base.py index d73d34a3..a2eb4a90 100644 --- a/langchain/chains/sql_database/base.py +++ b/langchain/chains/sql_database/base.py @@ -12,7 +12,11 @@ from langchain.chains.base import Chain from langchain.chains.llm import LLMChain from langchain.chains.sql_database.prompt import DECIDER_PROMPT, PROMPT, SQL_PROMPTS from langchain.prompts.base import BasePromptTemplate +from langchain.prompts.prompt import PromptTemplate from langchain.sql_database import SQLDatabase +from langchain.tools.sql_database.prompt import QUERY_CHECKER + +INTERMEDIATE_STEPS_KEY = "intermediate_steps" class SQLDatabaseChain(Chain): @@ -41,6 +45,11 @@ class SQLDatabaseChain(Chain): """Whether or not to return the intermediate steps along with the final answer.""" return_direct: bool = False """Whether or not to return the result of querying the SQL table directly.""" + use_query_checker: bool = False + """Whether or not the query checker tool should be used to attempt + to fix the initial SQL from the LLM.""" + query_checker_prompt: Optional[BasePromptTemplate] = None + """The prompt template that should be used by the query checker""" class Config: """Configuration for this pydantic object.""" @@ -81,7 +90,7 @@ class SQLDatabaseChain(Chain): if not self.return_intermediate_steps: return [self.output_key] else: - return [self.output_key, "intermediate_steps"] + return [self.output_key, INTERMEDIATE_STEPS_KEY] def _call( self, @@ -96,36 +105,80 @@ class SQLDatabaseChain(Chain): table_info = self.database.get_table_info(table_names=table_names_to_use) llm_inputs = { "input": input_text, - "top_k": self.top_k, + "top_k": str(self.top_k), "dialect": self.database.dialect, "table_info": table_info, "stop": ["\nSQLResult:"], } - intermediate_steps = [] - sql_cmd = self.llm_chain.predict( - callbacks=_run_manager.get_child(), **llm_inputs - ) - intermediate_steps.append(sql_cmd) - _run_manager.on_text(sql_cmd, color="green", verbose=self.verbose) - result = self.database.run(sql_cmd) - intermediate_steps.append(result) - _run_manager.on_text("\nSQLResult: ", verbose=self.verbose) - _run_manager.on_text(result, color="yellow", verbose=self.verbose) - # If return direct, we just set the final result equal to the sql query - if self.return_direct: - final_result = result - else: - _run_manager.on_text("\nAnswer:", verbose=self.verbose) - input_text += f"{sql_cmd}\nSQLResult: {result}\nAnswer:" - llm_inputs["input"] = input_text - final_result = self.llm_chain.predict( - callbacks=_run_manager.get_child(), **llm_inputs - ) - _run_manager.on_text(final_result, color="green", verbose=self.verbose) - chain_result: Dict[str, Any] = {self.output_key: final_result} - if self.return_intermediate_steps: - chain_result["intermediate_steps"] = intermediate_steps - return chain_result + intermediate_steps: List = [] + try: + intermediate_steps.append(llm_inputs) # input: sql generation + sql_cmd = self.llm_chain.predict( + callbacks=_run_manager.get_child(), + **llm_inputs, + ).strip() + if not self.use_query_checker: + _run_manager.on_text(sql_cmd, color="green", verbose=self.verbose) + intermediate_steps.append( + sql_cmd + ) # output: sql generation (no checker) + intermediate_steps.append({"sql_cmd": sql_cmd}) # input: sql exec + result = self.database.run(sql_cmd) + intermediate_steps.append(str(result)) # output: sql exec + else: + query_checker_prompt = self.query_checker_prompt or PromptTemplate( + template=QUERY_CHECKER, input_variables=["query", "dialect"] + ) + query_checker_chain = LLMChain( + llm=self.llm, prompt=query_checker_prompt + ) + query_checker_inputs = { + "query": sql_cmd, + "dialect": self.database.dialect, + } + checked_sql_command: str = query_checker_chain.predict( + callbacks=_run_manager.get_child(), **query_checker_inputs + ).strip() + intermediate_steps.append( + checked_sql_command + ) # output: sql generation (checker) + _run_manager.on_text( + checked_sql_command, color="green", verbose=self.verbose + ) + intermediate_steps.append( + {"sql_cmd": checked_sql_command} + ) # input: sql exec + result = self.database.run(checked_sql_command) + intermediate_steps.append(str(result)) # output: sql exec + sql_cmd = checked_sql_command + + _run_manager.on_text("\nSQLResult: ", verbose=self.verbose) + _run_manager.on_text(result, color="yellow", verbose=self.verbose) + # If return direct, we just set the final result equal to + # the result of the sql query result, otherwise try to get a human readable + # final answer + if self.return_direct: + final_result = result + else: + _run_manager.on_text("\nAnswer:", verbose=self.verbose) + input_text += f"{sql_cmd}\nSQLResult: {result}\nAnswer:" + llm_inputs["input"] = input_text + intermediate_steps.append(llm_inputs) # input: final answer + final_result = self.llm_chain.predict( + callbacks=_run_manager.get_child(), + **llm_inputs, + ).strip() + intermediate_steps.append(final_result) # output: final answer + _run_manager.on_text(final_result, color="green", verbose=self.verbose) + chain_result: Dict[str, Any] = {self.output_key: final_result} + if self.return_intermediate_steps: + chain_result[INTERMEDIATE_STEPS_KEY] = intermediate_steps + return chain_result + except Exception as exc: + # Append intermediate steps to exception, to aid in logging and later + # improvement of few shot prompt seeds + exc.intermediate_steps = intermediate_steps # type: ignore + raise exc @property def _chain_type(self) -> str: @@ -195,7 +248,7 @@ class SQLDatabaseSequentialChain(Chain): if not self.return_intermediate_steps: return [self.output_key] else: - return [self.output_key, "intermediate_steps"] + return [self.output_key, INTERMEDIATE_STEPS_KEY] def _call( self, @@ -209,9 +262,13 @@ class SQLDatabaseSequentialChain(Chain): "query": inputs[self.input_key], "table_names": table_names, } - table_names_to_use = self.decider_chain.predict_and_parse( - callbacks=_run_manager.get_child(), **llm_inputs - ) + _lowercased_table_names = [name.lower() for name in _table_names] + table_names_from_chain = self.decider_chain.predict_and_parse(**llm_inputs) + table_names_to_use = [ + name + for name in table_names_from_chain + if name.lower() in _lowercased_table_names + ] _run_manager.on_text("Table names to use:", end="\n", verbose=self.verbose) _run_manager.on_text( str(table_names_to_use), color="yellow", verbose=self.verbose diff --git a/langchain/chains/sql_database/prompt.py b/langchain/chains/sql_database/prompt.py index a2b6f4c2..3b204093 100644 --- a/langchain/chains/sql_database/prompt.py +++ b/langchain/chains/sql_database/prompt.py @@ -3,6 +3,11 @@ from langchain.output_parsers.list import CommaSeparatedListOutputParser from langchain.prompts.prompt import PromptTemplate +PROMPT_SUFFIX = """Only use the following tables: +{table_info} + +Question: {input}""" + _DEFAULT_TEMPLATE = """Given an input question, first create a syntactically correct {dialect} query to run, then look at the results of the query and return the answer. Unless the user specifies in his question a specific number of examples he wishes to obtain, always limit your query to at most {top_k} results. You can order the results by a relevant column to return the most interesting examples in the database. Never query for all the columns from a specific table, only ask for a the few relevant columns given the question. @@ -16,17 +21,14 @@ SQLQuery: SQL Query to run SQLResult: Result of the SQLQuery Answer: Final answer here -Only use the tables listed below. - -{table_info} - -Question: {input}""" +""" PROMPT = PromptTemplate( input_variables=["input", "table_info", "dialect", "top_k"], - template=_DEFAULT_TEMPLATE, + template=_DEFAULT_TEMPLATE + PROMPT_SUFFIX, ) + _DECIDER_TEMPLATE = """Given the below input question and list of potential tables, output a comma separated list of the table names that may be necessary to answer this question. Question: {query} @@ -53,14 +55,11 @@ SQLQuery: SQL Query to run SQLResult: Result of the SQLQuery Answer: Final answer here -Only use the following tables: -{table_info} - -Question: {input}""" +""" DUCKDB_PROMPT = PromptTemplate( input_variables=["input", "table_info", "top_k"], - template=_duckdb_prompt, + template=_duckdb_prompt + PROMPT_SUFFIX, ) _googlesql_prompt = """You are a GoogleSQL expert. Given an input question, first create a syntactically correct GoogleSQL query to run, then look at the results of the query and return the answer to the input question. @@ -76,14 +75,11 @@ SQLQuery: SQL Query to run SQLResult: Result of the SQLQuery Answer: Final answer here -Only use the following tables: -{table_info} - -Question: {input}""" +""" GOOGLESQL_PROMPT = PromptTemplate( input_variables=["input", "table_info", "top_k"], - template=_googlesql_prompt, + template=_googlesql_prompt + PROMPT_SUFFIX, ) @@ -100,13 +96,11 @@ SQLQuery: SQL Query to run SQLResult: Result of the SQLQuery Answer: Final answer here -Only use the following tables: -{table_info} - -Question: {input}""" +""" MSSQL_PROMPT = PromptTemplate( - input_variables=["input", "table_info", "top_k"], template=_mssql_prompt + input_variables=["input", "table_info", "top_k"], + template=_mssql_prompt + PROMPT_SUFFIX, ) @@ -123,14 +117,11 @@ SQLQuery: SQL Query to run SQLResult: Result of the SQLQuery Answer: Final answer here -Only use the following tables: -{table_info} - -Question: {input}""" +""" MYSQL_PROMPT = PromptTemplate( input_variables=["input", "table_info", "top_k"], - template=_mysql_prompt, + template=_mysql_prompt + PROMPT_SUFFIX, ) @@ -147,14 +138,11 @@ SQLQuery: SQL Query to run SQLResult: Result of the SQLQuery Answer: Final answer here -Only use the following tables: -{table_info} - -Question: {input}""" +""" MARIADB_PROMPT = PromptTemplate( input_variables=["input", "table_info", "top_k"], - template=_mariadb_prompt, + template=_mariadb_prompt + PROMPT_SUFFIX, ) @@ -171,14 +159,11 @@ SQLQuery: SQL Query to run SQLResult: Result of the SQLQuery Answer: Final answer here -Only use the following tables: -{table_info} - -Question: {input}""" +""" ORACLE_PROMPT = PromptTemplate( input_variables=["input", "table_info", "top_k"], - template=_oracle_prompt, + template=_oracle_prompt + PROMPT_SUFFIX, ) @@ -195,13 +180,11 @@ SQLQuery: SQL Query to run SQLResult: Result of the SQLQuery Answer: Final answer here -Only use the following tables: -{table_info} - -Question: {input}""" +""" POSTGRES_PROMPT = PromptTemplate( - input_variables=["input", "table_info", "top_k"], template=_postgres_prompt + input_variables=["input", "table_info", "top_k"], + template=_postgres_prompt + PROMPT_SUFFIX, ) @@ -218,14 +201,11 @@ SQLQuery: SQL Query to run SQLResult: Result of the SQLQuery Answer: Final answer here -Only use the following tables: -{table_info} - -Question: {input}""" +""" SQLITE_PROMPT = PromptTemplate( input_variables=["input", "table_info", "top_k"], - template=_sqlite_prompt, + template=_sqlite_prompt + PROMPT_SUFFIX, ) _clickhouse_prompt = """You are a ClickHouse expert. Given an input question, first create a syntactically correct Clic query to run, then look at the results of the query and return the answer to the input question. @@ -241,14 +221,11 @@ SQLQuery: "SQL Query to run" SQLResult: "Result of the SQLQuery" Answer: "Final answer here" -Only use the following tables: -{table_info} - -Question: {input}""" +""" CLICKHOUSE_PROMPT = PromptTemplate( input_variables=["input", "table_info", "top_k"], - template=_clickhouse_prompt, + template=_clickhouse_prompt + PROMPT_SUFFIX, ) _prestodb_prompt = """You are a PrestoDB expert. Given an input question, first create a syntactically correct PrestoDB query to run, then look at the results of the query and return the answer to the input question. @@ -264,14 +241,11 @@ SQLQuery: "SQL Query to run" SQLResult: "Result of the SQLQuery" Answer: "Final answer here" -Only use the following tables: -{table_info} - -Question: {input}""" +""" PRESTODB_PROMPT = PromptTemplate( input_variables=["input", "table_info", "top_k"], - template=_prestodb_prompt, + template=_prestodb_prompt + PROMPT_SUFFIX, )