mirror of
https://github.com/hwchase17/langchain
synced 2024-10-29 17:07:25 +00:00
241 lines
103 KiB
Plaintext
241 lines
103 KiB
Plaintext
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "ddb208a0-617e-433e-b9de-69099bc456f8",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"# Image captions\n",
|
||
|
|
"This notebook shows how to use the ImageCaptionLoader tutorial to generate a query-able index of image captions"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 1,
|
||
|
|
"id": "ac0a2a76-c36a-4952-b511-7906ca840e08",
|
||
|
|
"metadata": {
|
||
|
|
"pycharm": {
|
||
|
|
"is_executing": true
|
||
|
|
},
|
||
|
|
"tags": []
|
||
|
|
},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"from langchain.document_loaders import ImageCaptionLoader"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "faefe80f-08f2-4683-a325-4efd61fae0bf",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Prepare a list of image urls from Wikimedia"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 2,
|
||
|
|
"id": "9a400568-5fea-47e6-8703-d9c1a1cc00ea",
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"list_image_urls = [\n",
|
||
|
|
" 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/5a/Hyla_japonica_sep01.jpg/260px-Hyla_japonica_sep01.jpg',\n",
|
||
|
|
" 'https://upload.wikimedia.org/wikipedia/commons/thumb/7/71/Tibur%C3%B3n_azul_%28Prionace_glauca%29%2C_canal_Fayal-Pico%2C_islas_Azores%2C_Portugal%2C_2020-07-27%2C_DD_14.jpg/270px-Tibur%C3%B3n_azul_%28Prionace_glauca%29%2C_canal_Fayal-Pico%2C_islas_Azores%2C_Portugal%2C_2020-07-27%2C_DD_14.jpg',\n",
|
||
|
|
" 'https://upload.wikimedia.org/wikipedia/commons/thumb/2/21/Thure_de_Thulstrup_-_Battle_of_Shiloh.jpg/251px-Thure_de_Thulstrup_-_Battle_of_Shiloh.jpg',\n",
|
||
|
|
" 'https://upload.wikimedia.org/wikipedia/commons/thumb/2/21/Passion_fruits_-_whole_and_halved.jpg/270px-Passion_fruits_-_whole_and_halved.jpg',\n",
|
||
|
|
" 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/5e/Messier83_-_Heic1403a.jpg/277px-Messier83_-_Heic1403a.jpg',\n",
|
||
|
|
" 'https://upload.wikimedia.org/wikipedia/commons/thumb/b/b6/2022-01-22_Men%27s_World_Cup_at_2021-22_St._Moritz%E2%80%93Celerina_Luge_World_Cup_and_European_Championships_by_Sandro_Halank%E2%80%93257.jpg/288px-2022-01-22_Men%27s_World_Cup_at_2021-22_St._Moritz%E2%80%93Celerina_Luge_World_Cup_and_European_Championships_by_Sandro_Halank%E2%80%93257.jpg',\n",
|
||
|
|
" 'https://upload.wikimedia.org/wikipedia/commons/thumb/9/99/Wiesen_Pippau_%28Crepis_biennis%29-20220624-RM-123950.jpg/224px-Wiesen_Pippau_%28Crepis_biennis%29-20220624-RM-123950.jpg',\n",
|
||
|
|
"]"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "be585acd-6e28-4400-9e8f-17fdde11e02c",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Create the loader"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 7,
|
||
|
|
"id": "fb392517-72d8-416e-852c-da90b77267ed",
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"/Users/saitosean/dev/langchain/.venv/lib/python3.10/site-packages/transformers/generation/utils.py:1313: UserWarning: Using `max_length`'s default (20) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
|
||
|
|
" warnings.warn(\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"[Document(page_content='an image of a frog on a flower [SEP]', metadata={'image_path': 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/5a/Hyla_japonica_sep01.jpg/260px-Hyla_japonica_sep01.jpg'}),\n",
|
||
|
|
" Document(page_content='an image of a shark swimming in the ocean [SEP]', metadata={'image_path': 'https://upload.wikimedia.org/wikipedia/commons/thumb/7/71/Tibur%C3%B3n_azul_%28Prionace_glauca%29%2C_canal_Fayal-Pico%2C_islas_Azores%2C_Portugal%2C_2020-07-27%2C_DD_14.jpg/270px-Tibur%C3%B3n_azul_%28Prionace_glauca%29%2C_canal_Fayal-Pico%2C_islas_Azores%2C_Portugal%2C_2020-07-27%2C_DD_14.jpg'}),\n",
|
||
|
|
" Document(page_content='an image of a painting of a battle scene [SEP]', metadata={'image_path': 'https://upload.wikimedia.org/wikipedia/commons/thumb/2/21/Thure_de_Thulstrup_-_Battle_of_Shiloh.jpg/251px-Thure_de_Thulstrup_-_Battle_of_Shiloh.jpg'}),\n",
|
||
|
|
" Document(page_content='an image of a passion fruit and a half cut passion [SEP]', metadata={'image_path': 'https://upload.wikimedia.org/wikipedia/commons/thumb/2/21/Passion_fruits_-_whole_and_halved.jpg/270px-Passion_fruits_-_whole_and_halved.jpg'}),\n",
|
||
|
|
" Document(page_content='an image of the spiral galaxy [SEP]', metadata={'image_path': 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/5e/Messier83_-_Heic1403a.jpg/277px-Messier83_-_Heic1403a.jpg'}),\n",
|
||
|
|
" Document(page_content='an image of a man on skis in the snow [SEP]', metadata={'image_path': 'https://upload.wikimedia.org/wikipedia/commons/thumb/b/b6/2022-01-22_Men%27s_World_Cup_at_2021-22_St._Moritz%E2%80%93Celerina_Luge_World_Cup_and_European_Championships_by_Sandro_Halank%E2%80%93257.jpg/288px-2022-01-22_Men%27s_World_Cup_at_2021-22_St._Moritz%E2%80%93Celerina_Luge_World_Cup_and_European_Championships_by_Sandro_Halank%E2%80%93257.jpg'}),\n",
|
||
|
|
" Document(page_content='an image of a flower in the dark [SEP]', metadata={'image_path': 'https://upload.wikimedia.org/wikipedia/commons/thumb/9/99/Wiesen_Pippau_%28Crepis_biennis%29-20220624-RM-123950.jpg/224px-Wiesen_Pippau_%28Crepis_biennis%29-20220624-RM-123950.jpg'})]"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"execution_count": 7,
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "execute_result"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"loader = ImageCaptionLoader(path_images=list_image_urls)\n",
|
||
|
|
"list_docs = loader.load()\n",
|
||
|
|
"list_docs"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 10,
|
||
|
|
"id": "0f56db67-99bb-4543-ba40-1871a58b2da5",
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAC0CAIAAAD918WcAAABJGlDQ1BJQ0MgUHJvZmlsZQAAeJxjYGAycHRxcmUSYGDIzSspCnJ3UoiIjFJgP8/AxsDMAAaJycUFjgEBPiB2Xn5eKgMG+HaNgRFEX9YFmYUpjxdwJRcUlQDpP0BslJJanMzAwGgAZGeXlxQAxRnnANkiSdlg9gYQuygkyBnIPgJk86VD2FdA7CQI+wmIXQT0BJD9BaQ+Hcxm4gCbA2HLgNglqRUgexmc8wsqizLTM0oUDC0tLRUcU/KTUhWCK4tLUnOLFTzzkvOLCvKLEktSU4BqIe4DA0GIQlCIaQA1WmiS6G+CABQPENbnQHD4MoqdQYghQHJpURmUychkTJiPMGOOBAOD/1IGBpY/CDGTXgaGBToMDPxTEWJqhgwMAvoMDPvmAADAxk/9GlU2EAABAABJREFUeJzM/XnQJcl2H4adk1nr3bdv33pfprtnn3kbHoAHgCAWmiQAgptpOcwIBMOWTPoPmaIshyNMWaYkyqRgOqiwacqWudikCTMEkgCeKALvkW/eOm/27um9v327+1pr5vEfmVW37v2+7pkHSBHKmbh9v7pVWVlZZ/mdJU/ir/6Jn0RExhgAIDFMGgAgIoBEBoyASBARoASQABKQACQiIhOIgEwASGSSMYYoGGPIiDFkDLgpEYkxQERihIwQkekvugdgyBioczgRIqrT1DBUy35XjYgAQBCjpElCIpISpJSxZEREkhGRJJ58R5BIhFIQAAAw9UlEQBwAEEwAQOQAQOoUYkSEJNQfAAAkiISUEkCCFFJKKWMiIhkDgKAYAJCApsOS6bWMAAAQIH06AkREBsgYMxhLj5ucAwAwRERi6esA0POAACDUTYClzx4TAIBUn1ISkQAppZSSpJT6TAnTT0FEJAGJSAogIqleir4LS+c5afNHpJRzR573XZ3OCAAlnGnpmepL+qdQX1BmjhMAqDdCyWeGPFjaZ0rGc8TzPLoypJRT0p8feuZ2JJOhKn4AQECUQAIRUBKgYAQAEhGIBAMAYESSQDJOUiJjICUwBsgkESIJRGJMjUU9KkeUDJBIIiGyzMCS2UmHrieLGMzzyPMbSkXxn3VO+r4/V9dqroko/VTjm85yZvzZRplznsfqmvto+l7Pe0Nnj83/mn6+4Llmxv9ZHX7mr3OnJX+ifqT/wbR0nIiYMgMDIKaEPXLGMCFQSUSkH0ACSilFwgxKORCgRJSoxCsRMkAkKYkxYJykEsiIjAExlJIYlwlXkJQSEZADMgIgREI9BkCClPJhyuXpUygKk8SUBsAXU6960wASgOvB03PPTibo8/JDtinunOkbEQAY4ZzI1MTHWJbn1UOdpUulpQEwkXxERNnpSAn9LLm/mHyzDRHVu1bqGWZ5SXX2gm6fJ93nzzl/5tPbaWmSHBc/0iP8vhsRpcwgM1KHiAhR6+BUJyAiEEqZniQBCZEABQBxjup8lJofiEhKYlwxBhFjCiYRITKR8gMiISBKAiLGQZCCTxyRUrym7n6u6EqwyPRPNZsJ9cPc92lDApIZrZq8BpQK0eiDKLMvT3O8JuVpz4iYvM75O2bP+fwvNasZFIknKucccnwBA6SwNzOGMzOhAdt5A3iujD//yLlXfZ6nfsH5RARnnv2/q5ZFHIaUwBgmLMERgTGSUjLGiGSG0hQFqL/0J2gmZ4gkkVDTAaBiHAmMAyAyYJIIiBEQA5QEDIBAAqgzFRsoSkIEQAYIwDiCgkzIQIurlBMoJSwCIKmFytxUzYGK5E95rryfxzkglS2BiIo9UrQ611IuBWCIMjsI3dEswIPZ2zNAIECmnwsAkAABiAFgeqZUUwUIWi3TVD8kAARBvw8CNWPJSBATaQLZB5yfsKydAOdQ5HSE2V9JplJA01UGEaVnpohRfiZBn897+nlFevBHVtnPbapnBAAjK1cpmaBkXpBIEqEi3IQNELR0SdQZIgAowAMAyEABHsbUaYxIKANd3YYxkIIhQymEspUV2REj4gTIeKKXEAGRKZmLqLVNYg/pGU85Ift8iJglBcp+n58Lmc4FQAZHzeHsz3qFCahQkyvnfsrK6fRmLzAY0payKCTTN3MEp6c9zzI5e3BGkz7/vtkL5021z3Fh9ukyiltR3osM6Bf1/N+DfpgxoJVjBJGnen/2DDV1SCAS0YgkQZOn6k2JZ8E0sWpvBAExZAIEoURiSAyIARESA2TECBVcRiLQU4dEBExKBERgHFKZg5oZQIk3BEwmlBD49O2eoarnIxNl9rA5CKSVw/ScLIh6wYRyIgkgEkY9h7hTakqnl0/lPCTjmNIcarBHWVU2pVHUyP4sQJqeBnLWp6W9SUQz0loJiFRMTFXudN5Yckl6A/YC7HR2wrPmh3ieJXa+RJBzE/+ct8lgShK/z4aIRtJ7CoVmdLqibyIgQklCjVlTCyEAIGNEUs8uAgNU7kNNnKTUgibpRJkAEqE2ZAGUswkZaIjFlLtWuxUREJEwyw8Z3x/KF5NpQkkJPWGWvtUTKssBAXEqt1ArnFl+AN3DrBKCqX2PREyR1jkDmZ9YDXowQ5Qz/SZ/CiBl7Sco9HzSf+7zSznlB417z2ehFD7N9XyWSj+P/skywOyRFwz2bGNnNa2ec2LJ2/yD8kD63YjjGBEZk4jIQOkHREQhFOlPFfTZx1MCJ+2UAIQAlICIjKEkREYMuEzIhYikEvlIJAkZMA6MkJAhojoCJEEAYyh11CI1o0nFLmZYAhgloAMAsnDmzAuTigKV2aAUQMLoqISfep6ZCSKmb61NTJH8iorIEnGuUCMHFJi41ZJpA644LcNTLDM0lvgG9OBIcSwIIRlLHjh5BGIIMvvuGUAKkEQKlkA/msxiqpT9piZgoiVIQVuNkrPkO++JTmTEDD2k/acgdk63zHyZhTrPA2CZbmFK8cRSs0FKqSYqM9NKl2ZnDJLnmok2ZGFzNrRgZMeduNVIJjOeuPmmI0xgAGlJnwxD6YeE2rRRQQIBCAmAGDKlPQhAASRA0mREkETZiFADBFCUgUxbC4gIEgkQeToeDdCmjz0Ddqc0lJoBAAAoUUXZtLpINQBO7dbMyQAccSYCQzoAB59HLKXXZt8Qm3WJzv06vTA9BwETFfEjGY4ZUDT9PgeTzt73XMGf0vpZEj97yfOsIEQkYgTpBM5YFy8YybR/rRAUsTBAmcWlWYr/zMGc/dWQUkXvKPM5o82JSFE4yRcoR8JEugARoPKQAiKQDsMRSiKGiEBcs7SOepNywgOi8oIo96tmQpTIGADqeDYRSamHqgaYleWzzPCCCZBZeXnmXEoimmzuKKCWAefDYvVelQgDQERGU9M2ywBZK36GGs4LzQKA9kglGiAdmxafOAN7Up2QJf258RLNCJHZpnx3lHpjtbwHroTsGdKf0RVzuCirNNLRzYexM/Apo5PVhXL2kikXfc72Yk4AUOJPe1wMSTFDJiUwxiRJxpjyq2a6m3n22Ykg5SwCUqZC+vCgZhOQGKUBNUUiQESMA8REHIhUrgcl0WgCTPI8FARgCTklMiJlGwWWsnOdzvd8FBklaMVDqYWcoHyJjAEplpCJcpiSC6JMXPzP5bSzRJCelhErmfMzvc0Js6kqP/vW1PkzpDwPXLMnv0ADpCJgVvTOWC/Zbs9lHpwl4jN0f247BxRlWe6soqAUPsxqCUyQ+dx4zo7w8zcjwZoKIzGNlmakJqbESJRqAE1400FLTLzIWbRHAKBiC8iUXFFxunRqJKnDDAAANRkDY2kIDxRjEkkpiAEohIVSw1POmBKOmfk7m1Gj+AGUsEZESkJp52mGtOlQAwASCG22ne/hRv28z3k9qCQPaM4/79d5T0BiREglYiSCihIklwtQpoGWRwSMq0ki0mMkEsnAzuXVZLaIKENt51Czms/ZcwBAv7PZEFD2C5zHJHr6ztxFE3hyozkWnuGEc17AdCR/kGYQ6RCblJIBSilZxqpIBjrPx8mXWZbQEQnMIGkiQimJMQQJkqSS6JBqc9C0jkSgUjmAEFFKzUKMo5SECIIoUVeKo5CxrMCgzDjmETlkiB5
|
||
|
|
"text/plain": [
|
||
|
|
"<PIL.Image.Image image mode=RGB size=260x180>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"execution_count": 10,
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "execute_result"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"from PIL import Image\n",
|
||
|
|
"import requests\n",
|
||
|
|
"\n",
|
||
|
|
"Image.open(requests.get(list_image_urls[0], stream=True).raw).convert('RGB')"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "52193308-e2c5-4757-8f86-a73c07510f73",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Create the index"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 4,
|
||
|
|
"id": "7b7a15ac-d2c7-4359-9c5c-a543c8eebf80",
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"/Users/saitosean/dev/langchain/.venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
|
||
|
|
" from .autonotebook import tqdm as notebook_tqdm\n",
|
||
|
|
"/Users/saitosean/dev/langchain/.venv/lib/python3.10/site-packages/transformers/generation/utils.py:1313: UserWarning: Using `max_length`'s default (20) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
|
||
|
|
" warnings.warn(\n",
|
||
|
|
"Using embedded DuckDB without persistence: data will be transient\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"from langchain.indexes import VectorstoreIndexCreator\n",
|
||
|
|
"index = VectorstoreIndexCreator().from_loaders([loader])"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "677398d8-6ab7-4224-8e4a-4b94a7fb2a94",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Query"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 8,
|
||
|
|
"id": "e03e31c6-3018-434d-bcad-5c25144509e1",
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"' The painting is about a battle scene.'"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"execution_count": 8,
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "execute_result"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"query = \"What's the painting about?\"\n",
|
||
|
|
"index.query(query)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 9,
|
||
|
|
"id": "c3ec2b5a-9c03-4e32-b571-be5af9a22223",
|
||
|
|
"metadata": {
|
||
|
|
"tags": []
|
||
|
|
},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"' There are images of a spiral galaxy, a painting of a battle scene, a flower in the dark, and a frog on a flower.'"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"execution_count": 9,
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "execute_result"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"query = \"What kind of images are there?\"\n",
|
||
|
|
"index.query(query)"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"kernelspec": {
|
||
|
|
"display_name": "Python 3 (ipykernel)",
|
||
|
|
"language": "python",
|
||
|
|
"name": "python3"
|
||
|
|
},
|
||
|
|
"language_info": {
|
||
|
|
"codemirror_mode": {
|
||
|
|
"name": "ipython",
|
||
|
|
"version": 3
|
||
|
|
},
|
||
|
|
"file_extension": ".py",
|
||
|
|
"mimetype": "text/x-python",
|
||
|
|
"name": "python",
|
||
|
|
"nbconvert_exporter": "python",
|
||
|
|
"pygments_lexer": "ipython3",
|
||
|
|
"version": "3.9.1"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"nbformat": 4,
|
||
|
|
"nbformat_minor": 5
|
||
|
|
}
|