Merge branch 'main' into katia/graphdb
jhills20
3 months ago
committed by
GitHub
commit
26c6fa64d8
@ -1,189 +0,0 @@
|
||||
from typing import List, Union
|
||||
|
||||
from smokey import Smokey
|
||||
|
||||
import openai
|
||||
|
||||
|
||||
def get_candidates(
|
||||
prompt: str,
|
||||
stop: List[str],
|
||||
temperature: float,
|
||||
priming_prefix: str,
|
||||
engine: str,
|
||||
n: int = 5,
|
||||
) -> List[str]:
|
||||
"""
|
||||
Generate N candidate completions based on the prompt, generated with a specific temperature.
|
||||
|
||||
:param prompt: The prompt to start the conversation with.
|
||||
:param stop: A list of tokens that indicate the end of the generation.
|
||||
:param temperature: The temperature of the generation.
|
||||
:param priming_prefix: The prefix to use for the priming.
|
||||
:param engine: The engine to use for the generation.
|
||||
:param n: The number of completions to generate.
|
||||
:return: A list of completions.
|
||||
"""
|
||||
response = openai.Completion.create(
|
||||
engine=engine,
|
||||
prompt=prompt,
|
||||
temperature=temperature,
|
||||
max_tokens=150,
|
||||
top_p=1,
|
||||
frequency_penalty=0,
|
||||
presence_penalty=0,
|
||||
stop=stop,
|
||||
n=n,
|
||||
)
|
||||
responses = [priming_prefix + choice.text for choice in response.choices]
|
||||
return responses
|
||||
|
||||
|
||||
def rindex(lst: List, value: str) -> int:
|
||||
"""
|
||||
Return the index of the last occurrence of a value in a list.
|
||||
|
||||
:param lst: The list to search in.
|
||||
:param value: The value to search for.
|
||||
:return: The index of the last occurrence of the value.
|
||||
"""
|
||||
try:
|
||||
return len(lst) - lst[::-1].index(value) - 1
|
||||
except ValueError:
|
||||
raise ValueError(f"Answer start token `{value}` not found in the eval template")
|
||||
|
||||
|
||||
def eval_candidate(
|
||||
candidate_answer: str,
|
||||
original_instruction: str,
|
||||
eval_template: str,
|
||||
answer_start_token: str,
|
||||
engine: str,
|
||||
) -> float:
|
||||
"""
|
||||
Evaluate a candidate answer by calculating the average log probability
|
||||
of the original instruction, given the candidate answer with a specific
|
||||
evaluation template, aimed at reconstructing the original instruction.
|
||||
|
||||
:param candidate_answer: The candidate answer to evaluate.
|
||||
:param original_instruction: The original instruction.
|
||||
:param eval_template: The template to use for the evaluation.
|
||||
:param answer_start_token: The token to use to indicate the start of the answer.
|
||||
:param engine: The engine to use for the evaluation.
|
||||
:return: The evaluation of the candidate answer.
|
||||
"""
|
||||
response = openai.Completion.create(
|
||||
engine=engine,
|
||||
prompt=eval_template.format(candidate_answer, original_instruction),
|
||||
temperature=0,
|
||||
max_tokens=0,
|
||||
top_p=1,
|
||||
frequency_penalty=0,
|
||||
presence_penalty=0,
|
||||
logprobs=1,
|
||||
echo=True,
|
||||
)
|
||||
|
||||
answer_start = rindex(
|
||||
response["choices"][0]["logprobs"]["tokens"], answer_start_token
|
||||
)
|
||||
logprobs = response["choices"][0]["logprobs"]["token_logprobs"][answer_start + 1 :]
|
||||
return sum(logprobs) / len(logprobs)
|
||||
|
||||
|
||||
def backtranslation(
|
||||
prompt_template: str,
|
||||
additional_info: str,
|
||||
instruction: str,
|
||||
eval_template: str,
|
||||
priming_prefix: str = "SELECT",
|
||||
stop1: List[str] = ["#", ";"],
|
||||
answer_start_token: str = "--",
|
||||
n: int = 5,
|
||||
temperature: float = 0.5,
|
||||
return_all_results: bool = False,
|
||||
engine: str = "davinci-codex",
|
||||
) -> Union[str, List[str, float]]:
|
||||
"""
|
||||
Generate a number of SQL queries given a natural language instruction,
|
||||
and pick the best one based on the average log probability of explaining the
|
||||
candidate SQL query with the exact original instruction, when prompted for
|
||||
a natural language explanation of the candidate SQL query.
|
||||
|
||||
:param prompt_template: The template to use for the prompt to generate SQL.
|
||||
:param additional_info: Additional information to include in the prompt
|
||||
(SQL Tables, and their properties).
|
||||
:param instruction: The instruction in natural language.
|
||||
:param eval_template: The template to use for the evaluation.
|
||||
:param priming_prefix: The prefix to use for the priming of the SQL query.
|
||||
:param stop1: A list of tokens that indicate the end of the generation.
|
||||
:param answer_start_token: The token to use to indicate the start of the
|
||||
natural answer.
|
||||
:param n: The number of candidates to generate.
|
||||
:param temperature: The temperature of the generation.
|
||||
:param return_all_results: Whether to return all results or just the best one.
|
||||
:param engine: The engine to use for the generation and evaluation.
|
||||
:return: The best SQL query, or a list of all scored generated SQL queries.
|
||||
"""
|
||||
prompt_template = prompt_template.format(
|
||||
additional_info, instruction, priming_prefix
|
||||
)
|
||||
|
||||
candidates = []
|
||||
responses = get_candidates(
|
||||
prompt_template, stop1, temperature, priming_prefix, engine=engine, n=n
|
||||
)
|
||||
for i in range(n):
|
||||
quality = eval_candidate(
|
||||
responses[i],
|
||||
instruction,
|
||||
eval_template,
|
||||
answer_start_token,
|
||||
engine=engine,
|
||||
)
|
||||
candidates.append((responses[i], quality))
|
||||
|
||||
candidates.sort(key=lambda x: x[1], reverse=True)
|
||||
if return_all_results:
|
||||
return candidates
|
||||
return candidates[0][0]
|
||||
|
||||
|
||||
def main(
|
||||
nl_query: str = "Return the name of each department that had more than 10 employees in June 2021",
|
||||
eval_template: str = "{};\n-- Explanation of the above query in human readable format\n-- {}",
|
||||
table_definitions: str = "# Employee(id, name, department_id)\n# Department(id, name, address)\n# Salary_Payments(id, employee_id, amount, date)\n",
|
||||
prompt_template: str = "### Postgres SQL tables, with their properties:\n#\n{}#\n### {}\n{}",
|
||||
n: int = 3,
|
||||
temperature: float = 0.3,
|
||||
engine: str = "davinci-codex",
|
||||
):
|
||||
"""
|
||||
Generate a number of SQL queries given a natural language instruction,
|
||||
and pick the best one based on the highest backtranslation score.
|
||||
|
||||
:param nl_query: The natural language query.
|
||||
:param eval_template: The template to use for the evaluation.
|
||||
:param table_definitions: The definitions of the tables used in the query.
|
||||
:param prompt_template: The template to use for the prompt to generate SQL.
|
||||
:param n: The number of candidates to generate.
|
||||
:param temperature: The temperature of the generation.
|
||||
:param engine: The engine to use for the generation and evaluation.
|
||||
:return: The best SQL query, or a list of all scored generated SQL queries.
|
||||
"""
|
||||
|
||||
result = backtranslation(
|
||||
prompt_template,
|
||||
table_definitions,
|
||||
nl_query,
|
||||
eval_template,
|
||||
priming_prefix="SELECT",
|
||||
temperature=temperature,
|
||||
n=n,
|
||||
engine=engine,
|
||||
)
|
||||
print(result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
Smokey(main)
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -1,320 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Azure completions example\n",
|
||||
"\n",
|
||||
"This example will cover completions using the Azure OpenAI service. It also includes information on content filtering."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Setup\n",
|
||||
"\n",
|
||||
"First, we install the necessary dependencies and import the libraries we will be using."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip install \"openai>=1.0.0,<2.0.0\"\n",
|
||||
"! pip install python-dotenv"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import openai\n",
|
||||
"import dotenv\n",
|
||||
"\n",
|
||||
"dotenv.load_dotenv()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Authentication\n",
|
||||
"\n",
|
||||
"The Azure OpenAI service supports multiple authentication mechanisms that include API keys and Azure Active Directory token credentials."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"use_azure_active_directory = False # Set this flag to True if you are using Azure Active Directory"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### Authentication using API key\n",
|
||||
"\n",
|
||||
"To set up the OpenAI SDK to use an *Azure API Key*, we need to set `api_key` to a key associated with your endpoint (you can find this key in *\"Keys and Endpoints\"* under *\"Resource Management\"* in the [Azure Portal](https://portal.azure.com)). You'll also find the endpoint for your resource here."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if not use_azure_active_directory:\n",
|
||||
" endpoint = os.environ[\"AZURE_OPENAI_ENDPOINT\"]\n",
|
||||
" api_key = os.environ[\"AZURE_OPENAI_API_KEY\"]\n",
|
||||
"\n",
|
||||
" client = openai.AzureOpenAI(\n",
|
||||
" azure_endpoint=endpoint,\n",
|
||||
" api_key=api_key,\n",
|
||||
" api_version=\"2023-09-01-preview\"\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### Authentication using Azure Active Directory\n",
|
||||
"Let's now see how we can autheticate via Azure Active Directory. We'll start by installing the `azure-identity` library. This library will provide the token credentials we need to authenticate and help us build a token credential provider through the `get_bearer_token_provider` helper function. It's recommended to use `get_bearer_token_provider` over providing a static token to `AzureOpenAI` because this API will automatically cache and refresh tokens for you. \n",
|
||||
"\n",
|
||||
"For more information on how to set up Azure Active Directory authentication with Azure OpenAI, see the [documentation](https://learn.microsoft.com/azure/ai-services/openai/how-to/managed-identity)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip install \"azure-identity>=1.15.0\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n",
|
||||
"\n",
|
||||
"if use_azure_active_directory:\n",
|
||||
" endpoint = os.environ[\"AZURE_OPENAI_ENDPOINT\"]\n",
|
||||
" api_key = os.environ[\"AZURE_OPENAI_API_KEY\"]\n",
|
||||
"\n",
|
||||
" client = openai.AzureOpenAI(\n",
|
||||
" azure_endpoint=endpoint,\n",
|
||||
" azure_ad_token_provider=get_bearer_token_provider(DefaultAzureCredential(), \"https://cognitiveservices.azure.com/.default\"),\n",
|
||||
" api_version=\"2023-09-01-preview\"\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"> Note: the AzureOpenAI infers the following arguments from their corresponding environment variables if they are not provided:\n",
|
||||
"\n",
|
||||
"- `api_key` from `AZURE_OPENAI_API_KEY`\n",
|
||||
"- `azure_ad_token` from `AZURE_OPENAI_AD_TOKEN`\n",
|
||||
"- `api_version` from `OPENAI_API_VERSION`\n",
|
||||
"- `azure_endpoint` from `AZURE_OPENAI_ENDPOINT`\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Deployments\n",
|
||||
"\n",
|
||||
"In this section we are going to create a deployment of a model that we can use to create completions."
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Deployments: Create in the Azure OpenAI Studio\n",
|
||||
"Let's deploy a model to use with completions. Go to https://portal.azure.com, find your Azure OpenAI resource, and then navigate to the Azure OpenAI Studio. Click on the \"Deployments\" tab and then create a deployment for the model you want to use for completions. The deployment name that you give the model will be used in the code below."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"deployment = \"\" # Fill in the deployment name from the portal here"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Completions\n",
|
||||
"\n",
|
||||
"Now let's create a completion using the client we built."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"prompt = \"The food was delicious and the waiter\"\n",
|
||||
"completion = client.completions.create(\n",
|
||||
" model=deployment,\n",
|
||||
" prompt=prompt,\n",
|
||||
" stop=\".\",\n",
|
||||
" temperature=0\n",
|
||||
")\n",
|
||||
" \n",
|
||||
"print(f\"{prompt}{completion.choices[0].text}.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Create a streaming completion\n",
|
||||
"\n",
|
||||
"We can also stream the response."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"prompt = \"The food was delicious and the waiter\"\n",
|
||||
"response = client.completions.create(\n",
|
||||
" model=deployment,\n",
|
||||
" prompt=prompt,\n",
|
||||
" stream=True,\n",
|
||||
")\n",
|
||||
"for completion in response:\n",
|
||||
" if len(completion.choices) > 0:\n",
|
||||
" print(f\"{completion.choices[0].text}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Content filtering\n",
|
||||
"\n",
|
||||
"Azure OpenAI service includes content filtering of prompts and completion responses. You can learn more about content filtering and how to configure it [here](https://learn.microsoft.com/azure/ai-services/openai/concepts/content-filter).\n",
|
||||
"\n",
|
||||
"If the prompt is flagged by the content filter, the library will raise a `BadRequestError` exception with a `content_filter` error code. Otherwise, you can access the `prompt_filter_results` and `content_filter_results` on the response to see the results of the content filtering and what categories were flagged."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### Prompt flagged by content filter"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" completion = client.completions.create(\n",
|
||||
" prompt=\"<text violating the content policy>\",\n",
|
||||
" model=deployment,\n",
|
||||
" )\n",
|
||||
"except openai.BadRequestError as e:\n",
|
||||
" err = json.loads(e.response.text)\n",
|
||||
" if err[\"error\"][\"code\"] == \"content_filter\":\n",
|
||||
" print(\"Content filter triggered!\")\n",
|
||||
" content_filter_result = err[\"error\"][\"innererror\"][\"content_filter_result\"]\n",
|
||||
" for category, details in content_filter_result.items():\n",
|
||||
" print(f\"{category}:\\n filtered={details['filtered']}\\n severity={details['severity']}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Checking the result of the content filter"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"completion = client.completions.create(\n",
|
||||
" prompt=\"What's the biggest city in Washington?\",\n",
|
||||
" model=deployment,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(f\"Answer: {completion.choices[0].text}\")\n",
|
||||
"\n",
|
||||
"# prompt content filter result in \"model_extra\" for azure\n",
|
||||
"prompt_filter_result = completion.model_extra[\"prompt_filter_results\"][0][\"content_filter_results\"]\n",
|
||||
"print(\"\\nPrompt content filter results:\")\n",
|
||||
"for category, details in prompt_filter_result.items():\n",
|
||||
" print(f\"{category}:\\n filtered={details['filtered']}\\n severity={details['severity']}\")\n",
|
||||
"\n",
|
||||
"# completion content filter result\n",
|
||||
"print(\"\\nCompletion content filter results:\")\n",
|
||||
"completion_filter_result = completion.choices[0].model_extra[\"content_filter_results\"]\n",
|
||||
"for category, details in completion_filter_result.items():\n",
|
||||
" print(f\"{category}:\\n filtered={details['filtered']}\\n severity={details['severity']}\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.0"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "3a5103089ab7e7c666b279eeded403fcec76de49a40685dbdfe9f9c78ad97c17"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
@ -0,0 +1,738 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Azure AI Search as a vector database for OpenAI embeddings"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This notebook provides step by step instuctions on using Azure AI Search (f.k.a Azure Cognitive Search) as a vector database with OpenAI embeddings. Azure AI Search is a cloud search service that gives developers infrastructure, APIs, and tools for building a rich search experience over private, heterogeneous content in web, mobile, and enterprise applications.\n",
|
||||
"\n",
|
||||
"## Prerequistites:\n",
|
||||
"For the purposes of this exercise you must have the following:\n",
|
||||
"- [Azure AI Search Service](https://learn.microsoft.com/azure/search/)\n",
|
||||
"- [OpenAI Key](https://platform.openai.com/account/api-keys) or [Azure OpenAI credentials](https://learn.microsoft.com/azure/cognitive-services/openai/)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip install wget\n",
|
||||
"! pip install azure-search-documents \n",
|
||||
"! pip install azure-identity\n",
|
||||
"! pip install openai"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Import required libraries"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json \n",
|
||||
"import wget\n",
|
||||
"import pandas as pd\n",
|
||||
"import zipfile\n",
|
||||
"from openai import AzureOpenAI\n",
|
||||
"from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n",
|
||||
"from azure.core.credentials import AzureKeyCredential \n",
|
||||
"from azure.search.documents import SearchClient, SearchIndexingBufferedSender \n",
|
||||
"from azure.search.documents.indexes import SearchIndexClient \n",
|
||||
"from azure.search.documents.models import (\n",
|
||||
" QueryAnswerType,\n",
|
||||
" QueryCaptionType,\n",
|
||||
" QueryType,\n",
|
||||
" VectorizedQuery,\n",
|
||||
")\n",
|
||||
"from azure.search.documents.indexes.models import (\n",
|
||||
" HnswAlgorithmConfiguration,\n",
|
||||
" HnswParameters,\n",
|
||||
" SearchField,\n",
|
||||
" SearchableField,\n",
|
||||
" SearchFieldDataType,\n",
|
||||
" SearchIndex,\n",
|
||||
" SemanticConfiguration,\n",
|
||||
" SemanticField,\n",
|
||||
" SemanticPrioritizedFields,\n",
|
||||
" SemanticSearch,\n",
|
||||
" SimpleField,\n",
|
||||
" VectorSearch,\n",
|
||||
" VectorSearchAlgorithmKind,\n",
|
||||
" VectorSearchAlgorithmMetric,\n",
|
||||
" VectorSearchProfile,\n",
|
||||
")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Configure OpenAI settings\n",
|
||||
"\n",
|
||||
"This section guides you through setting up authentication for Azure OpenAI, allowing you to securely interact with the service using either Azure Active Directory (AAD) or an API key. Before proceeding, ensure you have your Azure OpenAI endpoint and credentials ready. For detailed instructions on setting up AAD with Azure OpenAI, refer to the [official documentation](https://learn.microsoft.com/azure/ai-services/openai/how-to/managed-identity).\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"endpoint: str = \"YOUR_AZURE_OPENAI_ENDPOINT\"\n",
|
||||
"api_key: str = \"YOUR_AZURE_OPENAI_KEY\"\n",
|
||||
"api_version: str = \"2023-05-15\"\n",
|
||||
"deployment = \"YOUR_AZURE_OPENAI_DEPLOYMENT_NAME\"\n",
|
||||
"credential = DefaultAzureCredential()\n",
|
||||
"token_provider = get_bearer_token_provider(\n",
|
||||
" credential, \"https://cognitiveservices.azure.com/.default\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Set this flag to True if you are using Azure Active Directory\n",
|
||||
"use_aad_for_aoai = True \n",
|
||||
"\n",
|
||||
"if use_aad_for_aoai:\n",
|
||||
" # Use Azure Active Directory (AAD) authentication\n",
|
||||
" client = AzureOpenAI(\n",
|
||||
" azure_endpoint=endpoint,\n",
|
||||
" api_version=api_version,\n",
|
||||
" azure_ad_token_provider=token_provider,\n",
|
||||
" )\n",
|
||||
"else:\n",
|
||||
" # Use API key authentication\n",
|
||||
" client = AzureOpenAI(\n",
|
||||
" api_key=api_key,\n",
|
||||
" api_version=api_version,\n",
|
||||
" azure_endpoint=endpoint,\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Configure Azure AI Search Vector Store settings\n",
|
||||
"This section explains how to set up the Azure AI Search client for integrating with the Vector Store feature. You can locate your Azure AI Search service details in the Azure Portal or programmatically via the [Search Management SDK](https://learn.microsoft.com/rest/api/searchmanagement/).\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Configuration\n",
|
||||
"search_service_endpoint: str = \"YOUR_AZURE_SEARCH_ENDPOINT\"\n",
|
||||
"search_service_api_key: str = \"YOUR_AZURE_SEARCH_ADMIN_KEY\"\n",
|
||||
"index_name: str = \"azure-ai-search-openai-cookbook-demo\"\n",
|
||||
"\n",
|
||||
"# Set this flag to True if you are using Azure Active Directory\n",
|
||||
"use_aad_for_search = True \n",
|
||||
"\n",
|
||||
"if use_aad_for_search:\n",
|
||||
" # Use Azure Active Directory (AAD) authentication\n",
|
||||
" credential = DefaultAzureCredential()\n",
|
||||
"else:\n",
|
||||
" # Use API key authentication\n",
|
||||
" credential = AzureKeyCredential(search_service_api_key)\n",
|
||||
"\n",
|
||||
"# Initialize the SearchClient with the selected authentication method\n",
|
||||
"search_client = SearchClient(\n",
|
||||
" endpoint=search_service_endpoint, index_name=index_name, credential=credential\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Load data\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'vector_database_wikipedia_articles_embedded.zip'"
|
||||
]
|
||||
},
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"embeddings_url = \"https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip\"\n",
|
||||
"\n",
|
||||
"# The file is ~700 MB so this will take some time\n",
|
||||
"wget.download(embeddings_url)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"with zipfile.ZipFile(\"vector_database_wikipedia_articles_embedded.zip\", \"r\") as zip_ref:\n",
|
||||
" zip_ref.extractall(\"../../data\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>id</th>\n",
|
||||
" <th>url</th>\n",
|
||||
" <th>title</th>\n",
|
||||
" <th>text</th>\n",
|
||||
" <th>title_vector</th>\n",
|
||||
" <th>content_vector</th>\n",
|
||||
" <th>vector_id</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>https://simple.wikipedia.org/wiki/April</td>\n",
|
||||
" <td>April</td>\n",
|
||||
" <td>April is the fourth month of the year in the J...</td>\n",
|
||||
" <td>[0.001009464613161981, -0.020700545981526375, ...</td>\n",
|
||||
" <td>[-0.011253940872848034, -0.013491976074874401,...</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>2</td>\n",
|
||||
" <td>https://simple.wikipedia.org/wiki/August</td>\n",
|
||||
" <td>August</td>\n",
|
||||
" <td>August (Aug.) is the eighth month of the year ...</td>\n",
|
||||
" <td>[0.0009286514250561595, 0.000820168002974242, ...</td>\n",
|
||||
" <td>[0.0003609954728744924, 0.007262262050062418, ...</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>6</td>\n",
|
||||
" <td>https://simple.wikipedia.org/wiki/Art</td>\n",
|
||||
" <td>Art</td>\n",
|
||||
" <td>Art is a creative activity that expresses imag...</td>\n",
|
||||
" <td>[0.003393713850528002, 0.0061537534929811954, ...</td>\n",
|
||||
" <td>[-0.004959689453244209, 0.015772193670272827, ...</td>\n",
|
||||
" <td>2</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>8</td>\n",
|
||||
" <td>https://simple.wikipedia.org/wiki/A</td>\n",
|
||||
" <td>A</td>\n",
|
||||
" <td>A or a is the first letter of the English alph...</td>\n",
|
||||
" <td>[0.0153952119871974, -0.013759135268628597, 0....</td>\n",
|
||||
" <td>[0.024894846603274345, -0.022186409682035446, ...</td>\n",
|
||||
" <td>3</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>9</td>\n",
|
||||
" <td>https://simple.wikipedia.org/wiki/Air</td>\n",
|
||||
" <td>Air</td>\n",
|
||||
" <td>Air refers to the Earth's atmosphere. Air is a...</td>\n",
|
||||
" <td>[0.02224554680287838, -0.02044147066771984, -0...</td>\n",
|
||||
" <td>[0.021524671465158463, 0.018522677943110466, -...</td>\n",
|
||||
" <td>4</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" id url title \\\n",
|
||||
"0 1 https://simple.wikipedia.org/wiki/April April \n",
|
||||
"1 2 https://simple.wikipedia.org/wiki/August August \n",
|
||||
"2 6 https://simple.wikipedia.org/wiki/Art Art \n",
|
||||
"3 8 https://simple.wikipedia.org/wiki/A A \n",
|
||||
"4 9 https://simple.wikipedia.org/wiki/Air Air \n",
|
||||
"\n",
|
||||
" text \\\n",
|
||||
"0 April is the fourth month of the year in the J... \n",
|
||||
"1 August (Aug.) is the eighth month of the year ... \n",
|
||||
"2 Art is a creative activity that expresses imag... \n",
|
||||
"3 A or a is the first letter of the English alph... \n",
|
||||
"4 Air refers to the Earth's atmosphere. Air is a... \n",
|
||||
"\n",
|
||||
" title_vector \\\n",
|
||||
"0 [0.001009464613161981, -0.020700545981526375, ... \n",
|
||||
"1 [0.0009286514250561595, 0.000820168002974242, ... \n",
|
||||
"2 [0.003393713850528002, 0.0061537534929811954, ... \n",
|
||||
"3 [0.0153952119871974, -0.013759135268628597, 0.... \n",
|
||||
"4 [0.02224554680287838, -0.02044147066771984, -0... \n",
|
||||
"\n",
|
||||
" content_vector vector_id \n",
|
||||
"0 [-0.011253940872848034, -0.013491976074874401,... 0 \n",
|
||||
"1 [0.0003609954728744924, 0.007262262050062418, ... 1 \n",
|
||||
"2 [-0.004959689453244209, 0.015772193670272827, ... 2 \n",
|
||||
"3 [0.024894846603274345, -0.022186409682035446, ... 3 \n",
|
||||
"4 [0.021524671465158463, 0.018522677943110466, -... 4 "
|
||||
]
|
||||
},
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"article_df = pd.read_csv(\"../../data/vector_database_wikipedia_articles_embedded.csv\")\n",
|
||||
"\n",
|
||||
"# Read vectors from strings back into a list using json.loads\n",
|
||||
"article_df[\"title_vector\"] = article_df.title_vector.apply(json.loads)\n",
|
||||
"article_df[\"content_vector\"] = article_df.content_vector.apply(json.loads)\n",
|
||||
"article_df[\"vector_id\"] = article_df[\"vector_id\"].apply(str)\n",
|
||||
"article_df.head()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Create an index\n",
|
||||
"This code snippet demonstrates how to define and create a search index using the `SearchIndexClient` from the Azure AI Search Python SDK. The index incorporates both vector search and semantic ranker capabilities. For more details, visit our documentation on how to [Create a Vector Index](https://learn.microsoft.com/azure/search/vector-search-how-to-create-index?.tabs=config-2023-11-01%2Crest-2023-11-01%2Cpush%2Cportal-check-index)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"azure-ai-search-openai-cookbook-demo created\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Initialize the SearchIndexClient\n",
|
||||
"index_client = SearchIndexClient(\n",
|
||||
" endpoint=search_service_endpoint, credential=credential\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Define the fields for the index\n",
|
||||
"fields = [\n",
|
||||
" SimpleField(name=\"id\", type=SearchFieldDataType.String),\n",
|
||||
" SimpleField(name=\"vector_id\", type=SearchFieldDataType.String, key=True),\n",
|
||||
" SimpleField(name=\"url\", type=SearchFieldDataType.String),\n",
|
||||
" SearchableField(name=\"title\", type=SearchFieldDataType.String),\n",
|
||||
" SearchableField(name=\"text\", type=SearchFieldDataType.String),\n",
|
||||
" SearchField(\n",
|
||||
" name=\"title_vector\",\n",
|
||||
" type=SearchFieldDataType.Collection(SearchFieldDataType.Single),\n",
|
||||
" vector_search_dimensions=1536,\n",
|
||||
" vector_search_profile_name=\"my-vector-config\",\n",
|
||||
" ),\n",
|
||||
" SearchField(\n",
|
||||
" name=\"content_vector\",\n",
|
||||
" type=SearchFieldDataType.Collection(SearchFieldDataType.Single),\n",
|
||||
" vector_search_dimensions=1536,\n",
|
||||
" vector_search_profile_name=\"my-vector-config\",\n",
|
||||
" ),\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"# Configure the vector search configuration\n",
|
||||
"vector_search = VectorSearch(\n",
|
||||
" algorithms=[\n",
|
||||
" HnswAlgorithmConfiguration(\n",
|
||||
" name=\"my-hnsw\",\n",
|
||||
" kind=VectorSearchAlgorithmKind.HNSW,\n",
|
||||
" parameters=HnswParameters(\n",
|
||||
" m=4,\n",
|
||||
" ef_construction=400,\n",
|
||||
" ef_search=500,\n",
|
||||
" metric=VectorSearchAlgorithmMetric.COSINE,\n",
|
||||
" ),\n",
|
||||
" )\n",
|
||||
" ],\n",
|
||||
" profiles=[\n",
|
||||
" VectorSearchProfile(\n",
|
||||
" name=\"my-vector-config\",\n",
|
||||
" algorithm_configuration_name=\"my-hnsw\",\n",
|
||||
" )\n",
|
||||
" ],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Configure the semantic search configuration\n",
|
||||
"semantic_search = SemanticSearch(\n",
|
||||
" configurations=[\n",
|
||||
" SemanticConfiguration(\n",
|
||||
" name=\"my-semantic-config\",\n",
|
||||
" prioritized_fields=SemanticPrioritizedFields(\n",
|
||||
" title_field=SemanticField(field_name=\"title\"),\n",
|
||||
" keywords_fields=[SemanticField(field_name=\"url\")],\n",
|
||||
" content_fields=[SemanticField(field_name=\"text\")],\n",
|
||||
" ),\n",
|
||||
" )\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Create the search index with the vector search and semantic search configurations\n",
|
||||
"index = SearchIndex(\n",
|
||||
" name=index_name,\n",
|
||||
" fields=fields,\n",
|
||||
" vector_search=vector_search,\n",
|
||||
" semantic_search=semantic_search,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Create or update the index\n",
|
||||
"result = index_client.create_or_update_index(index)\n",
|
||||
"print(f\"{result.name} created\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Uploading Data to Azure AI Search Index\n",
|
||||
"\n",
|
||||
"The following code snippet outlines the process of uploading a batch of documents—specifically, Wikipedia articles with pre-computed embeddings—from a pandas DataFrame to an Azure AI Search index. For a detailed guide on data import strategies and best practices, refer to [Data Import in Azure AI Search](https://learn.microsoft.com/azure/search/search-what-is-data-import).\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Uploaded 25000 documents in total\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from azure.core.exceptions import HttpResponseError\n",
|
||||
"\n",
|
||||
"# Convert the 'id' and 'vector_id' columns to string so one of them can serve as our key field\n",
|
||||
"article_df[\"id\"] = article_df[\"id\"].astype(str)\n",
|
||||
"article_df[\"vector_id\"] = article_df[\"vector_id\"].astype(str)\n",
|
||||
"# Convert the DataFrame to a list of dictionaries\n",
|
||||
"documents = article_df.to_dict(orient=\"records\")\n",
|
||||
"\n",
|
||||
"# Create a SearchIndexingBufferedSender\n",
|
||||
"batch_client = SearchIndexingBufferedSender(\n",
|
||||
" search_service_endpoint, index_name, credential\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" # Add upload actions for all documents in a single call\n",
|
||||
" batch_client.upload_documents(documents=documents)\n",
|
||||
"\n",
|
||||
" # Manually flush to send any remaining documents in the buffer\n",
|
||||
" batch_client.flush()\n",
|
||||
"except HttpResponseError as e:\n",
|
||||
" print(f\"An error occurred: {e}\")\n",
|
||||
"finally:\n",
|
||||
" # Clean up resources\n",
|
||||
" batch_client.close()\n",
|
||||
"\n",
|
||||
"print(f\"Uploaded {len(documents)} documents in total\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"If your dataset didn't already contain pre-computed embeddings, you can create embeddings by using the below function using the `openai` python library. You'll also notice the same function and model are being used to generate query embeddings for performing vector searches."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Content: April is the fourth month of the year in the Julian and Gregorian calendars, and comes between March\n",
|
||||
"Content vector generated\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Example function to generate document embedding\n",
|
||||
"def generate_embeddings(text, model):\n",
|
||||
" # Generate embeddings for the provided text using the specified model\n",
|
||||
" embeddings_response = client.embeddings.create(model=model, input=text)\n",
|
||||
" # Extract the embedding data from the response\n",
|
||||
" embedding = embeddings_response.data[0].embedding\n",
|
||||
" return embedding\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"first_document_content = documents[0][\"text\"]\n",
|
||||
"print(f\"Content: {first_document_content[:100]}\")\n",
|
||||
"\n",
|
||||
"content_vector = generate_embeddings(first_document_content, deployment)\n",
|
||||
"print(\"Content vector generated\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Perform a vector similarity search"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Title: Documenta\n",
|
||||
"Score: 0.8599451\n",
|
||||
"URL: https://simple.wikipedia.org/wiki/Documenta\n",
|
||||
"\n",
|
||||
"Title: Museum of Modern Art\n",
|
||||
"Score: 0.85260946\n",
|
||||
"URL: https://simple.wikipedia.org/wiki/Museum%20of%20Modern%20Art\n",
|
||||
"\n",
|
||||
"Title: Expressionism\n",
|
||||
"Score: 0.852354\n",
|
||||
"URL: https://simple.wikipedia.org/wiki/Expressionism\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Pure Vector Search\n",
|
||||
"query = \"modern art in Europe\"\n",
|
||||
" \n",
|
||||
"search_client = SearchClient(search_service_endpoint, index_name, credential) \n",
|
||||
"vector_query = VectorizedQuery(vector=generate_embeddings(query, deployment), k_nearest_neighbors=3, fields=\"content_vector\")\n",
|
||||
" \n",
|
||||
"results = search_client.search( \n",
|
||||
" search_text=None, \n",
|
||||
" vector_queries= [vector_query], \n",
|
||||
" select=[\"title\", \"text\", \"url\"] \n",
|
||||
")\n",
|
||||
" \n",
|
||||
"for result in results: \n",
|
||||
" print(f\"Title: {result['title']}\") \n",
|
||||
" print(f\"Score: {result['@search.score']}\") \n",
|
||||
" print(f\"URL: {result['url']}\\n\") "
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Perform a Hybrid Search\n",
|
||||
"Hybrid search combines the capabilities of traditional keyword-based search with vector-based similarity search to provide more relevant and contextual results. This approach is particularly useful when dealing with complex queries that benefit from understanding the semantic meaning behind the text.\n",
|
||||
"\n",
|
||||
"The provided code snippet demonstrates how to execute a hybrid search query:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 61,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Title: Wars of Scottish Independence\n",
|
||||
"Score: 0.03306011110544205\n",
|
||||
"URL: https://simple.wikipedia.org/wiki/Wars%20of%20Scottish%20Independence\n",
|
||||
"\n",
|
||||
"Title: Battle of Bannockburn\n",
|
||||
"Score: 0.022253260016441345\n",
|
||||
"URL: https://simple.wikipedia.org/wiki/Battle%20of%20Bannockburn\n",
|
||||
"\n",
|
||||
"Title: Scottish\n",
|
||||
"Score: 0.016393441706895828\n",
|
||||
"URL: https://simple.wikipedia.org/wiki/Scottish\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Hybrid Search\n",
|
||||
"query = \"Famous battles in Scottish history\" \n",
|
||||
" \n",
|
||||
"search_client = SearchClient(search_service_endpoint, index_name, credential) \n",
|
||||
"vector_query = VectorizedQuery(vector=generate_embeddings(query, deployment), k_nearest_neighbors=3, fields=\"content_vector\")\n",
|
||||
" \n",
|
||||
"results = search_client.search( \n",
|
||||
" search_text=query, \n",
|
||||
" vector_queries= [vector_query], \n",
|
||||
" select=[\"title\", \"text\", \"url\"],\n",
|
||||
" top=3\n",
|
||||
")\n",
|
||||
" \n",
|
||||
"for result in results: \n",
|
||||
" print(f\"Title: {result['title']}\") \n",
|
||||
" print(f\"Score: {result['@search.score']}\") \n",
|
||||
" print(f\"URL: {result['url']}\\n\") "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Perform a Hybrid Search with Reranking (powered by Bing)\n",
|
||||
"[Semantic ranker](https://learn.microsoft.com/azure/search/semantic-search-overview) measurably improves search relevance by using language understanding to rerank search results. Additionally, you can get extractive captions, answers, and highlights. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 25,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Semantic Answer: Advancements During the industrial revolution, new technology brought many changes. For example:<em> Canals</em> were built to allow heavy goods to be moved easily where they were needed. The steam engine became the main source of power. It replaced horses and human labor. Cheap iron and steel became mass-produced.\n",
|
||||
"Semantic Answer Score: 0.90478515625\n",
|
||||
"\n",
|
||||
"Title: Industrial Revolution\n",
|
||||
"Reranker Score: 3.408700942993164\n",
|
||||
"URL: https://simple.wikipedia.org/wiki/Industrial%20Revolution\n",
|
||||
"Caption: Advancements During the industrial revolution, new technology brought many changes. For example: Canals were built to allow heavy goods to be moved easily where they were needed. The steam engine became the main source of power. It replaced horses and human labor. Cheap iron and steel became mass-produced.\n",
|
||||
"\n",
|
||||
"Title: Printing\n",
|
||||
"Reranker Score: 1.603400707244873\n",
|
||||
"URL: https://simple.wikipedia.org/wiki/Printing\n",
|
||||
"Caption: Machines to speed printing, cheaper paper, automatic stitching and binding all arrived in the 19th century during the industrial revolution. What had once been done by a few men by hand was now done by limited companies on huge machines. The result was much lower prices, and a much wider readership.\n",
|
||||
"\n",
|
||||
"Title: Industrialisation\n",
|
||||
"Reranker Score: 1.3238357305526733\n",
|
||||
"URL: https://simple.wikipedia.org/wiki/Industrialisation\n",
|
||||
"Caption: <em>Industrialisation</em> (or<em> industrialization)</em> is a process that happens in countries when they start to use machines to do work that was once done by people.<em> Industrialisation changes</em> the things people do.<em> Industrialisation</em> caused towns to grow larger. Many people left farming to take higher paid jobs in factories in towns.\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Semantic Hybrid Search\n",
|
||||
"query = \"What were the key technological advancements during the Industrial Revolution?\"\n",
|
||||
"\n",
|
||||
"search_client = SearchClient(search_service_endpoint, index_name, credential)\n",
|
||||
"vector_query = VectorizedQuery(\n",
|
||||
" vector=generate_embeddings(query, deployment),\n",
|
||||
" k_nearest_neighbors=3,\n",
|
||||
" fields=\"content_vector\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"results = search_client.search(\n",
|
||||
" search_text=query,\n",
|
||||
" vector_queries=[vector_query],\n",
|
||||
" select=[\"title\", \"text\", \"url\"],\n",
|
||||
" query_type=QueryType.SEMANTIC,\n",
|
||||
" semantic_configuration_name=\"my-semantic-config\",\n",
|
||||
" query_caption=QueryCaptionType.EXTRACTIVE,\n",
|
||||
" query_answer=QueryAnswerType.EXTRACTIVE,\n",
|
||||
" top=3,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"semantic_answers = results.get_answers()\n",
|
||||
"for answer in semantic_answers:\n",
|
||||
" if answer.highlights:\n",
|
||||
" print(f\"Semantic Answer: {answer.highlights}\")\n",
|
||||
" else:\n",
|
||||
" print(f\"Semantic Answer: {answer.text}\")\n",
|
||||
" print(f\"Semantic Answer Score: {answer.score}\\n\")\n",
|
||||
"\n",
|
||||
"for result in results:\n",
|
||||
" print(f\"Title: {result['title']}\")\n",
|
||||
" print(f\"Reranker Score: {result['@search.reranker_score']}\")\n",
|
||||
" print(f\"URL: {result['url']}\")\n",
|
||||
" captions = result[\"@search.captions\"]\n",
|
||||
" if captions:\n",
|
||||
" caption = captions[0]\n",
|
||||
" if caption.highlights:\n",
|
||||
" print(f\"Caption: {caption.highlights}\\n\")\n",
|
||||
" else:\n",
|
||||
" print(f\"Caption: {caption.text}\\n\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.7"
|
||||
},
|
||||
"orig_nbformat": 4
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue