mirror of https://github.com/hwchase17/langchain
Merge branch 'master' into deepsense/text-to-speech
mateusz.wosinski
1 year ago
commit
868db99b17
@ -0,0 +1,2 @@
|
||||
position: 0
|
||||
collapsed: false
|
||||
@ -1,9 +0,0 @@
|
||||
---
|
||||
sidebar_position: 3
|
||||
---
|
||||
|
||||
# Web Scraping
|
||||
|
||||
Web scraping has historically been a challenging endeavor due to the ever-changing nature of website structures, making it tedious for developers to maintain their scraping scripts. Traditional methods often rely on specific HTML tags and patterns which, when altered, can disrupt data extraction processes.
|
||||
|
||||
Enter the LLM-based method for parsing HTML: By leveraging the capabilities of LLMs, and especially OpenAI Functions in LangChain's extraction chain, developers can instruct the model to extract only the desired data in a specified format. This method not only streamlines the extraction process but also significantly reduces the time spent on manual debugging and script modifications. Its adaptability means that even if websites undergo significant design changes, the extraction remains consistent and robust. This level of resilience translates to reduced maintenance efforts, cost savings, and ensures a higher quality of extracted data. Compared to its predecessors, the LLM-based approach wins out in the web scraping domain by transforming a historically cumbersome task into a more automated and efficient process.
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,461 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Reversible data anonymization with Microsoft Presidio\n",
|
||||
"\n",
|
||||
"[](https://colab.research.google.com/github/langchain-ai/langchain/blob/master/docs/extras/guides/privacy/presidio_reversible_anonymization.ipynb)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Use case\n",
|
||||
"\n",
|
||||
"We have already written about the importance of anonymizing sensitive data in the previous section. **Reversible Anonymization** is an equally essential technology while sharing information with language models, as it balances data protection with data usability. This technique involves masking sensitive personally identifiable information (PII), yet it can be reversed and original data can be restored when authorized users need it. Its main advantage lies in the fact that while it conceals individual identities to prevent misuse, it also allows the concealed data to be accurately unmasked should it be necessary for legal or compliance purposes. \n",
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"We implemented the `PresidioReversibleAnonymizer`, which consists of two parts:\n",
|
||||
"\n",
|
||||
"1. anonymization - it works the same way as `PresidioAnonymizer`, plus the object itself stores a mapping of made-up values to original ones, for example:\n",
|
||||
"```\n",
|
||||
" {\n",
|
||||
" \"PERSON\": {\n",
|
||||
" \"<anonymized>\": \"<original>\",\n",
|
||||
" \"John Doe\": \"Slim Shady\"\n",
|
||||
" },\n",
|
||||
" \"PHONE_NUMBER\": {\n",
|
||||
" \"111-111-1111\": \"555-555-5555\"\n",
|
||||
" }\n",
|
||||
" ...\n",
|
||||
" }\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"2. deanonymization - using the mapping described above, it matches fake data with original data and then substitutes it.\n",
|
||||
"\n",
|
||||
"Between anonymization and deanonymization user can perform different operations, for example, passing the output to LLM.\n",
|
||||
"\n",
|
||||
"## Quickstart\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Install necessary packages\n",
|
||||
"# ! pip install langchain langchain-experimental openai presidio-analyzer presidio-anonymizer spacy Faker\n",
|
||||
"# ! python -m spacy download en_core_web_lg"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"`PresidioReversibleAnonymizer` is not significantly different from its predecessor (`PresidioAnonymizer`) in terms of anonymization:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'My name is Maria Lynch, call me at 7344131647 or email me at jamesmichael@example.com. By the way, my card number is: 4838637940262'"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain_experimental.data_anonymizer import PresidioReversibleAnonymizer\n",
|
||||
"\n",
|
||||
"anonymizer = PresidioReversibleAnonymizer(\n",
|
||||
" analyzed_fields=[\"PERSON\", \"PHONE_NUMBER\", \"EMAIL_ADDRESS\", \"CREDIT_CARD\"],\n",
|
||||
" # Faker seed is used here to make sure the same fake data is generated for the test purposes\n",
|
||||
" # In production, it is recommended to remove the faker_seed parameter (it will default to None)\n",
|
||||
" faker_seed=42,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"anonymizer.anonymize(\n",
|
||||
" \"My name is Slim Shady, call me at 313-666-7440 or email me at real.slim.shady@gmail.com. \"\n",
|
||||
" \"By the way, my card number is: 4916 0387 9536 0861\"\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This is what the full string we want to deanonymize looks like:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Maria Lynch recently lost his wallet. \n",
|
||||
"Inside is some cash and his credit card with the number 4838637940262. \n",
|
||||
"If you would find it, please call at 7344131647 or write an email here: jamesmichael@example.com.\n",
|
||||
"Maria Lynch would be very grateful!\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# We know this data, as we set the faker_seed parameter\n",
|
||||
"fake_name = \"Maria Lynch\"\n",
|
||||
"fake_phone = \"7344131647\"\n",
|
||||
"fake_email = \"jamesmichael@example.com\"\n",
|
||||
"fake_credit_card = \"4838637940262\"\n",
|
||||
"\n",
|
||||
"anonymized_text = f\"\"\"{fake_name} recently lost his wallet. \n",
|
||||
"Inside is some cash and his credit card with the number {fake_credit_card}. \n",
|
||||
"If you would find it, please call at {fake_phone} or write an email here: {fake_email}.\n",
|
||||
"{fake_name} would be very grateful!\"\"\"\n",
|
||||
"\n",
|
||||
"print(anonymized_text)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"And now, using the `deanonymize` method, we can reverse the process:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Slim Shady recently lost his wallet. \n",
|
||||
"Inside is some cash and his credit card with the number 4916 0387 9536 0861. \n",
|
||||
"If you would find it, please call at 313-666-7440 or write an email here: real.slim.shady@gmail.com.\n",
|
||||
"Slim Shady would be very grateful!\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(anonymizer.deanonymize(anonymized_text))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Using with LangChain Expression Language\n",
|
||||
"\n",
|
||||
"With LCEL we can easily chain together anonymization and deanonymization with the rest of our application. This is an example of using the anonymization mechanism with a query to LLM (without deanonymization for now):"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"text = f\"\"\"Slim Shady recently lost his wallet. \n",
|
||||
"Inside is some cash and his credit card with the number 4916 0387 9536 0861. \n",
|
||||
"If you would find it, please call at 313-666-7440 or write an email here: real.slim.shady@gmail.com.\"\"\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dear Sir/Madam,\n",
|
||||
"\n",
|
||||
"We regret to inform you that Mr. Dana Rhodes has reported the loss of his wallet. The wallet contains a sum of cash and his credit card, bearing the number 4397528473885757. \n",
|
||||
"\n",
|
||||
"If you happen to come across the aforementioned wallet, we kindly request that you contact us immediately at 258-481-7074x714 or via email at laurengoodman@example.com.\n",
|
||||
"\n",
|
||||
"Your prompt assistance in this matter would be greatly appreciated.\n",
|
||||
"\n",
|
||||
"Yours faithfully,\n",
|
||||
"\n",
|
||||
"[Your Name]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain.prompts.prompt import PromptTemplate\n",
|
||||
"from langchain.chat_models import ChatOpenAI\n",
|
||||
"\n",
|
||||
"anonymizer = PresidioReversibleAnonymizer()\n",
|
||||
"\n",
|
||||
"template = \"\"\"Rewrite this text into an official, short email:\n",
|
||||
"\n",
|
||||
"{anonymized_text}\"\"\"\n",
|
||||
"prompt = PromptTemplate.from_template(template)\n",
|
||||
"llm = ChatOpenAI(temperature=0)\n",
|
||||
"\n",
|
||||
"chain = {\"anonymized_text\": anonymizer.anonymize} | prompt | llm\n",
|
||||
"response = chain.invoke(text)\n",
|
||||
"print(response.content)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Now, let's add **deanonymization step** to our sequence:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dear Sir/Madam,\n",
|
||||
"\n",
|
||||
"We regret to inform you that Mr. Slim Shady has recently misplaced his wallet. The wallet contains a sum of cash and his credit card, bearing the number 4916 0387 9536 0861. \n",
|
||||
"\n",
|
||||
"If by any chance you come across the lost wallet, kindly contact us immediately at 313-666-7440 or send an email to real.slim.shady@gmail.com.\n",
|
||||
"\n",
|
||||
"Your prompt assistance in this matter would be greatly appreciated.\n",
|
||||
"\n",
|
||||
"Yours faithfully,\n",
|
||||
"\n",
|
||||
"[Your Name]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"chain = chain | (lambda ai_message: anonymizer.deanonymize(ai_message.content))\n",
|
||||
"response = chain.invoke(text)\n",
|
||||
"print(response)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Anonymized data was given to the model itself, and therefore it was protected from being leaked to the outside world. Then, the model's response was processed, and the factual value was replaced with the real one."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Extra knowledge"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"`PresidioReversibleAnonymizer` stores the mapping of the fake values to the original values in the `deanonymizer_mapping` parameter, where key is fake PII and value is the original one: "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'PERSON': {'Maria Lynch': 'Slim Shady'},\n",
|
||||
" 'PHONE_NUMBER': {'7344131647': '313-666-7440'},\n",
|
||||
" 'EMAIL_ADDRESS': {'jamesmichael@example.com': 'real.slim.shady@gmail.com'},\n",
|
||||
" 'CREDIT_CARD': {'4838637940262': '4916 0387 9536 0861'}}"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain_experimental.data_anonymizer import PresidioReversibleAnonymizer\n",
|
||||
"\n",
|
||||
"anonymizer = PresidioReversibleAnonymizer(\n",
|
||||
" analyzed_fields=[\"PERSON\", \"PHONE_NUMBER\", \"EMAIL_ADDRESS\", \"CREDIT_CARD\"],\n",
|
||||
" # Faker seed is used here to make sure the same fake data is generated for the test purposes\n",
|
||||
" # In production, it is recommended to remove the faker_seed parameter (it will default to None)\n",
|
||||
" faker_seed=42,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"anonymizer.anonymize(\n",
|
||||
" \"My name is Slim Shady, call me at 313-666-7440 or email me at real.slim.shady@gmail.com. \"\n",
|
||||
" \"By the way, my card number is: 4916 0387 9536 0861\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"anonymizer.deanonymizer_mapping"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Anonymizing more texts will result in new mapping entries:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Do you have his VISA card number? Yep, it's 3537672423884966. I'm William Bowman by the way.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'PERSON': {'Maria Lynch': 'Slim Shady', 'William Bowman': 'John Doe'},\n",
|
||||
" 'PHONE_NUMBER': {'7344131647': '313-666-7440'},\n",
|
||||
" 'EMAIL_ADDRESS': {'jamesmichael@example.com': 'real.slim.shady@gmail.com'},\n",
|
||||
" 'CREDIT_CARD': {'4838637940262': '4916 0387 9536 0861',\n",
|
||||
" '3537672423884966': '4001 9192 5753 7193'}}"
|
||||
]
|
||||
},
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(\n",
|
||||
" anonymizer.anonymize(\n",
|
||||
" \"Do you have his VISA card number? Yep, it's 4001 9192 5753 7193. I'm John Doe by the way.\"\n",
|
||||
" )\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"anonymizer.deanonymizer_mapping"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We can save the mapping itself to a file for future use: "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# We can save the deanonymizer mapping as a JSON or YAML file\n",
|
||||
"\n",
|
||||
"anonymizer.save_deanonymizer_mapping(\"deanonymizer_mapping.json\")\n",
|
||||
"# anonymizer.save_deanonymizer_mapping(\"deanonymizer_mapping.yaml\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"And then, load it in another `PresidioReversibleAnonymizer` instance:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{}"
|
||||
]
|
||||
},
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"anonymizer = PresidioReversibleAnonymizer()\n",
|
||||
"\n",
|
||||
"anonymizer.deanonymizer_mapping"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'PERSON': {'Maria Lynch': 'Slim Shady', 'William Bowman': 'John Doe'},\n",
|
||||
" 'PHONE_NUMBER': {'7344131647': '313-666-7440'},\n",
|
||||
" 'EMAIL_ADDRESS': {'jamesmichael@example.com': 'real.slim.shady@gmail.com'},\n",
|
||||
" 'CREDIT_CARD': {'4838637940262': '4916 0387 9536 0861',\n",
|
||||
" '3537672423884966': '4001 9192 5753 7193'}}"
|
||||
]
|
||||
},
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"anonymizer.load_deanonymizer_mapping(\"deanonymizer_mapping.json\")\n",
|
||||
"\n",
|
||||
"anonymizer.deanonymizer_mapping"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Future works\n",
|
||||
"\n",
|
||||
"- **instance anonymization** - at this point, each occurrence of PII is treated as a separate entity and separately anonymized. Therefore, two occurrences of the name John Doe in the text will be changed to two different names. It is therefore worth introducing support for full instance detection, so that repeated occurrences are treated as a single object.\n",
|
||||
"- **better matching and substitution of fake values for real ones** - currently the strategy is based on matching full strings and then substituting them. Due to the indeterminism of language models, it may happen that the value in the answer is slightly changed (e.g. *John Doe* -> *John* or *Main St, New York* -> *New York*) and such a substitution is then no longer possible. Therefore, it is worth adjusting the matching for your needs."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
@ -1,79 +1,72 @@
|
||||
# Banana
|
||||
|
||||
This page covers how to use the Banana ecosystem within LangChain.
|
||||
It is broken into two parts: installation and setup, and then references to specific Banana wrappers.
|
||||
Banana provided serverless GPU inference for AI models, including a CI/CD build pipeline and a simple Python framework (Potassium) to server your models.
|
||||
|
||||
This page covers how to use the [Banana](https://www.banana.dev) ecosystem within LangChain.
|
||||
|
||||
It is broken into two parts:
|
||||
* installation and setup,
|
||||
* and then references to specific Banana wrappers.
|
||||
|
||||
## Installation and Setup
|
||||
|
||||
- Install with `pip install banana-dev`
|
||||
- Get an Banana api key and set it as an environment variable (`BANANA_API_KEY`)
|
||||
- Get an Banana api key from the [Banana.dev dashboard](https://app.banana.dev) and set it as an environment variable (`BANANA_API_KEY`)
|
||||
- Get your model's key and url slug from the model's details page
|
||||
|
||||
## Define your Banana Template
|
||||
|
||||
If you want to use an available language model template you can find one [here](https://app.banana.dev/templates/conceptofmind/serverless-template-palmyra-base).
|
||||
This template uses the Palmyra-Base model by [Writer](https://writer.com/product/api/).
|
||||
You can check out an example Banana repository [here](https://github.com/conceptofmind/serverless-template-palmyra-base).
|
||||
You'll need to set up a Github repo for your Banana app. You can get started in 5 minutes using [this guide](https://docs.banana.dev/banana-docs/).
|
||||
|
||||
Alternatively, for a ready-to-go LLM example, you can check out Banana's [CodeLlama-7B-Instruct-GPTQ](https://github.com/bananaml/demo-codellama-7b-instruct-gptq) GitHub repository. Just fork it and deploy it within Banana.
|
||||
|
||||
Other starter repos are available [here](https://github.com/orgs/bananaml/repositories?q=demo-&type=all&language=&sort=).
|
||||
|
||||
## Build the Banana app
|
||||
|
||||
Banana Apps must include the "output" key in the return json.
|
||||
There is a rigid response structure.
|
||||
To use Banana apps within Langchain, they must include the `outputs` key
|
||||
in the returned json, and the value must be a string.
|
||||
|
||||
```python
|
||||
# Return the results as a dictionary
|
||||
result = {'output': result}
|
||||
result = {'outputs': result}
|
||||
```
|
||||
|
||||
An example inference function would be:
|
||||
|
||||
```python
|
||||
def inference(model_inputs:dict) -> dict:
|
||||
global model
|
||||
global tokenizer
|
||||
|
||||
# Parse out your arguments
|
||||
prompt = model_inputs.get('prompt', None)
|
||||
if prompt == None:
|
||||
return {'message': "No prompt provided"}
|
||||
|
||||
# Run the model
|
||||
input_ids = tokenizer.encode(prompt, return_tensors='pt').cuda()
|
||||
output = model.generate(
|
||||
input_ids,
|
||||
max_length=100,
|
||||
do_sample=True,
|
||||
top_k=50,
|
||||
top_p=0.95,
|
||||
num_return_sequences=1,
|
||||
temperature=0.9,
|
||||
early_stopping=True,
|
||||
no_repeat_ngram_size=3,
|
||||
num_beams=5,
|
||||
length_penalty=1.5,
|
||||
repetition_penalty=1.5,
|
||||
bad_words_ids=[[tokenizer.encode(' ', add_prefix_space=True)[0]]]
|
||||
)
|
||||
|
||||
result = tokenizer.decode(output[0], skip_special_tokens=True)
|
||||
# Return the results as a dictionary
|
||||
result = {'output': result}
|
||||
return result
|
||||
@app.handler("/")
|
||||
def handler(context: dict, request: Request) -> Response:
|
||||
"""Handle a request to generate code from a prompt."""
|
||||
model = context.get("model")
|
||||
tokenizer = context.get("tokenizer")
|
||||
max_new_tokens = request.json.get("max_new_tokens", 512)
|
||||
temperature = request.json.get("temperature", 0.7)
|
||||
prompt = request.json.get("prompt")
|
||||
prompt_template=f'''[INST] Write code to solve the following coding problem that obeys the constraints and passes the example test cases. Please wrap your code answer using ```:
|
||||
{prompt}
|
||||
[/INST]
|
||||
'''
|
||||
input_ids = tokenizer(prompt_template, return_tensors='pt').input_ids.cuda()
|
||||
output = model.generate(inputs=input_ids, temperature=temperature, max_new_tokens=max_new_tokens)
|
||||
result = tokenizer.decode(output[0])
|
||||
return Response(json={"outputs": result}, status=200)
|
||||
```
|
||||
|
||||
You can find a full example of a Banana app [here](https://github.com/conceptofmind/serverless-template-palmyra-base/blob/main/app.py).
|
||||
This example is from the `app.py` file in [CodeLlama-7B-Instruct-GPTQ](https://github.com/bananaml/demo-codellama-7b-instruct-gptq).
|
||||
|
||||
## Wrappers
|
||||
|
||||
### LLM
|
||||
|
||||
There exists an Banana LLM wrapper, which you can access with
|
||||
Within Langchain, there exists a Banana LLM wrapper, which you can access with
|
||||
|
||||
```python
|
||||
from langchain.llms import Banana
|
||||
```
|
||||
|
||||
You need to provide a model key located in the dashboard:
|
||||
You need to provide a model key and model url slug, which you can get from the model's details page in the [Banana.dev dashboard](https://app.banana.dev).
|
||||
|
||||
```python
|
||||
llm = Banana(model_key="YOUR_MODEL_KEY")
|
||||
llm = Banana(model_key="YOUR_MODEL_KEY", model_url_slug="YOUR_MODEL_URL_SLUG")
|
||||
```
|
||||
|
||||
@ -0,0 +1,126 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# NucliaDB\n",
|
||||
"\n",
|
||||
"You can use a local NucliaDB instance or use [Nuclia Cloud](https://nuclia.cloud).\n",
|
||||
"\n",
|
||||
"When using a local instance, you need a Nuclia Understanding API key, so your texts are properly vectorized and indexed. You can get a key by creating a free account at [https://nuclia.cloud](https://nuclia.cloud), and then [create a NUA key](https://docs.nuclia.dev/docs/docs/using/understanding/intro)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#!pip install langchain nuclia"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Usage with nuclia.cloud"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.vectorstores.nucliadb import NucliaDB\n",
|
||||
"API_KEY = \"YOUR_API_KEY\"\n",
|
||||
"\n",
|
||||
"ndb = NucliaDB(knowledge_box=\"YOUR_KB_ID\", local=False, api_key=API_KEY)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Usage with a local instance\n",
|
||||
"\n",
|
||||
"Note: By default `backend` is set to `http://localhost:8080`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.vectorstores.nucliadb import NucliaDB\n",
|
||||
"\n",
|
||||
"ndb = NucliaDB(knowledge_box=\"YOUR_KB_ID\", local=True, backend=\"http://my-local-server\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Add and delete texts to your Knowledge Box"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"ids = ndb.add_texts([\"This is a new test\", \"This is a second test\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"ndb.delete(ids=ids)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Search in your Knowledge Box"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"results = ndb.similarity_search(\"Who was inspired by Ada Lovelace?\")\n",
|
||||
"print(res.page_content)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.1"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
@ -1,2 +1,2 @@
|
||||
label: 'More'
|
||||
position: 1
|
||||
position: 2
|
||||
@ -0,0 +1,307 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7f0b0c06-ee70-468c-8bf5-b023f9e5e0a2",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Diffbot Graph Transformer\n",
|
||||
"\n",
|
||||
"[](https://colab.research.google.com/github/langchain-ai/langchain/blob/master/docs/extras/use_cases/more/graph/diffbot_transformer.ipynb)\n",
|
||||
"\n",
|
||||
"## Use case\n",
|
||||
"\n",
|
||||
"Text data often contain rich relationships and insights that can be useful for various analytics, recommendation engines, or knowledge management applications.\n",
|
||||
"\n",
|
||||
"Diffbot's NLP API allows for the extraction of entities, relationships, and semantic meaning from unstructured text data.\n",
|
||||
"\n",
|
||||
"By coupling Diffbot's NLP API with Neo4j, a graph database, you can create powerful, dynamic graph structures based on the information extracted from text. These graph structures are fully queryable and can be integrated into various applications.\n",
|
||||
"\n",
|
||||
"This combination allows for use cases such as:\n",
|
||||
"\n",
|
||||
"* Building knowledge graphs from textual documents, websites, or social media feeds.\n",
|
||||
"* Generating recommendations based on semantic relationships in the data.\n",
|
||||
"* Creating advanced search features that understand the relationships between entities.\n",
|
||||
"* Building analytics dashboards that allow users to explore the hidden relationships in data.\n",
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"LangChain provides tools to interact with Graph Databases:\n",
|
||||
"\n",
|
||||
"1. `Construct knowledge graphs from text` using graph transformer and store integrations \n",
|
||||
"2. `Query a graph database` using chains for query creation and execution\n",
|
||||
"3. `Interact with a graph database` using agents for robust and flexible querying \n",
|
||||
"\n",
|
||||
"## Quickstart\n",
|
||||
"\n",
|
||||
"First, get required packages and set environment variables:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "975648da-b24f-4164-a671-6772179e12df",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install langchain langchain-experimental openai neo4j wikipedia"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "77718977-629e-46c2-b091-f9191b9ec569",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Diffbot NLP Service\n",
|
||||
"\n",
|
||||
"Diffbot's NLP service is a tool for extracting entities, relationships, and semantic context from unstructured text data.\n",
|
||||
"This extracted information can be used to construct a knowledge graph.\n",
|
||||
"To use their service, you'll need to obtain an API key from [Diffbot](https://www.diffbot.com/products/natural-language/)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "2cbf97d0-3682-439b-8750-b695ff726789",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_experimental.graph_transformers.diffbot import DiffbotGraphTransformer\n",
|
||||
"\n",
|
||||
"diffbot_api_key = \"DIFFBOT_API_KEY\"\n",
|
||||
"diffbot_nlp = DiffbotGraphTransformer(diffbot_api_key=diffbot_api_key)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5e3b894a-e3ee-46c7-8116-f8377f8f0159",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This code fetches Wikipedia articles about \"Baldur's Gate 3\" and then uses `DiffbotGraphTransformer` to extract entities and relationships.\n",
|
||||
"The `DiffbotGraphTransformer` outputs a structured data `GraphDocument`, which can be used to populate a graph database.\n",
|
||||
"Note that text chunking is avoided due to Diffbot's [character limit per API request](https://docs.diffbot.com/reference/introduction-to-natural-language-api)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "53f8df86-47a1-44a1-9a0f-6725b90703bc",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.document_loaders import WikipediaLoader\n",
|
||||
"\n",
|
||||
"query = \"Warren Buffett\"\n",
|
||||
"raw_documents = WikipediaLoader(query=query).load()\n",
|
||||
"graph_documents = diffbot_nlp.convert_to_graph_documents(raw_documents)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "31bb851a-aab4-4b97-a6b7-fce397d32b47",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Loading the data into a knowledge graph\n",
|
||||
"\n",
|
||||
"You will need to have a running Neo4j instance. One option is to create a [free Neo4j database instance in their Aura cloud service](https://neo4j.com/cloud/platform/aura-graph-database/). You can also run the database locally using the [Neo4j Desktop application](https://neo4j.com/download/), or running a docker container. You can run a local docker container by running the executing the following script:\n",
|
||||
"```\n",
|
||||
"docker run \\\n",
|
||||
" --name neo4j \\\n",
|
||||
" -p 7474:7474 -p 7687:7687 \\\n",
|
||||
" -d \\\n",
|
||||
" -e NEO4J_AUTH=neo4j/pleaseletmein \\\n",
|
||||
" -e NEO4J_PLUGINS=\\[\\\"apoc\\\"\\] \\\n",
|
||||
" neo4j:latest\n",
|
||||
"``` \n",
|
||||
"If you are using the docker container, you need to wait a couple of second for the database to start."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "0b2b6641-5a5d-467c-b148-e6aad5e4baa7",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.graphs import Neo4jGraph\n",
|
||||
"\n",
|
||||
"url=\"bolt://localhost:7687\"\n",
|
||||
"username=\"neo4j\"\n",
|
||||
"password=\"pleaseletmein\"\n",
|
||||
"\n",
|
||||
"graph = Neo4jGraph(\n",
|
||||
" url=url,\n",
|
||||
" username=username, \n",
|
||||
" password=password\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "0b15e840-fe6f-45db-9193-1b4e2df5c12c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The `GraphDocuments` can be loaded into a knowledge graph using the `add_graph_documents` method."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "1a67c4a8-955c-42a2-9c5d-de3ac0e640ec",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"graph.add_graph_documents(graph_documents)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ed411e05-2b03-460d-997e-938482774f40",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Refresh graph schema information\n",
|
||||
"If the schema of database changes, you can refresh the schema information needed to generate Cypher statements"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "904c9ee3-787c-403f-857d-459ce5ad5a1b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"graph.refresh_schema()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f19d1387-5899-4258-8c94-8ef5fa7db464",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Querying the graph\n",
|
||||
"We can now use the graph cypher QA chain to ask question of the graph. It is advisable to use **gpt-4** to construct Cypher queries to get the best experience."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "9393b732-67c8-45c1-9ec2-089f49c62448",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.chains import GraphCypherQAChain\n",
|
||||
"from langchain.chat_models import ChatOpenAI\n",
|
||||
"\n",
|
||||
"chain = GraphCypherQAChain.from_llm(\n",
|
||||
" cypher_llm=ChatOpenAI(temperature=0, model_name=\"gpt-4\"),\n",
|
||||
" qa_llm=ChatOpenAI(temperature=0, model_name=\"gpt-3.5-turbo\"),\n",
|
||||
" graph=graph, verbose=True,\n",
|
||||
" \n",
|
||||
")\n",
|
||||
" "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "1a9b3652-b436-404d-aa25-5fb576f23dc0",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[1m> Entering new GraphCypherQAChain chain...\u001b[0m\n",
|
||||
"Generated Cypher:\n",
|
||||
"\u001b[32;1m\u001b[1;3mMATCH (p:Person {name: \"Warren Buffett\"})-[:EDUCATED_AT]->(o:Organization)\n",
|
||||
"RETURN o.name\u001b[0m\n",
|
||||
"Full Context:\n",
|
||||
"\u001b[32;1m\u001b[1;3m[{'o.name': 'New York Institute of Finance'}, {'o.name': 'Alice Deal Junior High School'}, {'o.name': 'Woodrow Wilson High School'}, {'o.name': 'University of Nebraska'}]\u001b[0m\n",
|
||||
"\n",
|
||||
"\u001b[1m> Finished chain.\u001b[0m\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'Warren Buffett attended the University of Nebraska.'"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"chain.run(\"Which university did Warren Buffett attend?\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "adc0ba0f-a62c-4875-89ce-da717f3ab148",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[1m> Entering new GraphCypherQAChain chain...\u001b[0m\n",
|
||||
"Generated Cypher:\n",
|
||||
"\u001b[32;1m\u001b[1;3mMATCH (p:Person)-[r:EMPLOYEE_OR_MEMBER_OF]->(o:Organization) WHERE o.name = 'Berkshire Hathaway' RETURN p.name\u001b[0m\n",
|
||||
"Full Context:\n",
|
||||
"\u001b[32;1m\u001b[1;3m[{'p.name': 'Charlie Munger'}, {'p.name': 'Oliver Chace'}, {'p.name': 'Howard Buffett'}, {'p.name': 'Howard'}, {'p.name': 'Susan Buffett'}, {'p.name': 'Warren Buffett'}]\u001b[0m\n",
|
||||
"\n",
|
||||
"\u001b[1m> Finished chain.\u001b[0m\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'Charlie Munger, Oliver Chace, Howard Buffett, Susan Buffett, and Warren Buffett are or were working at Berkshire Hathaway.'"
|
||||
]
|
||||
},
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"chain.run(\"Who is or was working at Berkshire Hathaway?\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d636954b-d967-4e96-9489-92e11c74af35",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -0,0 +1,3 @@
|
||||
label: 'QA over structured data'
|
||||
collapsed: false
|
||||
position: 0.5
|
||||
@ -0,0 +1 @@
|
||||
label: 'Integration-specific'
|
||||
@ -0,0 +1,158 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Elasticsearch\n",
|
||||
"\n",
|
||||
"[](https://colab.research.google.com/github/langchain-ai/langchain/blob/master/docs/extras/use_cases/qa_structured/integrations/elasticsearch.ipynb)\n",
|
||||
"\n",
|
||||
"We can use LLMs to interact with Elasticsearch analytics databases in natural language.\n",
|
||||
"\n",
|
||||
"This chain builds search queries via the Elasticsearch DSL API (filters and aggregations).\n",
|
||||
"\n",
|
||||
"The Elasticsearch client must have permissions for index listing, mapping description and search queries.\n",
|
||||
"\n",
|
||||
"See [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html) for instructions on how to run Elasticsearch locally."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip install langchain langchain-experimental openai elasticsearch\n",
|
||||
"\n",
|
||||
"# Set env var OPENAI_API_KEY or load from a .env file\n",
|
||||
"# import dotenv\n",
|
||||
"\n",
|
||||
"# dotenv.load_dotenv()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from elasticsearch import Elasticsearch\n",
|
||||
"\n",
|
||||
"from langchain.chat_models import ChatOpenAI\n",
|
||||
"from langchain.chains.elasticsearch_database import ElasticsearchDatabaseChain"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Initialize Elasticsearch python client.\n",
|
||||
"# See https://elasticsearch-py.readthedocs.io/en/v8.8.2/api.html#elasticsearch.Elasticsearch\n",
|
||||
"ELASTIC_SEARCH_SERVER = \"https://elastic:pass@localhost:9200\"\n",
|
||||
"db = Elasticsearch(ELASTIC_SEARCH_SERVER)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Uncomment the next cell to initially populate your db."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# customers = [\n",
|
||||
"# {\"firstname\": \"Jennifer\", \"lastname\": \"Walters\"},\n",
|
||||
"# {\"firstname\": \"Monica\",\"lastname\":\"Rambeau\"},\n",
|
||||
"# {\"firstname\": \"Carol\",\"lastname\":\"Danvers\"},\n",
|
||||
"# {\"firstname\": \"Wanda\",\"lastname\":\"Maximoff\"},\n",
|
||||
"# {\"firstname\": \"Jennifer\",\"lastname\":\"Takeda\"},\n",
|
||||
"# ]\n",
|
||||
"# for i, customer in enumerate(customers):\n",
|
||||
"# db.create(index=\"customers\", document=customer, id=i)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"llm = ChatOpenAI(model_name=\"gpt-4\", temperature=0)\n",
|
||||
"chain = ElasticsearchDatabaseChain.from_llm(llm=llm, database=db, verbose=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"question = \"What are the first names of all the customers?\"\n",
|
||||
"chain.run(question)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We can customize the prompt."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.chains.elasticsearch_database.prompts import DEFAULT_DSL_TEMPLATE\n",
|
||||
"from langchain.prompts.prompt import PromptTemplate\n",
|
||||
"\n",
|
||||
"PROMPT_TEMPLATE = \"\"\"Given an input question, create a syntactically correct Elasticsearch query to run. Unless the user specifies in their question a specific number of examples they wish to obtain, always limit your query to at most {top_k} results. You can order the results by a relevant column to return the most interesting examples in the database.\n",
|
||||
"\n",
|
||||
"Unless told to do not query for all the columns from a specific index, only ask for a the few relevant columns given the question.\n",
|
||||
"\n",
|
||||
"Pay attention to use only the column names that you can see in the mapping description. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which index. Return the query as valid json.\n",
|
||||
"\n",
|
||||
"Use the following format:\n",
|
||||
"\n",
|
||||
"Question: Question here\n",
|
||||
"ESQuery: Elasticsearch Query formatted as json\n",
|
||||
"\"\"\"\n",
|
||||
"\n",
|
||||
"PROMPT = PromptTemplate.from_template(\n",
|
||||
" PROMPT_TEMPLATE,\n",
|
||||
")\n",
|
||||
"chain = ElasticsearchDatabaseChain.from_llm(llm=llm, database=db, query_prompt=PROMPT)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.1"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
@ -0,0 +1,200 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "245065c6",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Vector SQL Retriever with MyScale\n",
|
||||
"\n",
|
||||
">[MyScale](https://docs.myscale.com/en/) is an integrated vector database. You can access your database in SQL and also from here, LangChain. MyScale can make a use of [various data types and functions for filters](https://blog.myscale.com/2023/06/06/why-integrated-database-solution-can-boost-your-llm-apps/#filter-on-anything-without-constraints). It will boost up your LLM app no matter if you are scaling up your data or expand your system to broader application."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0246c5bf",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip3 install clickhouse-sqlalchemy InstructorEmbedding sentence_transformers openai langchain-experimental"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7585d2c3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"from os import environ\n",
|
||||
"import getpass\n",
|
||||
"from typing import Dict, Any\n",
|
||||
"from langchain import OpenAI, SQLDatabase, LLMChain\n",
|
||||
"from langchain_experimental.sql.vector_sql import VectorSQLDatabaseChain\n",
|
||||
"from sqlalchemy import create_engine, Column, MetaData\n",
|
||||
"from langchain import PromptTemplate\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"from sqlalchemy import create_engine\n",
|
||||
"\n",
|
||||
"MYSCALE_HOST = \"msc-1decbcc9.us-east-1.aws.staging.myscale.cloud\"\n",
|
||||
"MYSCALE_PORT = 443\n",
|
||||
"MYSCALE_USER = \"chatdata\"\n",
|
||||
"MYSCALE_PASSWORD = \"myscale_rocks\"\n",
|
||||
"OPENAI_API_KEY = getpass.getpass(\"OpenAI API Key:\")\n",
|
||||
"\n",
|
||||
"engine = create_engine(\n",
|
||||
" f\"clickhouse://{MYSCALE_USER}:{MYSCALE_PASSWORD}@{MYSCALE_HOST}:{MYSCALE_PORT}/default?protocol=https\"\n",
|
||||
")\n",
|
||||
"metadata = MetaData(bind=engine)\n",
|
||||
"environ[\"OPENAI_API_KEY\"] = OPENAI_API_KEY"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e08d9ddc",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.embeddings import HuggingFaceInstructEmbeddings\n",
|
||||
"from langchain_experimental.sql.vector_sql import VectorSQLOutputParser\n",
|
||||
"\n",
|
||||
"output_parser = VectorSQLOutputParser.from_embeddings(\n",
|
||||
" model=HuggingFaceInstructEmbeddings(\n",
|
||||
" model_name=\"hkunlp/instructor-xl\", model_kwargs={\"device\": \"cpu\"}\n",
|
||||
" )\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "84b705b2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"from langchain.llms import OpenAI\n",
|
||||
"from langchain.callbacks import StdOutCallbackHandler\n",
|
||||
"\n",
|
||||
"from langchain.utilities.sql_database import SQLDatabase\n",
|
||||
"from langchain_experimental.sql.prompt import MYSCALE_PROMPT\n",
|
||||
"from langchain_experimental.sql.vector_sql import VectorSQLDatabaseChain\n",
|
||||
"\n",
|
||||
"chain = VectorSQLDatabaseChain(\n",
|
||||
" llm_chain=LLMChain(\n",
|
||||
" llm=OpenAI(openai_api_key=OPENAI_API_KEY, temperature=0),\n",
|
||||
" prompt=MYSCALE_PROMPT,\n",
|
||||
" ),\n",
|
||||
" top_k=10,\n",
|
||||
" return_direct=True,\n",
|
||||
" sql_cmd_parser=output_parser,\n",
|
||||
" database=SQLDatabase(engine, None, metadata),\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"pd.DataFrame(\n",
|
||||
" chain.run(\n",
|
||||
" \"Please give me 10 papers to ask what is PageRank?\",\n",
|
||||
" callbacks=[StdOutCallbackHandler()],\n",
|
||||
" )\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6c09cda0",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## SQL Database as Retriever"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "734d7ff5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.chat_models import ChatOpenAI\n",
|
||||
"from langchain.chains.qa_with_sources.retrieval import RetrievalQAWithSourcesChain\n",
|
||||
"\n",
|
||||
"from langchain_experimental.sql.vector_sql import VectorSQLDatabaseChain\n",
|
||||
"from langchain_experimental.retrievers.vector_sql_database \\\n",
|
||||
" import VectorSQLDatabaseChainRetriever\n",
|
||||
"from langchain_experimental.sql.prompt import MYSCALE_PROMPT\n",
|
||||
"from langchain_experimental.sql.vector_sql import VectorSQLRetrieveAllOutputParser\n",
|
||||
"\n",
|
||||
"output_parser_retrieve_all = VectorSQLRetrieveAllOutputParser.from_embeddings(\n",
|
||||
" output_parser.model\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"chain = VectorSQLDatabaseChain.from_llm(\n",
|
||||
" llm=OpenAI(openai_api_key=OPENAI_API_KEY, temperature=0),\n",
|
||||
" prompt=MYSCALE_PROMPT,\n",
|
||||
" top_k=10,\n",
|
||||
" return_direct=True,\n",
|
||||
" db=SQLDatabase(engine, None, metadata),\n",
|
||||
" sql_cmd_parser=output_parser_retrieve_all,\n",
|
||||
" native_format=True,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# You need all those keys to get docs\n",
|
||||
"retriever = VectorSQLDatabaseChainRetriever(sql_db_chain=chain, page_content_key=\"abstract\")\n",
|
||||
"\n",
|
||||
"document_with_metadata_prompt = PromptTemplate(\n",
|
||||
" input_variables=[\"page_content\", \"id\", \"title\", \"authors\", \"pubdate\", \"categories\"],\n",
|
||||
" template=\"Content:\\n\\tTitle: {title}\\n\\tAbstract: {page_content}\\n\\tAuthors: {authors}\\n\\tDate of Publication: {pubdate}\\n\\tCategories: {categories}\\nSOURCE: {id}\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"chain = RetrievalQAWithSourcesChain.from_chain_type(\n",
|
||||
" ChatOpenAI(\n",
|
||||
" model_name=\"gpt-3.5-turbo-16k\", openai_api_key=OPENAI_API_KEY, temperature=0.6\n",
|
||||
" ),\n",
|
||||
" retriever=retriever,\n",
|
||||
" chain_type=\"stuff\",\n",
|
||||
" chain_type_kwargs={\n",
|
||||
" \"document_prompt\": document_with_metadata_prompt,\n",
|
||||
" },\n",
|
||||
" return_source_documents=True,\n",
|
||||
")\n",
|
||||
"ans = chain(\"Please give me 10 papers to ask what is PageRank?\",\n",
|
||||
" callbacks=[StdOutCallbackHandler()])\n",
|
||||
"print(ans[\"answer\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4948ff25",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -1,4 +1,7 @@
|
||||
"""Data anonymizer package"""
|
||||
from langchain_experimental.data_anonymizer.presidio import PresidioAnonymizer
|
||||
from langchain_experimental.data_anonymizer.presidio import (
|
||||
PresidioAnonymizer,
|
||||
PresidioReversibleAnonymizer,
|
||||
)
|
||||
|
||||
__all__ = ["PresidioAnonymizer"]
|
||||
__all__ = ["PresidioAnonymizer", "PresidioReversibleAnonymizer"]
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict
|
||||
|
||||
MappingDataType = Dict[str, Dict[str, str]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeanonymizerMapping:
|
||||
mapping: MappingDataType = field(
|
||||
default_factory=lambda: defaultdict(lambda: defaultdict(str))
|
||||
)
|
||||
|
||||
@property
|
||||
def data(self) -> MappingDataType:
|
||||
"""Return the deanonymizer mapping"""
|
||||
return {k: dict(v) for k, v in self.mapping.items()}
|
||||
|
||||
def update(self, new_mapping: MappingDataType) -> None:
|
||||
for entity_type, values in new_mapping.items():
|
||||
self.mapping[entity_type].update(values)
|
||||
@ -0,0 +1,17 @@
|
||||
from langchain_experimental.data_anonymizer.presidio import MappingDataType
|
||||
|
||||
|
||||
def default_matching_strategy(text: str, deanonymizer_mapping: MappingDataType) -> str:
|
||||
"""
|
||||
Default matching strategy for deanonymization.
|
||||
It replaces all the anonymized entities with the original ones.
|
||||
|
||||
Args:
|
||||
text: text to deanonymize
|
||||
deanonymizer_mapping: mapping between anonymized entities and original ones"""
|
||||
|
||||
# Iterate over all the entities (PERSON, EMAIL_ADDRESS, etc.)
|
||||
for entity_type in deanonymizer_mapping:
|
||||
for anonymized, original in deanonymizer_mapping[entity_type].items():
|
||||
text = text.replace(anonymized, original)
|
||||
return text
|
||||
@ -0,0 +1,5 @@
|
||||
from langchain_experimental.graph_transformers.diffbot import DiffbotGraphTransformer
|
||||
|
||||
__all__ = [
|
||||
"DiffbotGraphTransformer",
|
||||
]
|
||||
@ -0,0 +1,316 @@
|
||||
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
|
||||
|
||||
import requests
|
||||
from langchain.graphs.graph_document import GraphDocument, Node, Relationship
|
||||
from langchain.schema import Document
|
||||
from langchain.utils import get_from_env
|
||||
|
||||
|
||||
def format_property_key(s: str) -> str:
|
||||
words = s.split()
|
||||
if not words:
|
||||
return s
|
||||
first_word = words[0].lower()
|
||||
capitalized_words = [word.capitalize() for word in words[1:]]
|
||||
return "".join([first_word] + capitalized_words)
|
||||
|
||||
|
||||
class NodesList:
|
||||
"""
|
||||
Manages a list of nodes with associated properties.
|
||||
|
||||
Attributes:
|
||||
nodes (Dict[Tuple, Any]): Stores nodes as keys and their properties as values.
|
||||
Each key is a tuple where the first element is the
|
||||
node ID and the second is the node type.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.nodes: Dict[Tuple[Union[str, int], str], Any] = dict()
|
||||
|
||||
def add_node_property(
|
||||
self, node: Tuple[Union[str, int], str], properties: Dict[str, Any]
|
||||
) -> None:
|
||||
"""
|
||||
Adds or updates node properties.
|
||||
|
||||
If the node does not exist in the list, it's added along with its properties.
|
||||
If the node already exists, its properties are updated with the new values.
|
||||
|
||||
Args:
|
||||
node (Tuple): A tuple containing the node ID and node type.
|
||||
properties (Dict): A dictionary of properties to add or update for the node.
|
||||
"""
|
||||
if node not in self.nodes:
|
||||
self.nodes[node] = properties
|
||||
else:
|
||||
self.nodes[node].update(properties)
|
||||
|
||||
def return_node_list(self) -> List[Node]:
|
||||
"""
|
||||
Returns the nodes as a list of Node objects.
|
||||
|
||||
Each Node object will have its ID, type, and properties populated.
|
||||
|
||||
Returns:
|
||||
List[Node]: A list of Node objects.
|
||||
"""
|
||||
nodes = [
|
||||
Node(id=key[0], type=key[1], properties=self.nodes[key])
|
||||
for key in self.nodes
|
||||
]
|
||||
return nodes
|
||||
|
||||
|
||||
# Properties that should be treated as node properties instead of relationships
|
||||
FACT_TO_PROPERTY_TYPE = [
|
||||
"Date",
|
||||
"Number",
|
||||
"Job title",
|
||||
"Cause of death",
|
||||
"Organization type",
|
||||
"Academic title",
|
||||
]
|
||||
|
||||
|
||||
schema_mapping = [
|
||||
("HEADQUARTERS", "ORGANIZATION_LOCATIONS"),
|
||||
("RESIDENCE", "PERSON_LOCATION"),
|
||||
("ALL_PERSON_LOCATIONS", "PERSON_LOCATION"),
|
||||
("CHILD", "HAS_CHILD"),
|
||||
("PARENT", "HAS_PARENT"),
|
||||
("CUSTOMERS", "HAS_CUSTOMER"),
|
||||
("SKILLED_AT", "INTERESTED_IN"),
|
||||
]
|
||||
|
||||
|
||||
class SimplifiedSchema:
|
||||
"""
|
||||
Provides functionality for working with a simplified schema mapping.
|
||||
|
||||
Attributes:
|
||||
schema (Dict): A dictionary containing the mapping to simplified schema types.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initializes the schema dictionary based on the predefined list."""
|
||||
self.schema = dict()
|
||||
for row in schema_mapping:
|
||||
self.schema[row[0]] = row[1]
|
||||
|
||||
def get_type(self, type: str) -> str:
|
||||
"""
|
||||
Retrieves the simplified schema type for a given original type.
|
||||
|
||||
Args:
|
||||
type (str): The original schema type to find the simplified type for.
|
||||
|
||||
Returns:
|
||||
str: The simplified schema type if it exists;
|
||||
otherwise, returns the original type.
|
||||
"""
|
||||
try:
|
||||
return self.schema[type]
|
||||
except KeyError:
|
||||
return type
|
||||
|
||||
|
||||
class DiffbotGraphTransformer:
|
||||
"""Transforms documents into graph documents using Diffbot's NLP API.
|
||||
|
||||
A graph document transformation system takes a sequence of Documents and returns a
|
||||
sequence of Graph Documents.
|
||||
|
||||
Example:
|
||||
.. code-block:: python
|
||||
|
||||
class DiffbotGraphTransformer(BaseGraphDocumentTransformer):
|
||||
|
||||
def transform_documents(
|
||||
self, documents: Sequence[Document], **kwargs: Any
|
||||
) -> Sequence[GraphDocument]:
|
||||
results = []
|
||||
|
||||
for document in documents:
|
||||
raw_results = self.nlp_request(document.page_content)
|
||||
graph_document = self.process_response(raw_results, document)
|
||||
results.append(graph_document)
|
||||
return results
|
||||
|
||||
async def atransform_documents(
|
||||
self, documents: Sequence[Document], **kwargs: Any
|
||||
) -> Sequence[Document]:
|
||||
raise NotImplementedError
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
diffbot_api_key: Optional[str] = None,
|
||||
fact_confidence_threshold: float = 0.7,
|
||||
include_qualifiers: bool = True,
|
||||
include_evidence: bool = True,
|
||||
simplified_schema: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the graph transformer with various options.
|
||||
|
||||
Args:
|
||||
diffbot_api_key (str):
|
||||
The API key for Diffbot's NLP services.
|
||||
|
||||
fact_confidence_threshold (float):
|
||||
Minimum confidence level for facts to be included.
|
||||
include_qualifiers (bool):
|
||||
Whether to include qualifiers in the relationships.
|
||||
include_evidence (bool):
|
||||
Whether to include evidence for the relationships.
|
||||
simplified_schema (bool):
|
||||
Whether to use a simplified schema for relationships.
|
||||
"""
|
||||
self.diffbot_api_key = diffbot_api_key or get_from_env(
|
||||
"diffbot_api_key", "DIFFBOT_API_KEY"
|
||||
)
|
||||
self.fact_threshold_confidence = fact_confidence_threshold
|
||||
self.include_qualifiers = include_qualifiers
|
||||
self.include_evidence = include_evidence
|
||||
self.simplified_schema = None
|
||||
if simplified_schema:
|
||||
self.simplified_schema = SimplifiedSchema()
|
||||
|
||||
def nlp_request(self, text: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Make an API request to the Diffbot NLP endpoint.
|
||||
|
||||
Args:
|
||||
text (str): The text to be processed.
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: The JSON response from the API.
|
||||
"""
|
||||
|
||||
# Relationship extraction only works for English
|
||||
payload = {
|
||||
"content": text,
|
||||
"lang": "en",
|
||||
}
|
||||
|
||||
FIELDS = "facts"
|
||||
HOST = "nl.diffbot.com"
|
||||
url = (
|
||||
f"https://{HOST}/v1/?fields={FIELDS}&"
|
||||
f"token={self.diffbot_api_key}&language=en"
|
||||
)
|
||||
result = requests.post(url, data=payload)
|
||||
return result.json()
|
||||
|
||||
def process_response(
|
||||
self, payload: Dict[str, Any], document: Document
|
||||
) -> GraphDocument:
|
||||
"""
|
||||
Transform the Diffbot NLP response into a GraphDocument.
|
||||
|
||||
Args:
|
||||
payload (Dict[str, Any]): The JSON response from Diffbot's NLP API.
|
||||
document (Document): The original document.
|
||||
|
||||
Returns:
|
||||
GraphDocument: The transformed document as a graph.
|
||||
"""
|
||||
|
||||
# Return empty result if there are no facts
|
||||
if "facts" not in payload or not payload["facts"]:
|
||||
return GraphDocument(nodes=[], relationships=[], source=document)
|
||||
|
||||
# Nodes are a custom class because we need to deduplicate
|
||||
nodes_list = NodesList()
|
||||
# Relationships are a list because we don't deduplicate nor anything else
|
||||
relationships = list()
|
||||
for record in payload["facts"]:
|
||||
# Skip if the fact is below the threshold confidence
|
||||
if record["confidence"] < self.fact_threshold_confidence:
|
||||
continue
|
||||
|
||||
# TODO: It should probably be treated as a node property
|
||||
if not record["value"]["allTypes"]:
|
||||
continue
|
||||
|
||||
# Define source node
|
||||
source_id = (
|
||||
record["entity"]["allUris"][0]
|
||||
if record["entity"]["allUris"]
|
||||
else record["entity"]["name"]
|
||||
)
|
||||
source_label = record["entity"]["allTypes"][0]["name"].capitalize()
|
||||
source_name = record["entity"]["name"]
|
||||
source_node = Node(id=source_id, type=source_label)
|
||||
nodes_list.add_node_property(
|
||||
(source_id, source_label), {"name": source_name}
|
||||
)
|
||||
|
||||
# Define target node
|
||||
target_id = (
|
||||
record["value"]["allUris"][0]
|
||||
if record["value"]["allUris"]
|
||||
else record["value"]["name"]
|
||||
)
|
||||
target_label = record["value"]["allTypes"][0]["name"].capitalize()
|
||||
target_name = record["value"]["name"]
|
||||
# Some facts are better suited as node properties
|
||||
if target_label in FACT_TO_PROPERTY_TYPE:
|
||||
nodes_list.add_node_property(
|
||||
(source_id, source_label),
|
||||
{format_property_key(record["property"]["name"]): target_name},
|
||||
)
|
||||
else: # Define relationship
|
||||
# Define target node object
|
||||
target_node = Node(id=target_id, type=target_label)
|
||||
nodes_list.add_node_property(
|
||||
(target_id, target_label), {"name": target_name}
|
||||
)
|
||||
# Define relationship type
|
||||
rel_type = record["property"]["name"].replace(" ", "_").upper()
|
||||
if self.simplified_schema:
|
||||
rel_type = self.simplified_schema.get_type(rel_type)
|
||||
|
||||
# Relationship qualifiers/properties
|
||||
rel_properties = dict()
|
||||
relationship_evidence = [el["passage"] for el in record["evidence"]][0]
|
||||
if self.include_evidence:
|
||||
rel_properties.update({"evidence": relationship_evidence})
|
||||
if self.include_qualifiers and record.get("qualifiers"):
|
||||
for property in record["qualifiers"]:
|
||||
prop_key = format_property_key(property["property"]["name"])
|
||||
rel_properties[prop_key] = property["value"]["name"]
|
||||
|
||||
relationship = Relationship(
|
||||
source=source_node,
|
||||
target=target_node,
|
||||
type=rel_type,
|
||||
properties=rel_properties,
|
||||
)
|
||||
relationships.append(relationship)
|
||||
|
||||
return GraphDocument(
|
||||
nodes=nodes_list.return_node_list(),
|
||||
relationships=relationships,
|
||||
source=document,
|
||||
)
|
||||
|
||||
def convert_to_graph_documents(
|
||||
self, documents: Sequence[Document]
|
||||
) -> List[GraphDocument]:
|
||||
"""Convert a sequence of documents into graph documents.
|
||||
|
||||
Args:
|
||||
documents (Sequence[Document]): The original documents.
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
||||
Returns:
|
||||
Sequence[GraphDocument]: The transformed documents as graphs.
|
||||
"""
|
||||
results = []
|
||||
for document in documents:
|
||||
raw_results = self.nlp_request(document.page_content)
|
||||
graph_document = self.process_response(raw_results, document)
|
||||
results.append(graph_document)
|
||||
return results
|
||||
@ -0,0 +1,38 @@
|
||||
"""Vector SQL Database Chain Retriever"""
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from langchain.callbacks.manager import (
|
||||
AsyncCallbackManagerForRetrieverRun,
|
||||
CallbackManagerForRetrieverRun,
|
||||
)
|
||||
from langchain.schema import BaseRetriever, Document
|
||||
|
||||
from langchain_experimental.sql.vector_sql import VectorSQLDatabaseChain
|
||||
|
||||
|
||||
class VectorSQLDatabaseChainRetriever(BaseRetriever):
|
||||
"""Retriever that uses SQLDatabase as Retriever"""
|
||||
|
||||
sql_db_chain: VectorSQLDatabaseChain
|
||||
"""SQL Database Chain"""
|
||||
page_content_key: str = "content"
|
||||
"""column name for page content of documents"""
|
||||
|
||||
def _get_relevant_documents(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
run_manager: CallbackManagerForRetrieverRun,
|
||||
**kwargs: Any,
|
||||
) -> List[Document]:
|
||||
ret: List[Dict[str, Any]] = self.sql_db_chain(
|
||||
query, callbacks=run_manager.get_child(), **kwargs
|
||||
)["result"]
|
||||
return [
|
||||
Document(page_content=r[self.page_content_key], metadata=r) for r in ret
|
||||
]
|
||||
|
||||
async def _aget_relevant_documents(
|
||||
self, query: str, *, run_manager: AsyncCallbackManagerForRetrieverRun
|
||||
) -> List[Document]:
|
||||
raise NotImplementedError
|
||||
@ -0,0 +1,85 @@
|
||||
# flake8: noqa
|
||||
from langchain.prompts.prompt import PromptTemplate
|
||||
|
||||
|
||||
PROMPT_SUFFIX = """Only use the following tables:
|
||||
{table_info}
|
||||
|
||||
Question: {input}"""
|
||||
|
||||
_VECTOR_SQL_DEFAULT_TEMPLATE = """You are a {dialect} expert. Given an input question, first create a syntactically correct {dialect} query to run, then look at the results of the query and return the answer to the input question.
|
||||
{dialect} queries has a vector distance function called `DISTANCE(column, array)` to compute relevance to the user's question and sort the feature array column by the relevance.
|
||||
When the query is asking for {top_k} closest row, you have to use this distance function to calculate distance to entity's array on vector column and order by the distance to retrieve relevant rows.
|
||||
|
||||
*NOTICE*: `DISTANCE(column, array)` only accept an array column as its first argument and a `NeuralArray(entity)` as its second argument. You also need a user defined function called `NeuralArray(entity)` to retrieve the entity's array.
|
||||
|
||||
Unless the user specifies in the question a specific number of examples to obtain, query for at most {top_k} results using the LIMIT clause as per {dialect}. You should only order according to the distance function.
|
||||
Never query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in double quotes (") to denote them as delimited identifiers.
|
||||
Pay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table.
|
||||
Pay attention to use today() function to get the current date, if the question involves "today". `ORDER BY` clause should always be after `WHERE` clause. DO NOT add semicolon to the end of SQL. Pay attention to the comment in table schema.
|
||||
|
||||
Use the following format:
|
||||
|
||||
Question: "Question here"
|
||||
SQLQuery: "SQL Query to run"
|
||||
SQLResult: "Result of the SQLQuery"
|
||||
Answer: "Final answer here"
|
||||
"""
|
||||
|
||||
VECTOR_SQL_PROMPT = PromptTemplate(
|
||||
input_variables=["input", "table_info", "dialect", "top_k"],
|
||||
template=_VECTOR_SQL_DEFAULT_TEMPLATE + PROMPT_SUFFIX,
|
||||
)
|
||||
|
||||
|
||||
_myscale_prompt = """You are a MyScale expert. Given an input question, first create a syntactically correct MyScale query to run, then look at the results of the query and return the answer to the input question.
|
||||
MyScale queries has a vector distance function called `DISTANCE(column, array)` to compute relevance to the user's question and sort the feature array column by the relevance.
|
||||
When the query is asking for {top_k} closest row, you have to use this distance function to calculate distance to entity's array on vector column and order by the distance to retrieve relevant rows.
|
||||
|
||||
*NOTICE*: `DISTANCE(column, array)` only accept an array column as its first argument and a `NeuralArray(entity)` as its second argument. You also need a user defined function called `NeuralArray(entity)` to retrieve the entity's array.
|
||||
|
||||
Unless the user specifies in the question a specific number of examples to obtain, query for at most {top_k} results using the LIMIT clause as per MyScale. You should only order according to the distance function.
|
||||
Never query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in double quotes (") to denote them as delimited identifiers.
|
||||
Pay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table.
|
||||
Pay attention to use today() function to get the current date, if the question involves "today". `ORDER BY` clause should always be after `WHERE` clause. DO NOT add semicolon to the end of SQL. Pay attention to the comment in table schema.
|
||||
|
||||
Use the following format:
|
||||
|
||||
======== table info ========
|
||||
<some table infos>
|
||||
|
||||
Question: "Question here"
|
||||
SQLQuery: "SQL Query to run"
|
||||
|
||||
|
||||
Here are some examples:
|
||||
|
||||
======== table info ========
|
||||
CREATE TABLE "ChatPaper" (
|
||||
abstract String,
|
||||
id String,
|
||||
vector Array(Float32),
|
||||
) ENGINE = ReplicatedReplacingMergeTree()
|
||||
ORDER BY id
|
||||
PRIMARY KEY id
|
||||
|
||||
Question: What is Feartue Pyramid Network?
|
||||
SQLQuery: SELECT ChatPaper.title, ChatPaper.id, ChatPaper.authors FROM ChatPaper ORDER BY DISTANCE(vector, NeuralArray(PaperRank contribution)) LIMIT {top_k}
|
||||
|
||||
|
||||
Let's begin:
|
||||
======== table info ========
|
||||
{table_info}
|
||||
|
||||
Question: {input}
|
||||
SQLQuery: """
|
||||
|
||||
MYSCALE_PROMPT = PromptTemplate(
|
||||
input_variables=["input", "table_info", "top_k"],
|
||||
template=_myscale_prompt + PROMPT_SUFFIX,
|
||||
)
|
||||
|
||||
|
||||
VECTOR_SQL_PROMPTS = {
|
||||
"myscale": MYSCALE_PROMPT,
|
||||
}
|
||||
@ -0,0 +1,237 @@
|
||||
"""Vector SQL Database Chain Retriever"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from langchain.callbacks.manager import CallbackManagerForChainRun
|
||||
from langchain.chains.llm import LLMChain
|
||||
from langchain.chains.sql_database.prompt import PROMPT, SQL_PROMPTS
|
||||
from langchain.embeddings.base import Embeddings
|
||||
from langchain.prompts.prompt import PromptTemplate
|
||||
from langchain.schema import BaseOutputParser, BasePromptTemplate
|
||||
from langchain.schema.language_model import BaseLanguageModel
|
||||
from langchain.tools.sql_database.prompt import QUERY_CHECKER
|
||||
from langchain.utilities.sql_database import SQLDatabase
|
||||
|
||||
from langchain_experimental.sql.base import INTERMEDIATE_STEPS_KEY, SQLDatabaseChain
|
||||
|
||||
|
||||
class VectorSQLOutputParser(BaseOutputParser[str]):
|
||||
"""Output Parser for Vector SQL
|
||||
1. finds for `NeuralArray()` and replace it with the embedding
|
||||
2. finds for `DISTANCE()` and replace it with the distance name in backend SQL
|
||||
"""
|
||||
|
||||
model: Embeddings
|
||||
"""Embedding model to extract embedding for entity"""
|
||||
distance_func_name: str = "distance"
|
||||
"""Distance name for Vector SQL"""
|
||||
|
||||
class Config:
|
||||
arbitrary_types_allowed = 1
|
||||
|
||||
@property
|
||||
def _type(self) -> str:
|
||||
return "vector_sql_parser"
|
||||
|
||||
@classmethod
|
||||
def from_embeddings(
|
||||
cls, model: Embeddings, distance_func_name: str = "distance", **kwargs: Any
|
||||
) -> BaseOutputParser:
|
||||
return cls(model=model, distance_func_name=distance_func_name, **kwargs)
|
||||
|
||||
def parse(self, text: str) -> str:
|
||||
text = text.strip()
|
||||
start = text.find("NeuralArray(")
|
||||
_sql_str_compl = text
|
||||
if start > 0:
|
||||
_matched = text[text.find("NeuralArray(") + len("NeuralArray(") :]
|
||||
end = _matched.find(")") + start + len("NeuralArray(") + 1
|
||||
entity = _matched[: _matched.find(")")]
|
||||
vecs = self.model.embed_query(entity)
|
||||
vecs_str = "[" + ",".join(map(str, vecs)) + "]"
|
||||
_sql_str_compl = text.replace("DISTANCE", self.distance_func_name).replace(
|
||||
text[start:end], vecs_str
|
||||
)
|
||||
if _sql_str_compl[-1] == ";":
|
||||
_sql_str_compl = _sql_str_compl[:-1]
|
||||
return _sql_str_compl
|
||||
|
||||
|
||||
class VectorSQLRetrieveAllOutputParser(VectorSQLOutputParser):
|
||||
"""Based on VectorSQLOutputParser
|
||||
It also modify the SQL to get all columns
|
||||
"""
|
||||
|
||||
@property
|
||||
def _type(self) -> str:
|
||||
return "vector_sql_retrieve_all_parser"
|
||||
|
||||
def parse(self, text: str) -> str:
|
||||
text = text.strip()
|
||||
start = text.upper().find("SELECT")
|
||||
if start >= 0:
|
||||
end = text.upper().find("FROM")
|
||||
text = text.replace(text[start + len("SELECT") + 1 : end - 1], "*")
|
||||
return super().parse(text)
|
||||
|
||||
|
||||
def _try_eval(x: Any) -> Any:
|
||||
try:
|
||||
return eval(x)
|
||||
except Exception:
|
||||
return x
|
||||
|
||||
|
||||
def get_result_from_sqldb(
|
||||
db: SQLDatabase, cmd: str
|
||||
) -> Union[str, List[Dict[str, Any]], Dict[str, Any]]:
|
||||
result = db._execute(cmd, fetch="all") # type: ignore
|
||||
if isinstance(result, list):
|
||||
return [{k: _try_eval(v) for k, v in dict(d._asdict()).items()} for d in result]
|
||||
else:
|
||||
return {
|
||||
k: _try_eval(v) for k, v in dict(result._asdict()).items() # type: ignore
|
||||
}
|
||||
|
||||
|
||||
class VectorSQLDatabaseChain(SQLDatabaseChain):
|
||||
"""Chain for interacting with Vector SQL Database.
|
||||
|
||||
Example:
|
||||
.. code-block:: python
|
||||
|
||||
from langchain_experimental.sql import SQLDatabaseChain
|
||||
from langchain import OpenAI, SQLDatabase, OpenAIEmbeddings
|
||||
db = SQLDatabase(...)
|
||||
db_chain = VectorSQLDatabaseChain.from_llm(OpenAI(), db, OpenAIEmbeddings())
|
||||
|
||||
*Security note*: Make sure that the database connection uses credentials
|
||||
that are narrowly-scoped to only include the permissions this chain needs.
|
||||
Failure to do so may result in data corruption or loss, since this chain may
|
||||
attempt commands like `DROP TABLE` or `INSERT` if appropriately prompted.
|
||||
The best way to guard against such negative outcomes is to (as appropriate)
|
||||
limit the permissions granted to the credentials used with this chain.
|
||||
This issue shows an example negative outcome if these steps are not taken:
|
||||
https://github.com/langchain-ai/langchain/issues/5923
|
||||
"""
|
||||
|
||||
sql_cmd_parser: VectorSQLOutputParser
|
||||
"""Parser for Vector SQL"""
|
||||
native_format: bool = False
|
||||
"""If return_direct, controls whether to return in python native format"""
|
||||
|
||||
def _call(
|
||||
self,
|
||||
inputs: Dict[str, Any],
|
||||
run_manager: Optional[CallbackManagerForChainRun] = None,
|
||||
) -> Dict[str, Any]:
|
||||
_run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
|
||||
input_text = f"{inputs[self.input_key]}\nSQLQuery:"
|
||||
_run_manager.on_text(input_text, verbose=self.verbose)
|
||||
# If not present, then defaults to None which is all tables.
|
||||
table_names_to_use = inputs.get("table_names_to_use")
|
||||
table_info = self.database.get_table_info(table_names=table_names_to_use)
|
||||
llm_inputs = {
|
||||
"input": input_text,
|
||||
"top_k": str(self.top_k),
|
||||
"dialect": self.database.dialect,
|
||||
"table_info": table_info,
|
||||
"stop": ["\nSQLResult:"],
|
||||
}
|
||||
intermediate_steps: List = []
|
||||
try:
|
||||
intermediate_steps.append(llm_inputs) # input: sql generation
|
||||
llm_out = self.llm_chain.predict(
|
||||
callbacks=_run_manager.get_child(),
|
||||
**llm_inputs,
|
||||
)
|
||||
sql_cmd = self.sql_cmd_parser.parse(llm_out)
|
||||
if self.return_sql:
|
||||
return {self.output_key: sql_cmd}
|
||||
if not self.use_query_checker:
|
||||
_run_manager.on_text(llm_out, color="green", verbose=self.verbose)
|
||||
intermediate_steps.append(
|
||||
llm_out
|
||||
) # output: sql generation (no checker)
|
||||
intermediate_steps.append({"sql_cmd": llm_out}) # input: sql exec
|
||||
result = get_result_from_sqldb(self.database, sql_cmd)
|
||||
intermediate_steps.append(str(result)) # output: sql exec
|
||||
else:
|
||||
query_checker_prompt = self.query_checker_prompt or PromptTemplate(
|
||||
template=QUERY_CHECKER, input_variables=["query", "dialect"]
|
||||
)
|
||||
query_checker_chain = LLMChain(
|
||||
llm=self.llm_chain.llm,
|
||||
prompt=query_checker_prompt,
|
||||
output_parser=self.llm_chain.output_parser,
|
||||
)
|
||||
query_checker_inputs = {
|
||||
"query": llm_out,
|
||||
"dialect": self.database.dialect,
|
||||
}
|
||||
checked_llm_out = query_checker_chain.predict(
|
||||
callbacks=_run_manager.get_child(), **query_checker_inputs
|
||||
)
|
||||
checked_sql_command = self.sql_cmd_parser.parse(checked_llm_out)
|
||||
intermediate_steps.append(
|
||||
checked_llm_out
|
||||
) # output: sql generation (checker)
|
||||
_run_manager.on_text(
|
||||
checked_llm_out, color="green", verbose=self.verbose
|
||||
)
|
||||
intermediate_steps.append(
|
||||
{"sql_cmd": checked_llm_out}
|
||||
) # input: sql exec
|
||||
result = get_result_from_sqldb(self.database, checked_sql_command)
|
||||
intermediate_steps.append(str(result)) # output: sql exec
|
||||
llm_out = checked_llm_out
|
||||
sql_cmd = checked_sql_command
|
||||
|
||||
_run_manager.on_text("\nSQLResult: ", verbose=self.verbose)
|
||||
_run_manager.on_text(str(result), color="yellow", verbose=self.verbose)
|
||||
# If return direct, we just set the final result equal to
|
||||
# the result of the sql query result, otherwise try to get a human readable
|
||||
# final answer
|
||||
if self.return_direct:
|
||||
final_result = result
|
||||
else:
|
||||
_run_manager.on_text("\nAnswer:", verbose=self.verbose)
|
||||
input_text += f"{llm_out}\nSQLResult: {result}\nAnswer:"
|
||||
llm_inputs["input"] = input_text
|
||||
intermediate_steps.append(llm_inputs) # input: final answer
|
||||
final_result = self.llm_chain.predict(
|
||||
callbacks=_run_manager.get_child(),
|
||||
**llm_inputs,
|
||||
).strip()
|
||||
intermediate_steps.append(final_result) # output: final answer
|
||||
_run_manager.on_text(final_result, color="green", verbose=self.verbose)
|
||||
chain_result: Dict[str, Any] = {self.output_key: final_result}
|
||||
if self.return_intermediate_steps:
|
||||
chain_result[INTERMEDIATE_STEPS_KEY] = intermediate_steps
|
||||
return chain_result
|
||||
except Exception as exc:
|
||||
# Append intermediate steps to exception, to aid in logging and later
|
||||
# improvement of few shot prompt seeds
|
||||
exc.intermediate_steps = intermediate_steps # type: ignore
|
||||
raise exc
|
||||
|
||||
@property
|
||||
def _chain_type(self) -> str:
|
||||
return "vector_sql_database_chain"
|
||||
|
||||
@classmethod
|
||||
def from_llm(
|
||||
cls,
|
||||
llm: BaseLanguageModel,
|
||||
db: SQLDatabase,
|
||||
prompt: Optional[BasePromptTemplate] = None,
|
||||
sql_cmd_parser: Optional[VectorSQLOutputParser] = None,
|
||||
**kwargs: Any,
|
||||
) -> VectorSQLDatabaseChain:
|
||||
assert sql_cmd_parser, "`sql_cmd_parser` must be set in VectorSQLDatabaseChain."
|
||||
prompt = prompt or SQL_PROMPTS.get(db.dialect, PROMPT)
|
||||
llm_chain = LLMChain(llm=llm, prompt=prompt)
|
||||
return cls(
|
||||
llm_chain=llm_chain, database=db, sql_cmd_parser=sql_cmd_parser, **kwargs
|
||||
)
|
||||
@ -0,0 +1,154 @@
|
||||
import os
|
||||
from typing import Iterator, List
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def check_spacy_model() -> Iterator[None]:
|
||||
import spacy
|
||||
|
||||
if not spacy.util.is_package("en_core_web_lg"):
|
||||
pytest.skip(reason="Spacy model 'en_core_web_lg' not installed")
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
||||
@pytest.mark.parametrize(
|
||||
"analyzed_fields,should_contain",
|
||||
[(["PERSON"], False), (["PHONE_NUMBER"], True), (None, False)],
|
||||
)
|
||||
def test_anonymize(analyzed_fields: List[str], should_contain: bool) -> None:
|
||||
"""Test anonymizing a name in a simple sentence"""
|
||||
from langchain_experimental.data_anonymizer import PresidioReversibleAnonymizer
|
||||
|
||||
text = "Hello, my name is John Doe."
|
||||
anonymizer = PresidioReversibleAnonymizer(analyzed_fields=analyzed_fields)
|
||||
anonymized_text = anonymizer.anonymize(text)
|
||||
assert ("John Doe" in anonymized_text) == should_contain
|
||||
|
||||
|
||||
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
||||
def test_anonymize_multiple() -> None:
|
||||
"""Test anonymizing multiple items in a sentence"""
|
||||
from langchain_experimental.data_anonymizer import PresidioReversibleAnonymizer
|
||||
|
||||
text = "John Smith's phone number is 313-666-7440 and email is johnsmith@gmail.com"
|
||||
anonymizer = PresidioReversibleAnonymizer()
|
||||
anonymized_text = anonymizer.anonymize(text)
|
||||
for phrase in ["John Smith", "313-666-7440", "johnsmith@gmail.com"]:
|
||||
assert phrase not in anonymized_text
|
||||
|
||||
|
||||
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
||||
def test_anonymize_with_custom_operator() -> None:
|
||||
"""Test anonymize a name with a custom operator"""
|
||||
from presidio_anonymizer.entities import OperatorConfig
|
||||
|
||||
from langchain_experimental.data_anonymizer import PresidioReversibleAnonymizer
|
||||
|
||||
custom_operator = {"PERSON": OperatorConfig("replace", {"new_value": "<name>"})}
|
||||
anonymizer = PresidioReversibleAnonymizer(operators=custom_operator)
|
||||
|
||||
text = "Jane Doe was here."
|
||||
|
||||
anonymized_text = anonymizer.anonymize(text)
|
||||
assert anonymized_text == "<name> was here."
|
||||
|
||||
|
||||
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
||||
def test_add_recognizer_operator() -> None:
|
||||
"""
|
||||
Test add recognizer and anonymize a new type of entity and with a custom operator
|
||||
"""
|
||||
from presidio_analyzer import PatternRecognizer
|
||||
from presidio_anonymizer.entities import OperatorConfig
|
||||
|
||||
from langchain_experimental.data_anonymizer import PresidioReversibleAnonymizer
|
||||
|
||||
anonymizer = PresidioReversibleAnonymizer(analyzed_fields=[])
|
||||
titles_list = ["Sir", "Madam", "Professor"]
|
||||
custom_recognizer = PatternRecognizer(
|
||||
supported_entity="TITLE", deny_list=titles_list
|
||||
)
|
||||
anonymizer.add_recognizer(custom_recognizer)
|
||||
|
||||
# anonymizing with custom recognizer
|
||||
text = "Madam Jane Doe was here."
|
||||
anonymized_text = anonymizer.anonymize(text)
|
||||
assert anonymized_text == "<TITLE> Jane Doe was here."
|
||||
|
||||
# anonymizing with custom recognizer and operator
|
||||
custom_operator = {"TITLE": OperatorConfig("replace", {"new_value": "Dear"})}
|
||||
anonymizer.add_operators(custom_operator)
|
||||
anonymized_text = anonymizer.anonymize(text)
|
||||
assert anonymized_text == "Dear Jane Doe was here."
|
||||
|
||||
|
||||
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
||||
def test_deanonymizer_mapping() -> None:
|
||||
"""Test if deanonymizer mapping is correctly populated"""
|
||||
from langchain_experimental.data_anonymizer import PresidioReversibleAnonymizer
|
||||
|
||||
anonymizer = PresidioReversibleAnonymizer(
|
||||
analyzed_fields=["PERSON", "PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD"]
|
||||
)
|
||||
|
||||
anonymizer.anonymize("Hello, my name is John Doe and my number is 444 555 6666.")
|
||||
|
||||
# ["PERSON", "PHONE_NUMBER"]
|
||||
assert len(anonymizer.deanonymizer_mapping.keys()) == 2
|
||||
assert "John Doe" in anonymizer.deanonymizer_mapping.get("PERSON", {}).values()
|
||||
assert (
|
||||
"444 555 6666"
|
||||
in anonymizer.deanonymizer_mapping.get("PHONE_NUMBER", {}).values()
|
||||
)
|
||||
|
||||
text_to_anonymize = (
|
||||
"And my name is Jane Doe, my email is jane@gmail.com and "
|
||||
"my credit card is 4929 5319 6292 5362."
|
||||
)
|
||||
anonymizer.anonymize(text_to_anonymize)
|
||||
|
||||
# ["PERSON", "PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD"]
|
||||
assert len(anonymizer.deanonymizer_mapping.keys()) == 4
|
||||
assert "Jane Doe" in anonymizer.deanonymizer_mapping.get("PERSON", {}).values()
|
||||
assert (
|
||||
"jane@gmail.com"
|
||||
in anonymizer.deanonymizer_mapping.get("EMAIL_ADDRESS", {}).values()
|
||||
)
|
||||
assert (
|
||||
"4929 5319 6292 5362"
|
||||
in anonymizer.deanonymizer_mapping.get("CREDIT_CARD", {}).values()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
||||
def test_deanonymize() -> None:
|
||||
"""Test deanonymizing a name in a simple sentence"""
|
||||
from langchain_experimental.data_anonymizer import PresidioReversibleAnonymizer
|
||||
|
||||
text = "Hello, my name is John Doe."
|
||||
anonymizer = PresidioReversibleAnonymizer(analyzed_fields=["PERSON"])
|
||||
anonymized_text = anonymizer.anonymize(text)
|
||||
deanonymized_text = anonymizer.deanonymize(anonymized_text)
|
||||
assert deanonymized_text == text
|
||||
|
||||
|
||||
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
||||
def test_save_load_deanonymizer_mapping() -> None:
|
||||
from langchain_experimental.data_anonymizer import PresidioReversibleAnonymizer
|
||||
|
||||
anonymizer = PresidioReversibleAnonymizer(analyzed_fields=["PERSON"])
|
||||
anonymizer.anonymize("Hello, my name is John Doe.")
|
||||
try:
|
||||
anonymizer.save_deanonymizer_mapping("test_file.json")
|
||||
assert os.path.isfile("test_file.json")
|
||||
|
||||
anonymizer = PresidioReversibleAnonymizer()
|
||||
anonymizer.load_deanonymizer_mapping("test_file.json")
|
||||
|
||||
assert "John Doe" in anonymizer.deanonymizer_mapping.get("PERSON", {}).values()
|
||||
|
||||
finally:
|
||||
os.remove("test_file.json")
|
||||
@ -0,0 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Union
|
||||
|
||||
from langchain.load.serializable import Serializable
|
||||
from langchain.pydantic_v1 import Field
|
||||
from langchain.schema import Document
|
||||
|
||||
|
||||
class Node(Serializable):
|
||||
"""Represents a node in a graph with associated properties.
|
||||
|
||||
Attributes:
|
||||
id (Union[str, int]): A unique identifier for the node.
|
||||
type (str): The type or label of the node, default is "Node".
|
||||
properties (dict): Additional properties and metadata associated with the node.
|
||||
"""
|
||||
|
||||
id: Union[str, int]
|
||||
type: str = "Node"
|
||||
properties: dict = Field(default_factory=dict)
|
||||
|
||||
|
||||
class Relationship(Serializable):
|
||||
"""Represents a directed relationship between two nodes in a graph.
|
||||
|
||||
Attributes:
|
||||
source (Node): The source node of the relationship.
|
||||
target (Node): The target node of the relationship.
|
||||
type (str): The type of the relationship.
|
||||
properties (dict): Additional properties associated with the relationship.
|
||||
"""
|
||||
|
||||
source: Node
|
||||
target: Node
|
||||
type: str
|
||||
properties: dict = Field(default_factory=dict)
|
||||
|
||||
|
||||
class GraphDocument(Serializable):
|
||||
"""Represents a graph document consisting of nodes and relationships.
|
||||
|
||||
Attributes:
|
||||
nodes (List[Node]): A list of nodes in the graph.
|
||||
relationships (List[Relationship]): A list of relationships in the graph.
|
||||
source (Document): The document from which the graph information is derived.
|
||||
"""
|
||||
|
||||
nodes: List[Node]
|
||||
relationships: List[Relationship]
|
||||
source: Document
|
||||
@ -0,0 +1,159 @@
|
||||
import os
|
||||
from typing import Any, Dict, Iterable, List, Optional, Type
|
||||
|
||||
from langchain.embeddings.base import Embeddings
|
||||
from langchain.schema.document import Document
|
||||
from langchain.vectorstores.base import VST, VectorStore
|
||||
|
||||
FIELD_TYPES = {
|
||||
"f": "files",
|
||||
"t": "texts",
|
||||
"l": "links",
|
||||
}
|
||||
|
||||
|
||||
class NucliaDB(VectorStore):
|
||||
"""NucliaDB vector store."""
|
||||
|
||||
_config: Dict[str, Any] = {}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
knowledge_box: str,
|
||||
local: bool,
|
||||
api_key: Optional[str] = None,
|
||||
backend: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Initialize the NucliaDB client.
|
||||
|
||||
Args:
|
||||
knowledge_box: the Knowledge Box id.
|
||||
local: Whether to use a local NucliaDB instance or Nuclia Cloud
|
||||
api_key: A contributor API key for the kb (needed when local is False)
|
||||
backend: The backend url to use when local is True, defaults to
|
||||
http://localhost:8080
|
||||
"""
|
||||
try:
|
||||
from nuclia.sdk import NucliaAuth
|
||||
except ImportError:
|
||||
raise ValueError(
|
||||
"nuclia python package not found. "
|
||||
"Please install it with `pip install nuclia`."
|
||||
)
|
||||
self._config["LOCAL"] = local
|
||||
zone = os.environ.get("NUCLIA_ZONE", "europe-1")
|
||||
self._kb = knowledge_box
|
||||
if local:
|
||||
if not backend:
|
||||
backend = "http://localhost:8080"
|
||||
self._config["BACKEND"] = f"{backend}/api/v1"
|
||||
self._config["TOKEN"] = None
|
||||
NucliaAuth().nucliadb(url=backend)
|
||||
NucliaAuth().kb(url=self.kb_url, interactive=False)
|
||||
else:
|
||||
self._config["BACKEND"] = f"https://{zone}.nuclia.cloud/api/v1"
|
||||
self._config["TOKEN"] = api_key
|
||||
NucliaAuth().kb(
|
||||
url=self.kb_url, token=self._config["TOKEN"], interactive=False
|
||||
)
|
||||
|
||||
@property
|
||||
def is_local(self) -> str:
|
||||
return self._config["LOCAL"]
|
||||
|
||||
@property
|
||||
def kb_url(self) -> str:
|
||||
return f"{self._config['BACKEND']}/kb/{self._kb}"
|
||||
|
||||
def add_texts(
|
||||
self,
|
||||
texts: Iterable[str],
|
||||
metadatas: Optional[List[dict]] = None,
|
||||
**kwargs: Any,
|
||||
) -> List[str]:
|
||||
"""Upload texts to NucliaDB"""
|
||||
ids = []
|
||||
from nuclia.sdk import NucliaResource
|
||||
|
||||
factory = NucliaResource()
|
||||
for i, text in enumerate(texts):
|
||||
extra: Dict[str, Any] = {"metadata": ""}
|
||||
if metadatas:
|
||||
extra = {"metadata": metadatas[i]}
|
||||
id = factory.create(
|
||||
texts={"text": {"body": text}},
|
||||
extra=extra,
|
||||
url=self.kb_url,
|
||||
api_key=self._config["TOKEN"],
|
||||
)
|
||||
ids.append(id)
|
||||
return ids
|
||||
|
||||
def delete(self, ids: Optional[List[str]] = None, **kwargs: Any) -> Optional[bool]:
|
||||
if not ids:
|
||||
return None
|
||||
from nuclia.sdk import NucliaResource
|
||||
|
||||
factory = NucliaResource()
|
||||
results: List[bool] = []
|
||||
for id in ids:
|
||||
try:
|
||||
factory.delete(rid=id, url=self.kb_url, api_key=self._config["TOKEN"])
|
||||
results.append(True)
|
||||
except ValueError:
|
||||
results.append(False)
|
||||
return all(results)
|
||||
|
||||
def similarity_search(
|
||||
self, query: str, k: int = 4, **kwargs: Any
|
||||
) -> List[Document]:
|
||||
from nuclia.sdk import NucliaSearch
|
||||
from nucliadb_models.search import FindRequest, ResourceProperties
|
||||
|
||||
request = FindRequest(
|
||||
query=query,
|
||||
page_size=k,
|
||||
show=[ResourceProperties.VALUES, ResourceProperties.EXTRA],
|
||||
)
|
||||
search = NucliaSearch()
|
||||
results = search.find(
|
||||
query=request, url=self.kb_url, api_key=self._config["TOKEN"]
|
||||
)
|
||||
paragraphs = []
|
||||
for resource in results.resources.values():
|
||||
for field in resource.fields.values():
|
||||
for paragraph_id, paragraph in field.paragraphs.items():
|
||||
info = paragraph_id.split("/")
|
||||
field_type = FIELD_TYPES.get(info[1], None)
|
||||
field_id = info[2]
|
||||
if not field_type:
|
||||
continue
|
||||
value = getattr(resource.data, field_type, {}).get(field_id, None)
|
||||
paragraphs.append(
|
||||
{
|
||||
"text": paragraph.text,
|
||||
"metadata": {
|
||||
"extra": getattr(
|
||||
getattr(resource, "extra", {}), "metadata", None
|
||||
),
|
||||
"value": value,
|
||||
},
|
||||
"order": paragraph.order,
|
||||
}
|
||||
)
|
||||
sorted_paragraphs = sorted(paragraphs, key=lambda x: x["order"])
|
||||
return [
|
||||
Document(page_content=paragraph["text"], metadata=paragraph["metadata"])
|
||||
for paragraph in sorted_paragraphs
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def from_texts(
|
||||
cls: Type[VST],
|
||||
texts: List[str],
|
||||
embedding: Embeddings,
|
||||
metadatas: Optional[List[dict]] = None,
|
||||
**kwargs: Any,
|
||||
) -> VST:
|
||||
"""Return VectorStore initialized from texts and embeddings."""
|
||||
raise NotImplementedError
|
||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
@ -0,0 +1,98 @@
|
||||
from typing import Any
|
||||
from unittest import mock
|
||||
|
||||
from langchain.vectorstores.nucliadb import NucliaDB
|
||||
|
||||
|
||||
class attrdict(dict):
|
||||
def __getitem__(self, key: str) -> Any:
|
||||
value = dict.__getitem__(self, key)
|
||||
return attrdict(value) if isinstance(value, dict) else value
|
||||
|
||||
__getattr__ = __getitem__
|
||||
|
||||
|
||||
def FakeCreate(**args: Any) -> Any:
|
||||
def fn(self: Any, **kwargs: Any) -> str:
|
||||
return "fake_uuid"
|
||||
|
||||
return fn
|
||||
|
||||
|
||||
def FakeDelete(**args: Any) -> Any:
|
||||
def fn(self: Any, **kwargs: Any) -> None:
|
||||
return None
|
||||
|
||||
return fn
|
||||
|
||||
|
||||
def FakeFind(**args: Any) -> Any:
|
||||
def fn(self: Any, **kwargs: Any) -> Any:
|
||||
return attrdict(
|
||||
{
|
||||
"resources": {
|
||||
"123": attrdict(
|
||||
{
|
||||
"fields": {
|
||||
"456": attrdict(
|
||||
{
|
||||
"paragraphs": {
|
||||
"123/t/text/0-14": attrdict(
|
||||
{
|
||||
"text": "This is a test",
|
||||
"order": 0,
|
||||
}
|
||||
),
|
||||
}
|
||||
}
|
||||
)
|
||||
},
|
||||
"data": {
|
||||
"texts": {
|
||||
"text": {
|
||||
"body": "This is a test",
|
||||
}
|
||||
}
|
||||
},
|
||||
"extra": attrdict({"metadata": {"some": "metadata"}}),
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
return fn
|
||||
|
||||
|
||||
def test_add_texts() -> None:
|
||||
with mock.patch(
|
||||
"nuclia.sdk.resource.NucliaResource.create",
|
||||
new_callable=FakeCreate,
|
||||
):
|
||||
ndb = NucliaDB(knowledge_box="YOUR_KB_ID", local=False, api_key="YOUR_API_KEY")
|
||||
assert ndb.is_local is False
|
||||
ids = ndb.add_texts(["This is a new test", "This is a second test"])
|
||||
assert len(ids) == 2
|
||||
|
||||
|
||||
def test_delete() -> None:
|
||||
with mock.patch(
|
||||
"nuclia.sdk.resource.NucliaResource.delete",
|
||||
new_callable=FakeDelete,
|
||||
):
|
||||
ndb = NucliaDB(knowledge_box="YOUR_KB_ID", local=False, api_key="YOUR_API_KEY")
|
||||
success = ndb.delete(["123", "456"])
|
||||
assert success
|
||||
|
||||
|
||||
def test_search() -> None:
|
||||
with mock.patch(
|
||||
"nuclia.sdk.search.NucliaSearch.find",
|
||||
new_callable=FakeFind,
|
||||
):
|
||||
ndb = NucliaDB(knowledge_box="YOUR_KB_ID", local=False, api_key="YOUR_API_KEY")
|
||||
results = ndb.similarity_search("Who was inspired by Ada Lovelace?")
|
||||
assert len(results) == 1
|
||||
assert results[0].page_content == "This is a test"
|
||||
assert results[0].metadata["extra"]["some"] == "metadata"
|
||||
assert results[0].metadata["value"]["body"] == "This is a test"
|
||||
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue