mirror of
https://github.com/hwchase17/langchain
synced 2024-11-10 01:10:59 +00:00
a8f804a618
### Description The feature for anonymizing data has been implemented. In order to protect private data, such as when querying external APIs (OpenAI), it is worth pseudonymizing sensitive data to maintain full privacy. Anonynization consists of two steps: 1. **Identification:** Identify all data fields that contain personally identifiable information (PII). 2. **Replacement**: Replace all PIIs with pseudo values or codes that do not reveal any personal information about the individual but can be used for reference. We're not using regular encryption, because the language model won't be able to understand the meaning or context of the encrypted data. We use *Microsoft Presidio* together with *Faker* framework for anonymization purposes because of the wide range of functionalities they provide. The full implementation is available in `PresidioAnonymizer`. ### Future works - **deanonymization** - add the ability to reverse anonymization. For example, the workflow could look like this: `anonymize -> LLMChain -> deanonymize`. By doing this, we will retain anonymity in requests to, for example, OpenAI, and then be able restore the original data. - **instance anonymization** - at this point, each occurrence of PII is treated as a separate entity and separately anonymized. Therefore, two occurrences of the name John Doe in the text will be changed to two different names. It is therefore worth introducing support for full instance detection, so that repeated occurrences are treated as a single object. ### Twitter handle @deepsense_ai / @MaksOpp --------- Co-authored-by: MaksOpp <maks.operlejn@gmail.com> Co-authored-by: Bagatur <baskaryan@gmail.com>
85 lines
3.1 KiB
Python
85 lines
3.1 KiB
Python
from typing import Iterator, List
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture(scope="module", autouse=True)
|
|
def check_spacy_model() -> Iterator[None]:
|
|
import spacy
|
|
|
|
if not spacy.util.is_package("en_core_web_lg"):
|
|
pytest.skip(reason="Spacy model 'en_core_web_lg' not installed")
|
|
yield
|
|
|
|
|
|
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
|
@pytest.mark.parametrize(
|
|
"analyzed_fields,should_contain",
|
|
[(["PERSON"], False), (["PHONE_NUMBER"], True), (None, False)],
|
|
)
|
|
def test_anonymize(analyzed_fields: List[str], should_contain: bool) -> None:
|
|
"""Test anonymizing a name in a simple sentence"""
|
|
from langchain_experimental.data_anonymizer import PresidioAnonymizer
|
|
|
|
text = "Hello, my name is John Doe."
|
|
anonymizer = PresidioAnonymizer(analyzed_fields=analyzed_fields)
|
|
anonymized_text = anonymizer.anonymize(text)
|
|
assert ("John Doe" in anonymized_text) == should_contain
|
|
|
|
|
|
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
|
def test_anonymize_multiple() -> None:
|
|
"""Test anonymizing multiple items in a sentence"""
|
|
from langchain_experimental.data_anonymizer import PresidioAnonymizer
|
|
|
|
text = "John Smith's phone number is 313-666-7440 and email is johnsmith@gmail.com"
|
|
anonymizer = PresidioAnonymizer()
|
|
anonymized_text = anonymizer.anonymize(text)
|
|
for phrase in ["John Smith", "313-666-7440", "johnsmith@gmail.com"]:
|
|
assert phrase not in anonymized_text
|
|
|
|
|
|
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
|
def test_anonymize_with_custom_operator() -> None:
|
|
"""Test anonymize a name with a custom operator"""
|
|
from presidio_anonymizer.entities import OperatorConfig
|
|
|
|
from langchain_experimental.data_anonymizer import PresidioAnonymizer
|
|
|
|
custom_operator = {"PERSON": OperatorConfig("replace", {"new_value": "<name>"})}
|
|
anonymizer = PresidioAnonymizer(operators=custom_operator)
|
|
|
|
text = "Jane Doe was here."
|
|
|
|
anonymized_text = anonymizer.anonymize(text)
|
|
assert anonymized_text == "<name> was here."
|
|
|
|
|
|
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
|
def test_add_recognizer_operator() -> None:
|
|
"""
|
|
Test add recognizer and anonymize a new type of entity and with a custom operator
|
|
"""
|
|
from presidio_analyzer import PatternRecognizer
|
|
from presidio_anonymizer.entities import OperatorConfig
|
|
|
|
from langchain_experimental.data_anonymizer import PresidioAnonymizer
|
|
|
|
anonymizer = PresidioAnonymizer(analyzed_fields=[])
|
|
titles_list = ["Sir", "Madam", "Professor"]
|
|
custom_recognizer = PatternRecognizer(
|
|
supported_entity="TITLE", deny_list=titles_list
|
|
)
|
|
anonymizer.add_recognizer(custom_recognizer)
|
|
|
|
# anonymizing with custom recognizer
|
|
text = "Madam Jane Doe was here."
|
|
anonymized_text = anonymizer.anonymize(text)
|
|
assert anonymized_text == "<TITLE> Jane Doe was here."
|
|
|
|
# anonymizing with custom recognizer and operator
|
|
custom_operator = {"TITLE": OperatorConfig("replace", {"new_value": "Dear"})}
|
|
anonymizer.add_operators(custom_operator)
|
|
anonymized_text = anonymizer.anonymize(text)
|
|
assert anonymized_text == "Dear Jane Doe was here."
|