mirror of
https://github.com/hwchase17/langchain
synced 2024-11-10 01:10:59 +00:00
2aae1102b0
### Description
Add instance anonymization - if `John Doe` will appear twice in the
text, it will be treated as the same entity.
The difference between `PresidioAnonymizer` and
`PresidioReversibleAnonymizer` is that only the second one has a
built-in memory, so it will remember anonymization mapping for multiple
texts:
```
>>> anonymizer = PresidioAnonymizer()
>>> anonymizer.anonymize("My name is John Doe. Hi John Doe!")
'My name is Noah Rhodes. Hi Noah Rhodes!'
>>> anonymizer.anonymize("My name is John Doe. Hi John Doe!")
'My name is Brett Russell. Hi Brett Russell!'
```
```
>>> anonymizer = PresidioReversibleAnonymizer()
>>> anonymizer.anonymize("My name is John Doe. Hi John Doe!")
'My name is Noah Rhodes. Hi Noah Rhodes!'
>>> anonymizer.anonymize("My name is John Doe. Hi John Doe!")
'My name is Noah Rhodes. Hi Noah Rhodes!'
```
### Twitter handle
@deepsense_ai / @MaksOpp
### Tag maintainer
@baskaryan @hwchase17 @hinthornw
---------
Co-authored-by: Bagatur <baskaryan@gmail.com>
120 lines
4.5 KiB
Python
120 lines
4.5 KiB
Python
from typing import Iterator, List
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture(scope="module", autouse=True)
|
|
def check_spacy_model() -> Iterator[None]:
|
|
import spacy
|
|
|
|
if not spacy.util.is_package("en_core_web_lg"):
|
|
pytest.skip(reason="Spacy model 'en_core_web_lg' not installed")
|
|
yield
|
|
|
|
|
|
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
|
@pytest.mark.parametrize(
|
|
"analyzed_fields,should_contain",
|
|
[(["PERSON"], False), (["PHONE_NUMBER"], True), (None, False)],
|
|
)
|
|
def test_anonymize(analyzed_fields: List[str], should_contain: bool) -> None:
|
|
"""Test anonymizing a name in a simple sentence"""
|
|
from langchain_experimental.data_anonymizer import PresidioAnonymizer
|
|
|
|
text = "Hello, my name is John Doe."
|
|
anonymizer = PresidioAnonymizer(analyzed_fields=analyzed_fields)
|
|
anonymized_text = anonymizer.anonymize(text)
|
|
assert ("John Doe" in anonymized_text) == should_contain
|
|
|
|
|
|
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
|
def test_anonymize_multiple() -> None:
|
|
"""Test anonymizing multiple items in a sentence"""
|
|
from langchain_experimental.data_anonymizer import PresidioAnonymizer
|
|
|
|
text = "John Smith's phone number is 313-666-7440 and email is johnsmith@gmail.com"
|
|
anonymizer = PresidioAnonymizer()
|
|
anonymized_text = anonymizer.anonymize(text)
|
|
for phrase in ["John Smith", "313-666-7440", "johnsmith@gmail.com"]:
|
|
assert phrase not in anonymized_text
|
|
|
|
|
|
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
|
def test_check_instances() -> None:
|
|
"""Test anonymizing multiple items in a sentence"""
|
|
from langchain_experimental.data_anonymizer import PresidioAnonymizer
|
|
|
|
text = (
|
|
"This is John Smith. John Smith works in a bakery." "John Smith is a good guy"
|
|
)
|
|
anonymizer = PresidioAnonymizer(["PERSON"], faker_seed=42)
|
|
anonymized_text = anonymizer.anonymize(text)
|
|
assert anonymized_text.count("Connie Lawrence") == 3
|
|
|
|
# New name should be generated
|
|
anonymized_text = anonymizer.anonymize(text)
|
|
assert anonymized_text.count("Connie Lawrence") == 0
|
|
|
|
|
|
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
|
def test_anonymize_with_custom_operator() -> None:
|
|
"""Test anonymize a name with a custom operator"""
|
|
from presidio_anonymizer.entities import OperatorConfig
|
|
|
|
from langchain_experimental.data_anonymizer import PresidioAnonymizer
|
|
|
|
custom_operator = {"PERSON": OperatorConfig("replace", {"new_value": "NAME"})}
|
|
anonymizer = PresidioAnonymizer(operators=custom_operator)
|
|
|
|
text = "Jane Doe was here."
|
|
|
|
anonymized_text = anonymizer.anonymize(text)
|
|
assert anonymized_text == "NAME was here."
|
|
|
|
|
|
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
|
def test_add_recognizer_operator() -> None:
|
|
"""
|
|
Test add recognizer and anonymize a new type of entity and with a custom operator
|
|
"""
|
|
from presidio_analyzer import PatternRecognizer
|
|
from presidio_anonymizer.entities import OperatorConfig
|
|
|
|
from langchain_experimental.data_anonymizer import PresidioAnonymizer
|
|
|
|
anonymizer = PresidioAnonymizer(analyzed_fields=[])
|
|
titles_list = ["Sir", "Madam", "Professor"]
|
|
custom_recognizer = PatternRecognizer(
|
|
supported_entity="TITLE", deny_list=titles_list
|
|
)
|
|
anonymizer.add_recognizer(custom_recognizer)
|
|
|
|
# anonymizing with custom recognizer
|
|
text = "Madam Jane Doe was here."
|
|
anonymized_text = anonymizer.anonymize(text)
|
|
assert anonymized_text == "<TITLE> Jane Doe was here."
|
|
|
|
# anonymizing with custom recognizer and operator
|
|
custom_operator = {"TITLE": OperatorConfig("replace", {"new_value": "Dear"})}
|
|
anonymizer.add_operators(custom_operator)
|
|
anonymized_text = anonymizer.anonymize(text)
|
|
assert anonymized_text == "Dear Jane Doe was here."
|
|
|
|
|
|
@pytest.mark.requires("presidio_analyzer", "presidio_anonymizer", "faker")
|
|
def test_non_faker_values() -> None:
|
|
"""Test anonymizing multiple items in a sentence without faker values"""
|
|
from langchain_experimental.data_anonymizer import PresidioAnonymizer
|
|
|
|
text = (
|
|
"My name is John Smith. Your name is Adam Smith. Her name is Jane Smith."
|
|
"Our names are: John Smith, Adam Smith, Jane Smith."
|
|
)
|
|
expected_result = (
|
|
"My name is <PERSON>. Your name is <PERSON_2>. Her name is <PERSON_3>."
|
|
"Our names are: <PERSON>, <PERSON_2>, <PERSON_3>."
|
|
)
|
|
anonymizer = PresidioAnonymizer(add_default_faker_operators=False)
|
|
anonymized_text = anonymizer.anonymize(text)
|
|
assert anonymized_text == expected_result
|