forked from Archives/langchain
482611f426
This PR has two contributions: 1. Add test for when stop token is found in middle of text 2. Add code coverage tooling and instructions - Add pytest-cov via poetry - Add necessary config files - Add new make instruction for `coverage` - Update README with coverage guidance - Update minor README formatting/spelling Co-authored-by: Hunter Gerlach <hunter@huntergerlach.com>
23 lines
732 B
Python
23 lines
732 B
Python
"""Test LLM utility functions."""
|
|
from langchain.llms.utils import enforce_stop_tokens
|
|
|
|
|
|
def test_enforce_stop_tokens() -> None:
|
|
"""Test removing stop tokens when they occur."""
|
|
text = "foo bar baz"
|
|
output = enforce_stop_tokens(text, ["moo", "baz"])
|
|
assert output == "foo bar "
|
|
text = "foo bar baz"
|
|
output = enforce_stop_tokens(text, ["moo", "baz", "bar"])
|
|
assert output == "foo "
|
|
text = "foo bar baz"
|
|
output = enforce_stop_tokens(text, ["moo", "bar"])
|
|
assert output == "foo "
|
|
|
|
|
|
def test_enforce_stop_tokens_none() -> None:
|
|
"""Test removing stop tokens when they do not occur."""
|
|
text = "foo bar baz"
|
|
output = enforce_stop_tokens(text, ["moo"])
|
|
assert output == "foo bar baz"
|