mirror of
https://github.com/hwchase17/langchain
synced 2024-11-10 01:10:59 +00:00
29 lines
999 B
Python
29 lines
999 B
Python
|
"""Test text splitting functionality."""
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from langchain.text_splitter import CharacterTextSplitter
|
||
|
|
|
||
|
|
|
||
|
|
def test_character_text_splitter() -> None:
|
||
|
|
"""Test splitting by character count."""
|
||
|
|
text = "foo bar baz 123"
|
||
|
|
splitter = CharacterTextSplitter(separator=" ", chunk_size=5, chunk_overlap=3)
|
||
|
|
output = splitter.split_text(text)
|
||
|
|
expected_output = ["foo bar", "bar baz", "baz 123"]
|
||
|
|
assert output == expected_output
|
||
|
|
|
||
|
|
|
||
|
|
def test_character_text_splitter_longer_words() -> None:
|
||
|
|
"""Test splitting by characters when splits not found easily."""
|
||
|
|
text = "foo bar baz 123"
|
||
|
|
splitter = CharacterTextSplitter(separator=" ", chunk_size=1, chunk_overlap=1)
|
||
|
|
output = splitter.split_text(text)
|
||
|
|
expected_output = ["foo", "bar", "baz", "123"]
|
||
|
|
assert output == expected_output
|
||
|
|
|
||
|
|
|
||
|
|
def test_character_text_splitting_args() -> None:
|
||
|
|
"""Test invalid arguments."""
|
||
|
|
with pytest.raises(ValueError):
|
||
|
|
CharacterTextSplitter(chunk_size=2, chunk_overlap=4)
|