From d90379210a7533141dfdd08bd4002ff9da27269a Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 24 Jun 2024 11:15:03 -0400 Subject: [PATCH] standard-tests[minor]: Add standard tests for cache (#23357) Add standard tests for cache abstraction --- .../integration_tests/cache.py | 192 ++++++++++++++++++ libs/standard-tests/poetry.lock | 23 ++- libs/standard-tests/pyproject.toml | 22 ++ .../tests/unit_tests/test_in_memory_cache.py | 19 ++ 4 files changed, 253 insertions(+), 3 deletions(-) create mode 100644 libs/standard-tests/langchain_standard_tests/integration_tests/cache.py create mode 100644 libs/standard-tests/tests/unit_tests/test_in_memory_cache.py diff --git a/libs/standard-tests/langchain_standard_tests/integration_tests/cache.py b/libs/standard-tests/langchain_standard_tests/integration_tests/cache.py new file mode 100644 index 0000000000..fe84d8450c --- /dev/null +++ b/libs/standard-tests/langchain_standard_tests/integration_tests/cache.py @@ -0,0 +1,192 @@ +from abc import ABC, abstractmethod + +import pytest +from langchain_core.caches import BaseCache +from langchain_core.outputs import Generation + + +class SyncCacheTestSuite(ABC): + """Test suite for checking the BaseCache API of a caching layer for LLMs. + + This test suite verifies the basic caching API of a caching layer for LLMs. + + The test suite is designed for synchronous caching layers. + + Implementers should subclass this test suite and provide a fixture + that returns an empty cache for each test. + """ + + @abstractmethod + @pytest.fixture + def cache(self) -> BaseCache: + """Get the cache class to test. + + The returned cache should be EMPTY. + """ + + def get_sample_prompt(self) -> str: + """Return a sample prompt for testing.""" + return "Sample prompt for testing." + + def get_sample_llm_string(self) -> str: + """Return a sample LLM string for testing.""" + return "Sample LLM string configuration." + + def get_sample_generation(self) -> Generation: + """Return a sample Generation object for testing.""" + return Generation( + text="Sample generated text.", generation_info={"reason": "test"} + ) + + def test_cache_is_empty(self, cache: BaseCache) -> None: + """Test that the cache is empty.""" + assert ( + cache.lookup(self.get_sample_prompt(), self.get_sample_llm_string()) is None + ) + + def test_update_cache(self, cache: BaseCache) -> None: + """Test updating the cache.""" + prompt = self.get_sample_prompt() + llm_string = self.get_sample_llm_string() + generation = self.get_sample_generation() + cache.update(prompt, llm_string, [generation]) + assert cache.lookup(prompt, llm_string) == [generation] + + def test_cache_still_empty(self, cache: BaseCache) -> None: + """This test should follow a test that updates the cache. + + This just verifies that the fixture is set up properly to be empty + after each test. + """ + assert ( + cache.lookup(self.get_sample_prompt(), self.get_sample_llm_string()) is None + ) + + def test_clear_cache(self, cache: BaseCache) -> None: + """Test clearing the cache.""" + prompt = self.get_sample_prompt() + llm_string = self.get_sample_llm_string() + generation = self.get_sample_generation() + cache.update(prompt, llm_string, [generation]) + cache.clear() + assert cache.lookup(prompt, llm_string) is None + + def test_cache_miss(self, cache: BaseCache) -> None: + """Test cache miss.""" + assert cache.lookup("Nonexistent prompt", self.get_sample_llm_string()) is None + + def test_cache_hit(self, cache: BaseCache) -> None: + """Test cache hit.""" + prompt = self.get_sample_prompt() + llm_string = self.get_sample_llm_string() + generation = self.get_sample_generation() + cache.update(prompt, llm_string, [generation]) + assert cache.lookup(prompt, llm_string) == [generation] + + def test_update_cache_with_multiple_generations(self, cache: BaseCache) -> None: + """Test updating the cache with multiple Generation objects.""" + prompt = self.get_sample_prompt() + llm_string = self.get_sample_llm_string() + generations = [ + self.get_sample_generation(), + Generation(text="Another generated text."), + ] + cache.update(prompt, llm_string, generations) + assert cache.lookup(prompt, llm_string) == generations + + +class AsyncCacheTestSuite(ABC): + """Test suite for checking the BaseCache API of a caching layer for LLMs. + + This test suite verifies the basic caching API of a caching layer for LLMs. + + The test suite is designed for synchronous caching layers. + + Implementers should subclass this test suite and provide a fixture + that returns an empty cache for each test. + """ + + @abstractmethod + @pytest.fixture + async def cache(self) -> BaseCache: + """Get the cache class to test. + + The returned cache should be EMPTY. + """ + + def get_sample_prompt(self) -> str: + """Return a sample prompt for testing.""" + return "Sample prompt for testing." + + def get_sample_llm_string(self) -> str: + """Return a sample LLM string for testing.""" + return "Sample LLM string configuration." + + def get_sample_generation(self) -> Generation: + """Return a sample Generation object for testing.""" + return Generation( + text="Sample generated text.", generation_info={"reason": "test"} + ) + + async def test_cache_is_empty(self, cache: BaseCache) -> None: + """Test that the cache is empty.""" + assert ( + await cache.alookup(self.get_sample_prompt(), self.get_sample_llm_string()) + is None + ) + + async def test_update_cache(self, cache: BaseCache) -> None: + """Test updating the cache.""" + prompt = self.get_sample_prompt() + llm_string = self.get_sample_llm_string() + generation = self.get_sample_generation() + await cache.aupdate(prompt, llm_string, [generation]) + assert await cache.alookup(prompt, llm_string) == [generation] + + async def test_cache_still_empty(self, cache: BaseCache) -> None: + """This test should follow a test that updates the cache. + + This just verifies that the fixture is set up properly to be empty + after each test. + """ + assert ( + await cache.alookup(self.get_sample_prompt(), self.get_sample_llm_string()) + is None + ) + + async def test_clear_cache(self, cache: BaseCache) -> None: + """Test clearing the cache.""" + prompt = self.get_sample_prompt() + llm_string = self.get_sample_llm_string() + generation = self.get_sample_generation() + await cache.aupdate(prompt, llm_string, [generation]) + await cache.aclear() + assert await cache.alookup(prompt, llm_string) is None + + async def test_cache_miss(self, cache: BaseCache) -> None: + """Test cache miss.""" + assert ( + await cache.alookup("Nonexistent prompt", self.get_sample_llm_string()) + is None + ) + + async def test_cache_hit(self, cache: BaseCache) -> None: + """Test cache hit.""" + prompt = self.get_sample_prompt() + llm_string = self.get_sample_llm_string() + generation = self.get_sample_generation() + await cache.aupdate(prompt, llm_string, [generation]) + assert await cache.alookup(prompt, llm_string) == [generation] + + async def test_update_cache_with_multiple_generations( + self, cache: BaseCache + ) -> None: + """Test updating the cache with multiple Generation objects.""" + prompt = self.get_sample_prompt() + llm_string = self.get_sample_llm_string() + generations = [ + self.get_sample_generation(), + Generation(text="Another generated text."), + ] + await cache.aupdate(prompt, llm_string, generations) + assert await cache.alookup(prompt, llm_string) == generations diff --git a/libs/standard-tests/poetry.lock b/libs/standard-tests/poetry.lock index 91e3029387..f0c71bc886 100644 --- a/libs/standard-tests/poetry.lock +++ b/libs/standard-tests/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -605,6 +605,24 @@ tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-asyncio" +version = "0.23.7" +description = "Pytest support for asyncio" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest_asyncio-0.23.7-py3-none-any.whl", hash = "sha256:009b48127fbe44518a547bddd25611551b0e43ccdbf1e67d12479f569832c20b"}, + {file = "pytest_asyncio-0.23.7.tar.gz", hash = "sha256:5f5c72948f4c49e7db4f29f2521d4031f1c27f86e57b046126654083d4770268"}, +] + +[package.dependencies] +pytest = ">=7.0.0,<9" + +[package.extras] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] +testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] + [[package]] name = "pyyaml" version = "6.0.1" @@ -630,7 +648,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -780,4 +797,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "a27025e6afa0275f153a9fd98c890c16187f9d01f5ca0b60aae23cee1a7d9dcc" +content-hash = "fe8e04975482a0f8e67d07d186c401de1d321068dd1e595c836b156f7c4fcd9c" diff --git a/libs/standard-tests/pyproject.toml b/libs/standard-tests/pyproject.toml index d4d3b4472a..7b8a230f61 100644 --- a/libs/standard-tests/pyproject.toml +++ b/libs/standard-tests/pyproject.toml @@ -21,6 +21,7 @@ optional = true [tool.poetry.group.test.dependencies] langchain-core = { path = "../core", develop = true } +pytest-asyncio = "^0.23.7" [tool.poetry.group.test_integration] optional = true @@ -60,3 +61,24 @@ omit = ["tests/*"] [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +# --strict-markers will raise errors on unknown marks. +# https://docs.pytest.org/en/7.1.x/how-to/mark.html#raising-errors-on-unknown-marks +# +# https://docs.pytest.org/en/7.1.x/reference/reference.html +# --strict-config any warnings encountered while parsing the `pytest` +# section of the configuration file raise errors. +# +# https://github.com/tophat/syrupy +# --snapshot-warn-unused Prints a warning on unused snapshots rather than fail the test suite. +addopts = "--strict-markers --strict-config --durations=5 -vv" +# Registering custom markers. +# https://docs.pytest.org/en/7.1.x/example/markers.html#registering-markers +markers = [ + "requires: mark tests as requiring a specific library", + "scheduled: mark tests to run in scheduled testing", + "compile: mark placeholder test used to compile integration tests without running them", +] +asyncio_mode = "auto" + diff --git a/libs/standard-tests/tests/unit_tests/test_in_memory_cache.py b/libs/standard-tests/tests/unit_tests/test_in_memory_cache.py new file mode 100644 index 0000000000..4f67a87649 --- /dev/null +++ b/libs/standard-tests/tests/unit_tests/test_in_memory_cache.py @@ -0,0 +1,19 @@ +import pytest +from langchain_core.caches import InMemoryCache + +from langchain_standard_tests.integration_tests.cache import ( + AsyncCacheTestSuite, + SyncCacheTestSuite, +) + + +class TestInMemoryCache(SyncCacheTestSuite): + @pytest.fixture + def cache(self) -> InMemoryCache: + return InMemoryCache() + + +class TestInMemoryCacheAsync(AsyncCacheTestSuite): + @pytest.fixture + async def cache(self) -> InMemoryCache: + return InMemoryCache()