forked from Archives/langchain
5267ebce2d
Implementation of https://github.com/jagilley/fact-checker. Works pretty well. <img width="993" alt="Screenshot 2022-12-07 at 4 41 47 PM" src="https://user-images.githubusercontent.com/101075607/206302751-356a19ff-d000-4798-9aee-9c38b7f532b9.png"> Verifying this manually: 1. "Only two kinds of egg-laying mammals are left on the planet today—the duck-billed platypus and the echidna, or spiny anteater." https://www.scientificamerican.com/article/extreme-monotremes/ 2. "An [Echidna] egg weighs 1.5 to 2 grams (0.05 to 0.07 oz)[[19]](https://en.wikipedia.org/wiki/Echidna#cite_note-19) and is about 1.4 centimetres (0.55 in) long." https://en.wikipedia.org/wiki/Echidna#:~:text=sleep%20is%20suppressed.-,Reproduction,a%20reptile%2Dlike%20egg%20tooth. 3. "A [platypus] lays one to three (usually two) small, leathery eggs (similar to those of reptiles), about 11 mm (7⁄16 in) in diameter and slightly rounder than bird eggs." https://en.wikipedia.org/wiki/Platypus#:~:text=It%20lays%20one%20to%20three,slightly%20rounder%20than%20bird%20eggs. 4. Therefore, an Echidna is the mammal that lays the biggest eggs. cc @hwchase17
44 lines
1.8 KiB
Python
44 lines
1.8 KiB
Python
# flake8: noqa E501
|
|
|
|
"""Test LLMCheckerChain functionality."""
|
|
|
|
import pytest
|
|
|
|
from langchain.chains.llm_checker.base import LLMCheckerChain
|
|
from langchain.chains.llm_checker.prompt import (
|
|
_CHECK_ASSERTIONS_TEMPLATE,
|
|
_CREATE_DRAFT_ANSWER_TEMPLATE,
|
|
_LIST_ASSERTIONS_TEMPLATE,
|
|
_REVISED_ANSWER_TEMPLATE,
|
|
)
|
|
from tests.unit_tests.llms.fake_llm import FakeLLM
|
|
|
|
|
|
@pytest.fixture
|
|
def fake_llm_checker_chain() -> LLMCheckerChain:
|
|
"""Fake LLMCheckerChain for testing."""
|
|
queries = {
|
|
_CREATE_DRAFT_ANSWER_TEMPLATE.format(
|
|
question="Which mammal lays the biggest eggs?"
|
|
): "I don't know which mammal layers the biggest eggs.",
|
|
_LIST_ASSERTIONS_TEMPLATE.format(
|
|
statement="I don't know which mammal layers the biggest eggs.",
|
|
): "1) I know that mammals lay eggs.\n2) I know that birds lay eggs.\n3) I know that birds are mammals.",
|
|
_CHECK_ASSERTIONS_TEMPLATE.format(
|
|
assertions="1) I know that mammals lay eggs.\n2) I know that birds lay eggs.\n3) I know that birds are mammals.",
|
|
): "1) I know that mammals lay eggs. TRUE\n2) I know that birds lay eggs. TRUE\n3) I know that birds are mammals. TRUE",
|
|
_REVISED_ANSWER_TEMPLATE.format(
|
|
checked_assertions="1) I know that mammals lay eggs. TRUE\n2) I know that birds lay eggs. TRUE\n3) I know that birds are mammals. TRUE",
|
|
question="Which mammal lays the biggest eggs?",
|
|
): "I still don't know.",
|
|
}
|
|
fake_llm = FakeLLM(queries=queries)
|
|
return LLMCheckerChain(llm=fake_llm, input_key="q", output_key="a")
|
|
|
|
|
|
def test_simple_question(fake_llm_checker_chain: LLMCheckerChain) -> None:
|
|
"""Test simple question that should not need python."""
|
|
question = "Which mammal lays the biggest eggs?"
|
|
output = fake_llm_checker_chain.run(question)
|
|
assert output == "I still don't know."
|