mirror of https://github.com/hwchase17/langchain
model laboratory (#95)
parent
2ddab88c06
commit
db37bd089f
@ -0,0 +1,147 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "ab9e95ad",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain import LLMChain, OpenAI, Cohere, HuggingFaceHub, Prompt\n",
|
||||
"from langchain.model_laboratory import ModelLaboratory"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "32cb94e6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"llms = [OpenAI(temperature=0), Cohere(model=\"command-xlarge-20221108\", max_tokens=20, temperature=0), HuggingFaceHub(repo_id=\"google/flan-t5-xl\", model_kwargs={\"temperature\":1})]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "14cde09d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model_lab = ModelLaboratory(llms)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "f186c741",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[1mInput:\u001b[0m\n",
|
||||
"What color is a flamingo?\n",
|
||||
"\n",
|
||||
"\u001b[1mOpenAI\u001b[0m\n",
|
||||
"Params: {'model': 'text-davinci-002', 'temperature': 0.0, 'max_tokens': 256, 'top_p': 1, 'frequency_penalty': 0, 'presence_penalty': 0, 'n': 1, 'best_of': 1}\n",
|
||||
"\u001b[104m\n",
|
||||
"\n",
|
||||
"Flamingos are pink.\u001b[0m\n",
|
||||
"\n",
|
||||
"\u001b[1mCohere\u001b[0m\n",
|
||||
"Params: {'model': 'command-xlarge-20221108', 'max_tokens': 20, 'temperature': 0.0, 'k': 0, 'p': 1, 'frequency_penalty': 0, 'presence_penalty': 0}\n",
|
||||
"\u001b[103m\n",
|
||||
"\n",
|
||||
"Pink\u001b[0m\n",
|
||||
"\n",
|
||||
"\u001b[1mHuggingFaceHub\u001b[0m\n",
|
||||
"Params: {'repo_id': 'google/flan-t5-xl', 'temperature': 1}\n",
|
||||
"\u001b[101mpink\u001b[0m\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model_lab.compare(\"What color is a flamingo?\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "248b652a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"prompt = Prompt(template=\"What is the capital of {state}?\", input_variables=[\"state\"])\n",
|
||||
"model_lab_with_prompt = ModelLaboratory(llms, prompt=prompt)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "f64377ac",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[1mInput:\u001b[0m\n",
|
||||
"New York\n",
|
||||
"\n",
|
||||
"\u001b[1mOpenAI\u001b[0m\n",
|
||||
"Params: {'model': 'text-davinci-002', 'temperature': 0.0, 'max_tokens': 256, 'top_p': 1, 'frequency_penalty': 0, 'presence_penalty': 0, 'n': 1, 'best_of': 1}\n",
|
||||
"\u001b[104m\n",
|
||||
"\n",
|
||||
"The capital of New York is Albany.\u001b[0m\n",
|
||||
"\n",
|
||||
"\u001b[1mCohere\u001b[0m\n",
|
||||
"Params: {'model': 'command-xlarge-20221108', 'max_tokens': 20, 'temperature': 0.0, 'k': 0, 'p': 1, 'frequency_penalty': 0, 'presence_penalty': 0}\n",
|
||||
"\u001b[103m\n",
|
||||
"\n",
|
||||
"The capital of New York is Albany.\u001b[0m\n",
|
||||
"\n",
|
||||
"\u001b[1mHuggingFaceHub\u001b[0m\n",
|
||||
"Params: {'repo_id': 'google/flan-t5-xl', 'temperature': 1}\n",
|
||||
"\u001b[101mst john s\u001b[0m\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model_lab_with_prompt.compare(\"New York\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "54336dbf",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
@ -0,0 +1,50 @@
|
||||
"""Experiment with different models."""
|
||||
from typing import List, Optional
|
||||
|
||||
from langchain.chains.llm import LLMChain
|
||||
from langchain.input import get_color_mapping, print_text
|
||||
from langchain.llms.base import LLM
|
||||
from langchain.prompts.prompt import Prompt
|
||||
|
||||
|
||||
class ModelLaboratory:
|
||||
"""Experiment with different models."""
|
||||
|
||||
def __init__(self, llms: List[LLM], prompt: Optional[Prompt] = None):
|
||||
"""Initialize with LLMs to experiment with and optional prompt.
|
||||
|
||||
Args:
|
||||
llms: list of LLMs to experiment with
|
||||
prompt: Optional prompt to use to prompt the LLMs. Defaults to None.
|
||||
If a prompt was provided, it should only have one input variable.
|
||||
"""
|
||||
self.llms = llms
|
||||
llm_range = [str(i) for i in range(len(self.llms))]
|
||||
self.llm_colors = get_color_mapping(llm_range)
|
||||
if prompt is None:
|
||||
self.prompt = Prompt(input_variables=["_input"], template="{_input}")
|
||||
else:
|
||||
if len(prompt.input_variables) != 1:
|
||||
raise ValueError(
|
||||
"Currently only support prompts with one input variable, "
|
||||
f"got {prompt}"
|
||||
)
|
||||
self.prompt = prompt
|
||||
|
||||
def compare(self, text: str) -> None:
|
||||
"""Compare model outputs on an input text.
|
||||
|
||||
If a prompt was provided with starting the laboratory, then this text will be
|
||||
fed into the prompt. If no prompt was provided, then the input text is the
|
||||
entire prompt.
|
||||
|
||||
Args:
|
||||
text: input text to run all models on.
|
||||
"""
|
||||
print(f"\033[1mInput:\033[0m\n{text}\n")
|
||||
for i, llm in enumerate(self.llms):
|
||||
print_text(str(llm), end="\n")
|
||||
chain = LLMChain(llm=llm, prompt=self.prompt)
|
||||
llm_inputs = {self.prompt.input_variables[0]: text}
|
||||
output = chain.predict(**llm_inputs)
|
||||
print_text(output, color=self.llm_colors[str(i)], end="\n\n")
|
Loading…
Reference in New Issue