mirror of https://github.com/hwchase17/langchain
You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
257 lines
7.8 KiB
Plaintext
257 lines
7.8 KiB
Plaintext
2 years ago
|
{
|
||
|
"cells": [
|
||
2 years ago
|
{
|
||
|
"cell_type": "markdown",
|
||
|
"id": "920a3c1a",
|
||
|
"metadata": {},
|
||
|
"source": [
|
||
2 years ago
|
"# Model Comparison\n",
|
||
2 years ago
|
"\n",
|
||
2 years ago
|
"Constructing your language model application will likely involved choosing between many different options of prompts, models, and even chains to use. When doing so, you will want to compare these different options on different inputs in an easy, flexible, and intuitive way. \n",
|
||
|
"\n",
|
||
|
"LangChain provides the concept of a ModelLaboratory to test out and try different models."
|
||
2 years ago
|
]
|
||
|
},
|
||
2 years ago
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": 1,
|
||
|
"id": "ab9e95ad",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": [
|
||
2 years ago
|
"from langchain import LLMChain, OpenAI, Cohere, HuggingFaceHub, PromptTemplate\n",
|
||
2 years ago
|
"from langchain.model_laboratory import ModelLaboratory"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": 2,
|
||
|
"id": "32cb94e6",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": [
|
||
2 years ago
|
"llms = [\n",
|
||
|
" OpenAI(temperature=0), \n",
|
||
|
" Cohere(model=\"command-xlarge-20221108\", max_tokens=20, temperature=0), \n",
|
||
|
" HuggingFaceHub(repo_id=\"google/flan-t5-xl\", model_kwargs={\"temperature\":1})\n",
|
||
|
"]"
|
||
2 years ago
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": 3,
|
||
|
"id": "14cde09d",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": [
|
||
2 years ago
|
"model_lab = ModelLaboratory.from_llms(llms)"
|
||
2 years ago
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": 4,
|
||
|
"id": "f186c741",
|
||
|
"metadata": {},
|
||
|
"outputs": [
|
||
|
{
|
||
|
"name": "stdout",
|
||
|
"output_type": "stream",
|
||
|
"text": [
|
||
|
"\u001b[1mInput:\u001b[0m\n",
|
||
|
"What color is a flamingo?\n",
|
||
|
"\n",
|
||
|
"\u001b[1mOpenAI\u001b[0m\n",
|
||
|
"Params: {'model': 'text-davinci-002', 'temperature': 0.0, 'max_tokens': 256, 'top_p': 1, 'frequency_penalty': 0, 'presence_penalty': 0, 'n': 1, 'best_of': 1}\n",
|
||
2 years ago
|
"\u001b[36;1m\u001b[1;3m\n",
|
||
2 years ago
|
"\n",
|
||
|
"Flamingos are pink.\u001b[0m\n",
|
||
|
"\n",
|
||
|
"\u001b[1mCohere\u001b[0m\n",
|
||
|
"Params: {'model': 'command-xlarge-20221108', 'max_tokens': 20, 'temperature': 0.0, 'k': 0, 'p': 1, 'frequency_penalty': 0, 'presence_penalty': 0}\n",
|
||
2 years ago
|
"\u001b[33;1m\u001b[1;3m\n",
|
||
2 years ago
|
"\n",
|
||
|
"Pink\u001b[0m\n",
|
||
|
"\n",
|
||
|
"\u001b[1mHuggingFaceHub\u001b[0m\n",
|
||
|
"Params: {'repo_id': 'google/flan-t5-xl', 'temperature': 1}\n",
|
||
2 years ago
|
"\u001b[38;5;200m\u001b[1;3mpink\u001b[0m\n",
|
||
2 years ago
|
"\n"
|
||
|
]
|
||
|
}
|
||
|
],
|
||
|
"source": [
|
||
|
"model_lab.compare(\"What color is a flamingo?\")"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": 5,
|
||
|
"id": "248b652a",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": [
|
||
2 years ago
|
"prompt = PromptTemplate(template=\"What is the capital of {state}?\", input_variables=[\"state\"])\n",
|
||
2 years ago
|
"model_lab_with_prompt = ModelLaboratory.from_llms(llms, prompt=prompt)"
|
||
2 years ago
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": 6,
|
||
|
"id": "f64377ac",
|
||
|
"metadata": {},
|
||
|
"outputs": [
|
||
|
{
|
||
|
"name": "stdout",
|
||
|
"output_type": "stream",
|
||
|
"text": [
|
||
|
"\u001b[1mInput:\u001b[0m\n",
|
||
|
"New York\n",
|
||
|
"\n",
|
||
|
"\u001b[1mOpenAI\u001b[0m\n",
|
||
|
"Params: {'model': 'text-davinci-002', 'temperature': 0.0, 'max_tokens': 256, 'top_p': 1, 'frequency_penalty': 0, 'presence_penalty': 0, 'n': 1, 'best_of': 1}\n",
|
||
2 years ago
|
"\u001b[36;1m\u001b[1;3m\n",
|
||
2 years ago
|
"\n",
|
||
|
"The capital of New York is Albany.\u001b[0m\n",
|
||
|
"\n",
|
||
|
"\u001b[1mCohere\u001b[0m\n",
|
||
|
"Params: {'model': 'command-xlarge-20221108', 'max_tokens': 20, 'temperature': 0.0, 'k': 0, 'p': 1, 'frequency_penalty': 0, 'presence_penalty': 0}\n",
|
||
2 years ago
|
"\u001b[33;1m\u001b[1;3m\n",
|
||
2 years ago
|
"\n",
|
||
|
"The capital of New York is Albany.\u001b[0m\n",
|
||
|
"\n",
|
||
|
"\u001b[1mHuggingFaceHub\u001b[0m\n",
|
||
|
"Params: {'repo_id': 'google/flan-t5-xl', 'temperature': 1}\n",
|
||
2 years ago
|
"\u001b[38;5;200m\u001b[1;3mst john s\u001b[0m\n",
|
||
2 years ago
|
"\n"
|
||
|
]
|
||
|
}
|
||
|
],
|
||
|
"source": [
|
||
|
"model_lab_with_prompt.compare(\"New York\")"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
2 years ago
|
"execution_count": 7,
|
||
2 years ago
|
"id": "54336dbf",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
2 years ago
|
"source": [
|
||
2 years ago
|
"from langchain import SelfAskWithSearchChain, SerpAPIWrapper\n",
|
||
2 years ago
|
"\n",
|
||
|
"open_ai_llm = OpenAI(temperature=0)\n",
|
||
2 years ago
|
"search = SerpAPIWrapper()\n",
|
||
2 years ago
|
"self_ask_with_search_openai = SelfAskWithSearchChain(llm=open_ai_llm, search_chain=search, verbose=True)\n",
|
||
|
"\n",
|
||
|
"cohere_llm = Cohere(temperature=0, model=\"command-xlarge-20221108\")\n",
|
||
2 years ago
|
"search = SerpAPIWrapper()\n",
|
||
2 years ago
|
"self_ask_with_search_cohere = SelfAskWithSearchChain(llm=cohere_llm, search_chain=search, verbose=True)"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": 8,
|
||
|
"id": "6a50a9f1",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": [
|
||
|
"chains = [self_ask_with_search_openai, self_ask_with_search_cohere]\n",
|
||
|
"names = [str(open_ai_llm), str(cohere_llm)]"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": 9,
|
||
|
"id": "d3549e99",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
|
"source": [
|
||
|
"model_lab = ModelLaboratory(chains, names=names)"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": 10,
|
||
|
"id": "362f7f57",
|
||
|
"metadata": {},
|
||
|
"outputs": [
|
||
|
{
|
||
|
"name": "stdout",
|
||
|
"output_type": "stream",
|
||
|
"text": [
|
||
|
"\u001b[1mInput:\u001b[0m\n",
|
||
|
"What is the hometown of the reigning men's U.S. Open champion?\n",
|
||
|
"\n",
|
||
|
"\u001b[1mOpenAI\u001b[0m\n",
|
||
|
"Params: {'model': 'text-davinci-002', 'temperature': 0.0, 'max_tokens': 256, 'top_p': 1, 'frequency_penalty': 0, 'presence_penalty': 0, 'n': 1, 'best_of': 1}\n",
|
||
|
"\n",
|
||
|
"\n",
|
||
|
"\u001b[1m> Entering new chain...\u001b[0m\n",
|
||
|
"What is the hometown of the reigning men's U.S. Open champion?\n",
|
||
|
"Are follow up questions needed here:\u001b[32;1m\u001b[1;3m Yes.\n",
|
||
|
"Follow up: Who is the reigning men's U.S. Open champion?\u001b[0m\n",
|
||
|
"Intermediate answer: \u001b[33;1m\u001b[1;3mCarlos Alcaraz.\u001b[0m\u001b[32;1m\u001b[1;3m\n",
|
||
|
"Follow up: Where is Carlos Alcaraz from?\u001b[0m\n",
|
||
|
"Intermediate answer: \u001b[33;1m\u001b[1;3mEl Palmar, Spain.\u001b[0m\u001b[32;1m\u001b[1;3m\n",
|
||
|
"So the final answer is: El Palmar, Spain\u001b[0m\n",
|
||
|
"\u001b[1m> Finished chain.\u001b[0m\n",
|
||
|
"\u001b[36;1m\u001b[1;3m\n",
|
||
|
"So the final answer is: El Palmar, Spain\u001b[0m\n",
|
||
|
"\n",
|
||
|
"\u001b[1mCohere\u001b[0m\n",
|
||
|
"Params: {'model': 'command-xlarge-20221108', 'max_tokens': 256, 'temperature': 0.0, 'k': 0, 'p': 1, 'frequency_penalty': 0, 'presence_penalty': 0}\n",
|
||
|
"\n",
|
||
|
"\n",
|
||
|
"\u001b[1m> Entering new chain...\u001b[0m\n",
|
||
|
"What is the hometown of the reigning men's U.S. Open champion?\n",
|
||
|
"Are follow up questions needed here:\u001b[32;1m\u001b[1;3m Yes.\n",
|
||
|
"Follow up: Who is the reigning men's U.S. Open champion?\u001b[0m\n",
|
||
|
"Intermediate answer: \u001b[33;1m\u001b[1;3mCarlos Alcaraz.\u001b[0m\u001b[32;1m\u001b[1;3m\n",
|
||
|
"So the final answer is:\n",
|
||
|
"\n",
|
||
|
"Carlos Alcaraz\u001b[0m\n",
|
||
|
"\u001b[1m> Finished chain.\u001b[0m\n",
|
||
|
"\u001b[33;1m\u001b[1;3m\n",
|
||
|
"So the final answer is:\n",
|
||
|
"\n",
|
||
|
"Carlos Alcaraz\u001b[0m\n",
|
||
|
"\n"
|
||
|
]
|
||
|
}
|
||
|
],
|
||
|
"source": [
|
||
|
"model_lab.compare(\"What is the hometown of the reigning men's U.S. Open champion?\")"
|
||
|
]
|
||
|
},
|
||
|
{
|
||
|
"cell_type": "code",
|
||
|
"execution_count": null,
|
||
|
"id": "94159131",
|
||
|
"metadata": {},
|
||
|
"outputs": [],
|
||
2 years ago
|
"source": []
|
||
|
}
|
||
|
],
|
||
|
"metadata": {
|
||
|
"kernelspec": {
|
||
|
"display_name": "Python 3 (ipykernel)",
|
||
|
"language": "python",
|
||
|
"name": "python3"
|
||
|
},
|
||
|
"language_info": {
|
||
|
"codemirror_mode": {
|
||
|
"name": "ipython",
|
||
|
"version": 3
|
||
|
},
|
||
|
"file_extension": ".py",
|
||
|
"mimetype": "text/x-python",
|
||
|
"name": "python",
|
||
|
"nbconvert_exporter": "python",
|
||
|
"pygments_lexer": "ipython3",
|
||
2 years ago
|
"version": "3.10.9"
|
||
2 years ago
|
}
|
||
|
},
|
||
|
"nbformat": 4,
|
||
|
"nbformat_minor": 5
|
||
|
}
|