From 68f2363f5d7907836b7fef2b04cf07d49a11c04a Mon Sep 17 00:00:00 2001 From: Jayson Ng Date: Sun, 3 Sep 2023 23:40:06 -0400 Subject: [PATCH] Allow specifying arbitrary keyword arguments in `langchain.llms.VLLM` (#9683) Description: add arbitrary keyword arguments for VLLM Issue: https://github.com/langchain-ai/langchain/issues/9682 Dependencies: none Tag maintainer: @hwchase17, @baskaryan --- libs/langchain/langchain/llms/vllm.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libs/langchain/langchain/llms/vllm.py b/libs/langchain/langchain/llms/vllm.py index 9f456fde51..1a6e1a5910 100644 --- a/libs/langchain/langchain/llms/vllm.py +++ b/libs/langchain/langchain/llms/vllm.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional from langchain.callbacks.manager import CallbackManagerForLLMRun from langchain.llms.base import BaseLLM from langchain.llms.openai import BaseOpenAI -from langchain.pydantic_v1 import root_validator +from langchain.pydantic_v1 import Field, root_validator from langchain.schema.output import Generation, LLMResult @@ -62,6 +62,9 @@ class VLLM(BaseLLM): dtype: str = "auto" """The data type for the model weights and activations.""" + vllm_kwargs: Dict[str, Any] = Field(default_factory=dict) + """Holds any model parameters valid for `vllm.LLM` call not explicitly specified.""" + client: Any #: :meta private: @root_validator() @@ -81,6 +84,7 @@ class VLLM(BaseLLM): tensor_parallel_size=values["tensor_parallel_size"], trust_remote_code=values["trust_remote_code"], dtype=values["dtype"], + **values["vllm_kwargs"], ) return values