From e194dc53067537300367894d3b7249bc9be2a20e Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Sat, 17 Jun 2023 19:08:25 +0100 Subject: [PATCH 1/3] Make lckwargs private (#6344) Fixes # (issue) #### Before submitting #### Who can review? Tag maintainers/contributors who might be interested: --- langchain/chat_models/anthropic.py | 7 ------- langchain/chat_models/base.py | 3 +-- langchain/chat_models/openai.py | 3 +-- langchain/llms/anthropic.py | 7 +------ langchain/llms/base.py | 3 +-- langchain/llms/openai.py | 8 +------- langchain/load/serializable.py | 13 ++++++++----- langchain/prompts/base.py | 3 +-- langchain/prompts/prompt.py | 7 +------ langchain/schema.py | 3 +-- 10 files changed, 16 insertions(+), 41 deletions(-) diff --git a/langchain/chat_models/anthropic.py b/langchain/chat_models/anthropic.py index a9f3d0e0..023c9e41 100644 --- a/langchain/chat_models/anthropic.py +++ b/langchain/chat_models/anthropic.py @@ -1,7 +1,5 @@ from typing import Any, Dict, List, Optional -from pydantic import Extra - from langchain.callbacks.manager import ( AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun, @@ -34,11 +32,6 @@ class ChatAnthropic(BaseChatModel, _AnthropicCommon): model = ChatAnthropic(model="", anthropic_api_key="my-api-key") """ - class Config: - """Configuration for this pydantic object.""" - - extra = Extra.forbid - @property def _llm_type(self) -> str: """Return type of chat model.""" diff --git a/langchain/chat_models/base.py b/langchain/chat_models/base.py index 8a06c1d8..5fa949ef 100644 --- a/langchain/chat_models/base.py +++ b/langchain/chat_models/base.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from functools import partial from typing import Any, Dict, List, Mapping, Optional, Sequence -from pydantic import Extra, Field, root_validator +from pydantic import Field, root_validator import langchain from langchain.base_language import BaseLanguageModel @@ -56,7 +56,6 @@ class BaseChatModel(BaseLanguageModel, ABC): class Config: """Configuration for this pydantic object.""" - extra = Extra.forbid arbitrary_types_allowed = True def _combine_llm_outputs(self, llm_outputs: List[Optional[dict]]) -> dict: diff --git a/langchain/chat_models/openai.py b/langchain/chat_models/openai.py index 6c18b9c8..674bf5e9 100644 --- a/langchain/chat_models/openai.py +++ b/langchain/chat_models/openai.py @@ -15,7 +15,7 @@ from typing import ( Union, ) -from pydantic import Extra, Field, root_validator +from pydantic import Field, root_validator from tenacity import ( before_sleep_log, retry, @@ -182,7 +182,6 @@ class ChatOpenAI(BaseChatModel): class Config: """Configuration for this pydantic object.""" - extra = Extra.ignore allow_population_by_field_name = True @root_validator(pre=True) diff --git a/langchain/llms/anthropic.py b/langchain/llms/anthropic.py index 9554fd90..ec47b755 100644 --- a/langchain/llms/anthropic.py +++ b/langchain/llms/anthropic.py @@ -3,7 +3,7 @@ import re import warnings from typing import Any, Callable, Dict, Generator, List, Mapping, Optional, Tuple, Union -from pydantic import BaseModel, Extra, root_validator +from pydantic import BaseModel, root_validator from langchain.callbacks.manager import ( AsyncCallbackManagerForLLMRun, @@ -140,11 +140,6 @@ class Anthropic(LLM, _AnthropicCommon): ) return values - class Config: - """Configuration for this pydantic object.""" - - extra = Extra.forbid - @property def _llm_type(self) -> str: """Return type of llm.""" diff --git a/langchain/llms/base.py b/langchain/llms/base.py index 2647635d..46e3a81f 100644 --- a/langchain/llms/base.py +++ b/langchain/llms/base.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union import yaml -from pydantic import Extra, Field, root_validator, validator +from pydantic import Field, root_validator, validator import langchain from langchain.base_language import BaseLanguageModel @@ -85,7 +85,6 @@ class BaseLLM(BaseLanguageModel, ABC): class Config: """Configuration for this pydantic object.""" - extra = Extra.forbid arbitrary_types_allowed = True @root_validator() diff --git a/langchain/llms/openai.py b/langchain/llms/openai.py index c0d999d4..d6c9c5f2 100644 --- a/langchain/llms/openai.py +++ b/langchain/llms/openai.py @@ -20,7 +20,7 @@ from typing import ( Union, ) -from pydantic import Extra, Field, root_validator +from pydantic import Field, root_validator from tenacity import ( before_sleep_log, retry, @@ -187,7 +187,6 @@ class BaseOpenAI(BaseLLM): class Config: """Configuration for this pydantic object.""" - extra = Extra.ignore allow_population_by_field_name = True @root_validator(pre=True) @@ -686,11 +685,6 @@ class OpenAIChat(BaseLLM): disallowed_special: Union[Literal["all"], Collection[str]] = "all" """Set of special tokens that are not allowed。""" - class Config: - """Configuration for this pydantic object.""" - - extra = Extra.ignore - @root_validator(pre=True) def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]: """Build extra kwargs from additional params that were passed in.""" diff --git a/langchain/load/serializable.py b/langchain/load/serializable.py index 56c97dfa..88f2290e 100644 --- a/langchain/load/serializable.py +++ b/langchain/load/serializable.py @@ -1,7 +1,7 @@ from abc import ABC from typing import Any, Dict, List, Literal, TypedDict, Union, cast -from pydantic import BaseModel, Field +from pydantic import BaseModel, PrivateAttr class BaseSerialized(TypedDict): @@ -55,11 +55,14 @@ class Serializable(BaseModel, ABC): """ return {} - lc_kwargs: Dict[str, Any] = Field(default_factory=dict, exclude=True, repr=False) + class Config: + extra = "ignore" + + _lc_kwargs = PrivateAttr(default_factory=dict) def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) - self.lc_kwargs = kwargs + self._lc_kwargs = kwargs def to_json(self) -> Union[SerializedConstructor, SerializedNotImplemented]: if not self.lc_serializable: @@ -69,8 +72,8 @@ class Serializable(BaseModel, ABC): # Get latest values for kwargs if there is an attribute with same name lc_kwargs = { k: getattr(self, k, v) - for k, v in self.lc_kwargs.items() - if not self.__exclude_fields__.get(k, False) # type: ignore + for k, v in self._lc_kwargs.items() + if not (self.__exclude_fields__ or {}).get(k, False) # type: ignore } # Merge the lc_secrets and lc_attributes from every class in the MRO diff --git a/langchain/prompts/base.py b/langchain/prompts/base.py index 58e7339b..9e1f6515 100644 --- a/langchain/prompts/base.py +++ b/langchain/prompts/base.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Union import yaml -from pydantic import Extra, Field, root_validator +from pydantic import Field, root_validator from langchain.formatting import formatter from langchain.load.serializable import Serializable @@ -119,7 +119,6 @@ class BasePromptTemplate(Serializable, ABC): class Config: """Configuration for this pydantic object.""" - extra = Extra.forbid arbitrary_types_allowed = True @abstractmethod diff --git a/langchain/prompts/prompt.py b/langchain/prompts/prompt.py index c8ac2200..3532c610 100644 --- a/langchain/prompts/prompt.py +++ b/langchain/prompts/prompt.py @@ -5,7 +5,7 @@ from pathlib import Path from string import Formatter from typing import Any, Dict, List, Union -from pydantic import Extra, root_validator +from pydantic import root_validator from langchain.prompts.base import ( DEFAULT_FORMATTER_MAPPING, @@ -48,11 +48,6 @@ class PromptTemplate(StringPromptTemplate): """Return the prompt type key.""" return "prompt" - class Config: - """Configuration for this pydantic object.""" - - extra = Extra.forbid - def format(self, **kwargs: Any) -> str: """Format the prompt with the inputs. diff --git a/langchain/schema.py b/langchain/schema.py index ef675f6f..77fa6989 100644 --- a/langchain/schema.py +++ b/langchain/schema.py @@ -16,7 +16,7 @@ from typing import ( ) from uuid import UUID -from pydantic import BaseModel, Extra, Field, root_validator +from pydantic import BaseModel, Field, root_validator from langchain.load.serializable import Serializable @@ -229,7 +229,6 @@ class BaseMemory(Serializable, ABC): class Config: """Configuration for this pydantic object.""" - extra = Extra.forbid arbitrary_types_allowed = True @property From 2eec687474252cd8083da09ccc5ac5bf89a59611 Mon Sep 17 00:00:00 2001 From: Dhruvil Shah <48760936+jackfrost1411@users.noreply.github.com> Date: Sat, 17 Jun 2023 11:10:48 -0700 Subject: [PATCH 2/3] update web_base.py to have verify option (#6107) We propose an enhancement to the web-based loader initialize method by introducing a "verify" option. This enhancement addresses the issue of SSL verification errors encountered on certain web pages. By providing users with the option to set the verify parameter to False, we offer greater flexibility and control. ### Fixes #6079 #### Who can review? @eyurtsev @hwchase17 --------- Co-authored-by: Harrison Chase --- langchain/document_loaders/web_base.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/langchain/document_loaders/web_base.py b/langchain/document_loaders/web_base.py index cee218da..bd699eb8 100644 --- a/langchain/document_loaders/web_base.py +++ b/langchain/document_loaders/web_base.py @@ -51,7 +51,10 @@ class WebBaseLoader(BaseLoader): """kwargs for requests""" def __init__( - self, web_path: Union[str, List[str]], header_template: Optional[dict] = None + self, + web_path: Union[str, List[str]], + header_template: Optional[dict] = None, + verify: Optional[bool] = True, ): """Initialize with webpage path.""" @@ -71,6 +74,9 @@ class WebBaseLoader(BaseLoader): "bs4 package not found, please install it with " "`pip install bs4`" ) + # Choose to verify + self.verify = verify + headers = header_template or default_header_template if not headers.get("User-Agent"): try: @@ -98,7 +104,7 @@ class WebBaseLoader(BaseLoader): for i in range(retries): try: async with session.get( - url, headers=self.session.headers + url, headers=self.session.headers, verify=self.verify ) as response: return await response.text() except aiohttp.ClientConnectionError as e: @@ -173,7 +179,7 @@ class WebBaseLoader(BaseLoader): self._check_parser(parser) - html_doc = self.session.get(url, **self.requests_kwargs) + html_doc = self.session.get(url, verify=self.verify, **self.requests_kwargs) html_doc.encoding = html_doc.apparent_encoding return BeautifulSoup(html_doc.text, parser) From e30fdffd1e4765ba4fe46a59fcc46edf4f232328 Mon Sep 17 00:00:00 2001 From: Kenny Date: Sat, 17 Jun 2023 14:11:47 -0400 Subject: [PATCH 3/3] Add new openai 0613 model costs (#6110) Added costs for gpt-4-32k-0613, gpt-4-0613, gpt-3.5-turbo-16k, gpt-3.5-turbo-0613, and gpt-3.5-turbo-16k-0613 to openai_info callback based on this [OpenAI post](https://openai.com/blog/function-calling-and-other-api-updates) @agola11 --- langchain/callbacks/openai_info.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/langchain/callbacks/openai_info.py b/langchain/callbacks/openai_info.py index e8d2f3b6..765b8e34 100644 --- a/langchain/callbacks/openai_info.py +++ b/langchain/callbacks/openai_info.py @@ -13,8 +13,13 @@ MODEL_COST_PER_1K_TOKENS = { "gpt-4-32k-0314": 0.06, "gpt-4-32k-completion": 0.12, "gpt-4-32k-0314-completion": 0.12, + "gpt-4-0613": 0.06, + "gpt-4-32k-0613": 0.12, "gpt-3.5-turbo": 0.002, "gpt-3.5-turbo-0301": 0.002, + "gpt-3.5-turbo-16k": 0.004, + "gpt-3.5-turbo-0613": 0.002, + "gpt-3.5-turbo-16k-0613": 0.004, "text-ada-001": 0.0004, "ada": 0.0004, "text-babbage-001": 0.0005,