diff --git a/langchain/callbacks/openai_info.py b/langchain/callbacks/openai_info.py index e8d2f3b646..765b8e34f4 100644 --- a/langchain/callbacks/openai_info.py +++ b/langchain/callbacks/openai_info.py @@ -13,8 +13,13 @@ MODEL_COST_PER_1K_TOKENS = { "gpt-4-32k-0314": 0.06, "gpt-4-32k-completion": 0.12, "gpt-4-32k-0314-completion": 0.12, + "gpt-4-0613": 0.06, + "gpt-4-32k-0613": 0.12, "gpt-3.5-turbo": 0.002, "gpt-3.5-turbo-0301": 0.002, + "gpt-3.5-turbo-16k": 0.004, + "gpt-3.5-turbo-0613": 0.002, + "gpt-3.5-turbo-16k-0613": 0.004, "text-ada-001": 0.0004, "ada": 0.0004, "text-babbage-001": 0.0005, diff --git a/langchain/chat_models/anthropic.py b/langchain/chat_models/anthropic.py index a9f3d0e02d..023c9e4150 100644 --- a/langchain/chat_models/anthropic.py +++ b/langchain/chat_models/anthropic.py @@ -1,7 +1,5 @@ from typing import Any, Dict, List, Optional -from pydantic import Extra - from langchain.callbacks.manager import ( AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun, @@ -34,11 +32,6 @@ class ChatAnthropic(BaseChatModel, _AnthropicCommon): model = ChatAnthropic(model="", anthropic_api_key="my-api-key") """ - class Config: - """Configuration for this pydantic object.""" - - extra = Extra.forbid - @property def _llm_type(self) -> str: """Return type of chat model.""" diff --git a/langchain/chat_models/base.py b/langchain/chat_models/base.py index 8a06c1d800..5fa949ef0a 100644 --- a/langchain/chat_models/base.py +++ b/langchain/chat_models/base.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from functools import partial from typing import Any, Dict, List, Mapping, Optional, Sequence -from pydantic import Extra, Field, root_validator +from pydantic import Field, root_validator import langchain from langchain.base_language import BaseLanguageModel @@ -56,7 +56,6 @@ class BaseChatModel(BaseLanguageModel, ABC): class Config: """Configuration for this pydantic object.""" - extra = Extra.forbid arbitrary_types_allowed = True def _combine_llm_outputs(self, llm_outputs: List[Optional[dict]]) -> dict: diff --git a/langchain/chat_models/openai.py b/langchain/chat_models/openai.py index 6c18b9c869..674bf5e925 100644 --- a/langchain/chat_models/openai.py +++ b/langchain/chat_models/openai.py @@ -15,7 +15,7 @@ from typing import ( Union, ) -from pydantic import Extra, Field, root_validator +from pydantic import Field, root_validator from tenacity import ( before_sleep_log, retry, @@ -182,7 +182,6 @@ class ChatOpenAI(BaseChatModel): class Config: """Configuration for this pydantic object.""" - extra = Extra.ignore allow_population_by_field_name = True @root_validator(pre=True) diff --git a/langchain/document_loaders/web_base.py b/langchain/document_loaders/web_base.py index cee218dadb..bd699eb831 100644 --- a/langchain/document_loaders/web_base.py +++ b/langchain/document_loaders/web_base.py @@ -51,7 +51,10 @@ class WebBaseLoader(BaseLoader): """kwargs for requests""" def __init__( - self, web_path: Union[str, List[str]], header_template: Optional[dict] = None + self, + web_path: Union[str, List[str]], + header_template: Optional[dict] = None, + verify: Optional[bool] = True, ): """Initialize with webpage path.""" @@ -71,6 +74,9 @@ class WebBaseLoader(BaseLoader): "bs4 package not found, please install it with " "`pip install bs4`" ) + # Choose to verify + self.verify = verify + headers = header_template or default_header_template if not headers.get("User-Agent"): try: @@ -98,7 +104,7 @@ class WebBaseLoader(BaseLoader): for i in range(retries): try: async with session.get( - url, headers=self.session.headers + url, headers=self.session.headers, verify=self.verify ) as response: return await response.text() except aiohttp.ClientConnectionError as e: @@ -173,7 +179,7 @@ class WebBaseLoader(BaseLoader): self._check_parser(parser) - html_doc = self.session.get(url, **self.requests_kwargs) + html_doc = self.session.get(url, verify=self.verify, **self.requests_kwargs) html_doc.encoding = html_doc.apparent_encoding return BeautifulSoup(html_doc.text, parser) diff --git a/langchain/llms/anthropic.py b/langchain/llms/anthropic.py index 9554fd90ed..ec47b755b3 100644 --- a/langchain/llms/anthropic.py +++ b/langchain/llms/anthropic.py @@ -3,7 +3,7 @@ import re import warnings from typing import Any, Callable, Dict, Generator, List, Mapping, Optional, Tuple, Union -from pydantic import BaseModel, Extra, root_validator +from pydantic import BaseModel, root_validator from langchain.callbacks.manager import ( AsyncCallbackManagerForLLMRun, @@ -140,11 +140,6 @@ class Anthropic(LLM, _AnthropicCommon): ) return values - class Config: - """Configuration for this pydantic object.""" - - extra = Extra.forbid - @property def _llm_type(self) -> str: """Return type of llm.""" diff --git a/langchain/llms/base.py b/langchain/llms/base.py index 2647635db6..46e3a81fd8 100644 --- a/langchain/llms/base.py +++ b/langchain/llms/base.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union import yaml -from pydantic import Extra, Field, root_validator, validator +from pydantic import Field, root_validator, validator import langchain from langchain.base_language import BaseLanguageModel @@ -85,7 +85,6 @@ class BaseLLM(BaseLanguageModel, ABC): class Config: """Configuration for this pydantic object.""" - extra = Extra.forbid arbitrary_types_allowed = True @root_validator() diff --git a/langchain/llms/openai.py b/langchain/llms/openai.py index c0d999d42f..d6c9c5f26f 100644 --- a/langchain/llms/openai.py +++ b/langchain/llms/openai.py @@ -20,7 +20,7 @@ from typing import ( Union, ) -from pydantic import Extra, Field, root_validator +from pydantic import Field, root_validator from tenacity import ( before_sleep_log, retry, @@ -187,7 +187,6 @@ class BaseOpenAI(BaseLLM): class Config: """Configuration for this pydantic object.""" - extra = Extra.ignore allow_population_by_field_name = True @root_validator(pre=True) @@ -686,11 +685,6 @@ class OpenAIChat(BaseLLM): disallowed_special: Union[Literal["all"], Collection[str]] = "all" """Set of special tokens that are not allowed。""" - class Config: - """Configuration for this pydantic object.""" - - extra = Extra.ignore - @root_validator(pre=True) def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]: """Build extra kwargs from additional params that were passed in.""" diff --git a/langchain/load/serializable.py b/langchain/load/serializable.py index 56c97dfae3..88f2290e7d 100644 --- a/langchain/load/serializable.py +++ b/langchain/load/serializable.py @@ -1,7 +1,7 @@ from abc import ABC from typing import Any, Dict, List, Literal, TypedDict, Union, cast -from pydantic import BaseModel, Field +from pydantic import BaseModel, PrivateAttr class BaseSerialized(TypedDict): @@ -55,11 +55,14 @@ class Serializable(BaseModel, ABC): """ return {} - lc_kwargs: Dict[str, Any] = Field(default_factory=dict, exclude=True, repr=False) + class Config: + extra = "ignore" + + _lc_kwargs = PrivateAttr(default_factory=dict) def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) - self.lc_kwargs = kwargs + self._lc_kwargs = kwargs def to_json(self) -> Union[SerializedConstructor, SerializedNotImplemented]: if not self.lc_serializable: @@ -69,8 +72,8 @@ class Serializable(BaseModel, ABC): # Get latest values for kwargs if there is an attribute with same name lc_kwargs = { k: getattr(self, k, v) - for k, v in self.lc_kwargs.items() - if not self.__exclude_fields__.get(k, False) # type: ignore + for k, v in self._lc_kwargs.items() + if not (self.__exclude_fields__ or {}).get(k, False) # type: ignore } # Merge the lc_secrets and lc_attributes from every class in the MRO diff --git a/langchain/prompts/base.py b/langchain/prompts/base.py index 58e7339b8a..9e1f651588 100644 --- a/langchain/prompts/base.py +++ b/langchain/prompts/base.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Union import yaml -from pydantic import Extra, Field, root_validator +from pydantic import Field, root_validator from langchain.formatting import formatter from langchain.load.serializable import Serializable @@ -119,7 +119,6 @@ class BasePromptTemplate(Serializable, ABC): class Config: """Configuration for this pydantic object.""" - extra = Extra.forbid arbitrary_types_allowed = True @abstractmethod diff --git a/langchain/prompts/prompt.py b/langchain/prompts/prompt.py index c8ac2200b6..3532c610f5 100644 --- a/langchain/prompts/prompt.py +++ b/langchain/prompts/prompt.py @@ -5,7 +5,7 @@ from pathlib import Path from string import Formatter from typing import Any, Dict, List, Union -from pydantic import Extra, root_validator +from pydantic import root_validator from langchain.prompts.base import ( DEFAULT_FORMATTER_MAPPING, @@ -48,11 +48,6 @@ class PromptTemplate(StringPromptTemplate): """Return the prompt type key.""" return "prompt" - class Config: - """Configuration for this pydantic object.""" - - extra = Extra.forbid - def format(self, **kwargs: Any) -> str: """Format the prompt with the inputs. diff --git a/langchain/schema.py b/langchain/schema.py index ef675f6fc6..77fa6989d1 100644 --- a/langchain/schema.py +++ b/langchain/schema.py @@ -16,7 +16,7 @@ from typing import ( ) from uuid import UUID -from pydantic import BaseModel, Extra, Field, root_validator +from pydantic import BaseModel, Field, root_validator from langchain.load.serializable import Serializable @@ -229,7 +229,6 @@ class BaseMemory(Serializable, ABC): class Config: """Configuration for this pydantic object.""" - extra = Extra.forbid arbitrary_types_allowed = True @property