Merge branch 'master' of github.com:hwchase17/langchain

langchain/callbacks/openai_info.py

@@ -13,8 +13,13 @@ MODEL_COST_PER_1K_TOKENS = {
     "gpt-4-32k-0314": 0.06,
     "gpt-4-32k-completion": 0.12,
     "gpt-4-32k-0314-completion": 0.12,
+    "gpt-4-0613": 0.06,
+    "gpt-4-32k-0613": 0.12,
     "gpt-3.5-turbo": 0.002,
     "gpt-3.5-turbo-0301": 0.002,
+    "gpt-3.5-turbo-16k": 0.004,
+    "gpt-3.5-turbo-0613": 0.002,
+    "gpt-3.5-turbo-16k-0613": 0.004,
     "text-ada-001": 0.0004,
     "ada": 0.0004,
     "text-babbage-001": 0.0005,

langchain/chat_models/anthropic.py

@@ -1,7 +1,5 @@
 from typing import Any, Dict, List, Optional
 
-from pydantic import Extra
-
 from langchain.callbacks.manager import (
     AsyncCallbackManagerForLLMRun,
     CallbackManagerForLLMRun,
@@ -34,11 +32,6 @@ class ChatAnthropic(BaseChatModel, _AnthropicCommon):
             model = ChatAnthropic(model="<model_name>", anthropic_api_key="my-api-key")
     """
 
-    class Config:
-        """Configuration for this pydantic object."""
-
-        extra = Extra.forbid
-
     @property
     def _llm_type(self) -> str:
         """Return type of chat model."""

langchain/chat_models/base.py

@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
 from functools import partial
 from typing import Any, Dict, List, Mapping, Optional, Sequence
 
-from pydantic import Extra, Field, root_validator
+from pydantic import Field, root_validator
 
 import langchain
 from langchain.base_language import BaseLanguageModel
@@ -56,7 +56,6 @@ class BaseChatModel(BaseLanguageModel, ABC):
     class Config:
         """Configuration for this pydantic object."""
 
-        extra = Extra.forbid
         arbitrary_types_allowed = True
 
     def _combine_llm_outputs(self, llm_outputs: List[Optional[dict]]) -> dict:

langchain/chat_models/openai.py

@@ -15,7 +15,7 @@ from typing import (
     Union,
 )
 
-from pydantic import Extra, Field, root_validator
+from pydantic import Field, root_validator
 from tenacity import (
     before_sleep_log,
     retry,
@@ -182,7 +182,6 @@ class ChatOpenAI(BaseChatModel):
     class Config:
         """Configuration for this pydantic object."""
 
-        extra = Extra.ignore
         allow_population_by_field_name = True
 
     @root_validator(pre=True)

langchain/document_loaders/web_base.py

@@ -51,7 +51,10 @@ class WebBaseLoader(BaseLoader):
     """kwargs for requests"""
 
     def __init__(
-        self, web_path: Union[str, List[str]], header_template: Optional[dict] = None
+        self,
+        web_path: Union[str, List[str]],
+        header_template: Optional[dict] = None,
+        verify: Optional[bool] = True,
     ):
         """Initialize with webpage path."""
 
@@ -71,6 +74,9 @@ class WebBaseLoader(BaseLoader):
                 "bs4 package not found, please install it with " "`pip install bs4`"
             )
 
+        # Choose to verify
+        self.verify = verify
+
         headers = header_template or default_header_template
         if not headers.get("User-Agent"):
             try:
@@ -98,7 +104,7 @@ class WebBaseLoader(BaseLoader):
             for i in range(retries):
                 try:
                     async with session.get(
-                        url, headers=self.session.headers
+                        url, headers=self.session.headers, verify=self.verify
                     ) as response:
                         return await response.text()
                 except aiohttp.ClientConnectionError as e:
@@ -173,7 +179,7 @@ class WebBaseLoader(BaseLoader):
 
         self._check_parser(parser)
 
-        html_doc = self.session.get(url, **self.requests_kwargs)
+        html_doc = self.session.get(url, verify=self.verify, **self.requests_kwargs)
         html_doc.encoding = html_doc.apparent_encoding
         return BeautifulSoup(html_doc.text, parser)
 

langchain/llms/anthropic.py

@@ -3,7 +3,7 @@ import re
 import warnings
 from typing import Any, Callable, Dict, Generator, List, Mapping, Optional, Tuple, Union
 
-from pydantic import BaseModel, Extra, root_validator
+from pydantic import BaseModel, root_validator
 
 from langchain.callbacks.manager import (
     AsyncCallbackManagerForLLMRun,
@@ -140,11 +140,6 @@ class Anthropic(LLM, _AnthropicCommon):
         )
         return values
 
-    class Config:
-        """Configuration for this pydantic object."""
-
-        extra = Extra.forbid
-
     @property
     def _llm_type(self) -> str:
         """Return type of llm."""

langchain/llms/base.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union
 
 import yaml
-from pydantic import Extra, Field, root_validator, validator
+from pydantic import Field, root_validator, validator
 
 import langchain
 from langchain.base_language import BaseLanguageModel
@@ -85,7 +85,6 @@ class BaseLLM(BaseLanguageModel, ABC):
     class Config:
         """Configuration for this pydantic object."""
 
-        extra = Extra.forbid
         arbitrary_types_allowed = True
 
     @root_validator()

langchain/llms/openai.py

@@ -20,7 +20,7 @@ from typing import (
     Union,
 )
 
-from pydantic import Extra, Field, root_validator
+from pydantic import Field, root_validator
 from tenacity import (
     before_sleep_log,
     retry,
@@ -187,7 +187,6 @@ class BaseOpenAI(BaseLLM):
     class Config:
         """Configuration for this pydantic object."""
 
-        extra = Extra.ignore
         allow_population_by_field_name = True
 
     @root_validator(pre=True)
@@ -686,11 +685,6 @@ class OpenAIChat(BaseLLM):
     disallowed_special: Union[Literal["all"], Collection[str]] = "all"
     """Set of special tokens that are not allowed。"""
 
-    class Config:
-        """Configuration for this pydantic object."""
-
-        extra = Extra.ignore
-
     @root_validator(pre=True)
     def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]:
         """Build extra kwargs from additional params that were passed in."""

langchain/load/serializable.py

@@ -1,7 +1,7 @@
 from abc import ABC
 from typing import Any, Dict, List, Literal, TypedDict, Union, cast
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, PrivateAttr
 
 
 class BaseSerialized(TypedDict):
@@ -55,11 +55,14 @@ class Serializable(BaseModel, ABC):
         """
         return {}
 
-    lc_kwargs: Dict[str, Any] = Field(default_factory=dict, exclude=True, repr=False)
+    class Config:
+        extra = "ignore"
+
+    _lc_kwargs = PrivateAttr(default_factory=dict)
 
     def __init__(self, **kwargs: Any) -> None:
         super().__init__(**kwargs)
-        self.lc_kwargs = kwargs
+        self._lc_kwargs = kwargs
 
     def to_json(self) -> Union[SerializedConstructor, SerializedNotImplemented]:
         if not self.lc_serializable:
@@ -69,8 +72,8 @@ class Serializable(BaseModel, ABC):
         # Get latest values for kwargs if there is an attribute with same name
         lc_kwargs = {
             k: getattr(self, k, v)
-            for k, v in self.lc_kwargs.items()
+            for k, v in self._lc_kwargs.items()
-            if not self.__exclude_fields__.get(k, False)  # type: ignore
+            if not (self.__exclude_fields__ or {}).get(k, False)  # type: ignore
         }
 
         # Merge the lc_secrets and lc_attributes from every class in the MRO

langchain/prompts/base.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Union
 
 import yaml
-from pydantic import Extra, Field, root_validator
+from pydantic import Field, root_validator
 
 from langchain.formatting import formatter
 from langchain.load.serializable import Serializable
@@ -119,7 +119,6 @@ class BasePromptTemplate(Serializable, ABC):
     class Config:
         """Configuration for this pydantic object."""
 
-        extra = Extra.forbid
         arbitrary_types_allowed = True
 
     @abstractmethod

langchain/prompts/prompt.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from string import Formatter
 from typing import Any, Dict, List, Union
 
-from pydantic import Extra, root_validator
+from pydantic import root_validator
 
 from langchain.prompts.base import (
     DEFAULT_FORMATTER_MAPPING,
@@ -48,11 +48,6 @@ class PromptTemplate(StringPromptTemplate):
         """Return the prompt type key."""
         return "prompt"
 
-    class Config:
-        """Configuration for this pydantic object."""
-
-        extra = Extra.forbid
-
     def format(self, **kwargs: Any) -> str:
         """Format the prompt with the inputs.
 

langchain/schema.py

@@ -16,7 +16,7 @@ from typing import (
 )
 from uuid import UUID
 
-from pydantic import BaseModel, Extra, Field, root_validator
+from pydantic import BaseModel, Field, root_validator
 
 from langchain.load.serializable import Serializable
 
@@ -229,7 +229,6 @@ class BaseMemory(Serializable, ABC):
     class Config:
         """Configuration for this pydantic object."""
 
-        extra = Extra.forbid
         arbitrary_types_allowed = True
 
     @property