From 8c986221e4b5bd3de204ff9dd303454ca9a94786 Mon Sep 17 00:00:00 2001 From: Bagatur <22008038+baskaryan@users.noreply.github.com> Date: Thu, 17 Aug 2023 11:49:23 -0700 Subject: [PATCH] make openapi_schema_pydantic opt (#9408) --- .../chains/openai_functions/openapi.py | 8 ++- .../tools/openapi/utils/api_models.py | 46 +++++++++++++--- libs/langchain/langchain/utilities/openapi.py | 47 ++++++++++++----- libs/langchain/poetry.lock | 6 +-- libs/langchain/pyproject.toml | 3 +- .../tests/unit_tests/test_dependencies.py | 1 - .../tools/openapi/test_api_models.py | 52 ++++++++++++------- 7 files changed, 117 insertions(+), 46 deletions(-) diff --git a/libs/langchain/langchain/chains/openai_functions/openapi.py b/libs/langchain/langchain/chains/openai_functions/openapi.py index 511123a601..100e4be28a 100644 --- a/libs/langchain/langchain/chains/openai_functions/openapi.py +++ b/libs/langchain/langchain/chains/openai_functions/openapi.py @@ -1,10 +1,11 @@ +from __future__ import annotations + import json import re from collections import defaultdict -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union import requests -from openapi_schema_pydantic import Parameter from requests import Response from langchain import LLMChain @@ -20,6 +21,9 @@ from langchain.tools import APIOperation from langchain.utilities.openapi import OpenAPISpec from langchain.utils.input import get_colored_text +if TYPE_CHECKING: + from openapi_schema_pydantic import Parameter + def _get_description(o: Any, prefer_short: bool) -> Optional[str]: summary = getattr(o, "summary", None) diff --git a/libs/langchain/langchain/tools/openapi/utils/api_models.py b/libs/langchain/langchain/tools/openapi/utils/api_models.py index 5c0b8ba959..11606ca8e2 100644 --- a/libs/langchain/langchain/tools/openapi/utils/api_models.py +++ b/libs/langchain/langchain/tools/openapi/utils/api_models.py @@ -1,7 +1,19 @@ """Pydantic models for parsing an OpenAPI spec.""" +from __future__ import annotations + import logging from enum import Enum -from typing import Any, Dict, List, Optional, Sequence, Tuple, Type, Union +from typing import ( + TYPE_CHECKING, + Any, + Dict, + List, + Optional, + Sequence, + Tuple, + Type, + Union, +) from langchain.pydantic_v1 import _PYDANTIC_MAJOR_VERSION, BaseModel, Field from langchain.tools.openapi.utils.openapi_utils import HTTPVerb, OpenAPISpec @@ -84,13 +96,13 @@ class APIPropertyBase(BaseModel): if _PYDANTIC_MAJOR_VERSION == 1: - from openapi_schema_pydantic import ( - MediaType, - Parameter, - Reference, - RequestBody, - Schema, - ) + if TYPE_CHECKING: + from openapi_schema_pydantic import ( + MediaType, + Parameter, + RequestBody, + Schema, + ) class APIProperty(APIPropertyBase): """A model for a property in the query, path, header, or cookie params.""" @@ -118,6 +130,11 @@ if _PYDANTIC_MAJOR_VERSION == 1: def _get_schema_type_for_array( schema: Schema, ) -> Optional[Union[str, Tuple[str, ...]]]: + from openapi_schema_pydantic import ( + Reference, + Schema, + ) + items = schema.items if isinstance(items, Schema): schema_type = APIProperty._cast_schema_list_type(items) @@ -175,6 +192,11 @@ if _PYDANTIC_MAJOR_VERSION == 1: @staticmethod def _get_schema(parameter: Parameter, spec: OpenAPISpec) -> Optional[Schema]: + from openapi_schema_pydantic import ( + Reference, + Schema, + ) + schema = parameter.param_schema if isinstance(schema, Reference): schema = spec.get_referenced_schema(schema) @@ -231,6 +253,10 @@ if _PYDANTIC_MAJOR_VERSION == 1: def _process_object_schema( cls, schema: Schema, spec: OpenAPISpec, references_used: List[str] ) -> Tuple[Union[str, List[str], None], List["APIRequestBodyProperty"]]: + from openapi_schema_pydantic import ( + Reference, + ) + properties = [] required_props = schema.required or [] if schema.properties is None: @@ -265,6 +291,8 @@ if _PYDANTIC_MAJOR_VERSION == 1: spec: OpenAPISpec, references_used: List[str], ) -> str: + from openapi_schema_pydantic import Reference, Schema + items = schema.items if items is not None: if isinstance(items, Reference): @@ -352,6 +380,8 @@ if _PYDANTIC_MAJOR_VERSION == 1: spec: OpenAPISpec, ) -> List[APIRequestBodyProperty]: """Process the media type of the request body.""" + from openapi_schema_pydantic import Reference + references_used = [] schema = media_type_obj.media_type_schema if isinstance(schema, Reference): diff --git a/libs/langchain/langchain/utilities/openapi.py b/libs/langchain/langchain/utilities/openapi.py index 852879e11a..34224a2bbf 100644 --- a/libs/langchain/langchain/utilities/openapi.py +++ b/libs/langchain/langchain/utilities/openapi.py @@ -7,7 +7,7 @@ import logging import re from enum import Enum from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import TYPE_CHECKING, Dict, List, Optional, Union import requests import yaml @@ -39,17 +39,22 @@ class HTTPVerb(str, Enum): if _PYDANTIC_MAJOR_VERSION == 1: - from openapi_schema_pydantic import ( - Components, - OpenAPI, - Operation, - Parameter, - PathItem, - Paths, - Reference, - RequestBody, - Schema, - ) + if TYPE_CHECKING: + from openapi_schema_pydantic import ( + Components, + Operation, + Parameter, + PathItem, + Paths, + Reference, + RequestBody, + Schema, + ) + + try: + from openapi_schema_pydantic import OpenAPI + except ImportError: + OpenAPI = object class OpenAPISpec(OpenAPI): """OpenAPI Model that removes mis-formatted parts of the spec.""" @@ -109,6 +114,8 @@ if _PYDANTIC_MAJOR_VERSION == 1: def _get_root_referenced_parameter(self, ref: Reference) -> Parameter: """Get the root reference or err.""" + from openapi_schema_pydantic import Reference + parameter = self._get_referenced_parameter(ref) while isinstance(parameter, Reference): parameter = self._get_referenced_parameter(parameter) @@ -123,12 +130,16 @@ if _PYDANTIC_MAJOR_VERSION == 1: return schemas[ref_name] def get_schema(self, schema: Union[Reference, Schema]) -> Schema: + from openapi_schema_pydantic import Reference + if isinstance(schema, Reference): return self.get_referenced_schema(schema) return schema def _get_root_referenced_schema(self, ref: Reference) -> Schema: """Get the root reference or err.""" + from openapi_schema_pydantic import Reference + schema = self.get_referenced_schema(ref) while isinstance(schema, Reference): schema = self.get_referenced_schema(schema) @@ -148,6 +159,8 @@ if _PYDANTIC_MAJOR_VERSION == 1: self, ref: Reference ) -> Optional[RequestBody]: """Get the root request Body or err.""" + from openapi_schema_pydantic import Reference + request_body = self._get_referenced_request_body(ref) while isinstance(request_body, Reference): request_body = self._get_referenced_request_body(request_body) @@ -235,6 +248,8 @@ if _PYDANTIC_MAJOR_VERSION == 1: def get_methods_for_path(self, path: str) -> List[str]: """Return a list of valid methods for the specified path.""" + from openapi_schema_pydantic import Operation + path_item = self._get_path_strict(path) results = [] for method in HTTPVerb: @@ -244,6 +259,8 @@ if _PYDANTIC_MAJOR_VERSION == 1: return results def get_parameters_for_path(self, path: str) -> List[Parameter]: + from openapi_schema_pydantic import Reference + path_item = self._get_path_strict(path) parameters = [] if not path_item.parameters: @@ -256,6 +273,8 @@ if _PYDANTIC_MAJOR_VERSION == 1: def get_operation(self, path: str, method: str) -> Operation: """Get the operation object for a given path and HTTP method.""" + from openapi_schema_pydantic import Operation + path_item = self._get_path_strict(path) operation_obj = getattr(path_item, method, None) if not isinstance(operation_obj, Operation): @@ -264,6 +283,8 @@ if _PYDANTIC_MAJOR_VERSION == 1: def get_parameters_for_operation(self, operation: Operation) -> List[Parameter]: """Get the components for a given operation.""" + from openapi_schema_pydantic import Reference + parameters = [] if operation.parameters: for parameter in operation.parameters: @@ -276,6 +297,8 @@ if _PYDANTIC_MAJOR_VERSION == 1: self, operation: Operation ) -> Optional[RequestBody]: """Get the request body for a given operation.""" + from openapi_schema_pydantic import Reference + request_body = operation.requestBody if isinstance(request_body, Reference): request_body = self._get_root_referenced_request_body(request_body) diff --git a/libs/langchain/poetry.lock b/libs/langchain/poetry.lock index bb10e8ee27..9305a0838f 100644 --- a/libs/langchain/poetry.lock +++ b/libs/langchain/poetry.lock @@ -5637,7 +5637,7 @@ name = "openapi-schema-pydantic" version = "1.2.4" description = "OpenAPI (v3) specification schema as pydantic class" category = "main" -optional = false +optional = true python-versions = ">=3.6.1" files = [ {file = "openapi-schema-pydantic-1.2.4.tar.gz", hash = "sha256:3e22cf58b74a69f752cc7e5f1537f6e44164282db2700cbbcd3bb99ddd065196"}, @@ -10477,7 +10477,7 @@ clarifai = ["clarifai"] cohere = ["cohere"] docarray = ["docarray"] embeddings = ["sentence-transformers"] -extended-testing = ["amazon-textract-caller", "atlassian-python-api", "beautifulsoup4", "bibtexparser", "cassio", "chardet", "esprima", "faiss-cpu", "feedparser", "geopandas", "gitpython", "gql", "html2text", "jinja2", "jq", "lxml", "mwparserfromhell", "mwxml", "newspaper3k", "openai", "openai", "pandas", "pdfminer-six", "pgvector", "psychicapi", "py-trello", "pymupdf", "pypdf", "pypdfium2", "pyspark", "rank-bm25", "rapidfuzz", "requests-toolbelt", "scikit-learn", "streamlit", "sympy", "telethon", "tqdm", "xata", "xmltodict"] +extended-testing = ["amazon-textract-caller", "atlassian-python-api", "beautifulsoup4", "bibtexparser", "cassio", "chardet", "esprima", "faiss-cpu", "feedparser", "geopandas", "gitpython", "gql", "html2text", "jinja2", "jq", "lxml", "mwparserfromhell", "mwxml", "newspaper3k", "openai", "openai", "openapi-schema-pydantic", "pandas", "pdfminer-six", "pgvector", "psychicapi", "py-trello", "pymupdf", "pypdf", "pypdfium2", "pyspark", "rank-bm25", "rapidfuzz", "requests-toolbelt", "scikit-learn", "streamlit", "sympy", "telethon", "tqdm", "xata", "xmltodict"] javascript = ["esprima"] llms = ["clarifai", "cohere", "huggingface_hub", "manifest-ml", "nlpcloud", "openai", "openlm", "torch", "transformers"] openai = ["openai", "tiktoken"] @@ -10487,4 +10487,4 @@ text-helpers = ["chardet"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "594d1f6ea7a3e00f0ab6c74cab8b75245d112a84635af440df7ab1242d464140" +content-hash = "a5e3458dd0cabcefd83caec6eb33b6fb593c2c347ca1d33c1f182341e852a9c8" diff --git a/libs/langchain/pyproject.toml b/libs/langchain/pyproject.toml index 7cd99962b7..c928ad800d 100644 --- a/libs/langchain/pyproject.toml +++ b/libs/langchain/pyproject.toml @@ -19,7 +19,7 @@ PyYAML = ">=5.3" numpy = "^1" azure-core = {version = "^1.26.4", optional=true} tqdm = {version = ">=4.48.0", optional = true} -openapi-schema-pydantic = "^1.2" +openapi-schema-pydantic = {version = "^1.2", optional = true} faiss-cpu = {version = "^1", optional = true} wikipedia = {version = "^1", optional = true} elasticsearch = {version = "^8", optional = true} @@ -336,6 +336,7 @@ extended_testing = [ "xata", "xmltodict", "faiss-cpu", + "openapi-schema-pydantic", ] [tool.ruff] diff --git a/libs/langchain/tests/unit_tests/test_dependencies.py b/libs/langchain/tests/unit_tests/test_dependencies.py index 85c6efdfd1..e446485cac 100644 --- a/libs/langchain/tests/unit_tests/test_dependencies.py +++ b/libs/langchain/tests/unit_tests/test_dependencies.py @@ -41,7 +41,6 @@ def test_required_dependencies(poetry_conf: Mapping[str, Any]) -> None: "langsmith", "numexpr", "numpy", - "openapi-schema-pydantic", "pydantic", "python", "requests", diff --git a/libs/langchain/tests/unit_tests/tools/openapi/test_api_models.py b/libs/langchain/tests/unit_tests/tools/openapi/test_api_models.py index 2e7ce57abc..a68bc2787d 100644 --- a/libs/langchain/tests/unit_tests/tools/openapi/test_api_models.py +++ b/libs/langchain/tests/unit_tests/tools/openapi/test_api_models.py @@ -18,14 +18,6 @@ if _PYDANTIC_MAJOR_VERSION != 1: import pytest import yaml -from openapi_schema_pydantic import ( - Components, - Info, - MediaType, - Reference, - RequestBody, - Schema, -) from langchain.tools.openapi.utils.api_models import ( APIOperation, @@ -86,30 +78,38 @@ def http_paths_and_methods() -> List[Tuple[str, OpenAPISpec, str, str]]: return http_paths_and_methods -@pytest.mark.parametrize( - "spec_name, spec, path, method", - http_paths_and_methods(), -) -def test_parse_api_operations( - spec_name: str, spec: OpenAPISpec, path: str, method: str -) -> None: +@pytest.mark.requires("openapi_schema_pydantic") +def test_parse_api_operations() -> None: """Test the APIOperation class.""" - try: - APIOperation.from_openapi_spec(spec, path, method) - except Exception as e: - raise AssertionError(f"Error processing {spec_name}: {e} ") from e + for spec_name, spec, path, method in http_paths_and_methods(): + try: + APIOperation.from_openapi_spec(spec, path, method) + except Exception as e: + raise AssertionError(f"Error processing {spec_name}: {e} ") from e +@pytest.mark.requires("openapi_schema_pydantic") @pytest.fixture def raw_spec() -> OpenAPISpec: """Return a raw OpenAPI spec.""" + from openapi_schema_pydantic import Info + return OpenAPISpec( info=Info(title="Test API", version="1.0.0"), ) +@pytest.mark.requires("openapi_schema_pydantic") def test_api_request_body_from_request_body_with_ref(raw_spec: OpenAPISpec) -> None: """Test instantiating APIRequestBody from RequestBody with a reference.""" + from openapi_schema_pydantic import ( + Components, + MediaType, + Reference, + RequestBody, + Schema, + ) + raw_spec.components = Components( schemas={ "Foo": Schema( @@ -140,8 +140,15 @@ def test_api_request_body_from_request_body_with_ref(raw_spec: OpenAPISpec) -> N assert api_request_body.media_type == "application/json" +@pytest.mark.requires("openapi_schema_pydantic") def test_api_request_body_from_request_body_with_schema(raw_spec: OpenAPISpec) -> None: """Test instantiating APIRequestBody from RequestBody with a schema.""" + from openapi_schema_pydantic import ( + MediaType, + RequestBody, + Schema, + ) + request_body = RequestBody( content={ "application/json": MediaType( @@ -164,7 +171,14 @@ def test_api_request_body_from_request_body_with_schema(raw_spec: OpenAPISpec) - assert api_request_body.media_type == "application/json" +@pytest.mark.requires("openapi_schema_pydantic") def test_api_request_body_property_from_schema(raw_spec: OpenAPISpec) -> None: + from openapi_schema_pydantic import ( + Components, + Reference, + Schema, + ) + raw_spec.components = Components( schemas={ "Bar": Schema(