langchain[minor]: Code to handle dynamic imports (#20893)

Proposing to centralize code for handling dynamic imports. This allows treating langchain-community as an optional dependency.

---

The proposal is to scan the code base and to replace all existing imports with dynamic imports using this functionality.

libs/langchain/langchain/_api/__init__.py

@@ -16,6 +16,7 @@ from .deprecation import (
     surface_langchain_deprecation_warnings,
     warn_deprecated,
 )
+from .module_import import create_importer
 
 __all__ = [
     "deprecated",
@@ -23,4 +24,5 @@ __all__ = [
     "suppress_langchain_deprecation_warning",
     "surface_langchain_deprecation_warnings",
     "warn_deprecated",
+    "create_importer",
 ]

libs/langchain/langchain/_api/module_import.py

@@ -0,0 +1,128 @@
+import importlib
+import os
+import pathlib
+import warnings
+from typing import Any, Callable, Dict, Optional
+
+from langchain_core._api import LangChainDeprecationWarning
+
+from langchain.utils.interactive_env import is_interactive_env
+
+ALLOWED_TOP_LEVEL_PKGS = {
+    "langchain_community",
+    "langchain_core",
+    "langchain",
+}
+
+
+HERE = pathlib.Path(__file__).parent
+ROOT = HERE.parent.parent
+
+
+def _get_current_module(path: str) -> str:
+    """Convert a path to a module name."""
+    path_as_pathlib = pathlib.Path(os.path.abspath(path))
+    relative_path = path_as_pathlib.relative_to(ROOT).with_suffix("")
+    posix_path = relative_path.as_posix()
+    norm_path = os.path.normpath(str(posix_path))
+    fully_qualified_module = norm_path.replace("/", ".")
+    # Strip off __init__ if present
+    if fully_qualified_module.endswith(".__init__"):
+        return fully_qualified_module[:-9]
+    return fully_qualified_module
+
+
+def create_importer(
+    here: str,
+    *,
+    module_lookup: Optional[Dict[str, str]] = None,
+    fallback_module: Optional[str] = None,
+) -> Callable[[str], Any]:
+    """Create a function that helps retrieve objects from their new locations.
+
+    The goal of this function is to help users transition from deprecated
+    imports to new imports.
+
+    This function will raise warnings when the old imports are used and
+    suggest the new imports.
+
+    This function should ideally only be used with deprecated imports not with
+    existing imports that are valid, as in addition to raising deprecation warnings
+    the dynamic imports can create other issues for developers (e.g.,
+    loss of type information, IDE support for going to definition etc).
+
+    Args:
+        here: path of the current file. Use __file__
+        module_lookup: maps name of object to the module where it is defined.
+            e.g.,
+            {
+                "MyDocumentLoader": (
+                    "langchain_community.document_loaders.my_document_loader"
+                )
+            }
+        fallback_module: module to import from if the object is not found in
+            module_lookup or if module_lookup is not provided.
+
+    Returns:
+        A function that imports objects from the specified modules.
+    """
+    current_module = _get_current_module(here)
+
+    def import_by_name(name: str) -> Any:
+        """Import stores from langchain_community."""
+        # If not in interactive env, raise warning.
+        if module_lookup and name in module_lookup:
+            new_module = module_lookup[name]
+            if new_module.split(".")[0] not in ALLOWED_TOP_LEVEL_PKGS:
+                raise AssertionError(
+                    f"Importing from {new_module} is not allowed. "
+                    f"Allowed top-level packages are: {ALLOWED_TOP_LEVEL_PKGS}"
+                )
+
+            try:
+                module = importlib.import_module(new_module)
+            except ModuleNotFoundError as e:
+                if new_module.startswith("langchain_community"):
+                    raise ModuleNotFoundError(
+                        f"Module {new_module} not found. "
+                        "Please install langchain-community to access this module. "
+                        "You can install it using `pip install -U langchain-community`"
+                    ) from e
+                raise
+
+            try:
+                result = getattr(module, name)
+                if not is_interactive_env():
+                    warnings.warn(
+                        f"Importing {name} from {current_module} is deprecated. "
+                        "Please replace the import with the following:\n"
+                        f"from {new_module} import {name}",
+                        category=LangChainDeprecationWarning,
+                    )
+                return result
+            except Exception as e:
+                raise AttributeError(
+                    f"module {new_module} has no attribute {name}"
+                ) from e
+
+        if fallback_module:
+            try:
+                module = importlib.import_module(fallback_module)
+                result = getattr(module, name)
+                if not is_interactive_env():
+                    warnings.warn(
+                        f"Importing {name} from {current_module} is deprecated. "
+                        "Please replace the import with the following:\n"
+                        f"from {fallback_module} import {name}",
+                        category=LangChainDeprecationWarning,
+                    )
+                return result
+
+            except Exception as e:
+                raise AttributeError(
+                    f"module {fallback_module} has no attribute {name}"
+                ) from e
+
+        raise AttributeError(f"module {current_module} has no attribute {name}")
+
+    return import_by_name

libs/langchain/langchain/chains/__init__.py

@@ -17,9 +17,10 @@ The Chain interface makes it easy to create apps that are:
     Chain --> <name>Chain  # Examples: LLMChain, MapReduceChain, RouterChain
 """
 
-import importlib
 from typing import Any
 
+from langchain._api import create_importer
+
 _module_lookup = {
     "APIChain": "langchain.chains.api.base",
     "OpenAPIEndpointChain": "langchain.chains.api.openapi.chain",
@@ -84,12 +85,11 @@ _module_lookup = {
     "TransformChain": "langchain.chains.transform",
 }
 
+importer = create_importer(__file__, module_lookup=_module_lookup)
+
 
 def __getattr__(name: str) -> Any:
-    if name in _module_lookup:
-        module = importlib.import_module(_module_lookup[name])
-        return getattr(module, name)
-    raise AttributeError(f"module {__name__} has no attribute {name}")
+    return importer(name)
 
 
 __all__ = list(_module_lookup.keys())

libs/langchain/langchain/chat_loaders/facebook_messenger.py

@@ -1,6 +1,32 @@
-from langchain_community.chat_loaders.facebook_messenger import (
-    FolderFacebookMessengerChatLoader,
-    SingleFileFacebookMessengerChatLoader,
+from typing import TYPE_CHECKING, Any
+
+from langchain._api.module_import import create_importer
+
+if TYPE_CHECKING:
+    from langchain_community.chat_loaders.facebook_messenger import (
+        FolderFacebookMessengerChatLoader,
+        SingleFileFacebookMessengerChatLoader,
+    )
+
+module_lookup = {
+    "SingleFileFacebookMessengerChatLoader": (
+        "langchain_community.chat_loaders.facebook_messenger"
+    ),
+    "FolderFacebookMessengerChatLoader": (
+        "langchain_community.chat_loaders.facebook_messenger"
+    ),
+}
+
+# Temporary code for backwards compatibility for deprecated imports.
+# This will eventually be removed.
+import_lookup = create_importer(
+    __file__,
+    module_lookup=module_lookup,
 )
 
+
+def __getattr__(name: str) -> Any:
+    return import_lookup(name)
+
+
 __all__ = ["SingleFileFacebookMessengerChatLoader", "FolderFacebookMessengerChatLoader"]

libs/langchain/langchain/graphs/__init__.py

@@ -1,27 +1,13 @@
 """**Graphs** provide a natural language interface to graph databases."""
-import warnings
 from typing import Any
 
-from langchain_core._api import LangChainDeprecationWarning
+from langchain._api import create_importer
 
-from langchain.utils.interactive_env import is_interactive_env
+importer = create_importer(__file__, fallback_module="langchain_community.graphs")
 
 
 def __getattr__(name: str) -> Any:
-    from langchain_community import graphs
-
-    # If not in interactive env, raise warning.
-    if not is_interactive_env():
-        warnings.warn(
-            "Importing graphs from langchain is deprecated. Importing from "
-            "langchain will no longer be supported as of langchain==0.2.0. "
-            "Please import from langchain-community instead:\n\n"
-            f"`from langchain_community.graphs import {name}`.\n\n"
-            "To install langchain-community run `pip install -U langchain-community`.",
-            category=LangChainDeprecationWarning,
-        )
-
-    return getattr(graphs, name)
+    return importer(name)
 
 
 __all__ = [

libs/langchain/langchain/retrievers/__init__.py

@@ -17,11 +17,9 @@ the backbone of a retriever, but there are other types of retrievers as well.
     Document, Serializable, Callbacks,
     CallbackManagerForRetrieverRun, AsyncCallbackManagerForRetrieverRun
 """
-import warnings
 from typing import Any
 
-from langchain_core._api import LangChainDeprecationWarning
-
+from langchain._api.module_import import create_importer
 from langchain.retrievers.contextual_compression import ContextualCompressionRetriever
 from langchain.retrievers.ensemble import EnsembleRetriever
 from langchain.retrievers.merger_retriever import MergerRetriever
@@ -35,24 +33,15 @@ from langchain.retrievers.time_weighted_retriever import (
     TimeWeightedVectorStoreRetriever,
 )
 from langchain.retrievers.web_research import WebResearchRetriever
-from langchain.utils.interactive_env import is_interactive_env
-
 
-def __getattr__(name: str) -> Any:
-    from langchain_community import retrievers
+import_lookup = create_importer(
+    __file__, fallback_module="langchain_community.retrievers"
+)
 
-    # If not in interactive env, raise warning.
-    if not is_interactive_env():
-        warnings.warn(
-            "Importing this retriever from langchain is deprecated. Importing it from "
-            "langchain will no longer be supported as of langchain==0.2.0. "
-            "Please import from langchain-community instead:\n\n"
-            f"`from langchain_community.retrievers import {name}`.\n\n"
-            "To install langchain-community run `pip install -U langchain-community`.",
-            category=LangChainDeprecationWarning,
-        )
 
-    return getattr(retrievers, name)
+def __getattr__(name: str) -> Any:
+    """Import retrievers from langchain_community."""
+    return import_lookup(name)
 
 
 __all__ = [

libs/langchain/tests/unit_tests/_api/test_importing.py

@@ -0,0 +1,13 @@
+from langchain._api.module_import import create_importer
+
+
+def test_importing_module() -> None:
+    """Test importing all modules in langchain."""
+    module_lookup = {
+        "Document": "langchain_core.documents",
+    }
+    lookup = create_importer(__file__, module_lookup=module_lookup)
+    imported_doc = lookup("Document")
+    from langchain_core.documents import Document
+
+    assert imported_doc is Document